@wix/pathgrade 0.37.0 → 1.0.1

package/dist/sdk/index.d.ts

@@ -17,6 +17,7 @@ export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from '.
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
+export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
 export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
 export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
@@ -25,6 +26,7 @@ export type { ConversationWindow, ConversationWindowOptions } from './conversati
 export type { JudgePipelineOptions } from './judge-pipeline.js';
 export type { RunScorerOptions } from './run-scorer.js';
 export type { OnScorerErrorMode } from './evaluate.js';
+export type { EvalResultEvent, EvalResultObserver, ResultObserverHandle, ResultObserverOptions, ResultObserverOwner, } from './result-capture.js';
 export type { RunSnapshot } from './snapshots.js';
 export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, } from './mcp-evidence.js';
 export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';

package/dist/sdk/index.js

@@ -17,6 +17,7 @@ export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
+export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
 export { createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
 export { createMockLLM } from '../utils/llm-mocks.js';

package/dist/sdk/lifecycle.d.ts

@@ -0,0 +1,40 @@
+import type { Agent, PathgradeTestMeta, RecordedEvalResult } from './types.js';
+export type LifecycleAgentOwner = {
+    type: 'runner-case';
+    caseId: string;
+} | {
+    type: 'runner-suite-shared';
+    caseId: string;
+} | {
+    type: 'manual';
+};
+type ResultAttribution = {
+    type: 'runner-case';
+    caseId: string;
+} | {
+    type: 'unattributed-runner';
+} | {
+    type: 'legacy-unowned';
+};
+interface FlushCaseInput {
+    caseId: string;
+}
+declare function getAgentOwner(agent: Agent): LifecycleAgentOwner | undefined;
+declare function registerAgent(agent: Agent, owner?: LifecycleAgentOwner | null): void;
+declare function untrackAgent(agent: Agent): void;
+declare function releaseAgent(agent: Agent): void;
+declare function recordResult(result: RecordedEvalResult, agent: Agent, attribution?: ResultAttribution): void;
+declare function flushCase(input: FlushCaseInput): Promise<PathgradeTestMeta[]>;
+declare function cleanupAll(): Promise<void>;
+declare function reset(): void;
+export declare const lifecycleCore: {
+    registerAgent: typeof registerAgent;
+    untrackAgent: typeof untrackAgent;
+    releaseAgent: typeof releaseAgent;
+    getAgentOwner: typeof getAgentOwner;
+    recordResult: typeof recordResult;
+    flushCase: typeof flushCase;
+    cleanupAll: typeof cleanupAll;
+    reset: typeof reset;
+};
+export {};

package/dist/sdk/lifecycle.js

@@ -0,0 +1,170 @@
+import { getCurrentCaseContext } from './case-context.js';
+import { buildDiagnosticsReport } from './diagnostics.js';
+const pendingAgents = new Set();
+const agentOwners = new WeakMap();
+const agentResults = new WeakMap();
+function currentAgentOwner() {
+    const current = getCurrentCaseContext();
+    if (current.status !== 'active')
+        return { type: 'manual' };
+    return current.context.scope === 'runner-case'
+        ? { type: 'runner-case', caseId: current.context.caseId }
+        : { type: 'runner-suite-shared', caseId: current.context.caseId };
+}
+function belongsToCase(owner, caseId) {
+    return !owner || (owner.type === 'runner-case' && owner.caseId === caseId);
+}
+function canFlushResultsToCase(owner, caseId) {
+    if (!owner)
+        return true;
+    if (owner.type === 'runner-case')
+        return owner.caseId === caseId;
+    if (owner.type === 'runner-suite-shared')
+        return true;
+    return false;
+}
+function currentResultAttribution(owner) {
+    const current = getCurrentCaseContext();
+    if (current.status === 'active' && current.context.scope === 'runner-case') {
+        return { type: 'runner-case', caseId: current.context.caseId };
+    }
+    if (owner?.type === 'runner-case')
+        return { type: 'runner-case', caseId: owner.caseId };
+    if (!owner)
+        return { type: 'legacy-unowned' };
+    return { type: 'unattributed-runner' };
+}
+function getAgentOwner(agent) {
+    return agentOwners.get(agent);
+}
+function resultMatchesCase(entry, caseId) {
+    if (entry.attribution.type === 'runner-case')
+        return entry.attribution.caseId === caseId;
+    return entry.attribution.type === 'legacy-unowned';
+}
+function registerAgent(agent, owner) {
+    pendingAgents.add(agent);
+    const resolvedOwner = owner === undefined ? currentAgentOwner() : owner;
+    if (resolvedOwner) {
+        agentOwners.set(agent, resolvedOwner);
+    }
+}
+function untrackAgent(agent) {
+    pendingAgents.delete(agent);
+}
+function releaseAgent(agent) {
+    const owner = agentOwners.get(agent);
+    if (owner?.type !== 'manual')
+        return;
+    pendingAgents.delete(agent);
+    agentResults.delete(agent);
+    agentOwners.delete(agent);
+}
+function recordResult(result, agent, attribution) {
+    if (!agentResults.has(agent)) {
+        agentResults.set(agent, []);
+    }
+    const owner = agentOwners.get(agent);
+    const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
+    const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
+    agentResults.get(agent).push({
+        attribution: attribution ?? currentResultAttribution(owner),
+        meta: {
+            score: result.score,
+            scorers: result.scorers,
+            trial: result.trial,
+            diagnostics: buildDiagnosticsReport({
+                completionReason,
+                completionDetail: conversationEnd?.completion_detail,
+                turnDetails: conversationEnd?.turn_details,
+                reactionsFired: conversationEnd?.reactions_fired,
+                score: result.score,
+                scorers: result.scorers,
+                log: agent.log,
+            }),
+        },
+    });
+}
+async function flushCase(input) {
+    const results = [];
+    const toDispose = [];
+    for (const agent of pendingAgents) {
+        const owner = agentOwners.get(agent);
+        const belongsToThisCase = belongsToCase(owner, input.caseId);
+        const meta = agentResults.get(agent);
+        const matchingMeta = meta?.filter((entry) => resultMatchesCase(entry, input.caseId));
+        if (matchingMeta && matchingMeta.length > 0 && canFlushResultsToCase(owner, input.caseId)) {
+            results.push(...matchingMeta.map((entry) => entry.meta));
+            const remainingMeta = meta?.filter((entry) => !resultMatchesCase(entry, input.caseId)) ?? [];
+            if (remainingMeta.length > 0) {
+                agentResults.set(agent, remainingMeta);
+            }
+            else {
+                agentResults.delete(agent);
+            }
+        }
+        if (belongsToThisCase) {
+            const hasResultsForCase = matchingMeta && matchingMeta.length > 0;
+            if (!hasResultsForCase) {
+                const synthTrial = synthesizeTrialFromAgent(agent);
+                if (synthTrial) {
+                    results.push(synthTrial);
+                }
+            }
+            pendingAgents.delete(agent);
+            agentOwners.delete(agent);
+            toDispose.push(agent);
+        }
+    }
+    await Promise.all(toDispose.map((agent) => agent.dispose().catch(() => { })));
+    return results;
+}
+function synthesizeTrialFromAgent(agent) {
+    if (agent.log.length === 0)
+        return null;
+    const nCommands = agent.log.filter((entry) => entry.type === 'command').length;
+    const tokenUsage = agent.llm.tokenUsage;
+    const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
+    const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
+    return {
+        score: 1,
+        scorers: [],
+        trial: {
+            trial_id: 0,
+            reward: 1,
+            scorer_results: [],
+            duration_ms: 0,
+            n_commands: nCommands,
+            input_tokens: tokenUsage?.inputTokens ?? 0,
+            output_tokens: tokenUsage?.outputTokens ?? 0,
+            session_log: [...agent.log],
+        },
+        diagnostics: buildDiagnosticsReport({
+            completionReason,
+            completionDetail: conversationEnd?.completion_detail,
+            turnDetails: conversationEnd?.turn_details,
+            reactionsFired: conversationEnd?.reactions_fired,
+            score: 1,
+            scorers: [],
+            log: agent.log,
+        }),
+    };
+}
+async function cleanupAll() {
+    const toDispose = [...pendingAgents];
+    pendingAgents.clear();
+    await Promise.all(toDispose.map((agent) => agent.dispose().catch(() => { })));
+}
+function reset() {
+    pendingAgents.clear();
+}
+export const lifecycleCore = {
+    registerAgent,
+    untrackAgent,
+    releaseAgent,
+    getAgentOwner,
+    recordResult,
+    flushCase,
+    cleanupAll,
+    reset,
+};

package/dist/sdk/result-capture.d.ts

@@ -0,0 +1,18 @@
+import type { Agent, RecordedEvalResult } from './types.js';
+export interface EvalResultEvent {
+    readonly result: RecordedEvalResult;
+    readonly agent: Agent;
+}
+export type EvalResultObserver = (event: EvalResultEvent) => void;
+export type ResultObserverOwner = 'user' | 'adapter' | 'test';
+export interface ResultObserverOptions {
+    readonly owner?: ResultObserverOwner;
+    readonly key?: string;
+}
+export interface ResultObserverHandle {
+    unsubscribe(): void;
+}
+export declare function subscribeToEvalResults(observer: EvalResultObserver, options?: ResultObserverOptions): ResultObserverHandle;
+export declare function emitEvalResult(event: EvalResultEvent): void;
+export declare function resetUserResultObservers(): void;
+export declare function resetAllResultObserversForTests(): void;

package/dist/sdk/result-capture.js

@@ -0,0 +1,49 @@
+const observers = new Set();
+export function subscribeToEvalResults(observer, options = {}) {
+    if (options.owner === 'adapter' && options.key) {
+        removeObserverByOwnerAndKey(options.owner, options.key);
+    }
+    const subscription = {
+        observer,
+        owner: options.owner ?? 'user',
+        key: options.key,
+    };
+    observers.add(subscription);
+    let active = true;
+    return {
+        unsubscribe() {
+            if (!active)
+                return;
+            active = false;
+            observers.delete(subscription);
+        },
+    };
+}
+export function emitEvalResult(event) {
+    for (const subscription of [...observers]) {
+        try {
+            subscription.observer(event);
+        }
+        catch {
+            // Result capture should be best-effort for every observer; one
+            // broken hook must not block adapter-owned reporting.
+        }
+    }
+}
+export function resetUserResultObservers() {
+    for (const subscription of [...observers]) {
+        if (subscription.owner === 'user') {
+            observers.delete(subscription);
+        }
+    }
+}
+export function resetAllResultObserversForTests() {
+    observers.clear();
+}
+function removeObserverByOwnerAndKey(owner, key) {
+    for (const subscription of [...observers]) {
+        if (subscription.owner === owner && subscription.key === key) {
+            observers.delete(subscription);
+        }
+    }
+}

package/dist/sdk/types.d.ts

@@ -2,7 +2,7 @@ import type { CommandResult, LogEntry } from '../types.js';
 import type { ToolAction } from '../tool-events.js';
 import type { MockMcpServerDescriptor } from '../core/mcp-mock.types.js';
 import type { TrialResult } from '../types.js';
-import type { DiagnosticsReport } from '../reporters/diagnostics.js';
+import type { DiagnosticsReport } from './diagnostics.js';
 import type { LLMPort } from '../utils/llm-types.js';
 import type { McpSafetyOptions } from './mcp-safety.js';
 export type AgentName = 'claude' | 'codex' | 'cursor';

package/dist/types.d.ts

@@ -1,4 +1,4 @@
-import type { DiagnosticsReport } from './reporters/diagnostics.js';
+import type { DiagnosticsReport } from './sdk/diagnostics.js';
 import type { RuntimePolicyDescriptor } from './sdk/runtime-policy.js';
 import type { LLMPort } from './utils/llm-types.js';
 export interface CommandResult {
@@ -230,7 +230,7 @@ export interface PathgradeSelectionReport {
 export interface PathgradeReport {
     version: 1;
     timestamp: string;
-    /** `ci.threshold` from the plugin config, if configured. */
+    /** `ci.threshold` from Pathgrade config or legacy plugin config, if configured. */
     threshold?: number;
     /** Weighted average of every individual trial score across all groups. */
     overall_pass_rate: number;

package/dist/utils/llm-providers/anthropic.js

@@ -1,8 +1,15 @@
 function getApiKey(env) {
-    return env?.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY;
+    return env?.ANTHROPIC_API_KEY
+        || env?.APP_ANTHROPIC_API_KEY
+        || process.env.ANTHROPIC_API_KEY
+        || process.env.APP_ANTHROPIC_API_KEY;
 }
 function resolveBaseUrl(env) {
-    return env?.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
+    return env?.ANTHROPIC_BASE_URL
+        || env?.APP_ANTHROPIC_BASE_URL
+        || process.env.ANTHROPIC_BASE_URL
+        || process.env.APP_ANTHROPIC_BASE_URL
+        || 'https://api.anthropic.com';
 }
 function buildHeaders(apiKey, useCache) {
     const headers = {

package/package.json

@@ -1,12 +1,15 @@
 {
   "name": "@wix/pathgrade",
-  "version": "0.37.0",
+  "version": "1.0.1",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/wix-incubator/pathgrade.git"
   },
+  "installConfig": {
+    "selfReferences": true
+  },
   "exports": {
     ".": {
       "types": "./dist/sdk/index.d.ts",
@@ -16,6 +19,34 @@
       "types": "./dist/plugin/index.d.ts",
       "default": "./dist/plugin/index.js"
     },
+    "./plugin/vitest": {
+      "types": "./dist/adapters/vitest/index.d.ts",
+      "default": "./dist/adapters/vitest/index.js"
+    },
+    "./adapters/vitest": {
+      "types": "./dist/adapters/vitest/index.d.ts",
+      "default": "./dist/adapters/vitest/index.js"
+    },
+    "./adapters/node-test": {
+      "types": "./dist/adapters/node-test/index.d.ts",
+      "default": "./dist/adapters/node-test/index.js"
+    },
+    "./adapters/jest": {
+      "types": "./dist/adapters/jest/index.d.ts",
+      "default": "./dist/adapters/jest/index.js"
+    },
+    "./adapters/jest/setup": {
+      "types": "./dist/adapters/jest/setup.d.ts",
+      "default": "./dist/adapters/jest/setup.js"
+    },
+    "./adapters/jest/reporter": {
+      "types": "./dist/adapters/jest/reporter.d.ts",
+      "default": "./dist/adapters/jest/reporter.cjs"
+    },
+    "./adapter-kit": {
+      "types": "./dist/adapter-kit/index.d.ts",
+      "default": "./dist/adapter-kit/index.js"
+    },
     "./mcp-mock": {
       "types": "./dist/core/mcp-mock.d.ts",
       "default": "./dist/core/mcp-mock.js"
@@ -25,6 +56,7 @@
   "bin": "bin/pathgrade.js",
   "files": [
     "dist/**/*.js",
+    "dist/**/*.cjs",
     "dist/**/*.d.ts",
     "dist/viewer.html",
     "!dist/**/*.js.map",
@@ -35,11 +67,20 @@
     "LICENSE"
   ],
   "scripts": {
-    "test": "yarn build && vitest run",
+    "lint": "oxlint src tests evals examples scripts --deny-warnings --disable-unicorn-plugin --disable-oxc-plugin --allow no-unused-vars --allow no-underscore-dangle --allow no-shadow --allow preserve-caught-error --allow no-unsafe-optional-chaining --allow no-unreachable --ignore-pattern tests/fixtures --ignore-pattern .worktrees",
+    "knip": "yarn quality:deps",
+    "knip:unused": "knip --exclude binaries --no-config-hints --max-show-issues 20",
+    "quality:deps": "knip --dependencies --exclude binaries --no-config-hints --max-show-issues 20",
+    "quality:arch": "sentrux check .",
+    "quality:boundaries": "scripts/check-quality-boundaries.sh",
+    "quality:max-lines": "scripts/check-max-lines.sh .",
+    "quality": "yarn lint && yarn quality:deps && yarn quality:arch && yarn quality:boundaries && yarn quality:max-lines",
+    "test": "yarn build && vitest run && yarn test:runner-cli-smoke",
+    "test:runner-cli-smoke": "node tests/runner-cli-smoke/run-smokes.mjs",
     "test:evals": "vitest run --config evals/vitest.config.mts",
     "test:coverage": "vitest run --coverage",
     "dev": "tsx src/pathgrade.ts",
-    "build": "tsc -p tsconfig.build.json && cp src/viewer.html dist/viewer.html"
+    "build": "tsc -p tsconfig.build.json && cp src/viewer.html dist/viewer.html && cp src/adapters/jest/reporter.cjs dist/adapters/jest/reporter.cjs"
   },
   "keywords": [
     "ai",
@@ -61,13 +102,23 @@
     "node": ">=20.11.0"
   },
   "peerDependencies": {
+    "jest": "^30.0.0",
     "vitest": "^4.0.0"
   },
+  "peerDependenciesMeta": {
+    "jest": {
+      "optional": true
+    },
+    "vitest": {
+      "optional": true
+    }
+  },
   "devDependencies": {
     "@types/fs-extra": "^11.0.4",
+    "@types/jest": "^30.0.0",
     "@types/picomatch": "^4.0.2",
     "@vitest/coverage-v8": "4.1.7",
-    "tsx": "4.22.3",
+    "jest": "^30.0.0",
     "vitest": "4.1.7"
   },
   "dependencies": {
@@ -77,6 +128,7 @@
     "fs-extra": "11.3.3",
     "jiti": "2.6.1",
     "picomatch": "^4.0.4",
+    "tsx": "4.22.3",
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },