@wix/pathgrade 1.0.33 → 1.0.35

package/dist/adapter-kit/index.d.ts

@@ -1,6 +1,6 @@
 export { runnerAdapterContractVersion, type AdapterCaseContext, type AdapterDiagnostic, type AdapterDiscoveryInput, type AdapterDiscoveryResult, type AdapterEvalUnit, type AdapterInvocationInput, type AdapterLifecycleHooks, type AdapterRunHandle, type AdapterRunStatus, type EvalResultEvent, type RunnerAdapter, } from '../runners/adapter.js';
 export type { RunnerInvocationAdapter, RunnerInvocationInput, } from '../runners/invocation.js';
-export type { AttemptOutcome, AttemptRecord, AssertionRecord, Diagnostic, EvalUnitRecord, EvaluationRecord, GroupingHint, NativeReference, NormalizedRunModel, NormalizedRunSnapshot, RunCaseRecord, RunCaseState, RunRecord, RunStatus, ScoringPolicy, SnapshotCompleteness, } from '../runners/model.js';
+export type { AttemptOutcome, AttemptRecord, AssertionRecord, Diagnostic, EvalUnitRecord, EvalUnitCollection, EvaluationRecord, GroupingHint, NativeReference, NormalizedRunModel, NormalizedRunSnapshot, RunCaseRecord, RunCaseState, RunRecord, RunStatus, ScoringPolicy, SnapshotCompleteness, } from '../runners/model.js';
 export { buildNormalizedRunSnapshotFromReportGroups } from '../runners/model-builders.js';
 export { validateNormalizedRunSnapshot } from '../runners/model-validation.js';
 export { projectNormalizedRunSnapshotToReportInput } from '../runners/report-projection.js';

package/dist/reporting/reliability-contract.d.ts

@@ -0,0 +1,152 @@
+import type { PathgradeGroupReport, PathgradeReport, StrippedTrialResult } from '../types.js';
+export declare const PATHGRADE_REPORT_VERSION: 3;
+export declare const TASK_INVENTORY_VERSION: 1;
+export declare const COMPARISON_CONTRACT_VERSION: 2;
+export declare const NORMALIZED_RUN_MODEL_VERSION: 2;
+export type TaskNonScoringReason = 'skipped' | 'pending' | 'timed-out' | 'failed-before-evaluation';
+export type CollectionIncompleteReason = 'adapter-cannot-prove-completeness' | 'collection-failed' | 'run-incomplete';
+export type TaskInventoryEntry = {
+    task_key: string;
+    state: 'scored';
+} | {
+    task_key: string;
+    state: 'not-scored';
+    reason: TaskNonScoringReason;
+};
+export type TaskInventoryFile = {
+    eval_file: string;
+    tasks: readonly TaskInventoryEntry[];
+} & ({
+    completeness: 'complete';
+} | {
+    completeness: 'incomplete';
+    reason: CollectionIncompleteReason;
+});
+export interface TaskInventory {
+    version: typeof TASK_INVENTORY_VERSION;
+    files: readonly TaskInventoryFile[];
+}
+export type ComparisonInputsInvalidReason = 'malformed-declaration' | 'no-matches' | 'outside-repository' | 'unreadable-input';
+export type ComparisonInputsState = {
+    state: 'missing';
+} | {
+    state: 'invalid';
+    reason: ComparisonInputsInvalidReason;
+} | {
+    state: 'resolved';
+    declarations: readonly string[];
+    files: readonly {
+        path: string;
+        revision: string;
+    }[];
+    revision: string;
+};
+export type ScorerRevisionState = {
+    state: 'complete';
+    revision: string;
+} | {
+    state: 'incomplete';
+    reason: 'explicit-revision-required' | 'metadata-missing';
+};
+export interface RuntimeComponentRevisions {
+    agent_name: string;
+    model: string;
+    transport: string;
+    interaction_mode: string;
+}
+type DefinitionRevisionContract = {
+    comparison_inputs: Extract<ComparisonInputsState, {
+        state: 'resolved';
+    }>;
+    definition_revision: string;
+} | {
+    comparison_inputs: Exclude<ComparisonInputsState, {
+        state: 'resolved';
+    }>;
+    definition_revision?: never;
+};
+type ScorerRevisionContract = {
+    scorer_revision_state: {
+        state: 'complete';
+    };
+    scorer_revision: string;
+} | {
+    scorer_revision_state: {
+        state: 'incomplete';
+        reason: 'explicit-revision-required' | 'metadata-missing';
+    };
+    scorer_revision?: never;
+};
+type RuntimeRevisionContract = {
+    runtime_revision_state: {
+        state: 'complete';
+    };
+    runtime_revision: string;
+    runtime_components: RuntimeComponentRevisions;
+} | {
+    runtime_revision_state: {
+        state: 'incomplete';
+        reason: 'metadata-missing';
+    };
+    runtime_revision?: never;
+    runtime_components?: never;
+};
+type SamplingRevisionContract = {
+    sampling_revision_state: {
+        state: 'complete';
+    };
+    sampling_revision: string;
+} | {
+    sampling_revision_state: {
+        state: 'incomplete';
+        reason: 'sampling-incomplete';
+    };
+    sampling_revision?: never;
+};
+export type ComparisonContractV2 = {
+    version: typeof COMPARISON_CONTRACT_VERSION;
+    report_schema_revision: 'pathgrade-results-v3';
+} & DefinitionRevisionContract & ScorerRevisionContract & RuntimeRevisionContract & SamplingRevisionContract;
+export type AdapterCollectionState = {
+    state: 'complete';
+} | {
+    state: 'incomplete';
+    reason: CollectionIncompleteReason;
+};
+export interface NormalizedEvalUnitV2Contract {
+    version: typeof NORMALIZED_RUN_MODEL_VERSION;
+    id: string;
+    source_file: string;
+    collection: AdapterCollectionState;
+}
+type AdapterCollectionContractBase = {
+    adapter: 'vitest' | 'jest' | 'node-test';
+    observed_task_keys: readonly string[];
+    unaccounted_task_keys: readonly string[];
+};
+export type AdapterCollectionContract = AdapterCollectionContractBase & ({
+    unit: Omit<NormalizedEvalUnitV2Contract, 'collection'> & {
+        collection: {
+            state: 'complete';
+        };
+    };
+    disposition: 'replace-complete';
+} | {
+    unit: Omit<NormalizedEvalUnitV2Contract, 'collection'> & {
+        collection: {
+            state: 'incomplete';
+            reason: CollectionIncompleteReason;
+        };
+    };
+    disposition: 'retain-with-file-gap';
+});
+export type PathgradeGroupReportV3 = Omit<PathgradeGroupReport, 'comparison_contract' | 'trials'> & {
+    trials: readonly StrippedTrialResult[];
+    comparison_contract: ComparisonContractV2;
+};
+export type PathgradeReportV3 = Omit<PathgradeReport, 'version' | 'groups'> & {
+    version: typeof PATHGRADE_REPORT_VERSION;
+    groups: readonly PathgradeGroupReportV3[];
+    task_inventory: TaskInventory;
+};
+export {};

package/dist/reporting/reliability-contract.js

@@ -0,0 +1,4 @@
+export const PATHGRADE_REPORT_VERSION = 3;
+export const TASK_INVENTORY_VERSION = 1;
+export const COMPARISON_CONTRACT_VERSION = 2;
+export const NORMALIZED_RUN_MODEL_VERSION = 2;

package/dist/runners/model.d.ts

@@ -1,5 +1,6 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { TrialResult } from '../types.js';
+import type { CollectionIncompleteReason } from '../reporting/reliability-contract.js';
 export interface NormalizedRunSnapshot {
     version: 1;
     completeness: SnapshotCompleteness;
@@ -25,12 +26,19 @@ export interface EvalUnitRecord {
     id: string;
     runId: string;
     displayName: string;
+    collection?: EvalUnitCollection;
     sourceFile?: string;
     sourceRevision?: string;
     diagnostics?: Diagnostic[];
     nativeReferences?: NativeReference[];
     groupingHints?: GroupingHint[];
 }
+export type EvalUnitCollection = {
+    state: 'complete';
+} | {
+    state: 'incomplete';
+    reason: CollectionIncompleteReason;
+};
 export type RunCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
 export type ScoringPolicy = {
     kind: 'score';

package/dist/sdk/index.d.ts

@@ -42,6 +42,8 @@ export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvi
 export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';
 export type { ToolEvent, McpToolCallClassification } from '../tool-events.js';
 export type { ComparisonContract, ComparisonUnavailableReason, PathgradeGroupReport, PathgradeReport, StrippedTrialResult, } from '../types.js';
+export { COMPARISON_CONTRACT_VERSION, NORMALIZED_RUN_MODEL_VERSION, PATHGRADE_REPORT_VERSION, TASK_INVENTORY_VERSION, } from '../reporting/reliability-contract.js';
+export type { AdapterCollectionContract, AdapterCollectionState, CollectionIncompleteReason, ComparisonContractV2, ComparisonInputsInvalidReason, ComparisonInputsState, NormalizedEvalUnitV2Contract, PathgradeGroupReportV3, PathgradeReportV3, RuntimeComponentRevisions, ScorerRevisionState, TaskInventory, TaskInventoryEntry, TaskInventoryFile, TaskNonScoringReason, } from '../reporting/reliability-contract.js';
 export type { LLMPort, EvalRuntime } from './eval-runtime.js';
 export { createAgentLLM, createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
 export type { CreateLLMClientOptions, LLMProviderAdapter, TokenUsage as LLMTokenUsage } from '../utils/llm.js';

package/dist/sdk/index.js

@@ -25,5 +25,6 @@ export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
 export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
+export { COMPARISON_CONTRACT_VERSION, NORMALIZED_RUN_MODEL_VERSION, PATHGRADE_REPORT_VERSION, TASK_INVENTORY_VERSION, } from '../reporting/reliability-contract.js';
 export { createAgentLLM, createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
 export { createMockLLM } from '../utils/llm-mocks.js';

package/dist/sdk/types.d.ts

@@ -6,7 +6,7 @@ import type { DiagnosticsReport } from './diagnostics.js';
 import type { LLMPort } from '../utils/llm-types.js';
 import type { McpSafetyOptions } from './mcp-safety.js';
 import type { McpMockApprovalRule } from './mcp-mock-approvals.js';
-import type { EvidenceEnvelope } from '../internal/direct-mcp-v2/types.js';
+import type { EvidenceEnvelope, JsonValue } from '../internal/direct-mcp-v2/types.js';
 import type { ScenarioMachineV2 } from './scenario-machine-v2.js';
 export type AgentName = 'claude' | 'codex' | 'cursor' | 'opencode';
 export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
@@ -280,12 +280,14 @@ export interface CheckScorer {
     name: string;
     weight: number;
     fn: (ctx: ScorerContext) => boolean | Promise<boolean>;
+    revision?: JsonValue;
 }
 export interface ScoreScorer {
     type: 'score';
     name: string;
     weight: number;
     fn: (ctx: ScorerContext) => number | ScoreResult | Promise<number | ScoreResult>;
+    revision?: JsonValue;
 }
 export interface ScoreResult {
     score: number;
@@ -308,6 +310,7 @@ export interface JudgeScorer {
     maxRounds?: number;
     /** Enable Anthropic prompt caching for system + tool schemas. Default: true when tools is set. */
     cacheControl?: boolean;
+    revision?: JsonValue;
 }
 export interface ToolExpectation {
     action: ToolAction;
@@ -324,6 +327,7 @@ export interface ToolUsageScorer {
     name: string;
     weight: number;
     expectations: ToolExpectation[];
+    revision?: JsonValue;
 }
 export interface SessionArtifactMatchOptions {
     actions?: import('../tool-events.js').ToolAction[];
@@ -472,6 +476,8 @@ export interface PathgradeMeta {
     deps?: string[];
     /** Unioned with the auto-detected skill root (or with `deps`). */
     extraDeps?: string[];
+    /** Methodology inputs whose raw bytes participate in comparison identity. */
+    comparisonInputs?: string[];
     /** Unconditionally include this eval in every `pathgrade run --changed`. */
     alwaysRun?: boolean;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.33",
+  "version": "1.0.35",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -134,11 +134,12 @@
     "@types/node": "25.6.0",
     "ajv": "8.20.0",
     "fs-extra": "11.3.3",
+    "hono": "4.13.3",
     "jiti": "2.6.1",
     "picomatch": "^4.0.4",
     "tsx": "4.22.3",
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "044358f11c1feb83b7e0a5b40972dd14eaf2c5b0d1261ba1339cf9d2"
+  "falconPackageHash": "9eca97f7cef9dc643ad6eef1d5e6c58f3661f23d216f365cd956102c"
 }