@wix/pathgrade 1.0.38 → 1.0.40

package/dist/sdk/tool-event-secrets.js

@@ -1,12 +1,43 @@
-import { attachOriginalMcpInput, getOriginalMcpInput } from './mcp-event-input.js';
+import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError } from '../tool-event-results.js';
+import { attachOriginalMcpInput, clearOriginalMcpInput, getOriginalMcpInput, } from './mcp-event-input.js';
 const sensitiveValuesByEvent = new WeakMap();
 export function attachToolEventSensitiveValues(event, sensitiveValues) {
-    sensitiveValuesByEvent.set(event, [...sensitiveValues]);
+    sensitiveValuesByEvent.set(event, [...new Set([
+            ...getToolEventSensitiveValues(event),
+            ...sensitiveValues,
+        ])]);
     return event;
 }
 export function getToolEventSensitiveValues(event) {
     return sensitiveValuesByEvent.get(event) ?? [];
 }
+export function attachLiveMcpInput(event, input) {
+    let sensitiveValues;
+    try {
+        sensitiveValues = collectStructuredSensitiveValues(input);
+    }
+    catch (error) {
+        if (!isSensitiveValueScanLimitError(error))
+            throw error;
+        return redactToolEventPayload(event);
+    }
+    return attachToolEventSensitiveValues(attachOriginalMcpInput(event, input), sensitiveValues);
+}
+export function redactToolEventPayload(event) {
+    return {
+        ...event,
+        arguments: { redacted: true },
+        summary: `${event.action} via ${event.providerToolName}`,
+        rawSnippet: '<redacted>',
+    };
+}
+export function collectToolEventSensitiveValues(events) {
+    return [...new Set(events.flatMap((event) => getToolEventSensitiveValues(event)))];
+}
+export function clearToolEventRuntimeMetadata(event) {
+    sensitiveValuesByEvent.delete(event);
+    clearOriginalMcpInput(event);
+}
 export function cloneToolEventWithRuntimeMetadata(source, overrides) {
     const clone = attachToolEventSensitiveValues({ ...source, ...overrides }, getToolEventSensitiveValues(source));
     const originalInput = getOriginalMcpInput(source);

package/dist/sdk/types.d.ts

@@ -360,6 +360,8 @@ export interface EvaluateOptions {
     failFast?: boolean;
     llm?: LLMPort;
     onScorerError?: 'skip' | 'zero' | 'fail';
+    /** Evidence visible to deterministic scorers. Live mode is unavailable for snapshot replay. */
+    deterministicToolEvidence?: 'persisted' | 'live';
     /** Stable identity for this evaluation definition across separate runs. */
     evaluationDefinitionKey?: string;
 }

package/dist/tool-event-results.d.ts

@@ -7,4 +7,7 @@ export declare function collectSensitiveEnvValues(env?: Readonly<Record<string,
  * is important for summaries, snippets, traces, and provider error text.
  */
 export declare function sanitizePersistenceValue<T>(source: T, explicitSensitiveValues?: readonly string[]): T;
+export declare function sanitizeUntrustedPersistenceValue<T>(source: T, explicitSensitiveValues?: readonly string[]): T;
+export declare function isSensitiveValueScanLimitError(error: unknown): boolean;
 export declare function sanitizeToolEventResult(source: Readonly<ToolEventResult>, sensitiveValues: readonly string[]): ToolEventResult;
+export declare function collectStructuredSensitiveValues(source: unknown): string[];

package/dist/tool-event-results.js

@@ -24,6 +24,10 @@ const SECRET_KEY_NAMES = [
     'connectionstring',
     'credentials',
     'credential',
+    'grant',
+    'capability',
+    'capabilityurl',
+    'cursor',
     'svsession',
     'smsession',
     'wixsession',
@@ -32,6 +36,10 @@ const SECRET_KEY_NAMES = [
 const EXACT_ONLY_SECRET_KEY_NAMES = new Set([
     'auth',
     'token',
+    'grant',
+    'capability',
+    'capabilityurl',
+    'cursor',
     'svsession',
     'smsession',
     'wixsession',
@@ -39,6 +47,9 @@ const EXACT_ONLY_SECRET_KEY_NAMES = new Set([
 ]);
 const NON_SECRET_ENVIRONMENT_KEY_NAMES = new Set(['tokencount', 'tokenizersparallelism', 'tokenusage']);
 const BOUNDARY_ONLY_SENSITIVE_VALUE_MAX_LENGTH = 1;
+const MAX_SANITIZE_DEPTH = 64;
+const MAX_SENSITIVE_SCAN_NODES = 100_000;
+const MAX_SENSITIVE_JSON_CHARS = 1024 * 1024;
 export function collectSensitiveEnvValues(env) {
     return [...new Set(Object.entries(env ?? {})
             .filter((entry) => typeof entry[1] === 'string')
@@ -60,15 +71,36 @@ export function sanitizePersistenceValue(source, explicitSensitiveValues = []) {
         ])].sort((a, b) => b.length - a.length);
     return sanitizeValue(source, '', sensitiveValues, new Set(explicitValues));
 }
+export function sanitizeUntrustedPersistenceValue(source, explicitSensitiveValues = []) {
+    try {
+        return sanitizePersistenceValue(source, explicitSensitiveValues);
+    }
+    catch (error) {
+        if (!isSensitiveValueScanLimitError(error))
+            throw error;
+        if (Array.isArray(source))
+            return [];
+        if (isRecord(source))
+            return { redacted: true };
+        return '<redacted>';
+    }
+}
+export function isSensitiveValueScanLimitError(error) {
+    return error instanceof Error && error.message.startsWith('Sensitive-value scan exceeded');
+}
 export function sanitizeToolEventResult(source, sensitiveValues) {
     const result = {};
     let truncated = source.truncated === true;
     const addBounded = (key, value) => {
         if (typeof value !== 'string')
             return;
-        const bounded = boundText(sanitizePersistenceValue(value, sensitiveValues));
-        result[key] = bounded.value;
-        truncated ||= bounded.truncated;
+        const explicitValues = [...new Set(sensitiveValues.filter((entry) => entry.length > 0))];
+        const overlap = explicitValues.reduce((max, entry) => Math.max(max, entry.length), 0);
+        const prefix = value.slice(0, TOOL_RESULT_MAX_CHARS + Math.max(overlap, 1));
+        const bounded = boundText(redactSensitiveValues(prefix, explicitValues, new Set(explicitValues)));
+        const sanitized = boundText(sanitizePersistenceValue(bounded.value, sensitiveValues));
+        result[key] = sanitized.value;
+        truncated ||= value.length > prefix.length || bounded.truncated || sanitized.truncated;
     };
     addBounded('content', source.content);
     addBounded('stdout', source.stdout);
@@ -96,56 +128,93 @@ function redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues)
     }
     return redacted;
 }
-function collectStructuredSensitiveValues(source) {
+export function collectStructuredSensitiveValues(source) {
     const values = new Set();
-    const visit = (value, key) => {
-        if (typeof value === 'string' && !isNumericTokenTelemetry(key, value)
-            && isSecretKey(key) && value.length > 0) {
-            values.add(value);
-            return;
+    let scannedNodes = 0;
+    let parsedJsonChars = 0;
+    const pending = [
+        { value: source, key: '' },
+    ];
+    while (pending.length > 0) {
+        if (++scannedNodes > MAX_SENSITIVE_SCAN_NODES) {
+            throw new Error('Sensitive-value scan exceeded the input node limit');
+        }
+        const { value, key } = pending.pop();
+        if (typeof value === 'string') {
+            if (value.length > 0 && !isNumericTokenTelemetry(key, value) && isSecretKey(key)) {
+                values.add(value);
+            }
+            for (const match of value.matchAll(/[?&](?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)=([^&#\s]+)/gi)) {
+                if (match[1])
+                    values.add(match[1]);
+            }
+            for (const sensitiveValue of collectCredentialShapeSensitiveValues(value)) {
+                values.add(sensitiveValue);
+            }
+            const trimmed = value.trim();
+            const looksStructured = (trimmed.startsWith('{') && trimmed.endsWith('}'))
+                || (trimmed.startsWith('[') && trimmed.endsWith(']'));
+            if (looksStructured && (parsedJsonChars += value.length) > MAX_SENSITIVE_JSON_CHARS) {
+                throw new Error('Sensitive-value scan exceeded the nested JSON limit');
+            }
+            const structured = looksStructured ? parseStructuredJson(value) : undefined;
+            if (structured !== undefined)
+                pending.push({ value: structured, key });
+            continue;
+        }
+        if ((typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint')
+            && !isNumericTokenTelemetry(key, value) && isSecretKey(key)) {
+            values.add(String(value));
+            continue;
         }
         if (Array.isArray(value)) {
             for (const entry of value)
-                visit(entry, key);
-            return;
+                pending.push({ value: entry, key });
+            continue;
         }
         if (isRecord(value)) {
-            for (const [entryKey, entryValue] of Object.entries(value))
-                visit(entryValue, entryKey);
+            for (const [entryKey, entryValue] of Object.entries(value)) {
+                pending.push({ value: entryValue, key: entryKey });
+            }
         }
-    };
-    visit(source, '');
+    }
     return [...values];
 }
-function sanitizeValue(value, key, sensitiveValues, explicitSensitiveValues) {
+function sanitizeValue(value, key, sensitiveValues, explicitSensitiveValues, depth = 0) {
     if (!isNumericTokenTelemetry(key, value) && isSecretKey(key))
         return '<redacted>';
     if (typeof value === 'string') {
-        return redactCredentialShapes(redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues), sensitiveValues, explicitSensitiveValues);
+        return redactCredentialShapes(redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues), sensitiveValues, explicitSensitiveValues, depth);
     }
+    if (depth >= MAX_SANITIZE_DEPTH && (Array.isArray(value) || isRecord(value)))
+        return '<redacted>';
     if (Array.isArray(value)) {
-        return value.map((entry) => sanitizeValue(entry, key, sensitiveValues, explicitSensitiveValues));
+        return value.map((entry) => sanitizeValue(entry, key, sensitiveValues, explicitSensitiveValues, depth + 1));
     }
     if (isRecord(value)) {
         return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [
             entryKey,
-            sanitizeValue(entryValue, entryKey, sensitiveValues, explicitSensitiveValues),
+            sanitizeValue(entryValue, entryKey, sensitiveValues, explicitSensitiveValues, depth + 1),
         ]));
     }
     return value;
 }
-function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues) {
+function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues, depth) {
     let redacted = value;
+    const credentialShapeValues = collectCredentialShapeSensitiveValues(redacted);
+    redacted = redactSensitiveValues(redacted, credentialShapeValues, new Set(credentialShapeValues));
     const structured = parseStructuredJson(redacted);
     if (structured !== undefined) {
         const nestedSensitiveValues = [...new Set([
                 ...sensitiveValues,
                 ...collectStructuredSensitiveValues(structured),
             ])].sort((a, b) => b.length - a.length);
-        redacted = JSON.stringify(sanitizeValue(structured, '', nestedSensitiveValues, explicitSensitiveValues));
+        redacted = JSON.stringify(sanitizeValue(structured, '', nestedSensitiveValues, explicitSensitiveValues, depth + 1));
     }
     if (redacted.includes('PRIVATE KEY-----')) {
-        redacted = redacted.replace(/-----BEGIN [^-\n]*PRIVATE KEY-----[\s\S]*?-----END [^-\n]*PRIVATE KEY-----/g, '<redacted>');
+        for (const block of findPrivateKeyBlocks(redacted).reverse()) {
+            redacted = `${redacted.slice(0, block.start)}<redacted>${redacted.slice(block.end)}`;
+        }
     }
     if (/\b(?:bearer|basic)\s/i.test(redacted)) {
         redacted = redacted.replace(/(\b(?:bearer|basic)\s+)[^\s,;"']+/gi, '$1<redacted>');
@@ -154,10 +223,71 @@ function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues)
         redacted = redacted.replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s/@]+(@)/gi, '$1<redacted>$2');
     }
     if (redacted.includes('=') || redacted.includes(':')) {
-        redacted = redacted.replace(/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)\s*[=:]\s*)[^\s,;"'}]+/gi, '$1<redacted>');
+        redacted = redacted.replace(/((?:^|[\s{[(,;])["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)["']?\s*[=:]\s*["']?)[^\s,;"'}]+/gi, '$1<redacted>');
+        redacted = redacted.replace(/((?:^|[\s{[(,;])["'](?:grant|capability|capabilityurl|cursor)["']\s*:\s*["']?)[^\s,;"'}]+/gi, '$1<redacted>');
     }
     return redacted;
 }
+function collectCredentialShapeSensitiveValues(value) {
+    const values = new Set();
+    for (const match of value.matchAll(/https?:\/\/[^\s,;"']+\/(?:cap|capability)\/[^\s,;"']+/gi)) {
+        values.add(match[0]);
+    }
+    if (value.includes('PRIVATE KEY-----')) {
+        for (const block of findPrivateKeyBlocks(value)) {
+            values.add(value.slice(block.start, block.end));
+            if (block.body)
+                values.add(block.body);
+        }
+    }
+    if (/\b(?:bearer|basic)\s/i.test(value)) {
+        for (const match of value.matchAll(/\b(?:bearer|basic)\s+([^\s,;"']+)/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+    }
+    if (value.includes('://')) {
+        for (const match of value.matchAll(/[a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:([^\s/@]+)@/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+    }
+    if (value.includes('=') || value.includes(':')) {
+        for (const match of value.matchAll(/(?:^|[\s{[(,;])["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)["']?\s*[=:]\s*["']?([^\s,;"'}]+)/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+        for (const match of value.matchAll(/(?:^|[\s{[(,;])["'](?:grant|capability|capabilityurl|cursor)["']\s*:\s*["']?([^\s,;"'}]+)/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+    }
+    return [...values].sort((a, b) => b.length - a.length);
+}
+function findPrivateKeyBlocks(value) {
+    const blocks = [];
+    let open;
+    let cursor = 0;
+    while (cursor < value.length) {
+        const start = value.indexOf('-----', cursor);
+        if (start < 0)
+            break;
+        const end = value.indexOf('-----', start + 5);
+        if (end < 0)
+            break;
+        const markerEnd = end + 5;
+        const marker = value.slice(start, markerEnd);
+        if (!open && marker.startsWith('-----BEGIN ') && marker.endsWith('PRIVATE KEY-----')) {
+            open = { start, bodyStart: markerEnd };
+        }
+        else if (open && marker.startsWith('-----END ') && marker.endsWith('PRIVATE KEY-----')) {
+            blocks.push({ start: open.start, end: markerEnd, body: value.slice(open.bodyStart, start).trim() });
+            open = undefined;
+        }
+        cursor = markerEnd;
+    }
+    return blocks;
+}
 function isSecretKey(key) {
     const normalized = key.replace(/[^a-z0-9]/gi, '').toLowerCase();
     if (normalized === 'tokens')

package/dist/types.d.ts

@@ -185,6 +185,10 @@ export interface EvalReport {
     task: string;
     /** Canonical status for this report or consolidated group when available. */
     status?: 'pass' | 'fail';
+    /** Whether every reportable runner attempt passed. */
+    runner_status?: 'pass' | 'fail';
+    /** Whether mean reward meets the configured threshold. */
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
     /** Schema-v2 canonical average score. */
     mean_reward?: number;
     /** Binary success frequency, absent when attempts are ineligible. */
@@ -203,7 +207,7 @@ export interface EvalReport {
     skills_used: string[];
 }
 export type ComparisonUnavailableReason = 'definition_metadata_missing' | 'scorer_metadata_missing' | 'runtime_metadata_missing' | 'sampling_incomplete';
-export interface ComparisonContract {
+export interface ComparisonContractV1 {
     version: 1;
     definition_revision?: string;
     scorer_revision?: string;
@@ -212,6 +216,7 @@ export interface ComparisonContract {
     report_schema_revision: 'pathgrade-results-v2';
     unavailable_reasons?: ComparisonUnavailableReason[];
 }
+export type ComparisonContract = ComparisonContractV1 | import('./reporting/reliability-contract.js').ComparisonContractV2;
 /**
  * TrialResult with large trace fields stripped. These fields
  * live only in the per-group trace files; the consolidated results.json keeps
@@ -256,7 +261,7 @@ export interface PathgradeSelectionReport {
  * external consumers can gate on schema revisions.
  */
 export interface PathgradeReport {
-    version: 1 | 2;
+    version: 1 | 2 | 3;
     timestamp: string;
     /** `ci.threshold` from Pathgrade config or legacy plugin config, if configured. */
     threshold?: number;
@@ -266,15 +271,17 @@ export interface PathgradeReport {
     overall_mean_reward?: number;
     attempts_requested?: number;
     attempts_completed?: number;
-    /**
-     * Threshold check result. When `threshold` is set: `'pass'` iff
-     * `overall_pass_rate >= threshold`. Otherwise: `'pass'` iff every trial
-     * in every group passed its vitest test.
-     */
+    /** Whether every reportable runner attempt passed. */
+    runner_status?: 'pass' | 'fail';
+    /** Whether overall mean reward meets the configured threshold. */
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
+    /** Conjunction of runner assertions and the optional aggregate threshold. */
     status: 'pass' | 'fail';
     /** Agent Evals compatibility manifest when changed selection finds no runnable evals. */
     run_kind?: 'evaluation' | 'no-affected';
     groups: PathgradeGroupReport[];
+    /** Current selected-task accounting, required for schema-v3 reports. */
+    task_inventory?: import('./reporting/reliability-contract.js').TaskInventory;
     /**
      * Present when `pathgrade run --changed` produced the run. Absent on
      * plain `pathgrade run`. Backward compatible — older consumers ignore

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.38",
+  "version": "1.0.40",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -142,5 +142,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "199b36204420a97fad83bac6e53d8b614d2b6592cb701e7db41d3bad"
+  "falconPackageHash": "4c4d43a7c8d439f02756404accbc72b44db02c52db4d0c2b2d18452f"
 }