@wix/pathgrade 1.0.37 → 1.0.39

package/dist/tool-event-results.js

@@ -24,6 +24,10 @@ const SECRET_KEY_NAMES = [
     'connectionstring',
     'credentials',
     'credential',
+    'grant',
+    'capability',
+    'capabilityurl',
+    'cursor',
     'svsession',
     'smsession',
     'wixsession',
@@ -32,6 +36,10 @@ const SECRET_KEY_NAMES = [
 const EXACT_ONLY_SECRET_KEY_NAMES = new Set([
     'auth',
     'token',
+    'grant',
+    'capability',
+    'capabilityurl',
+    'cursor',
     'svsession',
     'smsession',
     'wixsession',
@@ -39,6 +47,9 @@ const EXACT_ONLY_SECRET_KEY_NAMES = new Set([
 ]);
 const NON_SECRET_ENVIRONMENT_KEY_NAMES = new Set(['tokencount', 'tokenizersparallelism', 'tokenusage']);
 const BOUNDARY_ONLY_SENSITIVE_VALUE_MAX_LENGTH = 1;
+const MAX_SANITIZE_DEPTH = 64;
+const MAX_SENSITIVE_SCAN_NODES = 100_000;
+const MAX_SENSITIVE_JSON_CHARS = 1024 * 1024;
 export function collectSensitiveEnvValues(env) {
     return [...new Set(Object.entries(env ?? {})
             .filter((entry) => typeof entry[1] === 'string')
@@ -60,15 +71,36 @@ export function sanitizePersistenceValue(source, explicitSensitiveValues = []) {
         ])].sort((a, b) => b.length - a.length);
     return sanitizeValue(source, '', sensitiveValues, new Set(explicitValues));
 }
+export function sanitizeUntrustedPersistenceValue(source, explicitSensitiveValues = []) {
+    try {
+        return sanitizePersistenceValue(source, explicitSensitiveValues);
+    }
+    catch (error) {
+        if (!isSensitiveValueScanLimitError(error))
+            throw error;
+        if (Array.isArray(source))
+            return [];
+        if (isRecord(source))
+            return { redacted: true };
+        return '<redacted>';
+    }
+}
+export function isSensitiveValueScanLimitError(error) {
+    return error instanceof Error && error.message.startsWith('Sensitive-value scan exceeded');
+}
 export function sanitizeToolEventResult(source, sensitiveValues) {
     const result = {};
     let truncated = source.truncated === true;
     const addBounded = (key, value) => {
         if (typeof value !== 'string')
             return;
-        const bounded = boundText(sanitizePersistenceValue(value, sensitiveValues));
-        result[key] = bounded.value;
-        truncated ||= bounded.truncated;
+        const explicitValues = [...new Set(sensitiveValues.filter((entry) => entry.length > 0))];
+        const overlap = explicitValues.reduce((max, entry) => Math.max(max, entry.length), 0);
+        const prefix = value.slice(0, TOOL_RESULT_MAX_CHARS + Math.max(overlap, 1));
+        const bounded = boundText(redactSensitiveValues(prefix, explicitValues, new Set(explicitValues)));
+        const sanitized = boundText(sanitizePersistenceValue(bounded.value, sensitiveValues));
+        result[key] = sanitized.value;
+        truncated ||= value.length > prefix.length || bounded.truncated || sanitized.truncated;
     };
     addBounded('content', source.content);
     addBounded('stdout', source.stdout);
@@ -96,56 +128,93 @@ function redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues)
     }
     return redacted;
 }
-function collectStructuredSensitiveValues(source) {
+export function collectStructuredSensitiveValues(source) {
     const values = new Set();
-    const visit = (value, key) => {
-        if (typeof value === 'string' && !isNumericTokenTelemetry(key, value)
-            && isSecretKey(key) && value.length > 0) {
-            values.add(value);
-            return;
+    let scannedNodes = 0;
+    let parsedJsonChars = 0;
+    const pending = [
+        { value: source, key: '' },
+    ];
+    while (pending.length > 0) {
+        if (++scannedNodes > MAX_SENSITIVE_SCAN_NODES) {
+            throw new Error('Sensitive-value scan exceeded the input node limit');
+        }
+        const { value, key } = pending.pop();
+        if (typeof value === 'string') {
+            if (value.length > 0 && !isNumericTokenTelemetry(key, value) && isSecretKey(key)) {
+                values.add(value);
+            }
+            for (const match of value.matchAll(/[?&](?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)=([^&#\s]+)/gi)) {
+                if (match[1])
+                    values.add(match[1]);
+            }
+            for (const sensitiveValue of collectCredentialShapeSensitiveValues(value)) {
+                values.add(sensitiveValue);
+            }
+            const trimmed = value.trim();
+            const looksStructured = (trimmed.startsWith('{') && trimmed.endsWith('}'))
+                || (trimmed.startsWith('[') && trimmed.endsWith(']'));
+            if (looksStructured && (parsedJsonChars += value.length) > MAX_SENSITIVE_JSON_CHARS) {
+                throw new Error('Sensitive-value scan exceeded the nested JSON limit');
+            }
+            const structured = looksStructured ? parseStructuredJson(value) : undefined;
+            if (structured !== undefined)
+                pending.push({ value: structured, key });
+            continue;
+        }
+        if ((typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint')
+            && !isNumericTokenTelemetry(key, value) && isSecretKey(key)) {
+            values.add(String(value));
+            continue;
         }
         if (Array.isArray(value)) {
             for (const entry of value)
-                visit(entry, key);
-            return;
+                pending.push({ value: entry, key });
+            continue;
         }
         if (isRecord(value)) {
-            for (const [entryKey, entryValue] of Object.entries(value))
-                visit(entryValue, entryKey);
+            for (const [entryKey, entryValue] of Object.entries(value)) {
+                pending.push({ value: entryValue, key: entryKey });
+            }
         }
-    };
-    visit(source, '');
+    }
     return [...values];
 }
-function sanitizeValue(value, key, sensitiveValues, explicitSensitiveValues) {
+function sanitizeValue(value, key, sensitiveValues, explicitSensitiveValues, depth = 0) {
     if (!isNumericTokenTelemetry(key, value) && isSecretKey(key))
         return '<redacted>';
     if (typeof value === 'string') {
-        return redactCredentialShapes(redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues), sensitiveValues, explicitSensitiveValues);
+        return redactCredentialShapes(redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues), sensitiveValues, explicitSensitiveValues, depth);
     }
+    if (depth >= MAX_SANITIZE_DEPTH && (Array.isArray(value) || isRecord(value)))
+        return '<redacted>';
     if (Array.isArray(value)) {
-        return value.map((entry) => sanitizeValue(entry, key, sensitiveValues, explicitSensitiveValues));
+        return value.map((entry) => sanitizeValue(entry, key, sensitiveValues, explicitSensitiveValues, depth + 1));
     }
     if (isRecord(value)) {
         return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [
             entryKey,
-            sanitizeValue(entryValue, entryKey, sensitiveValues, explicitSensitiveValues),
+            sanitizeValue(entryValue, entryKey, sensitiveValues, explicitSensitiveValues, depth + 1),
         ]));
     }
     return value;
 }
-function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues) {
+function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues, depth) {
     let redacted = value;
+    const credentialShapeValues = collectCredentialShapeSensitiveValues(redacted);
+    redacted = redactSensitiveValues(redacted, credentialShapeValues, new Set(credentialShapeValues));
     const structured = parseStructuredJson(redacted);
     if (structured !== undefined) {
         const nestedSensitiveValues = [...new Set([
                 ...sensitiveValues,
                 ...collectStructuredSensitiveValues(structured),
             ])].sort((a, b) => b.length - a.length);
-        redacted = JSON.stringify(sanitizeValue(structured, '', nestedSensitiveValues, explicitSensitiveValues));
+        redacted = JSON.stringify(sanitizeValue(structured, '', nestedSensitiveValues, explicitSensitiveValues, depth + 1));
     }
     if (redacted.includes('PRIVATE KEY-----')) {
-        redacted = redacted.replace(/-----BEGIN [^-\n]*PRIVATE KEY-----[\s\S]*?-----END [^-\n]*PRIVATE KEY-----/g, '<redacted>');
+        for (const block of findPrivateKeyBlocks(redacted).reverse()) {
+            redacted = `${redacted.slice(0, block.start)}<redacted>${redacted.slice(block.end)}`;
+        }
     }
     if (/\b(?:bearer|basic)\s/i.test(redacted)) {
         redacted = redacted.replace(/(\b(?:bearer|basic)\s+)[^\s,;"']+/gi, '$1<redacted>');
@@ -154,10 +223,71 @@ function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues)
         redacted = redacted.replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s/@]+(@)/gi, '$1<redacted>$2');
     }
     if (redacted.includes('=') || redacted.includes(':')) {
-        redacted = redacted.replace(/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)\s*[=:]\s*)[^\s,;"'}]+/gi, '$1<redacted>');
+        redacted = redacted.replace(/((?:^|[\s{[(,;])["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)["']?\s*[=:]\s*["']?)[^\s,;"'}]+/gi, '$1<redacted>');
+        redacted = redacted.replace(/((?:^|[\s{[(,;])["'](?:grant|capability|capabilityurl|cursor)["']\s*:\s*["']?)[^\s,;"'}]+/gi, '$1<redacted>');
     }
     return redacted;
 }
+function collectCredentialShapeSensitiveValues(value) {
+    const values = new Set();
+    for (const match of value.matchAll(/https?:\/\/[^\s,;"']+\/(?:cap|capability)\/[^\s,;"']+/gi)) {
+        values.add(match[0]);
+    }
+    if (value.includes('PRIVATE KEY-----')) {
+        for (const block of findPrivateKeyBlocks(value)) {
+            values.add(value.slice(block.start, block.end));
+            if (block.body)
+                values.add(block.body);
+        }
+    }
+    if (/\b(?:bearer|basic)\s/i.test(value)) {
+        for (const match of value.matchAll(/\b(?:bearer|basic)\s+([^\s,;"']+)/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+    }
+    if (value.includes('://')) {
+        for (const match of value.matchAll(/[a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:([^\s/@]+)@/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+    }
+    if (value.includes('=') || value.includes(':')) {
+        for (const match of value.matchAll(/(?:^|[\s{[(,;])["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)["']?\s*[=:]\s*["']?([^\s,;"'}]+)/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+        for (const match of value.matchAll(/(?:^|[\s{[(,;])["'](?:grant|capability|capabilityurl|cursor)["']\s*:\s*["']?([^\s,;"'}]+)/gi)) {
+            if (match[1])
+                values.add(match[1]);
+        }
+    }
+    return [...values].sort((a, b) => b.length - a.length);
+}
+function findPrivateKeyBlocks(value) {
+    const blocks = [];
+    let open;
+    let cursor = 0;
+    while (cursor < value.length) {
+        const start = value.indexOf('-----', cursor);
+        if (start < 0)
+            break;
+        const end = value.indexOf('-----', start + 5);
+        if (end < 0)
+            break;
+        const markerEnd = end + 5;
+        const marker = value.slice(start, markerEnd);
+        if (!open && marker.startsWith('-----BEGIN ') && marker.endsWith('PRIVATE KEY-----')) {
+            open = { start, bodyStart: markerEnd };
+        }
+        else if (open && marker.startsWith('-----END ') && marker.endsWith('PRIVATE KEY-----')) {
+            blocks.push({ start: open.start, end: markerEnd, body: value.slice(open.bodyStart, start).trim() });
+            open = undefined;
+        }
+        cursor = markerEnd;
+    }
+    return blocks;
+}
 function isSecretKey(key) {
     const normalized = key.replace(/[^a-z0-9]/gi, '').toLowerCase();
     if (normalized === 'tokens')

package/dist/types.d.ts

@@ -172,6 +172,8 @@ export interface TrialResult {
     scoring_duration_ms?: number;
     recorded_at?: string;
     agent?: import('./sdk/types.js').AgentExecutionMetadata;
+    flow_summary?: import('./sdk/types.js').AgentFlowSummary;
+    flow_trace?: import('./sdk/types.js').AgentFlowTrace;
     conversation?: {
         turns: ConversationTurn[];
         total_turns: number;
@@ -183,6 +185,10 @@ export interface EvalReport {
     task: string;
     /** Canonical status for this report or consolidated group when available. */
     status?: 'pass' | 'fail';
+    /** Whether every reportable runner attempt passed. */
+    runner_status?: 'pass' | 'fail';
+    /** Whether mean reward meets the configured threshold. */
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
     /** Schema-v2 canonical average score. */
     mean_reward?: number;
     /** Binary success frequency, absent when attempts are ineligible. */
@@ -211,12 +217,12 @@ export interface ComparisonContract {
     unavailable_reasons?: ComparisonUnavailableReason[];
 }
 /**
- * TrialResult with `session_log` and `conversation` stripped. These fields
+ * TrialResult with large trace fields stripped. These fields
  * live only in the per-group trace files; the consolidated results.json keeps
  * the rest so consumers (preview, `pathgrade report`) can compute summaries
  * without loading trace data.
  */
-export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversation'>;
+export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversation' | 'flow_trace'>;
 /**
  * Per-group entry in the consolidated `.pathgrade/results.json` report.
  */
@@ -264,11 +270,11 @@ export interface PathgradeReport {
     overall_mean_reward?: number;
     attempts_requested?: number;
     attempts_completed?: number;
-    /**
-     * Threshold check result. When `threshold` is set: `'pass'` iff
-     * `overall_pass_rate >= threshold`. Otherwise: `'pass'` iff every trial
-     * in every group passed its vitest test.
-     */
+    /** Whether every reportable runner attempt passed. */
+    runner_status?: 'pass' | 'fail';
+    /** Whether overall mean reward meets the configured threshold. */
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
+    /** Conjunction of runner assertions and the optional aggregate threshold. */
     status: 'pass' | 'fail';
     /** Agent Evals compatibility manifest when changed selection finds no runnable evals. */
     run_kind?: 'evaluation' | 'no-affected';

package/docs/agent-flow-evaluation.md

@@ -0,0 +1,31 @@
+# Evaluating agent flows
+
+Use `evaluateFlow()` when the subject is an already-observed interaction graph rather than a Pathgrade-managed coding agent. The input is protocol-neutral, so an adapter can project A2A or another orchestration protocol into the same evidence contract.
+
+```typescript
+import { check, evaluateFlow, type AgentFlowTrace } from '@wix/pathgrade';
+
+const flow: AgentFlowTrace = {
+    version: 1,
+    protocol: 'a2a',
+    rootParticipantId: 'planner',
+    participants: [{ id: 'planner' }, { id: 'flights' }],
+    interactions: [{
+        id: 'delegate-1', sequence: 1,
+        sourceParticipantId: 'planner', targetParticipantId: 'flights',
+        operation: 'message/send', state: 'completed',
+    }],
+    outcome: { state: 'completed' },
+    completeness: {
+        topology: 'complete', outcomes: 'partial', timing: 'unavailable',
+        usage: 'unavailable', runtimeIdentity: 'partial',
+    },
+};
+
+await evaluateFlow(flow, [
+    check('delegated flight search', ({ flow }) =>
+        flow?.interactions.some(item => item.targetParticipantId === 'flights') === true),
+]);
+```
+
+`check()`, `score()`, and `judge()` work with flow evidence through `ScorerContext.flow`. `toolUsage()` is agent-only and is rejected explicitly for flows. Consolidated reports retain a compact `flow_summary`; the full `flow_trace` stays in the trace artifact. Completeness fields describe evidence availability and must not be read as inferred downstream success or timing.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.37",
+  "version": "1.0.39",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -64,6 +64,7 @@
     "!dist/**/*.d.ts.map",
     "bin/",
     "docs/OPENAI_OAUTH_JUDGE.md",
+    "docs/agent-flow-evaluation.md",
     "templates/",
     "README.md"
   ],
@@ -141,5 +142,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "f09fc542f60a4327a1e66eed12cb32899949f10edc82ae397902743f"
+  "falconPackageHash": "df08d3b446a37c3224730fca89a408f6238eef420358705733692eee"
 }