agent-inspect 6.26.0 → 6.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +1 -1
- package/docs/LIMITATIONS.md +1 -1
- package/docs/TRACE-CONTRACTS.md +21 -3
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-HPR2OJPU.mjs → chunk-WQITHDXH.mjs} +2 -2
- package/packages/cli/dist/{chunk-HPR2OJPU.mjs.map → chunk-WQITHDXH.mjs.map} +1 -1
- package/packages/cli/dist/index.cjs +15 -4
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-J43ECFHC.mjs → src-T2ATN6XF.mjs} +3 -3
- package/packages/cli/dist/{src-J43ECFHC.mjs.map → src-T2ATN6XF.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +537 -152
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +64 -2
- package/packages/core/dist/checks.d.ts +64 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-LHSXRIYU.mjs → chunk-B6PH7A24.mjs} +390 -5
- package/packages/core/dist/chunk-B6PH7A24.mjs.map +1 -0
- package/packages/core/dist/chunk-LHSXRIYU.mjs.map +0 -1
|
@@ -272,6 +272,61 @@ interface TraceContractControlRules {
|
|
|
272
272
|
}>;
|
|
273
273
|
}
|
|
274
274
|
|
|
275
|
+
/**
|
|
276
|
+
* Bounded safe recovery operation contracts (6.27).
|
|
277
|
+
*
|
|
278
|
+
* Additive `retry.operations[]` rules for read-first recovery oracles.
|
|
279
|
+
* AgentInspect evaluates traces; it does not perform retries.
|
|
280
|
+
*
|
|
281
|
+
* @experimental
|
|
282
|
+
*/
|
|
283
|
+
|
|
284
|
+
/** How same-argument evidence may be compared across attempts. */
|
|
285
|
+
type RecoverySameArgumentsMode = "structured-or-digest";
|
|
286
|
+
/**
|
|
287
|
+
* Side-effect class for conservative write recovery semantics.
|
|
288
|
+
* Read-only tools may recover without write idempotency proof.
|
|
289
|
+
* Write tools treat timeout/unknown completion as fail/unevaluable
|
|
290
|
+
* unless authoritative idempotency evidence is present.
|
|
291
|
+
*/
|
|
292
|
+
type RecoverySideEffectClass = "read" | "write";
|
|
293
|
+
interface TraceContractRecoveryRetryableErrors {
|
|
294
|
+
/** Allowed error codes on failed attempts that precede a retry. */
|
|
295
|
+
codes?: readonly string[];
|
|
296
|
+
}
|
|
297
|
+
interface TraceContractRecoverySuccessfulResultDependency {
|
|
298
|
+
/** Consumer kind that must observe the successful tool result. */
|
|
299
|
+
consumerKind: "LLM";
|
|
300
|
+
/**
|
|
301
|
+
* When true, an LLM event must explicitly reference the successful tool
|
|
302
|
+
* event id (attributes / workflow metadata), not merely follow it in time.
|
|
303
|
+
*/
|
|
304
|
+
requireExplicitReference?: boolean;
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Per-tool bounded recovery oracle (additive under `retry.operations`).
|
|
308
|
+
*
|
|
309
|
+
* @experimental Additive in 6.27.
|
|
310
|
+
*/
|
|
311
|
+
interface TraceContractRecoveryOperation {
|
|
312
|
+
tool: string;
|
|
313
|
+
maxAttempts?: number;
|
|
314
|
+
retryableErrors?: TraceContractRecoveryRetryableErrors;
|
|
315
|
+
requireFailureBeforeRetry?: boolean;
|
|
316
|
+
/**
|
|
317
|
+
* When true or `"structured-or-digest"`, retries must share structured
|
|
318
|
+
* arguments or matching digests. Missing evidence fails closed.
|
|
319
|
+
*/
|
|
320
|
+
requireSameArguments?: boolean | RecoverySameArgumentsMode;
|
|
321
|
+
requireTerminalSuccess?: boolean;
|
|
322
|
+
requireRecoveredFailureVisible?: boolean;
|
|
323
|
+
successfulResultDependency?: TraceContractRecoverySuccessfulResultDependency;
|
|
324
|
+
/**
|
|
325
|
+
* Defaults to `"read"`. Write tools apply conservative timeout/unknown rules.
|
|
326
|
+
*/
|
|
327
|
+
sideEffectClass?: RecoverySideEffectClass;
|
|
328
|
+
}
|
|
329
|
+
|
|
275
330
|
/**
|
|
276
331
|
* Retry / side-effect safety checks for TraceContract (6.23+; corrected in 6.25.1).
|
|
277
332
|
*
|
|
@@ -313,6 +368,12 @@ interface TraceContractRetryRules {
|
|
|
313
368
|
* same operation/retry chain (not merely coexistence of ok+error).
|
|
314
369
|
*/
|
|
315
370
|
requireRecoveredFailureVisible?: boolean;
|
|
371
|
+
/**
|
|
372
|
+
* Per-tool bounded recovery oracles (read-first; write timeout/unknown fails closed).
|
|
373
|
+
*
|
|
374
|
+
* @experimental Additive in 6.27.
|
|
375
|
+
*/
|
|
376
|
+
operations?: readonly TraceContractRecoveryOperation[];
|
|
316
377
|
}
|
|
317
378
|
|
|
318
379
|
/**
|
|
@@ -518,8 +579,9 @@ type TraceContractBody = {
|
|
|
518
579
|
controls?: TraceContractControlRules;
|
|
519
580
|
/**
|
|
520
581
|
* Retry / side-effect safety using explicit attempt identity.
|
|
582
|
+
* Additive `retry.operations[]` recovery oracles land in 6.27.
|
|
521
583
|
*
|
|
522
|
-
* @experimental Additive in 6.23.
|
|
584
|
+
* @experimental Additive in 6.23; operations in 6.27.
|
|
523
585
|
*/
|
|
524
586
|
retry?: TraceContractRetryRules;
|
|
525
587
|
};
|
|
@@ -590,4 +652,4 @@ declare function lintTraceContract(contract: TraceContract): TraceContractLintDi
|
|
|
590
652
|
*/
|
|
591
653
|
declare function explainTraceContract(contract: TraceContract): string[];
|
|
592
654
|
|
|
593
|
-
export { type ControlStage, type DerivedFailureConfidence, type DerivedFailureFact, type DerivedFailureRole, type FailureRoleCounts, LogicalProjectionDiagnostic, LogicalTraceEvent, type SemanticParitySummary, type ToolArgumentCheck, type ToolArgumentOccurrence, type ToolArgumentOperator, TraceCheckDiagnostic, TraceCheckInput, TraceCheckResult, type TraceContract, type TraceContractAlternativeBranch, type TraceContractAlternatives, type TraceContractBody, type TraceContractControlRules, type TraceContractInput, type TraceContractLintDiagnostic, type TraceContractLlmRules, type TraceContractObservationProvenance, type TraceContractObservationRules, type TraceContractRetryRules, type TraceContractRunRules, type TraceContractScope, type TraceContractToolRules, type TraceFacts, type TraceRelationship, type TraceRelationshipConfidence, type TraceRelationshipDiagnostic, type TraceRelationshipType, buildTraceFacts, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, resolveJsonPointer, resolveTraceContractScope, summarizeSemanticParity, workflowMetadataForRun };
|
|
655
|
+
export { type ControlStage, type DerivedFailureConfidence, type DerivedFailureFact, type DerivedFailureRole, type FailureRoleCounts, LogicalProjectionDiagnostic, LogicalTraceEvent, type RecoverySameArgumentsMode, type RecoverySideEffectClass, type SemanticParitySummary, type ToolArgumentCheck, type ToolArgumentOccurrence, type ToolArgumentOperator, TraceCheckDiagnostic, TraceCheckInput, TraceCheckResult, type TraceContract, type TraceContractAlternativeBranch, type TraceContractAlternatives, type TraceContractBody, type TraceContractControlRules, type TraceContractInput, type TraceContractLintDiagnostic, type TraceContractLlmRules, type TraceContractObservationProvenance, type TraceContractObservationRules, type TraceContractRecoveryOperation, type TraceContractRecoveryRetryableErrors, type TraceContractRecoverySuccessfulResultDependency, type TraceContractRetryRules, type TraceContractRunRules, type TraceContractScope, type TraceContractToolRules, type TraceFacts, type TraceRelationship, type TraceRelationshipConfidence, type TraceRelationshipDiagnostic, type TraceRelationshipType, buildTraceFacts, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, resolveJsonPointer, resolveTraceContractScope, summarizeSemanticParity, workflowMetadataForRun };
|
|
@@ -272,6 +272,61 @@ interface TraceContractControlRules {
|
|
|
272
272
|
}>;
|
|
273
273
|
}
|
|
274
274
|
|
|
275
|
+
/**
|
|
276
|
+
* Bounded safe recovery operation contracts (6.27).
|
|
277
|
+
*
|
|
278
|
+
* Additive `retry.operations[]` rules for read-first recovery oracles.
|
|
279
|
+
* AgentInspect evaluates traces; it does not perform retries.
|
|
280
|
+
*
|
|
281
|
+
* @experimental
|
|
282
|
+
*/
|
|
283
|
+
|
|
284
|
+
/** How same-argument evidence may be compared across attempts. */
|
|
285
|
+
type RecoverySameArgumentsMode = "structured-or-digest";
|
|
286
|
+
/**
|
|
287
|
+
* Side-effect class for conservative write recovery semantics.
|
|
288
|
+
* Read-only tools may recover without write idempotency proof.
|
|
289
|
+
* Write tools treat timeout/unknown completion as fail/unevaluable
|
|
290
|
+
* unless authoritative idempotency evidence is present.
|
|
291
|
+
*/
|
|
292
|
+
type RecoverySideEffectClass = "read" | "write";
|
|
293
|
+
interface TraceContractRecoveryRetryableErrors {
|
|
294
|
+
/** Allowed error codes on failed attempts that precede a retry. */
|
|
295
|
+
codes?: readonly string[];
|
|
296
|
+
}
|
|
297
|
+
interface TraceContractRecoverySuccessfulResultDependency {
|
|
298
|
+
/** Consumer kind that must observe the successful tool result. */
|
|
299
|
+
consumerKind: "LLM";
|
|
300
|
+
/**
|
|
301
|
+
* When true, an LLM event must explicitly reference the successful tool
|
|
302
|
+
* event id (attributes / workflow metadata), not merely follow it in time.
|
|
303
|
+
*/
|
|
304
|
+
requireExplicitReference?: boolean;
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Per-tool bounded recovery oracle (additive under `retry.operations`).
|
|
308
|
+
*
|
|
309
|
+
* @experimental Additive in 6.27.
|
|
310
|
+
*/
|
|
311
|
+
interface TraceContractRecoveryOperation {
|
|
312
|
+
tool: string;
|
|
313
|
+
maxAttempts?: number;
|
|
314
|
+
retryableErrors?: TraceContractRecoveryRetryableErrors;
|
|
315
|
+
requireFailureBeforeRetry?: boolean;
|
|
316
|
+
/**
|
|
317
|
+
* When true or `"structured-or-digest"`, retries must share structured
|
|
318
|
+
* arguments or matching digests. Missing evidence fails closed.
|
|
319
|
+
*/
|
|
320
|
+
requireSameArguments?: boolean | RecoverySameArgumentsMode;
|
|
321
|
+
requireTerminalSuccess?: boolean;
|
|
322
|
+
requireRecoveredFailureVisible?: boolean;
|
|
323
|
+
successfulResultDependency?: TraceContractRecoverySuccessfulResultDependency;
|
|
324
|
+
/**
|
|
325
|
+
* Defaults to `"read"`. Write tools apply conservative timeout/unknown rules.
|
|
326
|
+
*/
|
|
327
|
+
sideEffectClass?: RecoverySideEffectClass;
|
|
328
|
+
}
|
|
329
|
+
|
|
275
330
|
/**
|
|
276
331
|
* Retry / side-effect safety checks for TraceContract (6.23+; corrected in 6.25.1).
|
|
277
332
|
*
|
|
@@ -313,6 +368,12 @@ interface TraceContractRetryRules {
|
|
|
313
368
|
* same operation/retry chain (not merely coexistence of ok+error).
|
|
314
369
|
*/
|
|
315
370
|
requireRecoveredFailureVisible?: boolean;
|
|
371
|
+
/**
|
|
372
|
+
* Per-tool bounded recovery oracles (read-first; write timeout/unknown fails closed).
|
|
373
|
+
*
|
|
374
|
+
* @experimental Additive in 6.27.
|
|
375
|
+
*/
|
|
376
|
+
operations?: readonly TraceContractRecoveryOperation[];
|
|
316
377
|
}
|
|
317
378
|
|
|
318
379
|
/**
|
|
@@ -518,8 +579,9 @@ type TraceContractBody = {
|
|
|
518
579
|
controls?: TraceContractControlRules;
|
|
519
580
|
/**
|
|
520
581
|
* Retry / side-effect safety using explicit attempt identity.
|
|
582
|
+
* Additive `retry.operations[]` recovery oracles land in 6.27.
|
|
521
583
|
*
|
|
522
|
-
* @experimental Additive in 6.23.
|
|
584
|
+
* @experimental Additive in 6.23; operations in 6.27.
|
|
523
585
|
*/
|
|
524
586
|
retry?: TraceContractRetryRules;
|
|
525
587
|
};
|
|
@@ -590,4 +652,4 @@ declare function lintTraceContract(contract: TraceContract): TraceContractLintDi
|
|
|
590
652
|
*/
|
|
591
653
|
declare function explainTraceContract(contract: TraceContract): string[];
|
|
592
654
|
|
|
593
|
-
export { type ControlStage, type DerivedFailureConfidence, type DerivedFailureFact, type DerivedFailureRole, type FailureRoleCounts, LogicalProjectionDiagnostic, LogicalTraceEvent, type SemanticParitySummary, type ToolArgumentCheck, type ToolArgumentOccurrence, type ToolArgumentOperator, TraceCheckDiagnostic, TraceCheckInput, TraceCheckResult, type TraceContract, type TraceContractAlternativeBranch, type TraceContractAlternatives, type TraceContractBody, type TraceContractControlRules, type TraceContractInput, type TraceContractLintDiagnostic, type TraceContractLlmRules, type TraceContractObservationProvenance, type TraceContractObservationRules, type TraceContractRetryRules, type TraceContractRunRules, type TraceContractScope, type TraceContractToolRules, type TraceFacts, type TraceRelationship, type TraceRelationshipConfidence, type TraceRelationshipDiagnostic, type TraceRelationshipType, buildTraceFacts, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, resolveJsonPointer, resolveTraceContractScope, summarizeSemanticParity, workflowMetadataForRun };
|
|
655
|
+
export { type ControlStage, type DerivedFailureConfidence, type DerivedFailureFact, type DerivedFailureRole, type FailureRoleCounts, LogicalProjectionDiagnostic, LogicalTraceEvent, type RecoverySameArgumentsMode, type RecoverySideEffectClass, type SemanticParitySummary, type ToolArgumentCheck, type ToolArgumentOccurrence, type ToolArgumentOperator, TraceCheckDiagnostic, TraceCheckInput, TraceCheckResult, type TraceContract, type TraceContractAlternativeBranch, type TraceContractAlternatives, type TraceContractBody, type TraceContractControlRules, type TraceContractInput, type TraceContractLintDiagnostic, type TraceContractLlmRules, type TraceContractObservationProvenance, type TraceContractObservationRules, type TraceContractRecoveryOperation, type TraceContractRecoveryRetryableErrors, type TraceContractRecoverySuccessfulResultDependency, type TraceContractRetryRules, type TraceContractRunRules, type TraceContractScope, type TraceContractToolRules, type TraceFacts, type TraceRelationship, type TraceRelationshipConfidence, type TraceRelationshipDiagnostic, type TraceRelationshipType, buildTraceFacts, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, resolveJsonPointer, resolveTraceContractScope, summarizeSemanticParity, workflowMetadataForRun };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, summarizeSemanticParity, workflowMetadataForRun } from './chunk-
|
|
1
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, summarizeSemanticParity, workflowMetadataForRun } from './chunk-B6PH7A24.mjs';
|
|
2
2
|
import './chunk-R4SSYU6D.mjs';
|
|
3
3
|
import './chunk-U25RHRSU.mjs';
|
|
4
4
|
import './chunk-VU6O5QAH.mjs';
|
|
@@ -3750,6 +3750,368 @@ function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
|
3750
3750
|
return findings;
|
|
3751
3751
|
}
|
|
3752
3752
|
|
|
3753
|
+
// packages/core/src/checks/recovery-operations.ts
|
|
3754
|
+
function fail3(ruleId, message, evidence, expected, actual) {
|
|
3755
|
+
return {
|
|
3756
|
+
ruleId,
|
|
3757
|
+
severity: "error",
|
|
3758
|
+
status: "fail",
|
|
3759
|
+
message,
|
|
3760
|
+
...expected !== void 0 ? { expected } : {},
|
|
3761
|
+
...actual !== void 0 ? { actual } : {},
|
|
3762
|
+
evidence: [...evidence]
|
|
3763
|
+
};
|
|
3764
|
+
}
|
|
3765
|
+
function workflowFor2(event) {
|
|
3766
|
+
const attrs2 = event.attributes;
|
|
3767
|
+
if (!attrs2 || typeof attrs2 !== "object") return {};
|
|
3768
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
3769
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
3770
|
+
return { ...nested, ...direct };
|
|
3771
|
+
}
|
|
3772
|
+
function eventEvidence3(event) {
|
|
3773
|
+
return {
|
|
3774
|
+
runId: event.runId,
|
|
3775
|
+
eventId: event.eventId,
|
|
3776
|
+
kind: event.kind,
|
|
3777
|
+
name: event.name,
|
|
3778
|
+
status: event.status
|
|
3779
|
+
};
|
|
3780
|
+
}
|
|
3781
|
+
function eventTime2(event) {
|
|
3782
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
3783
|
+
}
|
|
3784
|
+
function sortByTime2(events) {
|
|
3785
|
+
return [...events].sort((a, b) => eventTime2(a).localeCompare(eventTime2(b)));
|
|
3786
|
+
}
|
|
3787
|
+
function hasIdempotencyEvidence2(event) {
|
|
3788
|
+
const workflow = workflowFor2(event);
|
|
3789
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
3790
|
+
return true;
|
|
3791
|
+
}
|
|
3792
|
+
const attrs2 = event.attributes ?? {};
|
|
3793
|
+
if (attrs2.noSideEffect === true) return true;
|
|
3794
|
+
if (attrs2.sideEffect === false) return true;
|
|
3795
|
+
return false;
|
|
3796
|
+
}
|
|
3797
|
+
function errorCodeOf(event) {
|
|
3798
|
+
if (typeof event.error?.code === "string" && event.error.code.trim() !== "") {
|
|
3799
|
+
return event.error.code.trim();
|
|
3800
|
+
}
|
|
3801
|
+
const attrs2 = event.attributes ?? {};
|
|
3802
|
+
for (const key of ["errorCode", "code"]) {
|
|
3803
|
+
const value = attrs2[key];
|
|
3804
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
3805
|
+
}
|
|
3806
|
+
const nested = attrs2.error;
|
|
3807
|
+
if (nested && typeof nested === "object") {
|
|
3808
|
+
const code = nested.code;
|
|
3809
|
+
if (typeof code === "string" && code.trim() !== "") return code.trim();
|
|
3810
|
+
}
|
|
3811
|
+
return void 0;
|
|
3812
|
+
}
|
|
3813
|
+
function argumentDigestOf(event) {
|
|
3814
|
+
const attrs2 = event.attributes ?? {};
|
|
3815
|
+
for (const key of ["argumentsDigest", "inputDigest", "toolArgumentsDigest"]) {
|
|
3816
|
+
const value = attrs2[key];
|
|
3817
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
3818
|
+
}
|
|
3819
|
+
const commitment = attrs2.omittedPayloadCommitment;
|
|
3820
|
+
if (commitment && typeof commitment === "object") {
|
|
3821
|
+
const digest = commitment.digest;
|
|
3822
|
+
if (typeof digest === "string" && digest.trim() !== "") return digest.trim();
|
|
3823
|
+
}
|
|
3824
|
+
return void 0;
|
|
3825
|
+
}
|
|
3826
|
+
function argumentFingerprint(event) {
|
|
3827
|
+
const payload = extractToolArgumentPayload(event);
|
|
3828
|
+
if (payload.present) {
|
|
3829
|
+
try {
|
|
3830
|
+
return { kind: "structured", value: JSON.stringify(payload.value) };
|
|
3831
|
+
} catch {
|
|
3832
|
+
return { kind: "missing" };
|
|
3833
|
+
}
|
|
3834
|
+
}
|
|
3835
|
+
const digest = argumentDigestOf(event);
|
|
3836
|
+
if (digest) return { kind: "digest", value: digest };
|
|
3837
|
+
return { kind: "missing" };
|
|
3838
|
+
}
|
|
3839
|
+
function sameArguments(left, right) {
|
|
3840
|
+
const a = argumentFingerprint(left);
|
|
3841
|
+
const b = argumentFingerprint(right);
|
|
3842
|
+
if (a.kind === "missing" || b.kind === "missing") {
|
|
3843
|
+
return {
|
|
3844
|
+
ok: false,
|
|
3845
|
+
reason: "Structured or digest argument evidence unavailable for same-arguments check.",
|
|
3846
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE"
|
|
3847
|
+
};
|
|
3848
|
+
}
|
|
3849
|
+
if (a.kind !== b.kind) {
|
|
3850
|
+
return {
|
|
3851
|
+
ok: false,
|
|
3852
|
+
reason: "Argument evidence kinds differ across attempts (structured vs digest).",
|
|
3853
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_MISMATCH"
|
|
3854
|
+
};
|
|
3855
|
+
}
|
|
3856
|
+
if (a.value !== b.value) {
|
|
3857
|
+
return {
|
|
3858
|
+
ok: false,
|
|
3859
|
+
reason: "Retry arguments do not match prior attempt.",
|
|
3860
|
+
code: "AI_CHECK_RECOVERY_ARGUMENTS_DIFFER"
|
|
3861
|
+
};
|
|
3862
|
+
}
|
|
3863
|
+
return { ok: true };
|
|
3864
|
+
}
|
|
3865
|
+
function isUnevaluableWriteCompletion(event) {
|
|
3866
|
+
if (event.status === "running" || event.status === "unknown") return true;
|
|
3867
|
+
const attrs2 = event.attributes ?? {};
|
|
3868
|
+
if (attrs2.timeout === true) return true;
|
|
3869
|
+
if (attrs2.completionState === "unknown" || attrs2.completionState === "timeout") return true;
|
|
3870
|
+
const code = errorCodeOf(event)?.toUpperCase();
|
|
3871
|
+
if (code === "TIMEOUT" || code === "UNKNOWN_COMPLETION" || code === "DEADLINE_EXCEEDED") {
|
|
3872
|
+
return true;
|
|
3873
|
+
}
|
|
3874
|
+
return false;
|
|
3875
|
+
}
|
|
3876
|
+
function collectExplicitReferences(event) {
|
|
3877
|
+
const refs = /* @__PURE__ */ new Set();
|
|
3878
|
+
const attrs2 = event.attributes ?? {};
|
|
3879
|
+
const candidates = [
|
|
3880
|
+
attrs2.referencedEventId,
|
|
3881
|
+
attrs2.referencedEventIds,
|
|
3882
|
+
attrs2.toolResultEventId,
|
|
3883
|
+
attrs2.toolResultEventIds,
|
|
3884
|
+
attrs2.inputEventIds,
|
|
3885
|
+
attrs2.dependsOnEventIds,
|
|
3886
|
+
attrs2.evidence,
|
|
3887
|
+
workflowFor2(event).toolCallId
|
|
3888
|
+
];
|
|
3889
|
+
for (const candidate of candidates) {
|
|
3890
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
3891
|
+
refs.add(candidate.trim());
|
|
3892
|
+
continue;
|
|
3893
|
+
}
|
|
3894
|
+
if (Array.isArray(candidate)) {
|
|
3895
|
+
for (const item of candidate) {
|
|
3896
|
+
if (typeof item === "string" && item.trim() !== "") refs.add(item.trim());
|
|
3897
|
+
if (item && typeof item === "object") {
|
|
3898
|
+
const id = item.eventId;
|
|
3899
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
3900
|
+
}
|
|
3901
|
+
}
|
|
3902
|
+
continue;
|
|
3903
|
+
}
|
|
3904
|
+
if (candidate && typeof candidate === "object") {
|
|
3905
|
+
const obj = candidate;
|
|
3906
|
+
if (typeof obj.eventId === "string" && obj.eventId.trim() !== "") {
|
|
3907
|
+
refs.add(obj.eventId.trim());
|
|
3908
|
+
}
|
|
3909
|
+
if (Array.isArray(obj.eventIds)) {
|
|
3910
|
+
for (const id of obj.eventIds) {
|
|
3911
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
3912
|
+
}
|
|
3913
|
+
}
|
|
3914
|
+
}
|
|
3915
|
+
}
|
|
3916
|
+
return refs;
|
|
3917
|
+
}
|
|
3918
|
+
function llmReferencesTool(llm, toolEvent, requireExplicit) {
|
|
3919
|
+
if (!requireExplicit) {
|
|
3920
|
+
return eventTime2(llm) >= eventTime2(toolEvent);
|
|
3921
|
+
}
|
|
3922
|
+
const refs = collectExplicitReferences(llm);
|
|
3923
|
+
if (refs.has(toolEvent.eventId)) return true;
|
|
3924
|
+
const toolCallId = workflowFor2(toolEvent).toolCallId;
|
|
3925
|
+
if (toolCallId && refs.has(toolCallId)) return true;
|
|
3926
|
+
return false;
|
|
3927
|
+
}
|
|
3928
|
+
function groupToolAttempts(events, toolName2) {
|
|
3929
|
+
const groups = /* @__PURE__ */ new Map();
|
|
3930
|
+
for (const event of events) {
|
|
3931
|
+
if (event.kind !== "TOOL") continue;
|
|
3932
|
+
if (resolveCanonicalToolName(event) !== toolName2) continue;
|
|
3933
|
+
const workflow = workflowFor2(event);
|
|
3934
|
+
const key = workflow.operationId ?? `__tool__:${toolName2}`;
|
|
3935
|
+
const list = groups.get(key) ?? [];
|
|
3936
|
+
list.push(event);
|
|
3937
|
+
groups.set(key, list);
|
|
3938
|
+
}
|
|
3939
|
+
return groups;
|
|
3940
|
+
}
|
|
3941
|
+
function evaluateRecoveryOperations(events, operations) {
|
|
3942
|
+
const findings = [];
|
|
3943
|
+
const llmEvents = events.filter((event) => event.kind === "LLM");
|
|
3944
|
+
for (const operation of operations) {
|
|
3945
|
+
const sideEffectClass = operation.sideEffectClass ?? "read";
|
|
3946
|
+
const groups = groupToolAttempts(events, operation.tool);
|
|
3947
|
+
if (groups.size === 0) {
|
|
3948
|
+
if (operation.requireTerminalSuccess) {
|
|
3949
|
+
findings.push(
|
|
3950
|
+
fail3(
|
|
3951
|
+
"contract.retry.operations.terminal-success",
|
|
3952
|
+
`Recovery operation for tool ${operation.tool} requires a terminal success but no attempts were observed.`,
|
|
3953
|
+
[],
|
|
3954
|
+
"ok",
|
|
3955
|
+
"missing"
|
|
3956
|
+
)
|
|
3957
|
+
);
|
|
3958
|
+
}
|
|
3959
|
+
continue;
|
|
3960
|
+
}
|
|
3961
|
+
for (const [operationKey, members] of groups) {
|
|
3962
|
+
const ordered = sortByTime2(members);
|
|
3963
|
+
if (operation.maxAttempts !== void 0) {
|
|
3964
|
+
const count = countOperationAttempts(members);
|
|
3965
|
+
if (count > operation.maxAttempts) {
|
|
3966
|
+
findings.push(
|
|
3967
|
+
fail3(
|
|
3968
|
+
"contract.retry.operations.max-attempts",
|
|
3969
|
+
`Tool ${operation.tool} operation ${operationKey} exceeded maxAttempts ${operation.maxAttempts}.`,
|
|
3970
|
+
members.slice(0, 4).map(eventEvidence3),
|
|
3971
|
+
operation.maxAttempts,
|
|
3972
|
+
count
|
|
3973
|
+
)
|
|
3974
|
+
);
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
if (sideEffectClass === "write") {
|
|
3978
|
+
for (const event of ordered) {
|
|
3979
|
+
if (isUnevaluableWriteCompletion(event) && !hasIdempotencyEvidence2(event)) {
|
|
3980
|
+
findings.push(
|
|
3981
|
+
fail3(
|
|
3982
|
+
"contract.retry.operations.write-completion-unevaluable",
|
|
3983
|
+
`Write tool ${operation.tool} has timeout/unknown completion without authoritative idempotency evidence.`,
|
|
3984
|
+
[eventEvidence3(event)],
|
|
3985
|
+
"idempotencyKey|authoritative completion",
|
|
3986
|
+
{
|
|
3987
|
+
code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE",
|
|
3988
|
+
status: event.status
|
|
3989
|
+
}
|
|
3990
|
+
)
|
|
3991
|
+
);
|
|
3992
|
+
}
|
|
3993
|
+
}
|
|
3994
|
+
}
|
|
3995
|
+
const prior = [];
|
|
3996
|
+
let sawEarlierError = false;
|
|
3997
|
+
let sawLaterOkAfterError = false;
|
|
3998
|
+
let lastOk;
|
|
3999
|
+
for (const event of ordered) {
|
|
4000
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
4001
|
+
const isRetry = classification.kind === "retry";
|
|
4002
|
+
if (isRetry && operation.requireFailureBeforeRetry) {
|
|
4003
|
+
const earlierFailure = prior.some(
|
|
4004
|
+
(candidate) => candidate.status === "error" && eventTime2(candidate) !== "" && eventTime2(event) !== "" && eventTime2(candidate) < eventTime2(event)
|
|
4005
|
+
);
|
|
4006
|
+
if (!earlierFailure) {
|
|
4007
|
+
findings.push(
|
|
4008
|
+
fail3(
|
|
4009
|
+
"contract.retry.operations.failure-before-retry",
|
|
4010
|
+
`Retry of tool ${operation.tool} appeared without an earlier failure in the same operation.`,
|
|
4011
|
+
[eventEvidence3(event)],
|
|
4012
|
+
"earlier error attempt",
|
|
4013
|
+
{ operationKey }
|
|
4014
|
+
)
|
|
4015
|
+
);
|
|
4016
|
+
}
|
|
4017
|
+
}
|
|
4018
|
+
if (isRetry && operation.retryableErrors?.codes && operation.retryableErrors.codes.length > 0) {
|
|
4019
|
+
const allowed = new Set(operation.retryableErrors.codes);
|
|
4020
|
+
const priorErrors = prior.filter((candidate) => candidate.status === "error");
|
|
4021
|
+
const lastError = priorErrors[priorErrors.length - 1];
|
|
4022
|
+
if (lastError) {
|
|
4023
|
+
const code = errorCodeOf(lastError);
|
|
4024
|
+
if (!code || !allowed.has(code)) {
|
|
4025
|
+
findings.push(
|
|
4026
|
+
fail3(
|
|
4027
|
+
"contract.retry.operations.retryable-error",
|
|
4028
|
+
`Retry of tool ${operation.tool} followed a non-retryable or missing error code.`,
|
|
4029
|
+
[eventEvidence3(event), eventEvidence3(lastError)],
|
|
4030
|
+
[...allowed],
|
|
4031
|
+
code ?? null
|
|
4032
|
+
)
|
|
4033
|
+
);
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
}
|
|
4037
|
+
if (isRetry && (operation.requireSameArguments === true || operation.requireSameArguments === "structured-or-digest")) {
|
|
4038
|
+
const baseline = prior[0] ?? prior[prior.length - 1];
|
|
4039
|
+
if (baseline) {
|
|
4040
|
+
const comparison = sameArguments(baseline, event);
|
|
4041
|
+
if (!comparison.ok) {
|
|
4042
|
+
findings.push(
|
|
4043
|
+
fail3(
|
|
4044
|
+
"contract.retry.operations.same-arguments",
|
|
4045
|
+
comparison.reason,
|
|
4046
|
+
[eventEvidence3(baseline), eventEvidence3(event)],
|
|
4047
|
+
"matching structured args or digests",
|
|
4048
|
+
{ code: comparison.code }
|
|
4049
|
+
)
|
|
4050
|
+
);
|
|
4051
|
+
}
|
|
4052
|
+
}
|
|
4053
|
+
}
|
|
4054
|
+
if (event.status === "error") {
|
|
4055
|
+
sawEarlierError = true;
|
|
4056
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
4057
|
+
sawLaterOkAfterError = true;
|
|
4058
|
+
}
|
|
4059
|
+
if (event.status === "ok") {
|
|
4060
|
+
lastOk = event;
|
|
4061
|
+
}
|
|
4062
|
+
prior.push(event);
|
|
4063
|
+
}
|
|
4064
|
+
if (operation.requireTerminalSuccess) {
|
|
4065
|
+
const hasOk = ordered.some((event) => event.status === "ok");
|
|
4066
|
+
if (!hasOk) {
|
|
4067
|
+
findings.push(
|
|
4068
|
+
fail3(
|
|
4069
|
+
"contract.retry.operations.terminal-success",
|
|
4070
|
+
`Tool ${operation.tool} operation ${operationKey} has no terminal ok success.`,
|
|
4071
|
+
ordered.slice(0, 4).map(eventEvidence3),
|
|
4072
|
+
"ok",
|
|
4073
|
+
ordered.map((event) => event.status)
|
|
4074
|
+
)
|
|
4075
|
+
);
|
|
4076
|
+
}
|
|
4077
|
+
}
|
|
4078
|
+
if (operation.requireRecoveredFailureVisible) {
|
|
4079
|
+
const hasOk = ordered.some((event) => event.status === "ok");
|
|
4080
|
+
if (ordered.length > 1 && hasOk && !sawLaterOkAfterError) {
|
|
4081
|
+
findings.push(
|
|
4082
|
+
fail3(
|
|
4083
|
+
"contract.retry.operations.recovered-failure-visible",
|
|
4084
|
+
`Tool ${operation.tool} operation ${operationKey} does not show an earlier error before later success.`,
|
|
4085
|
+
ordered.slice(0, 4).map(eventEvidence3),
|
|
4086
|
+
"earlier error then later ok",
|
|
4087
|
+
{ sawEarlierError }
|
|
4088
|
+
)
|
|
4089
|
+
);
|
|
4090
|
+
}
|
|
4091
|
+
}
|
|
4092
|
+
const dependency = operation.successfulResultDependency;
|
|
4093
|
+
if (dependency?.consumerKind === "LLM" && lastOk) {
|
|
4094
|
+
const requireExplicit = dependency.requireExplicitReference === true;
|
|
4095
|
+
const referenced = llmEvents.some(
|
|
4096
|
+
(llmEvent) => llmReferencesTool(llmEvent, lastOk, requireExplicit)
|
|
4097
|
+
);
|
|
4098
|
+
if (!referenced) {
|
|
4099
|
+
findings.push(
|
|
4100
|
+
fail3(
|
|
4101
|
+
"contract.retry.operations.successful-result-dependency",
|
|
4102
|
+
requireExplicit ? `Successful ${operation.tool} result is not explicitly referenced by a later LLM event.` : `Successful ${operation.tool} result has no later LLM consumer.`,
|
|
4103
|
+
[eventEvidence3(lastOk)],
|
|
4104
|
+
"LLM reference to successful tool event",
|
|
4105
|
+
{ code: "AI_CHECK_RECOVERY_RESULT_DEPENDENCY_MISSING" }
|
|
4106
|
+
)
|
|
4107
|
+
);
|
|
4108
|
+
}
|
|
4109
|
+
}
|
|
4110
|
+
}
|
|
4111
|
+
}
|
|
4112
|
+
return findings;
|
|
4113
|
+
}
|
|
4114
|
+
|
|
3753
4115
|
// packages/core/src/checks/contract.ts
|
|
3754
4116
|
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
3755
4117
|
return {
|
|
@@ -3796,7 +4158,23 @@ function cloneBody(body) {
|
|
|
3796
4158
|
...body.retry ? {
|
|
3797
4159
|
retry: {
|
|
3798
4160
|
...body.retry,
|
|
3799
|
-
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {}
|
|
4161
|
+
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {},
|
|
4162
|
+
...body.retry.operations ? {
|
|
4163
|
+
operations: body.retry.operations.map((operation) => ({
|
|
4164
|
+
...operation,
|
|
4165
|
+
...operation.retryableErrors ? {
|
|
4166
|
+
retryableErrors: {
|
|
4167
|
+
...operation.retryableErrors,
|
|
4168
|
+
...operation.retryableErrors.codes ? { codes: [...operation.retryableErrors.codes] } : {}
|
|
4169
|
+
}
|
|
4170
|
+
} : {},
|
|
4171
|
+
...operation.successfulResultDependency ? {
|
|
4172
|
+
successfulResultDependency: {
|
|
4173
|
+
...operation.successfulResultDependency
|
|
4174
|
+
}
|
|
4175
|
+
} : {}
|
|
4176
|
+
}))
|
|
4177
|
+
} : {}
|
|
3800
4178
|
}
|
|
3801
4179
|
} : {}
|
|
3802
4180
|
};
|
|
@@ -4231,7 +4609,11 @@ function contractToRules(contract) {
|
|
|
4231
4609
|
name: context.selectedRun.name
|
|
4232
4610
|
}
|
|
4233
4611
|
] : [];
|
|
4234
|
-
|
|
4612
|
+
const base = evaluateRetrySafetyRules(events, retry);
|
|
4613
|
+
if (!retry.operations || retry.operations.length === 0) {
|
|
4614
|
+
return base;
|
|
4615
|
+
}
|
|
4616
|
+
return [...base, ...evaluateRecoveryOperations(events, retry.operations)];
|
|
4235
4617
|
}
|
|
4236
4618
|
});
|
|
4237
4619
|
}
|
|
@@ -4716,7 +5098,10 @@ function explainTraceContract(contract) {
|
|
|
4716
5098
|
lines.push("Base: declared-versus-enforced control checks enabled.");
|
|
4717
5099
|
}
|
|
4718
5100
|
if (contract.retry) {
|
|
4719
|
-
|
|
5101
|
+
const opCount = contract.retry.operations?.length ?? 0;
|
|
5102
|
+
lines.push(
|
|
5103
|
+
opCount > 0 ? `Base: retry/side-effect safety checks enabled (${opCount} recovery operation oracle(s)).` : "Base: retry/side-effect safety checks enabled."
|
|
5104
|
+
);
|
|
4720
5105
|
}
|
|
4721
5106
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
4722
5107
|
if (branches.length > 0) {
|
|
@@ -4735,5 +5120,5 @@ function explainTraceContract(contract) {
|
|
|
4735
5120
|
}
|
|
4736
5121
|
|
|
4737
5122
|
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
4738
|
-
//# sourceMappingURL=chunk-
|
|
4739
|
-
//# sourceMappingURL=chunk-
|
|
5123
|
+
//# sourceMappingURL=chunk-B6PH7A24.mjs.map
|
|
5124
|
+
//# sourceMappingURL=chunk-B6PH7A24.mjs.map
|