agent-inspect 6.29.6 → 6.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +1 -1
- package/docs/CLI.md +26 -4
- package/docs/TRACE-CONTRACTS.md +25 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-DBRMI7TC.mjs → chunk-Y5Z4BXUO.mjs} +2422 -101
- package/packages/cli/dist/chunk-Y5Z4BXUO.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +10313 -7195
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +866 -59
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-6CET2GW2.mjs → src-WTTH5G33.mjs} +3 -3
- package/packages/cli/dist/{src-6CET2GW2.mjs.map → src-WTTH5G33.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +12 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +172 -43
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-OOX25GTH.mjs → chunk-VEQKLWGJ.mjs} +174 -46
- package/packages/core/dist/chunk-VEQKLWGJ.mjs.map +1 -0
- package/packages/core/dist/{index-7hG_qggp.d.ts → index-CgOAGuM5.d.ts} +111 -1
- package/packages/core/dist/{index-CQJUNPPe.d.cts → index-EQihlEei.d.cts} +111 -1
- package/packages/cli/dist/chunk-DBRMI7TC.mjs.map +0 -1
- package/packages/core/dist/chunk-OOX25GTH.mjs.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-
|
|
2
|
-
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as
|
|
1
|
+
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-EQihlEei.cjs';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as StepOrderingEndpoint, r as StepOrderingKind, s as StepOrderingRuleOptions, t as StructureIncompleteRuleOptions, u as StructureOrphanRuleOptions, v as StructureParallelWidthRuleOptions, w as StructureRelationshipRuleOptions, T as ToolArgumentCheck, x as ToolArgumentOccurrence, y as ToolArgumentOperator, z as ToolFailureRuleOptions, A as ToolOrderingRuleOptions, E as ToolUsageRuleOptions, F as TraceCheckContext, H as TraceCheckDiagnostic, I as TraceCheckDiagnosticCode, J as TraceCheckEvidence, K as TraceCheckFacts, N as TraceCheckFinding, P as TraceCheckFindingStatus, Q as TraceCheckInput, U as TraceCheckResult, V as TraceCheckRule, W as TraceCheckRuleCategory, X as TraceCheckRuleExecution, Y as TraceCheckRuleExecutionStatus, Z as TraceCheckSeverity, _ as TraceCheckStatus, $ as TraceCheckSummary, a0 as TraceContract, a1 as TraceContractAlternativeBranch, a2 as TraceContractAlternatives, a3 as TraceContractBody, a4 as TraceContractControlRules, a5 as TraceContractInput, a6 as TraceContractLintDiagnostic, a7 as TraceContractLlmRules, a8 as TraceContractObservationProvenance, a9 as TraceContractObservationRules, aa as TraceContractRecoveryOperation, ab as TraceContractRecoveryRetryableErrors, ac as TraceContractRecoverySuccessfulResultDependency, ad as TraceContractRetryRules, ae as TraceContractRunRules, af as TraceContractScope, ag as TraceContractStepKind, ah as TraceContractStepOrderRelation, ai as TraceContractStepRef, aj as TraceContractStepRules, ak as TraceContractToolRules, al as TraceSignalRuleOptions, am as createBaselineRegressionRule, an as createDecisionRule, ao as createGuardrailRule, ap as createLlmUsageRule, aq as createMaxStepDurationRule, ar as createObservedOutcomeRule, as as createRequireCompletedRule, at as createRetrievalRule, au as createRunDepthRule, av as createRunDurationRule, aw as createRunEventCountRule, ax as createRunStatusRule, ay as createSafetyOversizedAttributeRule, az as createSafetyRawContentRule, aA as createSafetyRedactionRule, aB as createSafetySecretPatternRule, aC as createStallDetectionRule, aD as createStepOrderingRule, aE as createStructureCycleRule, aF as createStructureIncompleteRule, aG as createStructureOrphanRule, aH as createStructureParallelWidthRule, aI as createStructureRelationshipRule, aJ as createToolFailureRule, aK as createToolOrderingRule, aL as createToolUsageRule, aM as defineTraceContract, aN as evaluateToolArgumentValue, aO as evaluateTraceContract, aP as evaluateTraceContractRead, aQ as explainTraceContract, aR as extractToolArgumentPayload, aS as lintTraceContract, aT as projectLogicalEvents, aU as resolveCanonicalToolName, aV as resolveJsonPointer, aW as resolveTraceContractScope, aX as runTraceChecks, aY as workflowMetadataForRun } from './index-EQihlEei.cjs';
|
|
3
3
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.cjs';
|
|
4
4
|
export { P as PROGRAMMATIC_DIAGNOSTIC_SPECS, a as ProgrammaticDiagnosticCode, b as ProgrammaticDiagnosticSpec, f as formatProgrammaticDiagnostic } from './programmatic-gZTWLObB.cjs';
|
|
5
5
|
import { g as TraceReadResult } from './index-Cdww9h50.cjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-
|
|
2
|
-
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as
|
|
1
|
+
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-CgOAGuM5.js';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as StepOrderingEndpoint, r as StepOrderingKind, s as StepOrderingRuleOptions, t as StructureIncompleteRuleOptions, u as StructureOrphanRuleOptions, v as StructureParallelWidthRuleOptions, w as StructureRelationshipRuleOptions, T as ToolArgumentCheck, x as ToolArgumentOccurrence, y as ToolArgumentOperator, z as ToolFailureRuleOptions, A as ToolOrderingRuleOptions, E as ToolUsageRuleOptions, F as TraceCheckContext, H as TraceCheckDiagnostic, I as TraceCheckDiagnosticCode, J as TraceCheckEvidence, K as TraceCheckFacts, N as TraceCheckFinding, P as TraceCheckFindingStatus, Q as TraceCheckInput, U as TraceCheckResult, V as TraceCheckRule, W as TraceCheckRuleCategory, X as TraceCheckRuleExecution, Y as TraceCheckRuleExecutionStatus, Z as TraceCheckSeverity, _ as TraceCheckStatus, $ as TraceCheckSummary, a0 as TraceContract, a1 as TraceContractAlternativeBranch, a2 as TraceContractAlternatives, a3 as TraceContractBody, a4 as TraceContractControlRules, a5 as TraceContractInput, a6 as TraceContractLintDiagnostic, a7 as TraceContractLlmRules, a8 as TraceContractObservationProvenance, a9 as TraceContractObservationRules, aa as TraceContractRecoveryOperation, ab as TraceContractRecoveryRetryableErrors, ac as TraceContractRecoverySuccessfulResultDependency, ad as TraceContractRetryRules, ae as TraceContractRunRules, af as TraceContractScope, ag as TraceContractStepKind, ah as TraceContractStepOrderRelation, ai as TraceContractStepRef, aj as TraceContractStepRules, ak as TraceContractToolRules, al as TraceSignalRuleOptions, am as createBaselineRegressionRule, an as createDecisionRule, ao as createGuardrailRule, ap as createLlmUsageRule, aq as createMaxStepDurationRule, ar as createObservedOutcomeRule, as as createRequireCompletedRule, at as createRetrievalRule, au as createRunDepthRule, av as createRunDurationRule, aw as createRunEventCountRule, ax as createRunStatusRule, ay as createSafetyOversizedAttributeRule, az as createSafetyRawContentRule, aA as createSafetyRedactionRule, aB as createSafetySecretPatternRule, aC as createStallDetectionRule, aD as createStepOrderingRule, aE as createStructureCycleRule, aF as createStructureIncompleteRule, aG as createStructureOrphanRule, aH as createStructureParallelWidthRule, aI as createStructureRelationshipRule, aJ as createToolFailureRule, aK as createToolOrderingRule, aL as createToolUsageRule, aM as defineTraceContract, aN as evaluateToolArgumentValue, aO as evaluateTraceContract, aP as evaluateTraceContractRead, aQ as explainTraceContract, aR as extractToolArgumentPayload, aS as lintTraceContract, aT as projectLogicalEvents, aU as resolveCanonicalToolName, aV as resolveJsonPointer, aW as resolveTraceContractScope, aX as runTraceChecks, aY as workflowMetadataForRun } from './index-CgOAGuM5.js';
|
|
3
3
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.js';
|
|
4
4
|
export { P as PROGRAMMATIC_DIAGNOSTIC_SPECS, a as ProgrammaticDiagnosticCode, b as ProgrammaticDiagnosticSpec, f as formatProgrammaticDiagnostic } from './programmatic-gZTWLObB.js';
|
|
5
5
|
import { g as TraceReadResult } from './index-xUbdpcrP.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, summarizeSemanticParity, workflowMetadataForRun } from './chunk-
|
|
1
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStepOrderingRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, summarizeSemanticParity, workflowMetadataForRun } from './chunk-VEQKLWGJ.mjs';
|
|
2
2
|
import './chunk-BGF5JDJN.mjs';
|
|
3
3
|
import './chunk-PBNBQRXV.mjs';
|
|
4
4
|
import './chunk-VU6O5QAH.mjs';
|
|
@@ -2371,6 +2371,17 @@ function cloneBody(body) {
|
|
|
2371
2371
|
}
|
|
2372
2372
|
} : {},
|
|
2373
2373
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
2374
|
+
...body.steps ? {
|
|
2375
|
+
steps: {
|
|
2376
|
+
...body.steps.orderRelations ? {
|
|
2377
|
+
orderRelations: body.steps.orderRelations.map((item) => ({
|
|
2378
|
+
...item,
|
|
2379
|
+
before: { ...item.before },
|
|
2380
|
+
after: { ...item.after }
|
|
2381
|
+
}))
|
|
2382
|
+
} : {}
|
|
2383
|
+
}
|
|
2384
|
+
} : {},
|
|
2374
2385
|
...body.observations ? {
|
|
2375
2386
|
observations: {
|
|
2376
2387
|
...body.observations,
|
|
@@ -2410,7 +2421,7 @@ function cloneBody(body) {
|
|
|
2410
2421
|
};
|
|
2411
2422
|
}
|
|
2412
2423
|
function bodyHasRules(body) {
|
|
2413
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
2424
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.steps !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
2414
2425
|
}
|
|
2415
2426
|
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
2416
2427
|
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
@@ -2753,6 +2764,18 @@ function contractToRules(contract) {
|
|
|
2753
2764
|
})
|
|
2754
2765
|
);
|
|
2755
2766
|
}
|
|
2767
|
+
const orderRelations = contract.steps?.orderRelations ?? [];
|
|
2768
|
+
for (const [index, relation] of orderRelations.entries()) {
|
|
2769
|
+
rules.push(
|
|
2770
|
+
createStepOrderingRule({
|
|
2771
|
+
before: { kind: relation.before.kind, name: relation.before.name },
|
|
2772
|
+
after: { kind: relation.after.kind, name: relation.after.name },
|
|
2773
|
+
id: `contract.step.orderRelation.${index}`,
|
|
2774
|
+
mode: relation.mode ?? "first-occurrence",
|
|
2775
|
+
requireEndpoints: relation.requireEndpoints !== false
|
|
2776
|
+
})
|
|
2777
|
+
);
|
|
2778
|
+
}
|
|
2756
2779
|
if (contract.observations) {
|
|
2757
2780
|
const required = contract.observations.required ?? [];
|
|
2758
2781
|
if (required.length > 0) {
|
|
@@ -3306,6 +3329,13 @@ function explainTraceContract(contract) {
|
|
|
3306
3329
|
lines.push(`Base: LLM allowedModels [${contract.llm.allowedModels.join(", ")}].`);
|
|
3307
3330
|
}
|
|
3308
3331
|
}
|
|
3332
|
+
if ((contract.steps?.orderRelations?.length ?? 0) > 0) {
|
|
3333
|
+
const relations = contract.steps.orderRelations;
|
|
3334
|
+
const summary = relations.map(
|
|
3335
|
+
(relation) => `${relation.before.kind}:${relation.before.name} \u2192 ${relation.after.kind}:${relation.after.name} mode=${relation.mode ?? "first-occurrence"}`
|
|
3336
|
+
).join("; ");
|
|
3337
|
+
lines.push(`Base: ${relations.length} steps.orderRelations [${summary}].`);
|
|
3338
|
+
}
|
|
3309
3339
|
if (contract.observations?.required?.length) {
|
|
3310
3340
|
lines.push(`Base: required observations [${contract.observations.required.join(", ")}].`);
|
|
3311
3341
|
}
|
|
@@ -3720,6 +3750,31 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
3720
3750
|
function toolName(event) {
|
|
3721
3751
|
return resolveCanonicalToolName(event);
|
|
3722
3752
|
}
|
|
3753
|
+
function stepEndpointName(event, kind) {
|
|
3754
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event);
|
|
3755
|
+
for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
|
|
3756
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
3757
|
+
}
|
|
3758
|
+
return event.name;
|
|
3759
|
+
}
|
|
3760
|
+
function formatStepEndpoint(endpoint) {
|
|
3761
|
+
return `${endpoint.kind} ${endpoint.name}`;
|
|
3762
|
+
}
|
|
3763
|
+
function otherKindsForStepName(context, expectedKind, name) {
|
|
3764
|
+
return [
|
|
3765
|
+
...new Set(
|
|
3766
|
+
semanticEvents(context).filter((event) => {
|
|
3767
|
+
const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
|
|
3768
|
+
if (kind === expectedKind) return false;
|
|
3769
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event) === name;
|
|
3770
|
+
if (kind === "LLM") return stepEndpointName(event, "LLM") === name;
|
|
3771
|
+
return event.name === name || event.name === `tool:${name}` || event.name === `llm:${name}` || event.name.endsWith(`:${name}`);
|
|
3772
|
+
}).map(
|
|
3773
|
+
(event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
|
|
3774
|
+
)
|
|
3775
|
+
)
|
|
3776
|
+
].sort();
|
|
3777
|
+
}
|
|
3723
3778
|
function semanticEvents(context) {
|
|
3724
3779
|
return context.logicalEvents ?? context.events;
|
|
3725
3780
|
}
|
|
@@ -3746,14 +3801,6 @@ function finishedEvents(context, kind) {
|
|
|
3746
3801
|
function toolInvocationEvents(context) {
|
|
3747
3802
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
3748
3803
|
}
|
|
3749
|
-
function firstIndexByName(events) {
|
|
3750
|
-
const index = /* @__PURE__ */ new Map();
|
|
3751
|
-
for (const [position, event] of events.entries()) {
|
|
3752
|
-
const name = toolName(event);
|
|
3753
|
-
if (!index.has(name)) index.set(name, position);
|
|
3754
|
-
}
|
|
3755
|
-
return index;
|
|
3756
|
-
}
|
|
3757
3804
|
function isRecord3(value) {
|
|
3758
3805
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3759
3806
|
}
|
|
@@ -4263,18 +4310,85 @@ function createToolUsageRule(options) {
|
|
|
4263
4310
|
}
|
|
4264
4311
|
};
|
|
4265
4312
|
}
|
|
4266
|
-
function
|
|
4267
|
-
const
|
|
4313
|
+
function createStepOrderingRule(options) {
|
|
4314
|
+
const bothTools = options.before.kind === "TOOL" && options.after.kind === "TOOL";
|
|
4315
|
+
const ruleId = options.id ?? (bothTools ? "tool.order" : "step.order");
|
|
4268
4316
|
const mode = options.mode ?? "first-occurrence";
|
|
4317
|
+
const requireEndpoints = options.requireEndpoints === true;
|
|
4318
|
+
const beforeLabel = bothTools ? options.before.name : formatStepEndpoint(options.before);
|
|
4319
|
+
const afterLabel = bothTools ? options.after.name : formatStepEndpoint(options.after);
|
|
4320
|
+
const subject = bothTools ? "Tool" : "Step";
|
|
4269
4321
|
return {
|
|
4270
4322
|
id: ruleId,
|
|
4271
|
-
category: "tool",
|
|
4323
|
+
category: bothTools ? "tool" : "structure",
|
|
4272
4324
|
defaultSeverity: "error",
|
|
4273
4325
|
evaluate(context) {
|
|
4274
|
-
const
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
const
|
|
4326
|
+
const beforeEvents = finishedEvents(context, options.before.kind).filter(
|
|
4327
|
+
(event) => stepEndpointName(event, options.before.kind) === options.before.name
|
|
4328
|
+
);
|
|
4329
|
+
const afterEvents = finishedEvents(context, options.after.kind).filter(
|
|
4330
|
+
(event) => stepEndpointName(event, options.after.kind) === options.after.name
|
|
4331
|
+
);
|
|
4332
|
+
if (beforeEvents.length === 0 || afterEvents.length === 0) {
|
|
4333
|
+
if (!requireEndpoints) return [];
|
|
4334
|
+
const findings = [];
|
|
4335
|
+
if (beforeEvents.length === 0) {
|
|
4336
|
+
const otherKinds = otherKindsForStepName(
|
|
4337
|
+
context,
|
|
4338
|
+
options.before.kind,
|
|
4339
|
+
options.before.name
|
|
4340
|
+
);
|
|
4341
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.before.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
4342
|
+
findings.push(
|
|
4343
|
+
failFinding(
|
|
4344
|
+
ruleId,
|
|
4345
|
+
`Required ${formatStepEndpoint(options.before)} did not appear.${hint}`,
|
|
4346
|
+
runEvidence(context.selectedRun),
|
|
4347
|
+
options.before,
|
|
4348
|
+
{
|
|
4349
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
4350
|
+
endpoint: "before",
|
|
4351
|
+
otherKinds
|
|
4352
|
+
}
|
|
4353
|
+
)
|
|
4354
|
+
);
|
|
4355
|
+
}
|
|
4356
|
+
if (afterEvents.length === 0) {
|
|
4357
|
+
const otherKinds = otherKindsForStepName(
|
|
4358
|
+
context,
|
|
4359
|
+
options.after.kind,
|
|
4360
|
+
options.after.name
|
|
4361
|
+
);
|
|
4362
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.after.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
4363
|
+
findings.push(
|
|
4364
|
+
failFinding(
|
|
4365
|
+
ruleId,
|
|
4366
|
+
`Required ${formatStepEndpoint(options.after)} did not appear.${hint}`,
|
|
4367
|
+
runEvidence(context.selectedRun),
|
|
4368
|
+
options.after,
|
|
4369
|
+
{
|
|
4370
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
4371
|
+
endpoint: "after",
|
|
4372
|
+
otherKinds
|
|
4373
|
+
}
|
|
4374
|
+
)
|
|
4375
|
+
);
|
|
4376
|
+
}
|
|
4377
|
+
return findings;
|
|
4378
|
+
}
|
|
4379
|
+
const encounter = finishedEvents(context).filter(
|
|
4380
|
+
(event) => event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name || event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name
|
|
4381
|
+
);
|
|
4382
|
+
let beforeIndex;
|
|
4383
|
+
let afterIndex;
|
|
4384
|
+
for (const [position, event] of encounter.entries()) {
|
|
4385
|
+
if (beforeIndex === void 0 && event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name) {
|
|
4386
|
+
beforeIndex = position;
|
|
4387
|
+
}
|
|
4388
|
+
if (afterIndex === void 0 && event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name) {
|
|
4389
|
+
afterIndex = position;
|
|
4390
|
+
}
|
|
4391
|
+
}
|
|
4278
4392
|
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
4279
4393
|
return [];
|
|
4280
4394
|
}
|
|
@@ -4282,32 +4396,37 @@ function createToolOrderingRule(options) {
|
|
|
4282
4396
|
return [
|
|
4283
4397
|
failFinding(
|
|
4284
4398
|
ruleId,
|
|
4285
|
-
|
|
4286
|
-
[
|
|
4287
|
-
|
|
4288
|
-
|
|
4399
|
+
`${subject} ${beforeLabel} must appear before ${afterLabel}.`,
|
|
4400
|
+
[
|
|
4401
|
+
eventEvidence3(encounter[beforeIndex]),
|
|
4402
|
+
eventEvidence3(encounter[afterIndex])
|
|
4403
|
+
],
|
|
4404
|
+
bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after },
|
|
4405
|
+
bothTools ? finishedEvents(context, "TOOL").map(toolName) : encounter.map(
|
|
4406
|
+
(event) => event.kind === "TOOL" || event.kind === "LLM" ? `${event.kind}:${stepEndpointName(event, event.kind)}` : event.name
|
|
4407
|
+
)
|
|
4289
4408
|
)
|
|
4290
4409
|
];
|
|
4291
4410
|
}
|
|
4292
|
-
const beforeEvent =
|
|
4293
|
-
const afterEvent =
|
|
4411
|
+
const beforeEvent = beforeEvents[0];
|
|
4412
|
+
const afterEvent = afterEvents[0];
|
|
4294
4413
|
const beforeEnd = eventEndMs(beforeEvent);
|
|
4295
4414
|
const afterStart = eventStartMs(afterEvent);
|
|
4415
|
+
const endpointExpected = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
|
|
4296
4416
|
if (mode === "happens-before") {
|
|
4297
4417
|
const expected = {
|
|
4298
|
-
|
|
4299
|
-
after: options.after,
|
|
4418
|
+
...endpointExpected,
|
|
4300
4419
|
mode,
|
|
4301
4420
|
relation: "before.end <= after.start"
|
|
4302
4421
|
};
|
|
4303
|
-
if (options.before === options.after) {
|
|
4422
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
4304
4423
|
return [
|
|
4305
4424
|
failFinding(
|
|
4306
4425
|
ruleId,
|
|
4307
|
-
|
|
4426
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
4308
4427
|
[eventEvidence3(beforeEvent)],
|
|
4309
4428
|
expected,
|
|
4310
|
-
{ code: "tool.order.same-tool" }
|
|
4429
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
4311
4430
|
)
|
|
4312
4431
|
];
|
|
4313
4432
|
}
|
|
@@ -4315,11 +4434,11 @@ function createToolOrderingRule(options) {
|
|
|
4315
4434
|
return [
|
|
4316
4435
|
failFinding(
|
|
4317
4436
|
ruleId,
|
|
4318
|
-
|
|
4437
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish causal timing.`,
|
|
4319
4438
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4320
4439
|
expected,
|
|
4321
4440
|
{
|
|
4322
|
-
code: "tool.order.interval-unresolved",
|
|
4441
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4323
4442
|
beforeEndResolved: beforeEnd !== void 0,
|
|
4324
4443
|
afterStartResolved: afterStart !== void 0
|
|
4325
4444
|
}
|
|
@@ -4330,7 +4449,7 @@ function createToolOrderingRule(options) {
|
|
|
4330
4449
|
return [
|
|
4331
4450
|
failFinding(
|
|
4332
4451
|
ruleId,
|
|
4333
|
-
|
|
4452
|
+
`${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
|
|
4334
4453
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4335
4454
|
expected,
|
|
4336
4455
|
{
|
|
@@ -4343,22 +4462,19 @@ function createToolOrderingRule(options) {
|
|
|
4343
4462
|
return [];
|
|
4344
4463
|
}
|
|
4345
4464
|
if (mode === "all-occurrences") {
|
|
4346
|
-
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
4347
|
-
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
4348
4465
|
const expected = {
|
|
4349
|
-
|
|
4350
|
-
after: options.after,
|
|
4466
|
+
...endpointExpected,
|
|
4351
4467
|
mode,
|
|
4352
4468
|
relation: "max(before.end) <= min(after.start)"
|
|
4353
4469
|
};
|
|
4354
|
-
if (options.before === options.after) {
|
|
4470
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
4355
4471
|
return [
|
|
4356
4472
|
failFinding(
|
|
4357
4473
|
ruleId,
|
|
4358
|
-
|
|
4474
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
4359
4475
|
[eventEvidence3(beforeEvent)],
|
|
4360
4476
|
expected,
|
|
4361
|
-
{ code: "tool.order.same-tool" }
|
|
4477
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
4362
4478
|
)
|
|
4363
4479
|
];
|
|
4364
4480
|
}
|
|
@@ -4390,11 +4506,11 @@ function createToolOrderingRule(options) {
|
|
|
4390
4506
|
return [
|
|
4391
4507
|
failFinding(
|
|
4392
4508
|
ruleId,
|
|
4393
|
-
|
|
4509
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish all causal intervals.`,
|
|
4394
4510
|
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
|
|
4395
4511
|
expected,
|
|
4396
4512
|
{
|
|
4397
|
-
code: "tool.order.interval-unresolved",
|
|
4513
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4398
4514
|
missingBeforeEnds,
|
|
4399
4515
|
missingAfterStarts
|
|
4400
4516
|
}
|
|
@@ -4405,7 +4521,7 @@ function createToolOrderingRule(options) {
|
|
|
4405
4521
|
return [
|
|
4406
4522
|
failFinding(
|
|
4407
4523
|
ruleId,
|
|
4408
|
-
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
4524
|
+
bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
|
|
4409
4525
|
[eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
|
|
4410
4526
|
expected,
|
|
4411
4527
|
{
|
|
@@ -4423,10 +4539,13 @@ function createToolOrderingRule(options) {
|
|
|
4423
4539
|
ruleId,
|
|
4424
4540
|
severity: "warning",
|
|
4425
4541
|
status: "warning",
|
|
4426
|
-
message:
|
|
4427
|
-
expected: {
|
|
4542
|
+
message: `${subject} order ${beforeLabel} before ${afterLabel} satisfied start/encounter order, but intervals overlap so causal completion before the next ${bothTools ? "tool" : "step"} was not established.`,
|
|
4543
|
+
expected: {
|
|
4544
|
+
...endpointExpected,
|
|
4545
|
+
mode: "first-occurrence"
|
|
4546
|
+
},
|
|
4428
4547
|
actual: {
|
|
4429
|
-
code: "tool.order.overlap",
|
|
4548
|
+
code: bothTools ? "tool.order.overlap" : "step.order.overlap",
|
|
4430
4549
|
beforeEndedAt: beforeEvent.endedAt,
|
|
4431
4550
|
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
4432
4551
|
},
|
|
@@ -4438,6 +4557,15 @@ function createToolOrderingRule(options) {
|
|
|
4438
4557
|
}
|
|
4439
4558
|
};
|
|
4440
4559
|
}
|
|
4560
|
+
function createToolOrderingRule(options) {
|
|
4561
|
+
return createStepOrderingRule({
|
|
4562
|
+
before: { kind: "TOOL", name: options.before },
|
|
4563
|
+
after: { kind: "TOOL", name: options.after },
|
|
4564
|
+
...options.id !== void 0 ? { id: options.id } : {},
|
|
4565
|
+
...options.mode !== void 0 ? { mode: options.mode } : {},
|
|
4566
|
+
requireEndpoints: false
|
|
4567
|
+
});
|
|
4568
|
+
}
|
|
4441
4569
|
function createToolFailureRule(options) {
|
|
4442
4570
|
return {
|
|
4443
4571
|
id: "tool.failures",
|
|
@@ -5360,6 +5488,6 @@ function runTraceChecks(input, options = {}) {
|
|
|
5360
5488
|
};
|
|
5361
5489
|
}
|
|
5362
5490
|
|
|
5363
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
5364
|
-
//# sourceMappingURL=chunk-
|
|
5365
|
-
//# sourceMappingURL=chunk-
|
|
5491
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStepOrderingRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
5492
|
+
//# sourceMappingURL=chunk-VEQKLWGJ.mjs.map
|
|
5493
|
+
//# sourceMappingURL=chunk-VEQKLWGJ.mjs.map
|