agent-inspect 6.30.0 → 6.31.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/COMPARE.md +1 -1
- package/docs/FIRST-TRACE-IN-5-MINUTES.md +21 -17
- package/docs/GETTING-STARTED.md +1 -1
- package/docs/TRACE-CONTRACTS.md +25 -0
- package/package.json +4 -2
- package/packages/cli/dist/{chunk-BDRJENLM.mjs → chunk-SJ6FRNOE.mjs} +201 -49
- package/packages/cli/dist/chunk-SJ6FRNOE.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +288 -50
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +91 -5
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-ZUGAUXY6.mjs → src-YNEJOPRA.mjs} +3 -3
- package/packages/cli/dist/{src-ZUGAUXY6.mjs.map → src-YNEJOPRA.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +39 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +207 -47
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-OOX25GTH.mjs → chunk-CBPWLHNQ.mjs} +209 -50
- package/packages/core/dist/{chunk-OOX25GTH.mjs.map → chunk-CBPWLHNQ.mjs.map} +1 -1
- package/packages/core/dist/{index-7hG_qggp.d.ts → index-CgOAGuM5.d.ts} +111 -1
- package/packages/core/dist/{index-CQJUNPPe.d.cts → index-EQihlEei.d.cts} +111 -1
- package/packages/cli/dist/chunk-BDRJENLM.mjs.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-
|
|
2
|
-
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as
|
|
1
|
+
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-EQihlEei.cjs';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as StepOrderingEndpoint, r as StepOrderingKind, s as StepOrderingRuleOptions, t as StructureIncompleteRuleOptions, u as StructureOrphanRuleOptions, v as StructureParallelWidthRuleOptions, w as StructureRelationshipRuleOptions, T as ToolArgumentCheck, x as ToolArgumentOccurrence, y as ToolArgumentOperator, z as ToolFailureRuleOptions, A as ToolOrderingRuleOptions, E as ToolUsageRuleOptions, F as TraceCheckContext, H as TraceCheckDiagnostic, I as TraceCheckDiagnosticCode, J as TraceCheckEvidence, K as TraceCheckFacts, N as TraceCheckFinding, P as TraceCheckFindingStatus, Q as TraceCheckInput, U as TraceCheckResult, V as TraceCheckRule, W as TraceCheckRuleCategory, X as TraceCheckRuleExecution, Y as TraceCheckRuleExecutionStatus, Z as TraceCheckSeverity, _ as TraceCheckStatus, $ as TraceCheckSummary, a0 as TraceContract, a1 as TraceContractAlternativeBranch, a2 as TraceContractAlternatives, a3 as TraceContractBody, a4 as TraceContractControlRules, a5 as TraceContractInput, a6 as TraceContractLintDiagnostic, a7 as TraceContractLlmRules, a8 as TraceContractObservationProvenance, a9 as TraceContractObservationRules, aa as TraceContractRecoveryOperation, ab as TraceContractRecoveryRetryableErrors, ac as TraceContractRecoverySuccessfulResultDependency, ad as TraceContractRetryRules, ae as TraceContractRunRules, af as TraceContractScope, ag as TraceContractStepKind, ah as TraceContractStepOrderRelation, ai as TraceContractStepRef, aj as TraceContractStepRules, ak as TraceContractToolRules, al as TraceSignalRuleOptions, am as createBaselineRegressionRule, an as createDecisionRule, ao as createGuardrailRule, ap as createLlmUsageRule, aq as createMaxStepDurationRule, ar as createObservedOutcomeRule, as as createRequireCompletedRule, at as createRetrievalRule, au as createRunDepthRule, av as createRunDurationRule, aw as createRunEventCountRule, ax as createRunStatusRule, ay as createSafetyOversizedAttributeRule, az as createSafetyRawContentRule, aA as createSafetyRedactionRule, aB as createSafetySecretPatternRule, aC as createStallDetectionRule, aD as createStepOrderingRule, aE as createStructureCycleRule, aF as createStructureIncompleteRule, aG as createStructureOrphanRule, aH as createStructureParallelWidthRule, aI as createStructureRelationshipRule, aJ as createToolFailureRule, aK as createToolOrderingRule, aL as createToolUsageRule, aM as defineTraceContract, aN as evaluateToolArgumentValue, aO as evaluateTraceContract, aP as evaluateTraceContractRead, aQ as explainTraceContract, aR as extractToolArgumentPayload, aS as lintTraceContract, aT as projectLogicalEvents, aU as resolveCanonicalToolName, aV as resolveJsonPointer, aW as resolveTraceContractScope, aX as runTraceChecks, aY as workflowMetadataForRun } from './index-EQihlEei.cjs';
|
|
3
3
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.cjs';
|
|
4
4
|
export { P as PROGRAMMATIC_DIAGNOSTIC_SPECS, a as ProgrammaticDiagnosticCode, b as ProgrammaticDiagnosticSpec, f as formatProgrammaticDiagnostic } from './programmatic-gZTWLObB.cjs';
|
|
5
5
|
import { g as TraceReadResult } from './index-Cdww9h50.cjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-
|
|
2
|
-
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as
|
|
1
|
+
import { L as LogicalTraceEvent, a as LogicalProjectionDiagnostic } from './index-CgOAGuM5.js';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, C as ControlStage, D as DecisionRuleOptions, G as GuardrailRuleOptions, b as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RecoverySameArgumentsMode, c as RecoverySideEffectClass, d as RetrievalRuleOptions, e as RunDepthRuleOptions, f as RunDurationRuleOptions, g as RunEventCountRuleOptions, h as RunStatusRuleOptions, i as RunTraceChecksOptions, S as SafetyFindingCategory, j as SafetyFindingConfidence, k as SafetyOversizedAttributeRuleOptions, l as SafetyRawContentRuleOptions, m as SafetyRedactionRuleOptions, n as SafetySecretPattern, o as SafetySecretPatternRuleOptions, p as StallDetectionRuleOptions, q as StepOrderingEndpoint, r as StepOrderingKind, s as StepOrderingRuleOptions, t as StructureIncompleteRuleOptions, u as StructureOrphanRuleOptions, v as StructureParallelWidthRuleOptions, w as StructureRelationshipRuleOptions, T as ToolArgumentCheck, x as ToolArgumentOccurrence, y as ToolArgumentOperator, z as ToolFailureRuleOptions, A as ToolOrderingRuleOptions, E as ToolUsageRuleOptions, F as TraceCheckContext, H as TraceCheckDiagnostic, I as TraceCheckDiagnosticCode, J as TraceCheckEvidence, K as TraceCheckFacts, N as TraceCheckFinding, P as TraceCheckFindingStatus, Q as TraceCheckInput, U as TraceCheckResult, V as TraceCheckRule, W as TraceCheckRuleCategory, X as TraceCheckRuleExecution, Y as TraceCheckRuleExecutionStatus, Z as TraceCheckSeverity, _ as TraceCheckStatus, $ as TraceCheckSummary, a0 as TraceContract, a1 as TraceContractAlternativeBranch, a2 as TraceContractAlternatives, a3 as TraceContractBody, a4 as TraceContractControlRules, a5 as TraceContractInput, a6 as TraceContractLintDiagnostic, a7 as TraceContractLlmRules, a8 as TraceContractObservationProvenance, a9 as TraceContractObservationRules, aa as TraceContractRecoveryOperation, ab as TraceContractRecoveryRetryableErrors, ac as TraceContractRecoverySuccessfulResultDependency, ad as TraceContractRetryRules, ae as TraceContractRunRules, af as TraceContractScope, ag as TraceContractStepKind, ah as TraceContractStepOrderRelation, ai as TraceContractStepRef, aj as TraceContractStepRules, ak as TraceContractToolRules, al as TraceSignalRuleOptions, am as createBaselineRegressionRule, an as createDecisionRule, ao as createGuardrailRule, ap as createLlmUsageRule, aq as createMaxStepDurationRule, ar as createObservedOutcomeRule, as as createRequireCompletedRule, at as createRetrievalRule, au as createRunDepthRule, av as createRunDurationRule, aw as createRunEventCountRule, ax as createRunStatusRule, ay as createSafetyOversizedAttributeRule, az as createSafetyRawContentRule, aA as createSafetyRedactionRule, aB as createSafetySecretPatternRule, aC as createStallDetectionRule, aD as createStepOrderingRule, aE as createStructureCycleRule, aF as createStructureIncompleteRule, aG as createStructureOrphanRule, aH as createStructureParallelWidthRule, aI as createStructureRelationshipRule, aJ as createToolFailureRule, aK as createToolOrderingRule, aL as createToolUsageRule, aM as defineTraceContract, aN as evaluateToolArgumentValue, aO as evaluateTraceContract, aP as evaluateTraceContractRead, aQ as explainTraceContract, aR as extractToolArgumentPayload, aS as lintTraceContract, aT as projectLogicalEvents, aU as resolveCanonicalToolName, aV as resolveJsonPointer, aW as resolveTraceContractScope, aX as runTraceChecks, aY as workflowMetadataForRun } from './index-CgOAGuM5.js';
|
|
3
3
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.js';
|
|
4
4
|
export { P as PROGRAMMATIC_DIAGNOSTIC_SPECS, a as ProgrammaticDiagnosticCode, b as ProgrammaticDiagnosticSpec, f as formatProgrammaticDiagnostic } from './programmatic-gZTWLObB.js';
|
|
5
5
|
import { g as TraceReadResult } from './index-xUbdpcrP.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, summarizeSemanticParity, workflowMetadataForRun } from './chunk-
|
|
1
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStepOrderingRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, summarizeSemanticParity, workflowMetadataForRun } from './chunk-CBPWLHNQ.mjs';
|
|
2
2
|
import './chunk-BGF5JDJN.mjs';
|
|
3
3
|
import './chunk-PBNBQRXV.mjs';
|
|
4
4
|
import './chunk-VU6O5QAH.mjs';
|
|
@@ -2354,11 +2354,40 @@ function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
|
2354
2354
|
evidence: [...evidence]
|
|
2355
2355
|
};
|
|
2356
2356
|
}
|
|
2357
|
+
var ALLOWED_STATUS_ALIASES = {
|
|
2358
|
+
ok: "ok",
|
|
2359
|
+
error: "error",
|
|
2360
|
+
running: "running",
|
|
2361
|
+
success: "ok",
|
|
2362
|
+
failed: "error"
|
|
2363
|
+
};
|
|
2364
|
+
function parseAllowedStatus(status) {
|
|
2365
|
+
return ALLOWED_STATUS_ALIASES[status];
|
|
2366
|
+
}
|
|
2357
2367
|
function normalizeStatus(status) {
|
|
2358
|
-
|
|
2359
|
-
if (
|
|
2360
|
-
|
|
2361
|
-
|
|
2368
|
+
const parsed = parseAllowedStatus(status);
|
|
2369
|
+
if (parsed === void 0) {
|
|
2370
|
+
throw new TypeError(
|
|
2371
|
+
`Unknown run status ${JSON.stringify(status)}. Allowed: ok, error, running (aliases: success, failed).`
|
|
2372
|
+
);
|
|
2373
|
+
}
|
|
2374
|
+
return parsed;
|
|
2375
|
+
}
|
|
2376
|
+
function validateAllowedStatusesShape(allowedStatuses, pathPrefix = "run.allowedStatuses") {
|
|
2377
|
+
if (!allowedStatuses?.length) return [];
|
|
2378
|
+
const out = [];
|
|
2379
|
+
for (let i = 0; i < allowedStatuses.length; i++) {
|
|
2380
|
+
const status = allowedStatuses[i];
|
|
2381
|
+
if (parseAllowedStatus(status) === void 0) {
|
|
2382
|
+
out.push({
|
|
2383
|
+
code: "contract.run.allowedStatuses.unknown",
|
|
2384
|
+
severity: "error",
|
|
2385
|
+
message: `${pathPrefix}[${i}] has unknown status ${JSON.stringify(status)}. Allowed: ok, error, running (aliases: success, failed).`,
|
|
2386
|
+
path: `${pathPrefix}[${i}]`
|
|
2387
|
+
});
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
return out;
|
|
2362
2391
|
}
|
|
2363
2392
|
function cloneBody(body) {
|
|
2364
2393
|
return {
|
|
@@ -2371,6 +2400,17 @@ function cloneBody(body) {
|
|
|
2371
2400
|
}
|
|
2372
2401
|
} : {},
|
|
2373
2402
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
2403
|
+
...body.steps ? {
|
|
2404
|
+
steps: {
|
|
2405
|
+
...body.steps.orderRelations ? {
|
|
2406
|
+
orderRelations: body.steps.orderRelations.map((item) => ({
|
|
2407
|
+
...item,
|
|
2408
|
+
before: { ...item.before },
|
|
2409
|
+
after: { ...item.after }
|
|
2410
|
+
}))
|
|
2411
|
+
} : {}
|
|
2412
|
+
}
|
|
2413
|
+
} : {},
|
|
2374
2414
|
...body.observations ? {
|
|
2375
2415
|
observations: {
|
|
2376
2416
|
...body.observations,
|
|
@@ -2410,7 +2450,7 @@ function cloneBody(body) {
|
|
|
2410
2450
|
};
|
|
2411
2451
|
}
|
|
2412
2452
|
function bodyHasRules(body) {
|
|
2413
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
2453
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.steps !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
2414
2454
|
}
|
|
2415
2455
|
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
2416
2456
|
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
@@ -2753,6 +2793,18 @@ function contractToRules(contract) {
|
|
|
2753
2793
|
})
|
|
2754
2794
|
);
|
|
2755
2795
|
}
|
|
2796
|
+
const orderRelations = contract.steps?.orderRelations ?? [];
|
|
2797
|
+
for (const [index, relation] of orderRelations.entries()) {
|
|
2798
|
+
rules.push(
|
|
2799
|
+
createStepOrderingRule({
|
|
2800
|
+
before: { kind: relation.before.kind, name: relation.before.name },
|
|
2801
|
+
after: { kind: relation.after.kind, name: relation.after.name },
|
|
2802
|
+
id: `contract.step.orderRelation.${index}`,
|
|
2803
|
+
mode: relation.mode ?? "first-occurrence",
|
|
2804
|
+
requireEndpoints: relation.requireEndpoints !== false
|
|
2805
|
+
})
|
|
2806
|
+
);
|
|
2807
|
+
}
|
|
2756
2808
|
if (contract.observations) {
|
|
2757
2809
|
const required = contract.observations.required ?? [];
|
|
2758
2810
|
if (required.length > 0) {
|
|
@@ -3002,6 +3054,7 @@ function evaluateBody(input, body, options = {}) {
|
|
|
3002
3054
|
}
|
|
3003
3055
|
function defineTraceContract(input) {
|
|
3004
3056
|
const shapeErrors = [
|
|
3057
|
+
...validateAllowedStatusesShape(input.run?.allowedStatuses),
|
|
3005
3058
|
...validateAlternativesShape(input.alternatives),
|
|
3006
3059
|
...validateScopeShape(input.scope),
|
|
3007
3060
|
...validateProvenanceShape(input.observations)
|
|
@@ -3032,6 +3085,7 @@ function defineTraceContract(input) {
|
|
|
3032
3085
|
}
|
|
3033
3086
|
function evaluateTraceContract(input, contract, options = {}) {
|
|
3034
3087
|
const shapeErrors = [
|
|
3088
|
+
...validateAllowedStatusesShape(contract.run?.allowedStatuses),
|
|
3035
3089
|
...validateAlternativesShape(contract.alternatives),
|
|
3036
3090
|
...validateScopeShape(contract.scope),
|
|
3037
3091
|
...validateProvenanceShape(contract.observations)
|
|
@@ -3306,6 +3360,13 @@ function explainTraceContract(contract) {
|
|
|
3306
3360
|
lines.push(`Base: LLM allowedModels [${contract.llm.allowedModels.join(", ")}].`);
|
|
3307
3361
|
}
|
|
3308
3362
|
}
|
|
3363
|
+
if ((contract.steps?.orderRelations?.length ?? 0) > 0) {
|
|
3364
|
+
const relations = contract.steps.orderRelations;
|
|
3365
|
+
const summary = relations.map(
|
|
3366
|
+
(relation) => `${relation.before.kind}:${relation.before.name} \u2192 ${relation.after.kind}:${relation.after.name} mode=${relation.mode ?? "first-occurrence"}`
|
|
3367
|
+
).join("; ");
|
|
3368
|
+
lines.push(`Base: ${relations.length} steps.orderRelations [${summary}].`);
|
|
3369
|
+
}
|
|
3309
3370
|
if (contract.observations?.required?.length) {
|
|
3310
3371
|
lines.push(`Base: required observations [${contract.observations.required.join(", ")}].`);
|
|
3311
3372
|
}
|
|
@@ -3720,6 +3781,31 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
3720
3781
|
function toolName(event) {
|
|
3721
3782
|
return resolveCanonicalToolName(event);
|
|
3722
3783
|
}
|
|
3784
|
+
function stepEndpointName(event, kind) {
|
|
3785
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event);
|
|
3786
|
+
for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
|
|
3787
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
3788
|
+
}
|
|
3789
|
+
return event.name;
|
|
3790
|
+
}
|
|
3791
|
+
function formatStepEndpoint(endpoint) {
|
|
3792
|
+
return `${endpoint.kind} ${endpoint.name}`;
|
|
3793
|
+
}
|
|
3794
|
+
function otherKindsForStepName(context, expectedKind, name) {
|
|
3795
|
+
return [
|
|
3796
|
+
...new Set(
|
|
3797
|
+
semanticEvents(context).filter((event) => {
|
|
3798
|
+
const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
|
|
3799
|
+
if (kind === expectedKind) return false;
|
|
3800
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event) === name;
|
|
3801
|
+
if (kind === "LLM") return stepEndpointName(event, "LLM") === name;
|
|
3802
|
+
return event.name === name || event.name === `tool:${name}` || event.name === `llm:${name}` || event.name.endsWith(`:${name}`);
|
|
3803
|
+
}).map(
|
|
3804
|
+
(event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
|
|
3805
|
+
)
|
|
3806
|
+
)
|
|
3807
|
+
].sort();
|
|
3808
|
+
}
|
|
3723
3809
|
function semanticEvents(context) {
|
|
3724
3810
|
return context.logicalEvents ?? context.events;
|
|
3725
3811
|
}
|
|
@@ -3746,14 +3832,6 @@ function finishedEvents(context, kind) {
|
|
|
3746
3832
|
function toolInvocationEvents(context) {
|
|
3747
3833
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
3748
3834
|
}
|
|
3749
|
-
function firstIndexByName(events) {
|
|
3750
|
-
const index = /* @__PURE__ */ new Map();
|
|
3751
|
-
for (const [position, event] of events.entries()) {
|
|
3752
|
-
const name = toolName(event);
|
|
3753
|
-
if (!index.has(name)) index.set(name, position);
|
|
3754
|
-
}
|
|
3755
|
-
return index;
|
|
3756
|
-
}
|
|
3757
3835
|
function isRecord3(value) {
|
|
3758
3836
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3759
3837
|
}
|
|
@@ -4263,18 +4341,85 @@ function createToolUsageRule(options) {
|
|
|
4263
4341
|
}
|
|
4264
4342
|
};
|
|
4265
4343
|
}
|
|
4266
|
-
function
|
|
4267
|
-
const
|
|
4344
|
+
function createStepOrderingRule(options) {
|
|
4345
|
+
const bothTools = options.before.kind === "TOOL" && options.after.kind === "TOOL";
|
|
4346
|
+
const ruleId = options.id ?? (bothTools ? "tool.order" : "step.order");
|
|
4268
4347
|
const mode = options.mode ?? "first-occurrence";
|
|
4348
|
+
const requireEndpoints = options.requireEndpoints === true;
|
|
4349
|
+
const beforeLabel = bothTools ? options.before.name : formatStepEndpoint(options.before);
|
|
4350
|
+
const afterLabel = bothTools ? options.after.name : formatStepEndpoint(options.after);
|
|
4351
|
+
const subject = bothTools ? "Tool" : "Step";
|
|
4269
4352
|
return {
|
|
4270
4353
|
id: ruleId,
|
|
4271
|
-
category: "tool",
|
|
4354
|
+
category: bothTools ? "tool" : "structure",
|
|
4272
4355
|
defaultSeverity: "error",
|
|
4273
4356
|
evaluate(context) {
|
|
4274
|
-
const
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
const
|
|
4357
|
+
const beforeEvents = finishedEvents(context, options.before.kind).filter(
|
|
4358
|
+
(event) => stepEndpointName(event, options.before.kind) === options.before.name
|
|
4359
|
+
);
|
|
4360
|
+
const afterEvents = finishedEvents(context, options.after.kind).filter(
|
|
4361
|
+
(event) => stepEndpointName(event, options.after.kind) === options.after.name
|
|
4362
|
+
);
|
|
4363
|
+
if (beforeEvents.length === 0 || afterEvents.length === 0) {
|
|
4364
|
+
if (!requireEndpoints) return [];
|
|
4365
|
+
const findings = [];
|
|
4366
|
+
if (beforeEvents.length === 0) {
|
|
4367
|
+
const otherKinds = otherKindsForStepName(
|
|
4368
|
+
context,
|
|
4369
|
+
options.before.kind,
|
|
4370
|
+
options.before.name
|
|
4371
|
+
);
|
|
4372
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.before.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
4373
|
+
findings.push(
|
|
4374
|
+
failFinding(
|
|
4375
|
+
ruleId,
|
|
4376
|
+
`Required ${formatStepEndpoint(options.before)} did not appear.${hint}`,
|
|
4377
|
+
runEvidence(context.selectedRun),
|
|
4378
|
+
options.before,
|
|
4379
|
+
{
|
|
4380
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
4381
|
+
endpoint: "before",
|
|
4382
|
+
otherKinds
|
|
4383
|
+
}
|
|
4384
|
+
)
|
|
4385
|
+
);
|
|
4386
|
+
}
|
|
4387
|
+
if (afterEvents.length === 0) {
|
|
4388
|
+
const otherKinds = otherKindsForStepName(
|
|
4389
|
+
context,
|
|
4390
|
+
options.after.kind,
|
|
4391
|
+
options.after.name
|
|
4392
|
+
);
|
|
4393
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.after.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
4394
|
+
findings.push(
|
|
4395
|
+
failFinding(
|
|
4396
|
+
ruleId,
|
|
4397
|
+
`Required ${formatStepEndpoint(options.after)} did not appear.${hint}`,
|
|
4398
|
+
runEvidence(context.selectedRun),
|
|
4399
|
+
options.after,
|
|
4400
|
+
{
|
|
4401
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
4402
|
+
endpoint: "after",
|
|
4403
|
+
otherKinds
|
|
4404
|
+
}
|
|
4405
|
+
)
|
|
4406
|
+
);
|
|
4407
|
+
}
|
|
4408
|
+
return findings;
|
|
4409
|
+
}
|
|
4410
|
+
const encounter = finishedEvents(context).filter(
|
|
4411
|
+
(event) => event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name || event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name
|
|
4412
|
+
);
|
|
4413
|
+
let beforeIndex;
|
|
4414
|
+
let afterIndex;
|
|
4415
|
+
for (const [position, event] of encounter.entries()) {
|
|
4416
|
+
if (beforeIndex === void 0 && event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name) {
|
|
4417
|
+
beforeIndex = position;
|
|
4418
|
+
}
|
|
4419
|
+
if (afterIndex === void 0 && event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name) {
|
|
4420
|
+
afterIndex = position;
|
|
4421
|
+
}
|
|
4422
|
+
}
|
|
4278
4423
|
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
4279
4424
|
return [];
|
|
4280
4425
|
}
|
|
@@ -4282,32 +4427,37 @@ function createToolOrderingRule(options) {
|
|
|
4282
4427
|
return [
|
|
4283
4428
|
failFinding(
|
|
4284
4429
|
ruleId,
|
|
4285
|
-
|
|
4286
|
-
[
|
|
4287
|
-
|
|
4288
|
-
|
|
4430
|
+
`${subject} ${beforeLabel} must appear before ${afterLabel}.`,
|
|
4431
|
+
[
|
|
4432
|
+
eventEvidence3(encounter[beforeIndex]),
|
|
4433
|
+
eventEvidence3(encounter[afterIndex])
|
|
4434
|
+
],
|
|
4435
|
+
bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after },
|
|
4436
|
+
bothTools ? finishedEvents(context, "TOOL").map(toolName) : encounter.map(
|
|
4437
|
+
(event) => event.kind === "TOOL" || event.kind === "LLM" ? `${event.kind}:${stepEndpointName(event, event.kind)}` : event.name
|
|
4438
|
+
)
|
|
4289
4439
|
)
|
|
4290
4440
|
];
|
|
4291
4441
|
}
|
|
4292
|
-
const beforeEvent =
|
|
4293
|
-
const afterEvent =
|
|
4442
|
+
const beforeEvent = beforeEvents[0];
|
|
4443
|
+
const afterEvent = afterEvents[0];
|
|
4294
4444
|
const beforeEnd = eventEndMs(beforeEvent);
|
|
4295
4445
|
const afterStart = eventStartMs(afterEvent);
|
|
4446
|
+
const endpointExpected = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
|
|
4296
4447
|
if (mode === "happens-before") {
|
|
4297
4448
|
const expected = {
|
|
4298
|
-
|
|
4299
|
-
after: options.after,
|
|
4449
|
+
...endpointExpected,
|
|
4300
4450
|
mode,
|
|
4301
4451
|
relation: "before.end <= after.start"
|
|
4302
4452
|
};
|
|
4303
|
-
if (options.before === options.after) {
|
|
4453
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
4304
4454
|
return [
|
|
4305
4455
|
failFinding(
|
|
4306
4456
|
ruleId,
|
|
4307
|
-
|
|
4457
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
4308
4458
|
[eventEvidence3(beforeEvent)],
|
|
4309
4459
|
expected,
|
|
4310
|
-
{ code: "tool.order.same-tool" }
|
|
4460
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
4311
4461
|
)
|
|
4312
4462
|
];
|
|
4313
4463
|
}
|
|
@@ -4315,11 +4465,11 @@ function createToolOrderingRule(options) {
|
|
|
4315
4465
|
return [
|
|
4316
4466
|
failFinding(
|
|
4317
4467
|
ruleId,
|
|
4318
|
-
|
|
4468
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish causal timing.`,
|
|
4319
4469
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4320
4470
|
expected,
|
|
4321
4471
|
{
|
|
4322
|
-
code: "tool.order.interval-unresolved",
|
|
4472
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4323
4473
|
beforeEndResolved: beforeEnd !== void 0,
|
|
4324
4474
|
afterStartResolved: afterStart !== void 0
|
|
4325
4475
|
}
|
|
@@ -4330,7 +4480,7 @@ function createToolOrderingRule(options) {
|
|
|
4330
4480
|
return [
|
|
4331
4481
|
failFinding(
|
|
4332
4482
|
ruleId,
|
|
4333
|
-
|
|
4483
|
+
`${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
|
|
4334
4484
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4335
4485
|
expected,
|
|
4336
4486
|
{
|
|
@@ -4343,22 +4493,19 @@ function createToolOrderingRule(options) {
|
|
|
4343
4493
|
return [];
|
|
4344
4494
|
}
|
|
4345
4495
|
if (mode === "all-occurrences") {
|
|
4346
|
-
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
4347
|
-
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
4348
4496
|
const expected = {
|
|
4349
|
-
|
|
4350
|
-
after: options.after,
|
|
4497
|
+
...endpointExpected,
|
|
4351
4498
|
mode,
|
|
4352
4499
|
relation: "max(before.end) <= min(after.start)"
|
|
4353
4500
|
};
|
|
4354
|
-
if (options.before === options.after) {
|
|
4501
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
4355
4502
|
return [
|
|
4356
4503
|
failFinding(
|
|
4357
4504
|
ruleId,
|
|
4358
|
-
|
|
4505
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
4359
4506
|
[eventEvidence3(beforeEvent)],
|
|
4360
4507
|
expected,
|
|
4361
|
-
{ code: "tool.order.same-tool" }
|
|
4508
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
4362
4509
|
)
|
|
4363
4510
|
];
|
|
4364
4511
|
}
|
|
@@ -4390,11 +4537,11 @@ function createToolOrderingRule(options) {
|
|
|
4390
4537
|
return [
|
|
4391
4538
|
failFinding(
|
|
4392
4539
|
ruleId,
|
|
4393
|
-
|
|
4540
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish all causal intervals.`,
|
|
4394
4541
|
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
|
|
4395
4542
|
expected,
|
|
4396
4543
|
{
|
|
4397
|
-
code: "tool.order.interval-unresolved",
|
|
4544
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4398
4545
|
missingBeforeEnds,
|
|
4399
4546
|
missingAfterStarts
|
|
4400
4547
|
}
|
|
@@ -4405,7 +4552,7 @@ function createToolOrderingRule(options) {
|
|
|
4405
4552
|
return [
|
|
4406
4553
|
failFinding(
|
|
4407
4554
|
ruleId,
|
|
4408
|
-
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
4555
|
+
bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
|
|
4409
4556
|
[eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
|
|
4410
4557
|
expected,
|
|
4411
4558
|
{
|
|
@@ -4423,10 +4570,13 @@ function createToolOrderingRule(options) {
|
|
|
4423
4570
|
ruleId,
|
|
4424
4571
|
severity: "warning",
|
|
4425
4572
|
status: "warning",
|
|
4426
|
-
message:
|
|
4427
|
-
expected: {
|
|
4573
|
+
message: `${subject} order ${beforeLabel} before ${afterLabel} satisfied start/encounter order, but intervals overlap so causal completion before the next ${bothTools ? "tool" : "step"} was not established.`,
|
|
4574
|
+
expected: {
|
|
4575
|
+
...endpointExpected,
|
|
4576
|
+
mode: "first-occurrence"
|
|
4577
|
+
},
|
|
4428
4578
|
actual: {
|
|
4429
|
-
code: "tool.order.overlap",
|
|
4579
|
+
code: bothTools ? "tool.order.overlap" : "step.order.overlap",
|
|
4430
4580
|
beforeEndedAt: beforeEvent.endedAt,
|
|
4431
4581
|
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
4432
4582
|
},
|
|
@@ -4438,6 +4588,15 @@ function createToolOrderingRule(options) {
|
|
|
4438
4588
|
}
|
|
4439
4589
|
};
|
|
4440
4590
|
}
|
|
4591
|
+
function createToolOrderingRule(options) {
|
|
4592
|
+
return createStepOrderingRule({
|
|
4593
|
+
before: { kind: "TOOL", name: options.before },
|
|
4594
|
+
after: { kind: "TOOL", name: options.after },
|
|
4595
|
+
...options.id !== void 0 ? { id: options.id } : {},
|
|
4596
|
+
...options.mode !== void 0 ? { mode: options.mode } : {},
|
|
4597
|
+
requireEndpoints: false
|
|
4598
|
+
});
|
|
4599
|
+
}
|
|
4441
4600
|
function createToolFailureRule(options) {
|
|
4442
4601
|
return {
|
|
4443
4602
|
id: "tool.failures",
|
|
@@ -5360,6 +5519,6 @@ function runTraceChecks(input, options = {}) {
|
|
|
5360
5519
|
};
|
|
5361
5520
|
}
|
|
5362
5521
|
|
|
5363
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
5364
|
-
//# sourceMappingURL=chunk-
|
|
5365
|
-
//# sourceMappingURL=chunk-
|
|
5522
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStepOrderingRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
5523
|
+
//# sourceMappingURL=chunk-CBPWLHNQ.mjs.map
|
|
5524
|
+
//# sourceMappingURL=chunk-CBPWLHNQ.mjs.map
|