agent-inspect 6.30.0 → 6.31.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/COMPARE.md +1 -1
- package/docs/FIRST-TRACE-IN-5-MINUTES.md +21 -17
- package/docs/GETTING-STARTED.md +1 -1
- package/docs/TRACE-CONTRACTS.md +25 -0
- package/package.json +4 -2
- package/packages/cli/dist/{chunk-BDRJENLM.mjs → chunk-SJ6FRNOE.mjs} +201 -49
- package/packages/cli/dist/chunk-SJ6FRNOE.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +288 -50
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +91 -5
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-ZUGAUXY6.mjs → src-YNEJOPRA.mjs} +3 -3
- package/packages/cli/dist/{src-ZUGAUXY6.mjs.map → src-YNEJOPRA.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +39 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +207 -47
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-OOX25GTH.mjs → chunk-CBPWLHNQ.mjs} +209 -50
- package/packages/core/dist/{chunk-OOX25GTH.mjs.map → chunk-CBPWLHNQ.mjs.map} +1 -1
- package/packages/core/dist/{index-7hG_qggp.d.ts → index-CgOAGuM5.d.ts} +111 -1
- package/packages/core/dist/{index-CQJUNPPe.d.cts → index-EQihlEei.d.cts} +111 -1
- package/packages/cli/dist/chunk-BDRJENLM.mjs.map +0 -1
|
@@ -6,8 +6,8 @@ import { a as InspectRunTree } from './inspect-event-ClM5Z8om.cjs';
|
|
|
6
6
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, E as EventSource, d as InspectEvent, I as InspectKind, e as InspectNode, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.cjs';
|
|
7
7
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-q2ViiBW6.cjs';
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
|
-
import {
|
|
10
|
-
export {
|
|
9
|
+
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-EQihlEei.cjs';
|
|
10
|
+
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-EQihlEei.cjs';
|
|
11
11
|
import { b as ObservedOutcome } from './types-BA_IOdg4.cjs';
|
|
12
12
|
export { a as ObservedOutcomeStatus } from './types-BA_IOdg4.cjs';
|
|
13
13
|
import './writers.cjs';
|
|
@@ -6,8 +6,8 @@ import { a as InspectRunTree } from './inspect-event-ClM5Z8om.js';
|
|
|
6
6
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, E as EventSource, d as InspectEvent, I as InspectKind, e as InspectNode, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.js';
|
|
7
7
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-DG02LhEP.js';
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
|
-
import {
|
|
10
|
-
export {
|
|
9
|
+
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-CgOAGuM5.js';
|
|
10
|
+
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-CgOAGuM5.js';
|
|
11
11
|
import { b as ObservedOutcome } from './types-BA_IOdg4.js';
|
|
12
12
|
export { a as ObservedOutcomeStatus } from './types-BA_IOdg4.js';
|
|
13
13
|
import './writers.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { extractSessionWorkflowMetadata, sessionKeyForRun, defineTraceContract, runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-
|
|
2
|
-
export { extractSessionWorkflowMetadata, sessionKeyForRun } from './chunk-
|
|
1
|
+
import { extractSessionWorkflowMetadata, sessionKeyForRun, defineTraceContract, runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-CBPWLHNQ.mjs';
|
|
2
|
+
export { extractSessionWorkflowMetadata, sessionKeyForRun } from './chunk-CBPWLHNQ.mjs';
|
|
3
3
|
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-ZFJ2ICTC.mjs';
|
|
4
4
|
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-KWBO36PH.mjs';
|
|
5
5
|
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-KWBO36PH.mjs';
|
|
@@ -2490,11 +2490,40 @@ function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
|
2490
2490
|
evidence: [...evidence]
|
|
2491
2491
|
};
|
|
2492
2492
|
}
|
|
2493
|
+
var ALLOWED_STATUS_ALIASES = {
|
|
2494
|
+
ok: "ok",
|
|
2495
|
+
error: "error",
|
|
2496
|
+
running: "running",
|
|
2497
|
+
success: "ok",
|
|
2498
|
+
failed: "error"
|
|
2499
|
+
};
|
|
2500
|
+
function parseAllowedStatus(status) {
|
|
2501
|
+
return ALLOWED_STATUS_ALIASES[status];
|
|
2502
|
+
}
|
|
2493
2503
|
function normalizeStatus(status) {
|
|
2494
|
-
|
|
2495
|
-
if (
|
|
2496
|
-
|
|
2497
|
-
|
|
2504
|
+
const parsed = parseAllowedStatus(status);
|
|
2505
|
+
if (parsed === void 0) {
|
|
2506
|
+
throw new TypeError(
|
|
2507
|
+
`Unknown run status ${JSON.stringify(status)}. Allowed: ok, error, running (aliases: success, failed).`
|
|
2508
|
+
);
|
|
2509
|
+
}
|
|
2510
|
+
return parsed;
|
|
2511
|
+
}
|
|
2512
|
+
function validateAllowedStatusesShape(allowedStatuses, pathPrefix = "run.allowedStatuses") {
|
|
2513
|
+
if (!allowedStatuses?.length) return [];
|
|
2514
|
+
const out = [];
|
|
2515
|
+
for (let i = 0; i < allowedStatuses.length; i++) {
|
|
2516
|
+
const status = allowedStatuses[i];
|
|
2517
|
+
if (parseAllowedStatus(status) === void 0) {
|
|
2518
|
+
out.push({
|
|
2519
|
+
code: "contract.run.allowedStatuses.unknown",
|
|
2520
|
+
severity: "error",
|
|
2521
|
+
message: `${pathPrefix}[${i}] has unknown status ${JSON.stringify(status)}. Allowed: ok, error, running (aliases: success, failed).`,
|
|
2522
|
+
path: `${pathPrefix}[${i}]`
|
|
2523
|
+
});
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
return out;
|
|
2498
2527
|
}
|
|
2499
2528
|
function cloneBody(body) {
|
|
2500
2529
|
return {
|
|
@@ -2507,6 +2536,17 @@ function cloneBody(body) {
|
|
|
2507
2536
|
}
|
|
2508
2537
|
} : {},
|
|
2509
2538
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
2539
|
+
...body.steps ? {
|
|
2540
|
+
steps: {
|
|
2541
|
+
...body.steps.orderRelations ? {
|
|
2542
|
+
orderRelations: body.steps.orderRelations.map((item) => ({
|
|
2543
|
+
...item,
|
|
2544
|
+
before: { ...item.before },
|
|
2545
|
+
after: { ...item.after }
|
|
2546
|
+
}))
|
|
2547
|
+
} : {}
|
|
2548
|
+
}
|
|
2549
|
+
} : {},
|
|
2510
2550
|
...body.observations ? {
|
|
2511
2551
|
observations: {
|
|
2512
2552
|
...body.observations,
|
|
@@ -2546,7 +2586,7 @@ function cloneBody(body) {
|
|
|
2546
2586
|
};
|
|
2547
2587
|
}
|
|
2548
2588
|
function bodyHasRules(body) {
|
|
2549
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
2589
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.steps !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
2550
2590
|
}
|
|
2551
2591
|
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
2552
2592
|
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
@@ -2889,6 +2929,18 @@ function contractToRules(contract) {
|
|
|
2889
2929
|
})
|
|
2890
2930
|
);
|
|
2891
2931
|
}
|
|
2932
|
+
const orderRelations = contract.steps?.orderRelations ?? [];
|
|
2933
|
+
for (const [index, relation] of orderRelations.entries()) {
|
|
2934
|
+
rules.push(
|
|
2935
|
+
createStepOrderingRule({
|
|
2936
|
+
before: { kind: relation.before.kind, name: relation.before.name },
|
|
2937
|
+
after: { kind: relation.after.kind, name: relation.after.name },
|
|
2938
|
+
id: `contract.step.orderRelation.${index}`,
|
|
2939
|
+
mode: relation.mode ?? "first-occurrence",
|
|
2940
|
+
requireEndpoints: relation.requireEndpoints !== false
|
|
2941
|
+
})
|
|
2942
|
+
);
|
|
2943
|
+
}
|
|
2892
2944
|
if (contract.observations) {
|
|
2893
2945
|
const required = contract.observations.required ?? [];
|
|
2894
2946
|
if (required.length > 0) {
|
|
@@ -3138,6 +3190,7 @@ function evaluateBody(input, body, options = {}) {
|
|
|
3138
3190
|
}
|
|
3139
3191
|
function defineTraceContract(input) {
|
|
3140
3192
|
const shapeErrors = [
|
|
3193
|
+
...validateAllowedStatusesShape(input.run?.allowedStatuses),
|
|
3141
3194
|
...validateAlternativesShape(input.alternatives),
|
|
3142
3195
|
...validateScopeShape(input.scope),
|
|
3143
3196
|
...validateProvenanceShape(input.observations)
|
|
@@ -3168,6 +3221,7 @@ function defineTraceContract(input) {
|
|
|
3168
3221
|
}
|
|
3169
3222
|
function evaluateTraceContract(input, contract, options = {}) {
|
|
3170
3223
|
const shapeErrors = [
|
|
3224
|
+
...validateAllowedStatusesShape(contract.run?.allowedStatuses),
|
|
3171
3225
|
...validateAlternativesShape(contract.alternatives),
|
|
3172
3226
|
...validateScopeShape(contract.scope),
|
|
3173
3227
|
...validateProvenanceShape(contract.observations)
|
|
@@ -3442,6 +3496,13 @@ function explainTraceContract(contract) {
|
|
|
3442
3496
|
lines.push(`Base: LLM allowedModels [${contract.llm.allowedModels.join(", ")}].`);
|
|
3443
3497
|
}
|
|
3444
3498
|
}
|
|
3499
|
+
if ((contract.steps?.orderRelations?.length ?? 0) > 0) {
|
|
3500
|
+
const relations = contract.steps.orderRelations;
|
|
3501
|
+
const summary = relations.map(
|
|
3502
|
+
(relation) => `${relation.before.kind}:${relation.before.name} \u2192 ${relation.after.kind}:${relation.after.name} mode=${relation.mode ?? "first-occurrence"}`
|
|
3503
|
+
).join("; ");
|
|
3504
|
+
lines.push(`Base: ${relations.length} steps.orderRelations [${summary}].`);
|
|
3505
|
+
}
|
|
3445
3506
|
if (contract.observations?.required?.length) {
|
|
3446
3507
|
lines.push(`Base: required observations [${contract.observations.required.join(", ")}].`);
|
|
3447
3508
|
}
|
|
@@ -3856,6 +3917,31 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
3856
3917
|
function toolName(event) {
|
|
3857
3918
|
return resolveCanonicalToolName(event);
|
|
3858
3919
|
}
|
|
3920
|
+
function stepEndpointName(event, kind) {
|
|
3921
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event);
|
|
3922
|
+
for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
|
|
3923
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
3924
|
+
}
|
|
3925
|
+
return event.name;
|
|
3926
|
+
}
|
|
3927
|
+
function formatStepEndpoint(endpoint) {
|
|
3928
|
+
return `${endpoint.kind} ${endpoint.name}`;
|
|
3929
|
+
}
|
|
3930
|
+
function otherKindsForStepName(context, expectedKind, name) {
|
|
3931
|
+
return [
|
|
3932
|
+
...new Set(
|
|
3933
|
+
semanticEvents(context).filter((event) => {
|
|
3934
|
+
const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
|
|
3935
|
+
if (kind === expectedKind) return false;
|
|
3936
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event) === name;
|
|
3937
|
+
if (kind === "LLM") return stepEndpointName(event, "LLM") === name;
|
|
3938
|
+
return event.name === name || event.name === `tool:${name}` || event.name === `llm:${name}` || event.name.endsWith(`:${name}`);
|
|
3939
|
+
}).map(
|
|
3940
|
+
(event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
|
|
3941
|
+
)
|
|
3942
|
+
)
|
|
3943
|
+
].sort();
|
|
3944
|
+
}
|
|
3859
3945
|
function semanticEvents(context) {
|
|
3860
3946
|
return context.logicalEvents ?? context.events;
|
|
3861
3947
|
}
|
|
@@ -3882,14 +3968,6 @@ function finishedEvents(context, kind) {
|
|
|
3882
3968
|
function toolInvocationEvents(context) {
|
|
3883
3969
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
3884
3970
|
}
|
|
3885
|
-
function firstIndexByName(events) {
|
|
3886
|
-
const index = /* @__PURE__ */ new Map();
|
|
3887
|
-
for (const [position, event] of events.entries()) {
|
|
3888
|
-
const name = toolName(event);
|
|
3889
|
-
if (!index.has(name)) index.set(name, position);
|
|
3890
|
-
}
|
|
3891
|
-
return index;
|
|
3892
|
-
}
|
|
3893
3971
|
function isRecord3(value) {
|
|
3894
3972
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3895
3973
|
}
|
|
@@ -4399,18 +4477,85 @@ function createToolUsageRule(options) {
|
|
|
4399
4477
|
}
|
|
4400
4478
|
};
|
|
4401
4479
|
}
|
|
4402
|
-
function
|
|
4403
|
-
const
|
|
4480
|
+
function createStepOrderingRule(options) {
|
|
4481
|
+
const bothTools = options.before.kind === "TOOL" && options.after.kind === "TOOL";
|
|
4482
|
+
const ruleId = options.id ?? (bothTools ? "tool.order" : "step.order");
|
|
4404
4483
|
const mode = options.mode ?? "first-occurrence";
|
|
4484
|
+
const requireEndpoints = options.requireEndpoints === true;
|
|
4485
|
+
const beforeLabel = bothTools ? options.before.name : formatStepEndpoint(options.before);
|
|
4486
|
+
const afterLabel = bothTools ? options.after.name : formatStepEndpoint(options.after);
|
|
4487
|
+
const subject = bothTools ? "Tool" : "Step";
|
|
4405
4488
|
return {
|
|
4406
4489
|
id: ruleId,
|
|
4407
|
-
category: "tool",
|
|
4490
|
+
category: bothTools ? "tool" : "structure",
|
|
4408
4491
|
defaultSeverity: "error",
|
|
4409
4492
|
evaluate(context) {
|
|
4410
|
-
const
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
const
|
|
4493
|
+
const beforeEvents = finishedEvents(context, options.before.kind).filter(
|
|
4494
|
+
(event) => stepEndpointName(event, options.before.kind) === options.before.name
|
|
4495
|
+
);
|
|
4496
|
+
const afterEvents = finishedEvents(context, options.after.kind).filter(
|
|
4497
|
+
(event) => stepEndpointName(event, options.after.kind) === options.after.name
|
|
4498
|
+
);
|
|
4499
|
+
if (beforeEvents.length === 0 || afterEvents.length === 0) {
|
|
4500
|
+
if (!requireEndpoints) return [];
|
|
4501
|
+
const findings = [];
|
|
4502
|
+
if (beforeEvents.length === 0) {
|
|
4503
|
+
const otherKinds = otherKindsForStepName(
|
|
4504
|
+
context,
|
|
4505
|
+
options.before.kind,
|
|
4506
|
+
options.before.name
|
|
4507
|
+
);
|
|
4508
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.before.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
4509
|
+
findings.push(
|
|
4510
|
+
failFinding(
|
|
4511
|
+
ruleId,
|
|
4512
|
+
`Required ${formatStepEndpoint(options.before)} did not appear.${hint}`,
|
|
4513
|
+
runEvidence(context.selectedRun),
|
|
4514
|
+
options.before,
|
|
4515
|
+
{
|
|
4516
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
4517
|
+
endpoint: "before",
|
|
4518
|
+
otherKinds
|
|
4519
|
+
}
|
|
4520
|
+
)
|
|
4521
|
+
);
|
|
4522
|
+
}
|
|
4523
|
+
if (afterEvents.length === 0) {
|
|
4524
|
+
const otherKinds = otherKindsForStepName(
|
|
4525
|
+
context,
|
|
4526
|
+
options.after.kind,
|
|
4527
|
+
options.after.name
|
|
4528
|
+
);
|
|
4529
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.after.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
4530
|
+
findings.push(
|
|
4531
|
+
failFinding(
|
|
4532
|
+
ruleId,
|
|
4533
|
+
`Required ${formatStepEndpoint(options.after)} did not appear.${hint}`,
|
|
4534
|
+
runEvidence(context.selectedRun),
|
|
4535
|
+
options.after,
|
|
4536
|
+
{
|
|
4537
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
4538
|
+
endpoint: "after",
|
|
4539
|
+
otherKinds
|
|
4540
|
+
}
|
|
4541
|
+
)
|
|
4542
|
+
);
|
|
4543
|
+
}
|
|
4544
|
+
return findings;
|
|
4545
|
+
}
|
|
4546
|
+
const encounter = finishedEvents(context).filter(
|
|
4547
|
+
(event) => event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name || event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name
|
|
4548
|
+
);
|
|
4549
|
+
let beforeIndex;
|
|
4550
|
+
let afterIndex;
|
|
4551
|
+
for (const [position, event] of encounter.entries()) {
|
|
4552
|
+
if (beforeIndex === void 0 && event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name) {
|
|
4553
|
+
beforeIndex = position;
|
|
4554
|
+
}
|
|
4555
|
+
if (afterIndex === void 0 && event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name) {
|
|
4556
|
+
afterIndex = position;
|
|
4557
|
+
}
|
|
4558
|
+
}
|
|
4414
4559
|
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
4415
4560
|
return [];
|
|
4416
4561
|
}
|
|
@@ -4418,32 +4563,37 @@ function createToolOrderingRule(options) {
|
|
|
4418
4563
|
return [
|
|
4419
4564
|
failFinding(
|
|
4420
4565
|
ruleId,
|
|
4421
|
-
|
|
4422
|
-
[
|
|
4423
|
-
|
|
4424
|
-
|
|
4566
|
+
`${subject} ${beforeLabel} must appear before ${afterLabel}.`,
|
|
4567
|
+
[
|
|
4568
|
+
eventEvidence3(encounter[beforeIndex]),
|
|
4569
|
+
eventEvidence3(encounter[afterIndex])
|
|
4570
|
+
],
|
|
4571
|
+
bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after },
|
|
4572
|
+
bothTools ? finishedEvents(context, "TOOL").map(toolName) : encounter.map(
|
|
4573
|
+
(event) => event.kind === "TOOL" || event.kind === "LLM" ? `${event.kind}:${stepEndpointName(event, event.kind)}` : event.name
|
|
4574
|
+
)
|
|
4425
4575
|
)
|
|
4426
4576
|
];
|
|
4427
4577
|
}
|
|
4428
|
-
const beforeEvent =
|
|
4429
|
-
const afterEvent =
|
|
4578
|
+
const beforeEvent = beforeEvents[0];
|
|
4579
|
+
const afterEvent = afterEvents[0];
|
|
4430
4580
|
const beforeEnd = eventEndMs(beforeEvent);
|
|
4431
4581
|
const afterStart = eventStartMs(afterEvent);
|
|
4582
|
+
const endpointExpected = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
|
|
4432
4583
|
if (mode === "happens-before") {
|
|
4433
4584
|
const expected = {
|
|
4434
|
-
|
|
4435
|
-
after: options.after,
|
|
4585
|
+
...endpointExpected,
|
|
4436
4586
|
mode,
|
|
4437
4587
|
relation: "before.end <= after.start"
|
|
4438
4588
|
};
|
|
4439
|
-
if (options.before === options.after) {
|
|
4589
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
4440
4590
|
return [
|
|
4441
4591
|
failFinding(
|
|
4442
4592
|
ruleId,
|
|
4443
|
-
|
|
4593
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
4444
4594
|
[eventEvidence3(beforeEvent)],
|
|
4445
4595
|
expected,
|
|
4446
|
-
{ code: "tool.order.same-tool" }
|
|
4596
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
4447
4597
|
)
|
|
4448
4598
|
];
|
|
4449
4599
|
}
|
|
@@ -4451,11 +4601,11 @@ function createToolOrderingRule(options) {
|
|
|
4451
4601
|
return [
|
|
4452
4602
|
failFinding(
|
|
4453
4603
|
ruleId,
|
|
4454
|
-
|
|
4604
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish causal timing.`,
|
|
4455
4605
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4456
4606
|
expected,
|
|
4457
4607
|
{
|
|
4458
|
-
code: "tool.order.interval-unresolved",
|
|
4608
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4459
4609
|
beforeEndResolved: beforeEnd !== void 0,
|
|
4460
4610
|
afterStartResolved: afterStart !== void 0
|
|
4461
4611
|
}
|
|
@@ -4466,7 +4616,7 @@ function createToolOrderingRule(options) {
|
|
|
4466
4616
|
return [
|
|
4467
4617
|
failFinding(
|
|
4468
4618
|
ruleId,
|
|
4469
|
-
|
|
4619
|
+
`${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
|
|
4470
4620
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4471
4621
|
expected,
|
|
4472
4622
|
{
|
|
@@ -4479,22 +4629,19 @@ function createToolOrderingRule(options) {
|
|
|
4479
4629
|
return [];
|
|
4480
4630
|
}
|
|
4481
4631
|
if (mode === "all-occurrences") {
|
|
4482
|
-
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
4483
|
-
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
4484
4632
|
const expected = {
|
|
4485
|
-
|
|
4486
|
-
after: options.after,
|
|
4633
|
+
...endpointExpected,
|
|
4487
4634
|
mode,
|
|
4488
4635
|
relation: "max(before.end) <= min(after.start)"
|
|
4489
4636
|
};
|
|
4490
|
-
if (options.before === options.after) {
|
|
4637
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
4491
4638
|
return [
|
|
4492
4639
|
failFinding(
|
|
4493
4640
|
ruleId,
|
|
4494
|
-
|
|
4641
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
4495
4642
|
[eventEvidence3(beforeEvent)],
|
|
4496
4643
|
expected,
|
|
4497
|
-
{ code: "tool.order.same-tool" }
|
|
4644
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
4498
4645
|
)
|
|
4499
4646
|
];
|
|
4500
4647
|
}
|
|
@@ -4526,11 +4673,11 @@ function createToolOrderingRule(options) {
|
|
|
4526
4673
|
return [
|
|
4527
4674
|
failFinding(
|
|
4528
4675
|
ruleId,
|
|
4529
|
-
|
|
4676
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish all causal intervals.`,
|
|
4530
4677
|
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
|
|
4531
4678
|
expected,
|
|
4532
4679
|
{
|
|
4533
|
-
code: "tool.order.interval-unresolved",
|
|
4680
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4534
4681
|
missingBeforeEnds,
|
|
4535
4682
|
missingAfterStarts
|
|
4536
4683
|
}
|
|
@@ -4541,7 +4688,7 @@ function createToolOrderingRule(options) {
|
|
|
4541
4688
|
return [
|
|
4542
4689
|
failFinding(
|
|
4543
4690
|
ruleId,
|
|
4544
|
-
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
4691
|
+
bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
|
|
4545
4692
|
[eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
|
|
4546
4693
|
expected,
|
|
4547
4694
|
{
|
|
@@ -4559,10 +4706,13 @@ function createToolOrderingRule(options) {
|
|
|
4559
4706
|
ruleId,
|
|
4560
4707
|
severity: "warning",
|
|
4561
4708
|
status: "warning",
|
|
4562
|
-
message:
|
|
4563
|
-
expected: {
|
|
4709
|
+
message: `${subject} order ${beforeLabel} before ${afterLabel} satisfied start/encounter order, but intervals overlap so causal completion before the next ${bothTools ? "tool" : "step"} was not established.`,
|
|
4710
|
+
expected: {
|
|
4711
|
+
...endpointExpected,
|
|
4712
|
+
mode: "first-occurrence"
|
|
4713
|
+
},
|
|
4564
4714
|
actual: {
|
|
4565
|
-
code: "tool.order.overlap",
|
|
4715
|
+
code: bothTools ? "tool.order.overlap" : "step.order.overlap",
|
|
4566
4716
|
beforeEndedAt: beforeEvent.endedAt,
|
|
4567
4717
|
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
4568
4718
|
},
|
|
@@ -4574,6 +4724,15 @@ function createToolOrderingRule(options) {
|
|
|
4574
4724
|
}
|
|
4575
4725
|
};
|
|
4576
4726
|
}
|
|
4727
|
+
function createToolOrderingRule(options) {
|
|
4728
|
+
return createStepOrderingRule({
|
|
4729
|
+
before: { kind: "TOOL", name: options.before },
|
|
4730
|
+
after: { kind: "TOOL", name: options.after },
|
|
4731
|
+
...options.id !== void 0 ? { id: options.id } : {},
|
|
4732
|
+
...options.mode !== void 0 ? { mode: options.mode } : {},
|
|
4733
|
+
requireEndpoints: false
|
|
4734
|
+
});
|
|
4735
|
+
}
|
|
4577
4736
|
function createToolFailureRule(options) {
|
|
4578
4737
|
return {
|
|
4579
4738
|
id: "tool.failures",
|
|
@@ -5517,6 +5676,7 @@ exports.createSafetyRawContentRule = createSafetyRawContentRule;
|
|
|
5517
5676
|
exports.createSafetyRedactionRule = createSafetyRedactionRule;
|
|
5518
5677
|
exports.createSafetySecretPatternRule = createSafetySecretPatternRule;
|
|
5519
5678
|
exports.createStallDetectionRule = createStallDetectionRule;
|
|
5679
|
+
exports.createStepOrderingRule = createStepOrderingRule;
|
|
5520
5680
|
exports.createStructureCycleRule = createStructureCycleRule;
|
|
5521
5681
|
exports.createStructureIncompleteRule = createStructureIncompleteRule;
|
|
5522
5682
|
exports.createStructureOrphanRule = createStructureOrphanRule;
|