agent-inspect 6.30.0 → 6.31.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 6.31.0
+
+### Minor Changes
+
+- 11a8b90: Add TraceContract `steps.orderRelations` for typed TOOL↔LLM step ordering (additive; TOOL-only `requiredOrder` / `orderRules` unchanged).
+
 ## 6.30.0
 
 ### Minor Changes

package/README.md

@@ -212,7 +212,7 @@ The root package is enough for custom capture, the CLI, checks, and Evidence wor
 
 ## Status and documentation
 
-**Current published baseline:** **6.30.0** · persisted schema `1.0` · Node.js `>=20` · MIT.
+**Current published baseline:** **6.31.0** · persisted schema `1.0` · Node.js `>=20` · MIT.
 
 Legacy v0.1 and v0.2 traces remain readable. Check the npm badge and [changelog](CHANGELOG.md) for the current published version.
 

package/docs/TRACE-CONTRACTS.md

@@ -12,6 +12,7 @@ Contracts compile to deterministic check rules for common cases:
 - tool required / forbidden / allowed / maxCalls / order (`requiredTools` / `forbiddenTools` aliases)
 - selectable `requiredOrderMode` (`first-occurrence` | `happens-before` | `all-occurrences`)
 - additive `tools.orderRules` with per-rule occurrence modes
+- typed cross-kind `steps.orderRelations` (TOOL ↔ LLM endpoints; additive in 6.31)
 - bounded `tools.arguments` JSON Pointer checks (`exists` | `type` | `equals` | `oneOf`)
 - `controls` declared-versus-enforced invariants
 - `retry` / side-effect safety using explicit attempt identity (additive `retry.operations[]` recovery oracles in 6.27)
@@ -61,6 +62,30 @@ For overlapping first calls, omitted / `first-occurrence` warns while `happens-b
 
 Immediate or positional `all-pairs` matching is not implemented.
 
+## `steps.orderRelations` (shipped — experimental, 6.31)
+
+Typed before/after relations with explicit `kind: "TOOL" | "LLM"` endpoints. Use this when a TOOL must precede an LLM (or vice versa). It does **not** change TOOL-only `tools.requiredOrder` / `orderRules` semantics.
+
+```ts
+defineTraceContract({
+  steps: {
+    orderRelations: [
+      {
+        before: { kind: "TOOL", name: "retrieve_policy" },
+        after: { kind: "LLM", name: "generate_answer" },
+        // mode?: "first-occurrence" | "happens-before" | "all-occurrences"
+        // requireEndpoints?: boolean // default true
+      },
+    ],
+  },
+});
+```
+
+- Default `mode` is `first-occurrence` (same three modes as tool order).
+- Default `requireEndpoints: true` — missing kind+name fails; a same display name under the other kind does **not** satisfy the endpoint.
+- LLM names match finished LLM events after stripping common prefixes (`llm:`, `generation:`, …).
+- Low-level `createStepOrderingRule` is compositional when `requireEndpoints` is omitted/false.
+
 ### Experimental Vitest / Jest matchers (shipped)
 
 | Package | Export | Matchers |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-inspect",
-  "version": "6.30.0",
+  "version": "6.31.0",
   "license": "MIT",
   "type": "module",
   "description": "Local evidence debugger and trajectory-test toolkit for TypeScript AI agents — execution trees, TraceContract checks, Evidence v2, and read-only MCP",

package/packages/cli/dist/{chunk-BDRJENLM.mjs → chunk-Y5Z4BXUO.mjs}

@@ -6368,6 +6368,17 @@ function cloneBody(body) {
       }
     } : {},
     ...body.llm ? { llm: { ...body.llm } } : {},
+    ...body.steps ? {
+      steps: {
+        ...body.steps.orderRelations ? {
+          orderRelations: body.steps.orderRelations.map((item) => ({
+            ...item,
+            before: { ...item.before },
+            after: { ...item.after }
+          }))
+        } : {}
+      }
+    } : {},
     ...body.observations ? {
       observations: {
         ...body.observations,
@@ -6407,7 +6418,7 @@ function cloneBody(body) {
   };
 }
 function bodyHasRules(body) {
-  return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
+  return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.steps !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
 }
 var MAX_EVIDENCE_EVENT_IDS = 16;
 var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
@@ -6750,6 +6761,18 @@ function contractToRules(contract) {
       })
     );
   }
+  const orderRelations = contract.steps?.orderRelations ?? [];
+  for (const [index, relation] of orderRelations.entries()) {
+    rules.push(
+      createStepOrderingRule({
+        before: { kind: relation.before.kind, name: relation.before.name },
+        after: { kind: relation.after.kind, name: relation.after.name },
+        id: `contract.step.orderRelation.${index}`,
+        mode: relation.mode ?? "first-occurrence",
+        requireEndpoints: relation.requireEndpoints !== false
+      })
+    );
+  }
   if (contract.observations) {
     const required = contract.observations.required ?? [];
     if (required.length > 0) {
@@ -7536,6 +7559,31 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
 function toolName(event) {
   return resolveCanonicalToolName(event);
 }
+function stepEndpointName(event, kind) {
+  if (kind === "TOOL") return resolveCanonicalToolName(event);
+  for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
+    if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
+  }
+  return event.name;
+}
+function formatStepEndpoint(endpoint) {
+  return `${endpoint.kind} ${endpoint.name}`;
+}
+function otherKindsForStepName(context, expectedKind, name) {
+  return [
+    ...new Set(
+      semanticEvents(context).filter((event) => {
+        const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
+        if (kind === expectedKind) return false;
+        if (kind === "TOOL") return resolveCanonicalToolName(event) === name;
+        if (kind === "LLM") return stepEndpointName(event, "LLM") === name;
+        return event.name === name || event.name === `tool:${name}` || event.name === `llm:${name}` || event.name.endsWith(`:${name}`);
+      }).map(
+        (event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
+      )
+    )
+  ].sort();
+}
 function semanticEvents(context) {
   return context.logicalEvents ?? context.events;
 }
@@ -7562,14 +7610,6 @@ function finishedEvents(context, kind) {
 function toolInvocationEvents(context) {
   return semanticEvents(context).filter((event) => event.kind === "TOOL");
 }
-function firstIndexByName(events) {
-  const index = /* @__PURE__ */ new Map();
-  for (const [position, event] of events.entries()) {
-    const name = toolName(event);
-    if (!index.has(name)) index.set(name, position);
-  }
-  return index;
-}
 function isRecord9(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -8040,18 +8080,85 @@ function createToolUsageRule(options) {
     }
   };
 }
-function createToolOrderingRule(options) {
-  const ruleId = options.id ?? "tool.order";
+function createStepOrderingRule(options) {
+  const bothTools = options.before.kind === "TOOL" && options.after.kind === "TOOL";
+  const ruleId = options.id ?? (bothTools ? "tool.order" : "step.order");
   const mode = options.mode ?? "first-occurrence";
+  const requireEndpoints = options.requireEndpoints === true;
+  const beforeLabel = bothTools ? options.before.name : formatStepEndpoint(options.before);
+  const afterLabel = bothTools ? options.after.name : formatStepEndpoint(options.after);
+  const subject = bothTools ? "Tool" : "Step";
   return {
     id: ruleId,
-    category: "tool",
+    category: bothTools ? "tool" : "structure",
     defaultSeverity: "error",
     evaluate(context) {
-      const tools = finishedEvents(context, "TOOL");
-      const index = firstIndexByName(tools);
-      const beforeIndex = index.get(options.before);
-      const afterIndex = index.get(options.after);
+      const beforeEvents = finishedEvents(context, options.before.kind).filter(
+        (event) => stepEndpointName(event, options.before.kind) === options.before.name
+      );
+      const afterEvents = finishedEvents(context, options.after.kind).filter(
+        (event) => stepEndpointName(event, options.after.kind) === options.after.name
+      );
+      if (beforeEvents.length === 0 || afterEvents.length === 0) {
+        if (!requireEndpoints) return [];
+        const findings = [];
+        if (beforeEvents.length === 0) {
+          const otherKinds = otherKindsForStepName(
+            context,
+            options.before.kind,
+            options.before.name
+          );
+          const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.before.kind} kind(s): ${otherKinds.join(", ")}.` : "";
+          findings.push(
+            failFinding(
+              ruleId,
+              `Required ${formatStepEndpoint(options.before)} did not appear.${hint}`,
+              runEvidence(context.selectedRun),
+              options.before,
+              {
+                code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
+                endpoint: "before",
+                otherKinds
+              }
+            )
+          );
+        }
+        if (afterEvents.length === 0) {
+          const otherKinds = otherKindsForStepName(
+            context,
+            options.after.kind,
+            options.after.name
+          );
+          const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.after.kind} kind(s): ${otherKinds.join(", ")}.` : "";
+          findings.push(
+            failFinding(
+              ruleId,
+              `Required ${formatStepEndpoint(options.after)} did not appear.${hint}`,
+              runEvidence(context.selectedRun),
+              options.after,
+              {
+                code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
+                endpoint: "after",
+                otherKinds
+              }
+            )
+          );
+        }
+        return findings;
+      }
+      const encounter = finishedEvents(context).filter(
+        (event) => event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name || event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name
+      );
+      let beforeIndex;
+      let afterIndex;
+      for (const [position, event] of encounter.entries()) {
+        if (beforeIndex === void 0 && event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name) {
+          beforeIndex = position;
+        }
+        if (afterIndex === void 0 && event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name) {
+          afterIndex = position;
+        }
+      }
       if (beforeIndex === void 0 || afterIndex === void 0) {
         return [];
       }
@@ -8059,32 +8166,37 @@ function createToolOrderingRule(options) {
         return [
           failFinding(
             ruleId,
-            `Tool ${options.before} must appear before ${options.after}.`,
-            [eventEvidence3(tools[beforeIndex]), eventEvidence3(tools[afterIndex])],
-            { before: options.before, after: options.after },
-            tools.map(toolName)
+            `${subject} ${beforeLabel} must appear before ${afterLabel}.`,
+            [
+              eventEvidence3(encounter[beforeIndex]),
+              eventEvidence3(encounter[afterIndex])
+            ],
+            bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after },
+            bothTools ? finishedEvents(context, "TOOL").map(toolName) : encounter.map(
+              (event) => event.kind === "TOOL" || event.kind === "LLM" ? `${event.kind}:${stepEndpointName(event, event.kind)}` : event.name
+            )
           )
         ];
       }
-      const beforeEvent = tools[beforeIndex];
-      const afterEvent = tools[afterIndex];
+      const beforeEvent = beforeEvents[0];
+      const afterEvent = afterEvents[0];
       const beforeEnd = eventEndMs(beforeEvent);
       const afterStart = eventStartMs(afterEvent);
+      const endpointExpected = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
       if (mode === "happens-before") {
         const expected = {
-          before: options.before,
-          after: options.after,
+          ...endpointExpected,
           mode,
           relation: "before.end <= after.start"
         };
-        if (options.before === options.after) {
+        if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
           return [
             failFinding(
               ruleId,
-              `Tool ${options.before} cannot causally happen before itself.`,
+              `${subject} ${beforeLabel} cannot causally happen before itself.`,
               [eventEvidence3(beforeEvent)],
               expected,
-              { code: "tool.order.same-tool" }
+              { code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
             )
           ];
         }
@@ -8092,11 +8204,11 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Tool order ${options.before} before ${options.after} could not establish causal timing.`,
+              `${subject} order ${beforeLabel} before ${afterLabel} could not establish causal timing.`,
               [eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
               expected,
               {
-                code: "tool.order.interval-unresolved",
+                code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
                 beforeEndResolved: beforeEnd !== void 0,
                 afterStartResolved: afterStart !== void 0
               }
@@ -8107,7 +8219,7 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Tool ${options.before} must finish before ${options.after} starts.`,
+              `${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
               [eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
               expected,
               {
@@ -8120,22 +8232,19 @@ function createToolOrderingRule(options) {
         return [];
       }
       if (mode === "all-occurrences") {
-        const beforeEvents = tools.filter((event) => toolName(event) === options.before);
-        const afterEvents = tools.filter((event) => toolName(event) === options.after);
         const expected = {
-          before: options.before,
-          after: options.after,
+          ...endpointExpected,
           mode,
           relation: "max(before.end) <= min(after.start)"
         };
-        if (options.before === options.after) {
+        if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
           return [
             failFinding(
               ruleId,
-              `Tool ${options.before} cannot causally happen before itself.`,
+              `${subject} ${beforeLabel} cannot causally happen before itself.`,
               [eventEvidence3(beforeEvent)],
               expected,
-              { code: "tool.order.same-tool" }
+              { code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
             )
           ];
         }
@@ -8167,11 +8276,11 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
+              `${subject} order ${beforeLabel} before ${afterLabel} could not establish all causal intervals.`,
               [firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
               expected,
               {
-                code: "tool.order.interval-unresolved",
+                code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
                 missingBeforeEnds,
                 missingAfterStarts
               }
@@ -8182,7 +8291,7 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
+              bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
               [eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
               expected,
               {
@@ -8200,10 +8309,13 @@ function createToolOrderingRule(options) {
             ruleId,
             severity: "warning",
             status: "warning",
-            message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
-            expected: { before: options.before, after: options.after, mode: "first-occurrence" },
+            message: `${subject} order ${beforeLabel} before ${afterLabel} satisfied start/encounter order, but intervals overlap so causal completion before the next ${bothTools ? "tool" : "step"} was not established.`,
+            expected: {
+              ...endpointExpected,
+              mode: "first-occurrence"
+            },
             actual: {
-              code: "tool.order.overlap",
+              code: bothTools ? "tool.order.overlap" : "step.order.overlap",
               beforeEndedAt: beforeEvent.endedAt,
               afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
             },
@@ -8215,6 +8327,15 @@ function createToolOrderingRule(options) {
     }
   };
 }
+function createToolOrderingRule(options) {
+  return createStepOrderingRule({
+    before: { kind: "TOOL", name: options.before },
+    after: { kind: "TOOL", name: options.after },
+    ...options.id !== void 0 ? { id: options.id } : {},
+    ...options.mode !== void 0 ? { mode: options.mode } : {},
+    requireEndpoints: false
+  });
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -14733,5 +14854,5 @@ function renderGateReport(result, options = {}) {
 }
 
 export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
-//# sourceMappingURL=chunk-BDRJENLM.mjs.map
-//# sourceMappingURL=chunk-BDRJENLM.mjs.map
+//# sourceMappingURL=chunk-Y5Z4BXUO.mjs.map
+//# sourceMappingURL=chunk-Y5Z4BXUO.mjs.map