agent-inspect 6.30.0 → 6.31.0

package/packages/cli/dist/index.cjs

@@ -5700,6 +5700,31 @@ function failFinding(ruleId, message, evidence, expected, actual, meta2) {
 function toolName(event) {
   return resolveCanonicalToolName(event);
 }
+function stepEndpointName(event, kind) {
+  if (kind === "TOOL") return resolveCanonicalToolName(event);
+  for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
+    if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
+  }
+  return event.name;
+}
+function formatStepEndpoint(endpoint) {
+  return `${endpoint.kind} ${endpoint.name}`;
+}
+function otherKindsForStepName(context, expectedKind, name) {
+  return [
+    ...new Set(
+      semanticEvents(context).filter((event) => {
+        const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
+        if (kind === expectedKind) return false;
+        if (kind === "TOOL") return resolveCanonicalToolName(event) === name;
+        if (kind === "LLM") return stepEndpointName(event, "LLM") === name;
+        return event.name === name || event.name === `tool:${name}` || event.name === `llm:${name}` || event.name.endsWith(`:${name}`);
+      }).map(
+        (event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
+      )
+    )
+  ].sort();
+}
 function semanticEvents(context) {
   return context.logicalEvents ?? context.events;
 }
@@ -5726,14 +5751,6 @@ function finishedEvents(context, kind) {
 function toolInvocationEvents(context) {
   return semanticEvents(context).filter((event) => event.kind === "TOOL");
 }
-function firstIndexByName(events) {
-  const index = /* @__PURE__ */ new Map();
-  for (const [position, event] of events.entries()) {
-    const name = toolName(event);
-    if (!index.has(name)) index.set(name, position);
-  }
-  return index;
-}
 function isRecord9(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -6204,18 +6221,85 @@ function createToolUsageRule(options) {
     }
   };
 }
-function createToolOrderingRule(options) {
-  const ruleId = options.id ?? "tool.order";
+function createStepOrderingRule(options) {
+  const bothTools = options.before.kind === "TOOL" && options.after.kind === "TOOL";
+  const ruleId = options.id ?? (bothTools ? "tool.order" : "step.order");
   const mode = options.mode ?? "first-occurrence";
+  const requireEndpoints = options.requireEndpoints === true;
+  const beforeLabel = bothTools ? options.before.name : formatStepEndpoint(options.before);
+  const afterLabel = bothTools ? options.after.name : formatStepEndpoint(options.after);
+  const subject = bothTools ? "Tool" : "Step";
   return {
     id: ruleId,
-    category: "tool",
+    category: bothTools ? "tool" : "structure",
     defaultSeverity: "error",
     evaluate(context) {
-      const tools = finishedEvents(context, "TOOL");
-      const index = firstIndexByName(tools);
-      const beforeIndex = index.get(options.before);
-      const afterIndex = index.get(options.after);
+      const beforeEvents = finishedEvents(context, options.before.kind).filter(
+        (event) => stepEndpointName(event, options.before.kind) === options.before.name
+      );
+      const afterEvents = finishedEvents(context, options.after.kind).filter(
+        (event) => stepEndpointName(event, options.after.kind) === options.after.name
+      );
+      if (beforeEvents.length === 0 || afterEvents.length === 0) {
+        if (!requireEndpoints) return [];
+        const findings = [];
+        if (beforeEvents.length === 0) {
+          const otherKinds = otherKindsForStepName(
+            context,
+            options.before.kind,
+            options.before.name
+          );
+          const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.before.kind} kind(s): ${otherKinds.join(", ")}.` : "";
+          findings.push(
+            failFinding(
+              ruleId,
+              `Required ${formatStepEndpoint(options.before)} did not appear.${hint}`,
+              runEvidence(context.selectedRun),
+              options.before,
+              {
+                code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
+                endpoint: "before",
+                otherKinds
+              }
+            )
+          );
+        }
+        if (afterEvents.length === 0) {
+          const otherKinds = otherKindsForStepName(
+            context,
+            options.after.kind,
+            options.after.name
+          );
+          const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.after.kind} kind(s): ${otherKinds.join(", ")}.` : "";
+          findings.push(
+            failFinding(
+              ruleId,
+              `Required ${formatStepEndpoint(options.after)} did not appear.${hint}`,
+              runEvidence(context.selectedRun),
+              options.after,
+              {
+                code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
+                endpoint: "after",
+                otherKinds
+              }
+            )
+          );
+        }
+        return findings;
+      }
+      const encounter = finishedEvents(context).filter(
+        (event) => event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name || event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name
+      );
+      let beforeIndex;
+      let afterIndex;
+      for (const [position, event] of encounter.entries()) {
+        if (beforeIndex === void 0 && event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name) {
+          beforeIndex = position;
+        }
+        if (afterIndex === void 0 && event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name) {
+          afterIndex = position;
+        }
+      }
       if (beforeIndex === void 0 || afterIndex === void 0) {
         return [];
       }
@@ -6223,32 +6307,37 @@ function createToolOrderingRule(options) {
         return [
           failFinding(
             ruleId,
-            `Tool ${options.before} must appear before ${options.after}.`,
-            [eventEvidence(tools[beforeIndex]), eventEvidence(tools[afterIndex])],
-            { before: options.before, after: options.after },
-            tools.map(toolName)
+            `${subject} ${beforeLabel} must appear before ${afterLabel}.`,
+            [
+              eventEvidence(encounter[beforeIndex]),
+              eventEvidence(encounter[afterIndex])
+            ],
+            bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after },
+            bothTools ? finishedEvents(context, "TOOL").map(toolName) : encounter.map(
+              (event) => event.kind === "TOOL" || event.kind === "LLM" ? `${event.kind}:${stepEndpointName(event, event.kind)}` : event.name
+            )
           )
         ];
       }
-      const beforeEvent = tools[beforeIndex];
-      const afterEvent = tools[afterIndex];
+      const beforeEvent = beforeEvents[0];
+      const afterEvent = afterEvents[0];
       const beforeEnd = eventEndMs(beforeEvent);
       const afterStart = eventStartMs(afterEvent);
+      const endpointExpected = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
       if (mode === "happens-before") {
         const expected = {
-          before: options.before,
-          after: options.after,
+          ...endpointExpected,
           mode,
           relation: "before.end <= after.start"
         };
-        if (options.before === options.after) {
+        if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
           return [
             failFinding(
               ruleId,
-              `Tool ${options.before} cannot causally happen before itself.`,
+              `${subject} ${beforeLabel} cannot causally happen before itself.`,
               [eventEvidence(beforeEvent)],
               expected,
-              { code: "tool.order.same-tool" }
+              { code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
             )
           ];
         }
@@ -6256,11 +6345,11 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Tool order ${options.before} before ${options.after} could not establish causal timing.`,
+              `${subject} order ${beforeLabel} before ${afterLabel} could not establish causal timing.`,
               [eventEvidence(beforeEvent), eventEvidence(afterEvent)],
               expected,
               {
-                code: "tool.order.interval-unresolved",
+                code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
                 beforeEndResolved: beforeEnd !== void 0,
                 afterStartResolved: afterStart !== void 0
               }
@@ -6271,7 +6360,7 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Tool ${options.before} must finish before ${options.after} starts.`,
+              `${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
               [eventEvidence(beforeEvent), eventEvidence(afterEvent)],
               expected,
               {
@@ -6284,22 +6373,19 @@ function createToolOrderingRule(options) {
         return [];
       }
       if (mode === "all-occurrences") {
-        const beforeEvents = tools.filter((event) => toolName(event) === options.before);
-        const afterEvents = tools.filter((event) => toolName(event) === options.after);
         const expected = {
-          before: options.before,
-          after: options.after,
+          ...endpointExpected,
           mode,
           relation: "max(before.end) <= min(after.start)"
         };
-        if (options.before === options.after) {
+        if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
           return [
             failFinding(
               ruleId,
-              `Tool ${options.before} cannot causally happen before itself.`,
+              `${subject} ${beforeLabel} cannot causally happen before itself.`,
               [eventEvidence(beforeEvent)],
               expected,
-              { code: "tool.order.same-tool" }
+              { code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
             )
           ];
         }
@@ -6331,11 +6417,11 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
+              `${subject} order ${beforeLabel} before ${afterLabel} could not establish all causal intervals.`,
               [firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence(event)),
               expected,
               {
-                code: "tool.order.interval-unresolved",
+                code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
                 missingBeforeEnds,
                 missingAfterStarts
               }
@@ -6346,7 +6432,7 @@ function createToolOrderingRule(options) {
           return [
             failFinding(
               ruleId,
-              `Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
+              bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
               [eventEvidence(latestBefore.event), eventEvidence(earliestAfter.event)],
               expected,
               {
@@ -6364,10 +6450,13 @@ function createToolOrderingRule(options) {
             ruleId,
             severity: "warning",
             status: "warning",
-            message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
-            expected: { before: options.before, after: options.after, mode: "first-occurrence" },
+            message: `${subject} order ${beforeLabel} before ${afterLabel} satisfied start/encounter order, but intervals overlap so causal completion before the next ${bothTools ? "tool" : "step"} was not established.`,
+            expected: {
+              ...endpointExpected,
+              mode: "first-occurrence"
+            },
             actual: {
-              code: "tool.order.overlap",
+              code: bothTools ? "tool.order.overlap" : "step.order.overlap",
               beforeEndedAt: beforeEvent.endedAt,
               afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
             },
@@ -6379,6 +6468,15 @@ function createToolOrderingRule(options) {
     }
   };
 }
+function createToolOrderingRule(options) {
+  return createStepOrderingRule({
+    before: { kind: "TOOL", name: options.before },
+    after: { kind: "TOOL", name: options.after },
+    ...options.id !== void 0 ? { id: options.id } : {},
+    ...options.mode !== void 0 ? { mode: options.mode } : {},
+    requireEndpoints: false
+  });
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -8657,6 +8755,17 @@ function cloneBody(body) {
       }
     } : {},
     ...body.llm ? { llm: { ...body.llm } } : {},
+    ...body.steps ? {
+      steps: {
+        ...body.steps.orderRelations ? {
+          orderRelations: body.steps.orderRelations.map((item) => ({
+            ...item,
+            before: { ...item.before },
+            after: { ...item.after }
+          }))
+        } : {}
+      }
+    } : {},
     ...body.observations ? {
       observations: {
         ...body.observations,
@@ -8696,7 +8805,7 @@ function cloneBody(body) {
   };
 }
 function bodyHasRules(body) {
-  return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
+  return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.steps !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
 }
 function parseEvidenceEventIds(evidence) {
   if (evidence === void 0 || evidence === null) {
@@ -9037,6 +9146,18 @@ function contractToRules(contract) {
       })
     );
   }
+  const orderRelations = contract.steps?.orderRelations ?? [];
+  for (const [index, relation] of orderRelations.entries()) {
+    rules.push(
+      createStepOrderingRule({
+        before: { kind: relation.before.kind, name: relation.before.name },
+        after: { kind: relation.after.kind, name: relation.after.name },
+        id: `contract.step.orderRelation.${index}`,
+        mode: relation.mode ?? "first-occurrence",
+        requireEndpoints: relation.requireEndpoints !== false
+      })
+    );
+  }
   if (contract.observations) {
     const required = contract.observations.required ?? [];
     if (required.length > 0) {
@@ -16001,7 +16122,7 @@ var init_src = __esm({
 });
 
 // package.json
-var version = "6.30.0";
+var version = "6.31.0";
 
 // packages/cli/src/list.ts
 init_advanced();
@@ -22925,6 +23046,7 @@ var CONTRACT_KEYS = /* @__PURE__ */ new Set([
   "run",
   "tools",
   "llm",
+  "steps",
   "observations",
   "scope",
   "alternatives",
@@ -22935,6 +23057,7 @@ var CONTRACT_BODY_KEYS = /* @__PURE__ */ new Set([
   "run",
   "tools",
   "llm",
+  "steps",
   "observations",
   "controls",
   "retry"
@@ -23012,6 +23135,15 @@ var CONTRACT_ORDER_RULE_KEYS = /* @__PURE__ */ new Set([
   "occurrenceMode",
   "requireEndpoints"
 ]);
+var CONTRACT_STEPS_KEYS = /* @__PURE__ */ new Set(["orderRelations"]);
+var CONTRACT_STEP_ORDER_RELATION_KEYS = /* @__PURE__ */ new Set([
+  "before",
+  "after",
+  "mode",
+  "requireEndpoints"
+]);
+var CONTRACT_STEP_REF_KEYS = /* @__PURE__ */ new Set(["kind", "name"]);
+var CONTRACT_STEP_KINDS = /* @__PURE__ */ new Set(["TOOL", "LLM"]);
 var CONTRACT_ARGUMENT_KEYS = /* @__PURE__ */ new Set([
   "tool",
   "path",
@@ -23342,6 +23474,7 @@ function contractConfigHasEffect(contract) {
   if (sectionHasEffect(contract.run)) return true;
   if (sectionHasEffect(contract.tools)) return true;
   if (sectionHasEffect(contract.llm)) return true;
+  if (sectionHasEffect(contract.steps)) return true;
   if (sectionHasEffect(contract.observations)) {
     return true;
   }
@@ -23526,6 +23659,66 @@ function parseContractLlmSection(value, pathPrefix = "contract.llm") {
   }
   return out;
 }
+function parseStepRef(value, pathPrefix) {
+  const record = requireObject(value, pathPrefix);
+  rejectUnknownKeys(record, CONTRACT_STEP_REF_KEYS, pathPrefix);
+  if (typeof record.kind !== "string" || !CONTRACT_STEP_KINDS.has(record.kind)) {
+    throw new CheckConfigError(
+      "AI_CHECK_CONFIG_INVALID_VALUE",
+      `${pathPrefix}.kind must be "TOOL" or "LLM".`
+    );
+  }
+  return {
+    kind: record.kind,
+    name: requireNonEmptyString(record.name, `${pathPrefix}.name`)
+  };
+}
+function parseStepOrderRelation(value, pathPrefix) {
+  const record = requireObject(value, pathPrefix);
+  rejectUnknownKeys(record, CONTRACT_STEP_ORDER_RELATION_KEYS, pathPrefix);
+  if (record.before === void 0) {
+    throw new CheckConfigError(
+      "AI_CHECK_CONFIG_INVALID_VALUE",
+      `${pathPrefix}.before is required.`
+    );
+  }
+  if (record.after === void 0) {
+    throw new CheckConfigError(
+      "AI_CHECK_CONFIG_INVALID_VALUE",
+      `${pathPrefix}.after is required.`
+    );
+  }
+  const out = {
+    before: parseStepRef(record.before, `${pathPrefix}.before`),
+    after: parseStepRef(record.after, `${pathPrefix}.after`)
+  };
+  if (record.mode !== void 0) {
+    out.mode = requireOrderMode(record.mode, `${pathPrefix}.mode`);
+  }
+  if (record.requireEndpoints !== void 0) {
+    out.requireEndpoints = requireBoolean(
+      record.requireEndpoints,
+      `${pathPrefix}.requireEndpoints`
+    );
+  }
+  return out;
+}
+function parseContractStepsSection(value, pathPrefix = "contract.steps") {
+  const record = requireObject(value, pathPrefix);
+  rejectUnknownKeys(record, CONTRACT_STEPS_KEYS, pathPrefix);
+  const out = {};
+  if (record.orderRelations !== void 0) {
+    const items = requireBoundedArray(
+      record.orderRelations,
+      `${pathPrefix}.orderRelations`,
+      CONTRACT_MAX_ORDER_RULES
+    );
+    out.orderRelations = items.map(
+      (item, index) => parseStepOrderRelation(item, `${pathPrefix}.orderRelations[${index}]`)
+    );
+  }
+  return out;
+}
 function parseRequireProvenance(value, pathPrefix) {
   const record = requireObject(value, pathPrefix);
   rejectUnknownKeys(record, CONTRACT_PROVENANCE_KEYS, pathPrefix);
@@ -23811,6 +24004,9 @@ function parseContractBodySection(value, pathPrefix) {
     out.tools = parseContractToolsSection(record.tools, `${pathPrefix}.tools`);
   }
   if (record.llm !== void 0) out.llm = parseContractLlmSection(record.llm, `${pathPrefix}.llm`);
+  if (record.steps !== void 0) {
+    out.steps = parseContractStepsSection(record.steps, `${pathPrefix}.steps`);
+  }
   if (record.observations !== void 0) {
     out.observations = parseContractObservationsSection(
       record.observations,