agent-inspect 6.17.3 → 6.17.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/README.md +1 -1
  3. package/docs/ADAPTER-CONFORMANCE.md +24 -0
  4. package/docs/ADAPTERS.md +3 -3
  5. package/docs/API.md +13 -7
  6. package/docs/CI-ARTIFACTS.md +2 -1
  7. package/docs/CLI.md +2 -0
  8. package/docs/LIMITATIONS.md +1 -0
  9. package/docs/OPENAI-AGENTS-LOCAL.md +14 -0
  10. package/docs/SAFE-TRACE-SHARING.md +1 -0
  11. package/docs/SELF-HOSTING.md +2 -0
  12. package/docs/STANDARDS.md +15 -0
  13. package/docs/SUPPORT-LEVELS.md +23 -0
  14. package/docs/TRACE-CONTRACTS.md +90 -0
  15. package/package.json +7 -3
  16. package/packages/cli/dist/{chunk-5GGYDIZD.mjs → chunk-GOHYSE7W.mjs} +93 -19
  17. package/packages/cli/dist/chunk-GOHYSE7W.mjs.map +1 -0
  18. package/packages/cli/dist/index.cjs +505 -88
  19. package/packages/cli/dist/index.cjs.map +1 -1
  20. package/packages/cli/dist/index.mjs +417 -74
  21. package/packages/cli/dist/index.mjs.map +1 -1
  22. package/packages/cli/dist/{src-IDMCWKRH.mjs → src-OYGG7QAZ.mjs} +3 -3
  23. package/packages/cli/dist/{src-IDMCWKRH.mjs.map → src-OYGG7QAZ.mjs.map} +1 -1
  24. package/packages/core/dist/advanced.cjs +86 -17
  25. package/packages/core/dist/advanced.cjs.map +1 -1
  26. package/packages/core/dist/advanced.d.cts +1 -1
  27. package/packages/core/dist/advanced.d.ts +1 -1
  28. package/packages/core/dist/advanced.mjs +15 -5
  29. package/packages/core/dist/advanced.mjs.map +1 -1
  30. package/packages/core/dist/checks.cjs +126 -28
  31. package/packages/core/dist/checks.cjs.map +1 -1
  32. package/packages/core/dist/checks.d.cts +28 -2
  33. package/packages/core/dist/checks.d.ts +28 -2
  34. package/packages/core/dist/checks.mjs +1 -1
  35. package/packages/core/dist/{chunk-UFP54T7F.mjs → chunk-DIZPIPY2.mjs} +128 -30
  36. package/packages/core/dist/chunk-DIZPIPY2.mjs.map +1 -0
  37. package/packages/core/dist/{index-Xk9X-yjY.d.cts → index-DWu54Y28.d.cts} +107 -122
  38. package/packages/core/dist/{index-BsCcOKxy.d.ts → index-DlwbVqEs.d.ts} +107 -122
  39. package/packages/cli/dist/chunk-5GGYDIZD.mjs.map +0 -1
  40. package/packages/core/dist/chunk-UFP54T7F.mjs.map +0 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # Changelog
2
2
 
3
+ ## 6.17.5
4
+
5
+ ### Patch Changes
6
+
7
+ - 093811b: Harden deterministic TraceContract / check gates against fail-open empty configs (rule execution evidence, unique order IDs, requiredOrder implies presence, tool policy includes running invocations, ObservedOutcome requireAny), map #308–#311 release ownership, and make demo:verify / pack:smoke validation cross-platform without unnecessary shell invocation.
8
+
9
+ ## 6.17.4
10
+
11
+ ### Patch Changes
12
+
13
+ - c4b0f03: Fix camelCase / kebab / dot compound credential key redaction (`userPassword`, `clientSecret`) while keeping token-config keys and camelCase topic fields (`emailNote`) un-key-redacted.
14
+
3
15
  ## 6.17.3
4
16
 
5
17
  ### Patch Changes
package/README.md CHANGED
@@ -210,7 +210,7 @@ The root package is enough for custom capture, the CLI, checks, and Evidence wor
210
210
 
211
211
  ## Status and documentation
212
212
 
213
- **Current published baseline:** **6.17.2** · persisted schema `1.0` · Node.js `>=20` · MIT.
213
+ **Current published baseline:** **6.17.5** · persisted schema `1.0` · Node.js `>=20` · MIT.
214
214
 
215
215
  Legacy v0.1 and v0.2 traces remain readable. Check the npm badge and [changelog](CHANGELOG.md) for the current published version.
216
216
 
@@ -58,6 +58,30 @@ This is an internal official-adapter gate, not a third-party certification progr
58
58
 
59
59
  An adapter can be documented as supported only after no-network fixtures cover run, step, tool, LLM, error, streaming, and metadata-bound expectations for that framework path.
60
60
 
61
+ ## Third-party adapter conformance CI
62
+
63
+ The gate above is the internal official-adapter gate, not a third-party certification program. Third-party adapter authors run the **public** conformance helper from [`@agent-inspect/adapter-sdk`](https://www.npmjs.com/package/@agent-inspect/adapter-sdk) in their own CI:
64
+
65
+ ```ts
66
+ import { runAdapterConformance } from "@agent-inspect/adapter-sdk";
67
+
68
+ const result = await runAdapterConformance({
69
+ adapterId: "my-adapter",
70
+ events, // PersistedInspectEvent[] your adapter emits
71
+ expectedKinds: ["RUN", "LLM"],
72
+ forbiddenRawStrings: ["super-secret-key"], // must not leak into events
73
+ });
74
+ // assert result.ok in your test runner
75
+ ```
76
+
77
+ A copyable GitHub Actions template lives at [`examples/adapter-sdk/third-party-conformance-ci.yml`](../examples/adapter-sdk/third-party-conformance-ci.yml). Copy it to `.github/workflows/agent-inspect-conformance.yml` in your adapter repo and adjust the install/build/test commands. The template:
78
+
79
+ - runs on a Node LTS matrix (18 / 20 / 22),
80
+ - requires **no provider keys** and makes **no network calls** (conformance is metadata-only),
81
+ - keeps the framework SDK as a peer dependency of your adapter, never of AgentInspect core.
82
+
83
+ See the runnable examples under [`examples/adapter-sdk/`](../examples/adapter-sdk/) for adapters that call `runAdapterConformance` end to end.
84
+
61
85
  ## v3.2 adoption evidence refresh
62
86
 
63
87
  - [AI-SDK-ADOPTION.md](./AI-SDK-ADOPTION.md) — blessed AI SDK path (`generateText`, `streamText`, Next.js recipe, troubleshooting)
package/docs/ADAPTERS.md CHANGED
@@ -22,7 +22,7 @@ Reporters (`@agent-inspect/vitest` and `@agent-inspect/jest`) are public package
22
22
 
23
23
  **Status:** experimental adapter — optional package published in the aligned v2.2.0 package set and hardened in the v2.3 adapter train.
24
24
 
25
- The adapter has hardened lifecycle identity and parallel integration isolation. It remains metadata-only: `capture: "preview"` and preview-only redaction options emit diagnostics and fall back to metadata-only capture until bounded free-text previews are implemented.
25
+ The adapter has hardened lifecycle identity and parallel integration isolation. It remains metadata-only: `capture: "preview"` and preview-only redaction options emit diagnostics **and one visible** `AI_ADAPTER_PREVIEW_NOT_AVAILABLE` warning per adapter instance, then fall back to metadata-only capture. Bounded free-text preview capture is planned for a later train; it is not implemented here.
26
26
 
27
27
  ### Install
28
28
 
@@ -59,7 +59,7 @@ const result = await generateText({
59
59
  - **Metadata-only by default** — records model, finish reason, token usage, timing, and safe counts/summaries.
60
60
  - **Required safe telemetry settings** — set `recordInputs: false` and `recordOutputs: false` on every AI SDK call using this adapter.
61
61
  - **No raw payload capture by default** — prompts, messages, generated text, stream chunks, tool inputs/outputs, headers, request bodies, and response bodies are not persisted.
62
- - **Preview capture is not enabled yet** — `capture: "preview"`, `redactionProfile`, and `maxPreviewChars` are diagnosed through `getDiagnostics()` and do not persist raw previews.
62
+ - **Preview capture is not enabled yet** — requesting `capture: "preview"` emits one console warning with code `AI_ADAPTER_PREVIEW_NOT_AVAILABLE`, records diagnostics via `getDiagnostics()`, and does not persist raw previews. Effective capture remains metadata-only.
63
63
 
64
64
  ### Local no-network recipe
65
65
 
@@ -375,7 +375,7 @@ Integration modes:
375
375
  - **No upload behavior** — the processor writes only to an explicit local writer or `traceDir`.
376
376
  - **Metadata-only by default** — records trace/span IDs, parentage, names, timing, status, errors, safe model/tool names, token counts, and bounded summaries.
377
377
  - **No raw payload capture by default** — prompts, messages, generated text, function inputs/outputs, arbitrary custom data, trace exporter credentials, headers, request bodies, response bodies, and hosted tool payloads are not persisted.
378
- - **Preview capture is not enabled yet** — `capture: "preview"`, `redactionProfile`, and `maxPreviewChars` are diagnosed through `getDiagnostics()` and do not persist raw previews.
378
+ - **Preview capture is not enabled yet** — requesting `capture: "preview"` emits one console warning with code `AI_ADAPTER_PREVIEW_NOT_AVAILABLE`, records diagnostics via `getDiagnostics()`, and does not persist raw previews. Effective capture remains metadata-only.
379
379
  - **Fixture-backed lifecycle coverage** — local tests and the recipe cover agent, generation, function tool, handoff, guardrail, response, MCP tools, custom, transcription, and speech span shapes without provider calls.
380
380
 
381
381
  Full API: [API.md](./API.md) §14.
package/docs/API.md CHANGED
@@ -180,7 +180,11 @@ Diff is local and read-only. Programmatic diff surfaces are experimental until t
180
180
  - In-memory **`getEvents()`** / **`clear()`** unchanged when `persist` is false
181
181
  - Metadata helpers: `extractModelName`, `extractTokenUsage`, `safePreview`, `toPlainMetadata`
182
182
 
183
- Rationale: v1.x includes one official adapter and **zero production sinks**, so adapter surfaces remain experimental.
183
+ Rationale: one official adapter and **zero production sinks** keep adapter surfaces experimental; they remain additive in minors.
184
+
185
+ ### TraceContract `requiredOrder`
186
+
187
+ When defining `tools.requiredOrder`, the array expands into adjacent first-occurrence pair checks. Unlisted intermediate tools are allowed; later repetitions do not invalidate an earlier valid first-occurrence order. Missing tools are handled by required-tool rules, not ordering alone. See [TRACE-CONTRACTS.md](./TRACE-CONTRACTS.md).
184
188
 
185
189
  ## 10. Experimental `@agent-inspect/tui` APIs
186
190
 
@@ -249,7 +253,7 @@ The manifest records framework, generation time, bounded test results, artifact
249
253
 
250
254
  ## 12. Experimental `@agent-inspect/vitest` APIs
251
255
 
252
- `@agent-inspect/vitest` is an optional experimental workspace package for local Vitest failure artifacts. It remains private/unpublished pending maintainer first-publication setup. It does not add a Vitest dependency to root/core, does not upload artifacts, and does not infer trace relationships by timestamp.
256
+ `@agent-inspect/vitest` is an optional experimental workspace package for local Vitest failure artifacts. It is published with the fixed release group. It does not add a Vitest dependency to root/core, does not upload artifacts, and does not infer trace relationships by timestamp.
253
257
 
254
258
  Import from `@agent-inspect/vitest`:
255
259
 
@@ -282,7 +286,7 @@ Artifacts are safe structural summaries. The reporter writes a shared `schemaVer
282
286
 
283
287
  ## 13. Experimental `@agent-inspect/jest` APIs
284
288
 
285
- `@agent-inspect/jest` is an optional experimental workspace package for local Jest failure artifacts. It remains private/unpublished pending maintainer first-publication setup. It does not add a Jest dependency to root/core, does not upload artifacts, and does not infer trace relationships by timestamp.
289
+ `@agent-inspect/jest` is an optional experimental workspace package for local Jest failure artifacts. It is published with the fixed release group. It does not add a Jest dependency to root/core, does not upload artifacts, and does not infer trace relationships by timestamp.
286
290
 
287
291
  Import from `@agent-inspect/jest`:
288
292
 
@@ -423,8 +427,8 @@ import type {
423
427
  ```
424
428
 
425
429
  - **`TraceWriter`**: async `write(event)`, optional `flush()`, optional `close()`, optional `getStats()`.
426
- - **`fileWriter({ dir?, filePath? })`**: appends `PersistedInspectEvent` JSONL rows to local disk. `createInspector()` emits schema 1.0 rows by default; compatibility adapters may still pass readable v0.2 rows. By default it derives one file per `event.runId`; `filePath` writes all events to an explicit local file. Filesystem and serialization failures are reflected in writer stats instead of being thrown into application code.
427
- - **`bufferedFileWriter({ dir?, filePath?, maxQueueSize?, flushIntervalMs?, maxBatchSize?, overflow? })`**: buffers local JSONL writes with bounded queue behavior. Overflow supports `drop-oldest` and `drop-newest`; neither mode throws into application code.
430
+ - **`fileWriter({ dir?, filePath? })`**: appends `PersistedInspectEvent` JSONL rows to local disk. `createInspector()` emits schema 1.0 rows by default; compatibility adapters may still pass readable v0.2 rows. By default it derives one file per `event.runId`; `filePath` writes all events to an explicit local file. Filesystem and serialization failures are reflected in writer stats instead of being thrown into application code. Append-only local JSONL is diagnostic persistence — not a write-ahead log or fsync-guaranteed durable-before-effect journal.
431
+ - **`bufferedFileWriter({ dir?, filePath?, maxQueueSize?, flushIntervalMs?, maxBatchSize?, overflow? })`**: buffers local JSONL writes with bounded queue behavior. Overflow supports `drop-oldest` and `drop-newest`; neither mode throws into application code. On abrupt process exit, a bounded unflushed tail may be lost.
428
432
  - **`compositeWriter([...writers])`**: fans out events to multiple explicit local/custom writers. A failing child writer does not prevent other children from receiving events; failures are reflected in composite stats.
429
433
  - **`memoryWriter()`**: stores cloned `PersistedInspectEvent` rows in memory for tests, adapter fixtures, and eval harnesses.
430
434
  - **`nullWriter()`**: accepts events without retaining them for disabled mode, overhead comparisons, and no-output tests.
@@ -433,7 +437,7 @@ No network writer or vendor sink exists in this package.
433
437
 
434
438
  ## 19. Experimental inspector API/runtime (v1.6)
435
439
 
436
- `createInspector()` is the public instance API for local-first tracing with explicit writers. It owns an instance-specific runtime context, records schema 1.0 persisted inspect events, preserves application return values/errors, and exposes diagnostics plus deterministic `flush()`/`close()` lifecycle hooks.
440
+ `createInspector()` is the public instance API for local-first tracing with explicit writers. It owns an instance-specific runtime context, records schema 1.0 persisted inspect events, preserves application return values/errors, and exposes diagnostics plus deterministic `flush()`/`close()` lifecycle hooks (`flush`/`close` drain writer queues idempotently — they do not claim crash durability or fsync).
437
441
 
438
442
  Import from `agent-inspect`:
439
443
 
@@ -537,7 +541,9 @@ import type { TraceCheckRule, TraceCheckResult, TraceFacts } from "agent-inspect
537
541
  - **`TraceCheckRule`**: synchronous pure rule contract.
538
542
  - **`TraceCheckResult`**: deterministic aggregate result with findings, evidence, summary counts, and execution diagnostics.
539
543
 
540
- The checks API is experimental in v1.x. The `agent-inspect check` CLI uses this API for local reader-backed checks and deterministic JSON output; `agent-inspect artifacts` reuses the same safe findings for local CI artifact bundles and optional step-summary file output. Built-in rules operate on normalized event metadata, tree relationships, bounded summaries, token counts, and normalized baseline facts; safety and baseline findings identify event IDs and field paths rather than emitting raw prompts, outputs, secrets, headers, request/response bodies, or full tool payloads.
544
+ The checks API is experimental. The `agent-inspect check` CLI uses this API for local reader-backed checks and deterministic JSON output; `agent-inspect artifacts` reuses the same safe findings for local CI artifact bundles and optional step-summary file output. Built-in rules operate on normalized event metadata, tree relationships, bounded summaries, token counts, and normalized baseline facts; safety and baseline findings identify event IDs and field paths rather than emitting raw prompts, outputs, secrets, headers, request/response bodies, or full tool payloads.
545
+
546
+ @beta Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
541
547
 
542
548
  Recipes: [deterministic-ci-checks](../examples/recipes/deterministic-ci-checks/README.md) for check/baseline/artifact workflows, and [test-reporter-artifacts](../examples/recipes/test-reporter-artifacts/README.md) for Vitest/Jest reporter configuration patterns.
543
549
 
@@ -86,8 +86,9 @@ Recipes:
86
86
 
87
87
  - [examples/recipes/deterministic-ci-checks](../examples/recipes/deterministic-ci-checks/README.md) for v1.8 `check`, baseline, safe artifact, and step-summary workflows.
88
88
  - [examples/recipes/github-actions-artifact](../examples/recipes/github-actions-artifact/README.md) for share-safe trace exports and reporter manifest summaries.
89
+ - [examples/recipes/github-actions-gate](../examples/recipes/github-actions-gate/README.md) for a retained broken-to-fixed suite/gate pilot with separate Evidence v2 artifacts.
89
90
 
90
- Sample workflows: [deterministic checks workflow](../examples/recipes/deterministic-ci-checks/workflow-example.yml), [share-safe export workflow](../examples/recipes/github-actions-artifact/workflow-example.yml)
91
+ Sample workflows: [deterministic checks workflow](../examples/recipes/deterministic-ci-checks/workflow-example.yml), [share-safe export workflow](../examples/recipes/github-actions-artifact/workflow-example.yml), [retained gate workflow](../examples/recipes/github-actions-gate/workflow-example.yml)
91
92
 
92
93
  ```yaml
93
94
  - uses: actions/upload-artifact@v4
package/docs/CLI.md CHANGED
@@ -204,6 +204,8 @@ Options:
204
204
  - `--refresh <ms>`: minimum time between renders
205
205
  - `--json`: newline-delimited JSON updates
206
206
 
207
+ **Truncation recovery:** when a watched `--file` shrinks below the last read offset (for example a truncate-and-rewrite), `tail` resets the offset, discards any buffered partial line, and continues the same session from the start of the current file contents. Full inode-aware rename/recreate rotation is **not** claimed unless separately implemented and tested.
208
+
207
209
  Important: `tail` is a local developer tool, not a production monitor.
208
210
 
209
211
  ### 6.6 `export`
@@ -23,6 +23,7 @@ This document states what AgentInspect **does not** provide today. It complement
23
23
 
24
24
  - **Subpath APIs:** `agent-inspect/writers`, `agent-inspect/readers`, and advanced helpers are available for local adoption from their owning subpaths. `createInspector()` is part of the small root API.
25
25
  - **Explicit writer ownership:** `createInspector()` does not print terminal lifecycle output or implicitly choose a disk writer. Use `fileWriter()` / `bufferedFileWriter()` / custom writers when persistence is desired.
26
+ - **Diagnostic evidence, not an event-sourced runtime:** AgentInspect traces are local diagnostic evidence of what an agent did. They are **not** a write-ahead log, durable-before-effect journal, or application event-sourcing runtime. Buffered writers may lose a bounded tail of events on abrupt process termination. Call `flush()` / `close()` according to writer docs when you need the queue drained before exit; that lifecycle is deterministic and idempotent, but it is **not** an fsync / crash-durability guarantee.
26
27
  - **No standards upload:** OpenInference and OTLP JSON support is local read/export compatibility only. There is no OTLP gRPC/HTTP streaming sink, collector client, or hosted ingestion behavior.
27
28
  - **Conservative detection:** `agent-inspect open` does not silently accept arbitrary JSON. Unsupported or ambiguous inputs produce errors/warnings rather than guessed traces.
28
29
  - **Large inputs:** reader inputs are bounded and read into local memory. This is not a database index or production log warehouse.
@@ -41,6 +41,20 @@ setTraceProcessors([
41
41
 
42
42
  - [openai-agents-local-processor](../../examples/recipes/openai-agents-local-processor/)
43
43
 
44
+ ## No-key packed consumer check
45
+
46
+ The repository includes a packed-consumer check that installs the root and
47
+ OpenAI Agents adapter tarballs into a temporary project. It drives deterministic
48
+ tracing fixtures without `OPENAI_API_KEY` or a live provider call, writes a local
49
+ trace, and inspects it through the packed AgentInspect CLI.
50
+
51
+ ```bash
52
+ pnpm build
53
+ node scripts/packed-openai-agents-e2e.mjs
54
+ ```
55
+
56
+ The check is also included in `pnpm pack:smoke`.
57
+
44
58
  ## Troubleshooting
45
59
 
46
60
  | Symptom | Check |
@@ -36,6 +36,7 @@ When a maintainer or support responder needs reproducible evidence, follow the [
36
36
  - Inspect manual metadata passed to `inspectRun()`, `step()`, `step.tool()`, `step.llm()`, or `observe()`.
37
37
  - Inspect log-derived fields from `logs` / `tail` ingest configs, including custom `run-id`, `event`, `parent`, timestamp, and attribute mappings.
38
38
  - Avoid posting raw prompts, completions, tool inputs, or tool outputs in public threads unless the content is approved for public disclosure.
39
+ - For cross-system correlation, retain a bounded identifier instead of copying the external record or payload. Follow [External reference metadata](./EXTERNAL-REFERENCES.md), and expect `share` / `strict` profiles to redact the named correlation fields.
39
40
  - Prefer Markdown export for issue or PR sharing when a summarized tree is enough.
40
41
 
41
42
  ## Remove or replace sensitive values
@@ -125,6 +125,8 @@ Postgres URLs are reserved for team deployments and are **not required** for loc
125
125
 
126
126
  ## Related docs
127
127
 
128
+ - [STUDIO-IMPORT-BUNDLE.md](./STUDIO-IMPORT-BUNDLE.md) — walkthrough: share-safe bundle → local Studio
129
+ - [STUDIO-IMPORT-GITHUB-ARTIFACT.md](./STUDIO-IMPORT-GITHUB-ARTIFACT.md) — walkthrough: GitHub Actions artifact → local Studio
128
130
  - [SELF-HOSTED-STUDIO-V6.0.md](./proposals/SELF-HOSTED-STUDIO-V6.0.md)
129
131
  - [CLIENT-HOSTED-INGESTION-V6.1.md](./proposals/CLIENT-HOSTED-INGESTION-V6.1.md)
130
132
  - [LOCAL-TRACE-WORKSPACE.md](./proposals/LOCAL-TRACE-WORKSPACE.md)
package/docs/STANDARDS.md CHANGED
@@ -21,6 +21,8 @@ Shape validation is **compatible**; semantic checks add field-level warnings for
21
21
 
22
22
  Fixture: [fixtures/standards/openinference-basic.json](../fixtures/standards/openinference-basic.json)
23
23
 
24
+ The fixture's top-level `version` is an **AgentInspect reference fixture revision**, not an upstream OpenInference version.
25
+
24
26
  ## OTLP JSON (experimental)
25
27
 
26
28
  ```bash
@@ -31,10 +33,23 @@ GenAI attribute mapping follows `OTEL_GEN_AI_SEMCONV_PIN` (see exporters API). N
31
33
 
32
34
  Fixture: [fixtures/standards/otlp-basic.json](../fixtures/standards/otlp-basic.json)
33
35
 
36
+ The fixture's `scope.version` is an **AgentInspect test-scope fixture revision**, not the `OTEL_GEN_AI_SEMCONV_PIN` and not an upstream OpenTelemetry version.
37
+
34
38
  ## Graduation guide
35
39
 
36
40
  Full path from local export through review to optional customer-owned import: [STANDARDS-GRADUATION.md](./STANDARDS-GRADUATION.md).
37
41
 
42
+ That guide is the canonical source for standards known-loss boundaries, including kind degradation, bounded metadata, no chain-of-thought capture, and snapshot limitations.
43
+
44
+ ## Maintaining tested provenance
45
+
46
+ - When the OTLP mapping changes, update `OTEL_GEN_AI_SEMCONV_PIN` in [`packages/core/src/exporters/semconv.ts`](../packages/core/src/exporters/semconv.ts) and any explicit tested-version claims together.
47
+ - When the OTLP reference shape changes, update its test-scope fixture revision in [`fixtures/standards/otlp-basic.json`](../fixtures/standards/otlp-basic.json) and any repeated fixture revision together.
48
+ - When the OpenInference reference shape changes, update the top-level fixture revision in [`fixtures/standards/openinference-basic.json`](../fixtures/standards/openinference-basic.json) and any repeated fixture revision together. The value remains an AgentInspect fixture revision, not an upstream version.
49
+ - Keep known-loss behavior canonical in [`STANDARDS-GRADUATION.md`](./STANDARDS-GRADUATION.md) and update its validation alongside any intentional exporter behavior change.
50
+
51
+ Run `pnpm public-truth:check` and `pnpm docs:check` after changing these sources.
52
+
38
53
  ## Import recipes
39
54
 
40
55
  - [Phoenix / OpenInference](../examples/recipes/phoenix-openinference-import/)
@@ -37,6 +37,29 @@ Canonical maturity labels for AgentInspect public packages and major surfaces (6
37
37
 
38
38
  Part of the fixed AgentInspect release line — see the npm badge for the current version.
39
39
 
40
+ > Every level in this matrix must be one of the Definitions levels above, and `docs/product/PUBLIC-PRODUCT-FACTS.json` `matchers.status` must match the matchers row here. `pnpm public-truth:check` enforces both; update the doc and the facts file together.
41
+
42
+ ## Changing a level
43
+
44
+ A level is stated in two places, and they must not disagree: the row above, and
45
+ the `**Support level:**` line in the package's own README (which is what npm
46
+ shows). `pnpm package-readmes:check` enforces the agreement and runs as part of
47
+ `pnpm docs:check`.
48
+
49
+ To promote or demote a surface:
50
+
51
+ 1. Edit the row in the matrix above.
52
+ 2. Edit the `**Support level:**` line in each affected `packages/*/README.md`.
53
+ 3. Run `pnpm package-readmes:check`.
54
+
55
+ The check also reports packages whose level is **unenforced** — those the matrix
56
+ above does not name, so the README is their only source. Adding a row that names
57
+ the package (in backticks) puts it under enforcement.
58
+
59
+ It rejects an absolute "no network" claim from any surface
60
+ [NETWORK-BEHAVIOR.md](./NETWORK-BEHAVIOR.md) records as making network calls, so
61
+ a promotion that changes network behavior cannot leave a stale guarantee on npm.
62
+
40
63
  ## Public package groups (presentation only)
41
64
 
42
65
  Physical packages stay the fixed group of 18. Outreach/install kits group them as:
@@ -14,6 +14,36 @@ Contracts compile to deterministic check rules for common cases:
14
14
  - evidence-bearing findings on failures
15
15
  - evaluation over **logical** TraceFacts (raw events remain available)
16
16
 
17
+ ## `tools.requiredOrder` semantics
18
+
19
+ `requiredOrder` is expanded into **adjacent pair** ordering rules with unique ids:
20
+
21
+ ```text
22
+ [A, B, C]
23
+ → contract.tool.order.0: A before B
24
+ → contract.tool.order.1: B before C
25
+ ```
26
+
27
+ Each pair compares the **first occurrence** (start/encounter order in the evaluated event stream):
28
+
29
+ - unlisted intermediate tools are allowed;
30
+ - later repetitions do not invalidate an earlier valid first-occurrence order;
31
+ - TraceContract `requiredOrder` **implies presence** — every listed name is added to the effective required-tool set;
32
+ - this is **not** causal happens-before; overlapping intervals emit a non-failing `tool.order.overlap` warning;
33
+ - combine ordering with `maxCalls` or custom rules when repeated calls matter.
34
+
35
+ Examples for `requiredOrder: ["retrieve", "generate"]`:
36
+
37
+ | Trajectory | Result |
38
+ | --- | --- |
39
+ | `retrieve → generate` | PASS |
40
+ | `retrieve → rerank → generate` | PASS |
41
+ | `retrieve → generate → retrieve` | PASS (first-occurrence) |
42
+ | `generate → retrieve` | FAIL (order) |
43
+ | `cache_lookup → generate` | FAIL (missing `retrieve` via implied presence) |
44
+
45
+ Low-level `createToolOrderingRule({ before, after })` alone may still pass when an endpoint is missing (compositional). TraceContract `requiredOrder` does not.
46
+
17
47
  ### Experimental Vitest / Jest matchers (shipped)
18
48
 
19
49
  | Package | Export | Matchers |
@@ -25,6 +55,66 @@ These are **Experimental** — API names may evolve. There is no `expectTrace(..
25
55
 
26
56
  See [API.md](./API.md), [TRACE-FACTS.md](./TRACE-FACTS.md), and `packages/core/src/checks/contract.ts`.
27
57
 
58
+ ## Rule kinds (shipped vs planned)
59
+
60
+ TraceContract rules fall into distinct categories. Mixing them incorrectly is a common source of false failures (see GitHub #308 and #309).
61
+
62
+ ### `tools.required` (shipped)
63
+
64
+ Unconditional path invariant: every named tool must appear **at least once** in the trace.
65
+
66
+ - Use when the tool is always part of a valid execution path.
67
+ - **Do not** use for steps that legitimate shortcuts may skip (for example cache hits that bypass `retrieve`).
68
+ - When a shortcut is valid but you still need evidence of the outcome, prefer `observations.required` until `alternatives.anyOf` ships (6.20.0).
69
+
70
+ ### `tools.requiredOrder` (shipped — first-occurrence / start-encounter)
71
+
72
+ Legacy first-start / encounter ordering. The evaluator walks the trace and checks that each listed tool's **first occurrence** appears after the previous tool's first occurrence.
73
+
74
+ - TraceContract `requiredOrder` **implies presence** of every listed tool (unioned into `tools.required`).
75
+ - Default mode is **first-occurrence** start/encounter order — **not** causal happens-before.
76
+ - Overlapping intervals that still satisfy start order emit a non-failing overlap warning.
77
+ - **Planned (6.20.0, GitHub #308):**
78
+ - `requiredOrderMode: "happens-before"` — first before must **end** before first after **starts**
79
+ - `requiredOrderMode: "all-occurrences"` — every before must end before every after starts
80
+
81
+ ### `observations.required` (shipped)
82
+
83
+ Requires externally observed or effect evidence (for example HTTP status, file write, cache key) rather than a specific tool call. Prefer this when the invariant is about **outcome** rather than **which tool ran**.
84
+
85
+ ### Planned (6.20.0 — not shipped)
86
+
87
+ Document only; **do not** use these fields in contracts today:
88
+
89
+ | Planned field | Purpose | GitHub |
90
+ |---------------|---------|--------|
91
+ | `alternatives.anyOf` | One of several deterministic valid paths (one level, no nested groups, no predicates) | #309 |
92
+ | `requiredOrderMode: "happens-before"` | Causal completion-before-start ordering | #308 |
93
+ | `requiredOrderMode: "all-occurrences"` | Strict ordering across all tool occurrences | #308 |
94
+
95
+ API shape for both requires maintainer approval before external PR lands. @HsienW volunteered on #308 for `requiredOrderMode` implementation.
96
+
97
+ ## Workaround until 6.20.0
98
+
99
+ When a legitimate shortcut skips a tool you would otherwise require:
100
+
101
+ 1. **Remove** unconditional `tools.required` for that step.
102
+ 2. **Express** the verified outcome via `observations.required` when possible.
103
+ 3. **Document** the cache-hit or alternate path in contract comments for reviewers.
104
+
105
+ Example matching GitHub #309 (cache hit skips second `retrieve`):
106
+
107
+ ```yaml
108
+ contract:
109
+ tools:
110
+ required: [generate] # not retrieve — cache may skip it
111
+ requiredOrder: [generate] # ordering only among tools that ran
112
+ observations:
113
+ required: [cache_hit_or_retrieve_evidence]
114
+ ```
115
+
116
+ With first-occurrence ordering, `retrieve → generate → retrieve` still **passes** when both retrieves are present (see worked example below).
117
+
28
118
  ## What is not shipped (yet)
29
119
 
30
120
  Do **not** document these as available:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-inspect",
3
- "version": "6.17.3",
3
+ "version": "6.17.5",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "description": "Local evidence debugger and trajectory-test toolkit for TypeScript AI agents — execution trees, TraceContract checks, Evidence v2, and read-only MCP",
@@ -235,13 +235,16 @@
235
235
  "test:all": "pnpm run typecheck && pnpm run linked-versions:check && pnpm run build && pnpm run test && pnpm run size",
236
236
  "prepublish:checks": "pnpm run typecheck && pnpm run test && pnpm run test:coverage && pnpm run build && pnpm run fixtures:check && pnpm run recipes:check && pnpm run size && pnpm run linked-versions:check && pnpm run repo:health && pnpm run pack:smoke",
237
237
  "pack:dry-run": "pnpm run build && npm pack --dry-run",
238
- "pack:smoke": "pnpm run build && node scripts/package-smoke.mjs && node scripts/packed-quickstart-e2e.mjs && node scripts/packed-semantic-loop-e2e.mjs && node scripts/packed-swarm-loop-e2e.mjs && node scripts/evidence-ci-golden-paths.mjs",
238
+ "pack:smoke": "pnpm run build && node scripts/package-smoke.mjs && node scripts/packed-openai-agents-e2e.mjs && node scripts/packed-quickstart-e2e.mjs && node scripts/packed-semantic-loop-e2e.mjs && node scripts/packed-swarm-loop-e2e.mjs && node scripts/evidence-ci-golden-paths.mjs",
239
239
  "linked-versions:check": "node scripts/check-linked-versions.mjs",
240
240
  "docs:commands": "node scripts/validate-doc-commands.mjs",
241
241
  "docs:links": "node scripts/validate-doc-links.mjs",
242
+ "public-truth:sync": "node scripts/sync-public-truth.mjs",
242
243
  "public-truth:check": "node scripts/validate-public-truth.mjs",
243
244
  "ai-assets:check": "node scripts/validate-ai-assets.mjs",
244
- "docs:check": "pnpm run docs:commands && pnpm run docs:links && pnpm run public-truth:check && pnpm run ai-assets:check && pnpm run repo:health && pnpm run demo:verify",
245
+ "package-readmes:check": "node scripts/validate-package-readmes.mjs",
246
+ "docs:check": "pnpm run docs:commands && pnpm run docs:links && pnpm run public-truth:check && pnpm run ai-assets:check && pnpm run package-readmes:check && pnpm run repo:health && pnpm run build:cli && pnpm run demo:verify",
247
+ "build:cli": "pnpm exec tsup --config tsup.cli.config.ts",
245
248
  "repo:health": "node scripts/validate-repo-health.mjs",
246
249
  "demo:generate": "node scripts/demo-generate.mjs",
247
250
  "demo:verify": "node scripts/demo-verify.mjs",
@@ -251,6 +254,7 @@
251
254
  "perf:baseline": "node scripts/performance-baseline.mjs",
252
255
  "examples:check": "pnpm install && pnpm --filter agent-inspect-example-01-basic run start",
253
256
  "changeset": "changeset",
257
+ "version:packages": "changeset version && pnpm public-truth:sync && pnpm public-truth:check",
254
258
  "release": "changeset publish",
255
259
  "website:dev": "pnpm --filter @agent-inspect/website dev",
256
260
  "website:build": "pnpm --filter @agent-inspect/website build",