agent-inspect 6.17.3 → 6.17.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/ADAPTER-CONFORMANCE.md +24 -0
- package/docs/ADAPTERS.md +3 -3
- package/docs/API.md +13 -7
- package/docs/CI-ARTIFACTS.md +2 -1
- package/docs/CLI.md +2 -0
- package/docs/LIMITATIONS.md +1 -0
- package/docs/OPENAI-AGENTS-LOCAL.md +14 -0
- package/docs/SAFE-TRACE-SHARING.md +1 -0
- package/docs/SELF-HOSTING.md +2 -0
- package/docs/STANDARDS.md +15 -0
- package/docs/SUPPORT-LEVELS.md +23 -0
- package/docs/TRACE-CONTRACTS.md +90 -0
- package/package.json +7 -3
- package/packages/cli/dist/{chunk-5GGYDIZD.mjs → chunk-GOHYSE7W.mjs} +93 -19
- package/packages/cli/dist/chunk-GOHYSE7W.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +505 -88
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +417 -74
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-IDMCWKRH.mjs → src-OYGG7QAZ.mjs} +3 -3
- package/packages/cli/dist/{src-IDMCWKRH.mjs.map → src-OYGG7QAZ.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +86 -17
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +1 -1
- package/packages/core/dist/advanced.d.ts +1 -1
- package/packages/core/dist/advanced.mjs +15 -5
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +126 -28
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +28 -2
- package/packages/core/dist/checks.d.ts +28 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-UFP54T7F.mjs → chunk-DIZPIPY2.mjs} +128 -30
- package/packages/core/dist/chunk-DIZPIPY2.mjs.map +1 -0
- package/packages/core/dist/{index-Xk9X-yjY.d.cts → index-DWu54Y28.d.cts} +107 -122
- package/packages/core/dist/{index-BsCcOKxy.d.ts → index-DlwbVqEs.d.ts} +107 -122
- package/packages/cli/dist/chunk-5GGYDIZD.mjs.map +0 -1
- package/packages/core/dist/chunk-UFP54T7F.mjs.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 6.17.5
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 093811b: Harden deterministic TraceContract / check gates against fail-open empty configs (rule execution evidence, unique order IDs, requiredOrder implies presence, tool policy includes running invocations, ObservedOutcome requireAny), map #308–#311 release ownership, and make demo:verify / pack:smoke validation cross-platform without unnecessary shell invocation.
|
|
8
|
+
|
|
9
|
+
## 6.17.4
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- c4b0f03: Fix camelCase / kebab / dot compound credential key redaction (`userPassword`, `clientSecret`) while keeping token-config keys and camelCase topic fields (`emailNote`) un-key-redacted.
|
|
14
|
+
|
|
3
15
|
## 6.17.3
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -210,7 +210,7 @@ The root package is enough for custom capture, the CLI, checks, and Evidence wor
|
|
|
210
210
|
|
|
211
211
|
## Status and documentation
|
|
212
212
|
|
|
213
|
-
**Current published baseline:** **6.17.
|
|
213
|
+
**Current published baseline:** **6.17.5** · persisted schema `1.0` · Node.js `>=20` · MIT.
|
|
214
214
|
|
|
215
215
|
Legacy v0.1 and v0.2 traces remain readable. Check the npm badge and [changelog](CHANGELOG.md) for the current published version.
|
|
216
216
|
|
|
@@ -58,6 +58,30 @@ This is an internal official-adapter gate, not a third-party certification progr
|
|
|
58
58
|
|
|
59
59
|
An adapter can be documented as supported only after no-network fixtures cover run, step, tool, LLM, error, streaming, and metadata-bound expectations for that framework path.
|
|
60
60
|
|
|
61
|
+
## Third-party adapter conformance CI
|
|
62
|
+
|
|
63
|
+
The gate above is the internal official-adapter gate, not a third-party certification program. Third-party adapter authors run the **public** conformance helper from [`@agent-inspect/adapter-sdk`](https://www.npmjs.com/package/@agent-inspect/adapter-sdk) in their own CI:
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
import { runAdapterConformance } from "@agent-inspect/adapter-sdk";
|
|
67
|
+
|
|
68
|
+
const result = await runAdapterConformance({
|
|
69
|
+
adapterId: "my-adapter",
|
|
70
|
+
events, // PersistedInspectEvent[] your adapter emits
|
|
71
|
+
expectedKinds: ["RUN", "LLM"],
|
|
72
|
+
forbiddenRawStrings: ["super-secret-key"], // must not leak into events
|
|
73
|
+
});
|
|
74
|
+
// assert result.ok in your test runner
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
A copyable GitHub Actions template lives at [`examples/adapter-sdk/third-party-conformance-ci.yml`](../examples/adapter-sdk/third-party-conformance-ci.yml). Copy it to `.github/workflows/agent-inspect-conformance.yml` in your adapter repo and adjust the install/build/test commands. The template:
|
|
78
|
+
|
|
79
|
+
- runs on a Node LTS matrix (18 / 20 / 22),
|
|
80
|
+
- requires **no provider keys** and makes **no network calls** (conformance is metadata-only),
|
|
81
|
+
- keeps the framework SDK as a peer dependency of your adapter, never of AgentInspect core.
|
|
82
|
+
|
|
83
|
+
See the runnable examples under [`examples/adapter-sdk/`](../examples/adapter-sdk/) for adapters that call `runAdapterConformance` end to end.
|
|
84
|
+
|
|
61
85
|
## v3.2 adoption evidence refresh
|
|
62
86
|
|
|
63
87
|
- [AI-SDK-ADOPTION.md](./AI-SDK-ADOPTION.md) — blessed AI SDK path (`generateText`, `streamText`, Next.js recipe, troubleshooting)
|
package/docs/ADAPTERS.md
CHANGED
|
@@ -22,7 +22,7 @@ Reporters (`@agent-inspect/vitest` and `@agent-inspect/jest`) are public package
|
|
|
22
22
|
|
|
23
23
|
**Status:** experimental adapter — optional package published in the aligned v2.2.0 package set and hardened in the v2.3 adapter train.
|
|
24
24
|
|
|
25
|
-
The adapter has hardened lifecycle identity and parallel integration isolation. It remains metadata-only: `capture: "preview"` and preview-only redaction options emit diagnostics and fall back to metadata-only capture
|
|
25
|
+
The adapter has hardened lifecycle identity and parallel integration isolation. It remains metadata-only: `capture: "preview"` and preview-only redaction options emit diagnostics **and one visible** `AI_ADAPTER_PREVIEW_NOT_AVAILABLE` warning per adapter instance, then fall back to metadata-only capture. Bounded free-text preview capture is planned for a later train; it is not implemented here.
|
|
26
26
|
|
|
27
27
|
### Install
|
|
28
28
|
|
|
@@ -59,7 +59,7 @@ const result = await generateText({
|
|
|
59
59
|
- **Metadata-only by default** — records model, finish reason, token usage, timing, and safe counts/summaries.
|
|
60
60
|
- **Required safe telemetry settings** — set `recordInputs: false` and `recordOutputs: false` on every AI SDK call using this adapter.
|
|
61
61
|
- **No raw payload capture by default** — prompts, messages, generated text, stream chunks, tool inputs/outputs, headers, request bodies, and response bodies are not persisted.
|
|
62
|
-
- **Preview capture is not enabled yet** — `capture: "preview"
|
|
62
|
+
- **Preview capture is not enabled yet** — requesting `capture: "preview"` emits one console warning with code `AI_ADAPTER_PREVIEW_NOT_AVAILABLE`, records diagnostics via `getDiagnostics()`, and does not persist raw previews. Effective capture remains metadata-only.
|
|
63
63
|
|
|
64
64
|
### Local no-network recipe
|
|
65
65
|
|
|
@@ -375,7 +375,7 @@ Integration modes:
|
|
|
375
375
|
- **No upload behavior** — the processor writes only to an explicit local writer or `traceDir`.
|
|
376
376
|
- **Metadata-only by default** — records trace/span IDs, parentage, names, timing, status, errors, safe model/tool names, token counts, and bounded summaries.
|
|
377
377
|
- **No raw payload capture by default** — prompts, messages, generated text, function inputs/outputs, arbitrary custom data, trace exporter credentials, headers, request bodies, response bodies, and hosted tool payloads are not persisted.
|
|
378
|
-
- **Preview capture is not enabled yet** — `capture: "preview"
|
|
378
|
+
- **Preview capture is not enabled yet** — requesting `capture: "preview"` emits one console warning with code `AI_ADAPTER_PREVIEW_NOT_AVAILABLE`, records diagnostics via `getDiagnostics()`, and does not persist raw previews. Effective capture remains metadata-only.
|
|
379
379
|
- **Fixture-backed lifecycle coverage** — local tests and the recipe cover agent, generation, function tool, handoff, guardrail, response, MCP tools, custom, transcription, and speech span shapes without provider calls.
|
|
380
380
|
|
|
381
381
|
Full API: [API.md](./API.md) §14.
|
package/docs/API.md
CHANGED
|
@@ -180,7 +180,11 @@ Diff is local and read-only. Programmatic diff surfaces are experimental until t
|
|
|
180
180
|
- In-memory **`getEvents()`** / **`clear()`** unchanged when `persist` is false
|
|
181
181
|
- Metadata helpers: `extractModelName`, `extractTokenUsage`, `safePreview`, `toPlainMetadata`
|
|
182
182
|
|
|
183
|
-
Rationale:
|
|
183
|
+
Rationale: one official adapter and **zero production sinks** keep adapter surfaces experimental; they remain additive in minors.
|
|
184
|
+
|
|
185
|
+
### TraceContract `requiredOrder`
|
|
186
|
+
|
|
187
|
+
When defining `tools.requiredOrder`, the array expands into adjacent first-occurrence pair checks. Unlisted intermediate tools are allowed; later repetitions do not invalidate an earlier valid first-occurrence order. Missing tools are handled by required-tool rules, not ordering alone. See [TRACE-CONTRACTS.md](./TRACE-CONTRACTS.md).
|
|
184
188
|
|
|
185
189
|
## 10. Experimental `@agent-inspect/tui` APIs
|
|
186
190
|
|
|
@@ -249,7 +253,7 @@ The manifest records framework, generation time, bounded test results, artifact
|
|
|
249
253
|
|
|
250
254
|
## 12. Experimental `@agent-inspect/vitest` APIs
|
|
251
255
|
|
|
252
|
-
`@agent-inspect/vitest` is an optional experimental workspace package for local Vitest failure artifacts. It
|
|
256
|
+
`@agent-inspect/vitest` is an optional experimental workspace package for local Vitest failure artifacts. It is published with the fixed release group. It does not add a Vitest dependency to root/core, does not upload artifacts, and does not infer trace relationships by timestamp.
|
|
253
257
|
|
|
254
258
|
Import from `@agent-inspect/vitest`:
|
|
255
259
|
|
|
@@ -282,7 +286,7 @@ Artifacts are safe structural summaries. The reporter writes a shared `schemaVer
|
|
|
282
286
|
|
|
283
287
|
## 13. Experimental `@agent-inspect/jest` APIs
|
|
284
288
|
|
|
285
|
-
`@agent-inspect/jest` is an optional experimental workspace package for local Jest failure artifacts. It
|
|
289
|
+
`@agent-inspect/jest` is an optional experimental workspace package for local Jest failure artifacts. It is published with the fixed release group. It does not add a Jest dependency to root/core, does not upload artifacts, and does not infer trace relationships by timestamp.
|
|
286
290
|
|
|
287
291
|
Import from `@agent-inspect/jest`:
|
|
288
292
|
|
|
@@ -423,8 +427,8 @@ import type {
|
|
|
423
427
|
```
|
|
424
428
|
|
|
425
429
|
- **`TraceWriter`**: async `write(event)`, optional `flush()`, optional `close()`, optional `getStats()`.
|
|
426
|
-
- **`fileWriter({ dir?, filePath? })`**: appends `PersistedInspectEvent` JSONL rows to local disk. `createInspector()` emits schema 1.0 rows by default; compatibility adapters may still pass readable v0.2 rows. By default it derives one file per `event.runId`; `filePath` writes all events to an explicit local file. Filesystem and serialization failures are reflected in writer stats instead of being thrown into application code.
|
|
427
|
-
- **`bufferedFileWriter({ dir?, filePath?, maxQueueSize?, flushIntervalMs?, maxBatchSize?, overflow? })`**: buffers local JSONL writes with bounded queue behavior. Overflow supports `drop-oldest` and `drop-newest`; neither mode throws into application code.
|
|
430
|
+
- **`fileWriter({ dir?, filePath? })`**: appends `PersistedInspectEvent` JSONL rows to local disk. `createInspector()` emits schema 1.0 rows by default; compatibility adapters may still pass readable v0.2 rows. By default it derives one file per `event.runId`; `filePath` writes all events to an explicit local file. Filesystem and serialization failures are reflected in writer stats instead of being thrown into application code. Append-only local JSONL is diagnostic persistence — not a write-ahead log or fsync-guaranteed durable-before-effect journal.
|
|
431
|
+
- **`bufferedFileWriter({ dir?, filePath?, maxQueueSize?, flushIntervalMs?, maxBatchSize?, overflow? })`**: buffers local JSONL writes with bounded queue behavior. Overflow supports `drop-oldest` and `drop-newest`; neither mode throws into application code. On abrupt process exit, a bounded unflushed tail may be lost.
|
|
428
432
|
- **`compositeWriter([...writers])`**: fans out events to multiple explicit local/custom writers. A failing child writer does not prevent other children from receiving events; failures are reflected in composite stats.
|
|
429
433
|
- **`memoryWriter()`**: stores cloned `PersistedInspectEvent` rows in memory for tests, adapter fixtures, and eval harnesses.
|
|
430
434
|
- **`nullWriter()`**: accepts events without retaining them for disabled mode, overhead comparisons, and no-output tests.
|
|
@@ -433,7 +437,7 @@ No network writer or vendor sink exists in this package.
|
|
|
433
437
|
|
|
434
438
|
## 19. Experimental inspector API/runtime (v1.6)
|
|
435
439
|
|
|
436
|
-
`createInspector()` is the public instance API for local-first tracing with explicit writers. It owns an instance-specific runtime context, records schema 1.0 persisted inspect events, preserves application return values/errors, and exposes diagnostics plus deterministic `flush()`/`close()` lifecycle hooks.
|
|
440
|
+
`createInspector()` is the public instance API for local-first tracing with explicit writers. It owns an instance-specific runtime context, records schema 1.0 persisted inspect events, preserves application return values/errors, and exposes diagnostics plus deterministic `flush()`/`close()` lifecycle hooks (`flush`/`close` drain writer queues idempotently — they do not claim crash durability or fsync).
|
|
437
441
|
|
|
438
442
|
Import from `agent-inspect`:
|
|
439
443
|
|
|
@@ -537,7 +541,9 @@ import type { TraceCheckRule, TraceCheckResult, TraceFacts } from "agent-inspect
|
|
|
537
541
|
- **`TraceCheckRule`**: synchronous pure rule contract.
|
|
538
542
|
- **`TraceCheckResult`**: deterministic aggregate result with findings, evidence, summary counts, and execution diagnostics.
|
|
539
543
|
|
|
540
|
-
The checks API is experimental
|
|
544
|
+
The checks API is experimental. The `agent-inspect check` CLI uses this API for local reader-backed checks and deterministic JSON output; `agent-inspect artifacts` reuses the same safe findings for local CI artifact bundles and optional step-summary file output. Built-in rules operate on normalized event metadata, tree relationships, bounded summaries, token counts, and normalized baseline facts; safety and baseline findings identify event IDs and field paths rather than emitting raw prompts, outputs, secrets, headers, request/response bodies, or full tool payloads.
|
|
545
|
+
|
|
546
|
+
@beta Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
541
547
|
|
|
542
548
|
Recipes: [deterministic-ci-checks](../examples/recipes/deterministic-ci-checks/README.md) for check/baseline/artifact workflows, and [test-reporter-artifacts](../examples/recipes/test-reporter-artifacts/README.md) for Vitest/Jest reporter configuration patterns.
|
|
543
549
|
|
package/docs/CI-ARTIFACTS.md
CHANGED
|
@@ -86,8 +86,9 @@ Recipes:
|
|
|
86
86
|
|
|
87
87
|
- [examples/recipes/deterministic-ci-checks](../examples/recipes/deterministic-ci-checks/README.md) for v1.8 `check`, baseline, safe artifact, and step-summary workflows.
|
|
88
88
|
- [examples/recipes/github-actions-artifact](../examples/recipes/github-actions-artifact/README.md) for share-safe trace exports and reporter manifest summaries.
|
|
89
|
+
- [examples/recipes/github-actions-gate](../examples/recipes/github-actions-gate/README.md) for a retained broken-to-fixed suite/gate pilot with separate Evidence v2 artifacts.
|
|
89
90
|
|
|
90
|
-
Sample workflows: [deterministic checks workflow](../examples/recipes/deterministic-ci-checks/workflow-example.yml), [share-safe export workflow](../examples/recipes/github-actions-artifact/workflow-example.yml)
|
|
91
|
+
Sample workflows: [deterministic checks workflow](../examples/recipes/deterministic-ci-checks/workflow-example.yml), [share-safe export workflow](../examples/recipes/github-actions-artifact/workflow-example.yml), [retained gate workflow](../examples/recipes/github-actions-gate/workflow-example.yml)
|
|
91
92
|
|
|
92
93
|
```yaml
|
|
93
94
|
- uses: actions/upload-artifact@v4
|
package/docs/CLI.md
CHANGED
|
@@ -204,6 +204,8 @@ Options:
|
|
|
204
204
|
- `--refresh <ms>`: minimum time between renders
|
|
205
205
|
- `--json`: newline-delimited JSON updates
|
|
206
206
|
|
|
207
|
+
**Truncation recovery:** when a watched `--file` shrinks below the last read offset (for example a truncate-and-rewrite), `tail` resets the offset, discards any buffered partial line, and continues the same session from the start of the current file contents. Full inode-aware rename/recreate rotation is **not** claimed unless separately implemented and tested.
|
|
208
|
+
|
|
207
209
|
Important: `tail` is a local developer tool, not a production monitor.
|
|
208
210
|
|
|
209
211
|
### 6.6 `export`
|
package/docs/LIMITATIONS.md
CHANGED
|
@@ -23,6 +23,7 @@ This document states what AgentInspect **does not** provide today. It complement
|
|
|
23
23
|
|
|
24
24
|
- **Subpath APIs:** `agent-inspect/writers`, `agent-inspect/readers`, and advanced helpers are available for local adoption from their owning subpaths. `createInspector()` is part of the small root API.
|
|
25
25
|
- **Explicit writer ownership:** `createInspector()` does not print terminal lifecycle output or implicitly choose a disk writer. Use `fileWriter()` / `bufferedFileWriter()` / custom writers when persistence is desired.
|
|
26
|
+
- **Diagnostic evidence, not an event-sourced runtime:** AgentInspect traces are local diagnostic evidence of what an agent did. They are **not** a write-ahead log, durable-before-effect journal, or application event-sourcing runtime. Buffered writers may lose a bounded tail of events on abrupt process termination. Call `flush()` / `close()` according to writer docs when you need the queue drained before exit; that lifecycle is deterministic and idempotent, but it is **not** an fsync / crash-durability guarantee.
|
|
26
27
|
- **No standards upload:** OpenInference and OTLP JSON support is local read/export compatibility only. There is no OTLP gRPC/HTTP streaming sink, collector client, or hosted ingestion behavior.
|
|
27
28
|
- **Conservative detection:** `agent-inspect open` does not silently accept arbitrary JSON. Unsupported or ambiguous inputs produce errors/warnings rather than guessed traces.
|
|
28
29
|
- **Large inputs:** reader inputs are bounded and read into local memory. This is not a database index or production log warehouse.
|
|
@@ -41,6 +41,20 @@ setTraceProcessors([
|
|
|
41
41
|
|
|
42
42
|
- [openai-agents-local-processor](../../examples/recipes/openai-agents-local-processor/)
|
|
43
43
|
|
|
44
|
+
## No-key packed consumer check
|
|
45
|
+
|
|
46
|
+
The repository includes a packed-consumer check that installs the root and
|
|
47
|
+
OpenAI Agents adapter tarballs into a temporary project. It drives deterministic
|
|
48
|
+
tracing fixtures without `OPENAI_API_KEY` or a live provider call, writes a local
|
|
49
|
+
trace, and inspects it through the packed AgentInspect CLI.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pnpm build
|
|
53
|
+
node scripts/packed-openai-agents-e2e.mjs
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The check is also included in `pnpm pack:smoke`.
|
|
57
|
+
|
|
44
58
|
## Troubleshooting
|
|
45
59
|
|
|
46
60
|
| Symptom | Check |
|
|
@@ -36,6 +36,7 @@ When a maintainer or support responder needs reproducible evidence, follow the [
|
|
|
36
36
|
- Inspect manual metadata passed to `inspectRun()`, `step()`, `step.tool()`, `step.llm()`, or `observe()`.
|
|
37
37
|
- Inspect log-derived fields from `logs` / `tail` ingest configs, including custom `run-id`, `event`, `parent`, timestamp, and attribute mappings.
|
|
38
38
|
- Avoid posting raw prompts, completions, tool inputs, or tool outputs in public threads unless the content is approved for public disclosure.
|
|
39
|
+
- For cross-system correlation, retain a bounded identifier instead of copying the external record or payload. Follow [External reference metadata](./EXTERNAL-REFERENCES.md), and expect `share` / `strict` profiles to redact the named correlation fields.
|
|
39
40
|
- Prefer Markdown export for issue or PR sharing when a summarized tree is enough.
|
|
40
41
|
|
|
41
42
|
## Remove or replace sensitive values
|
package/docs/SELF-HOSTING.md
CHANGED
|
@@ -125,6 +125,8 @@ Postgres URLs are reserved for team deployments and are **not required** for loc
|
|
|
125
125
|
|
|
126
126
|
## Related docs
|
|
127
127
|
|
|
128
|
+
- [STUDIO-IMPORT-BUNDLE.md](./STUDIO-IMPORT-BUNDLE.md) — walkthrough: share-safe bundle → local Studio
|
|
129
|
+
- [STUDIO-IMPORT-GITHUB-ARTIFACT.md](./STUDIO-IMPORT-GITHUB-ARTIFACT.md) — walkthrough: GitHub Actions artifact → local Studio
|
|
128
130
|
- [SELF-HOSTED-STUDIO-V6.0.md](./proposals/SELF-HOSTED-STUDIO-V6.0.md)
|
|
129
131
|
- [CLIENT-HOSTED-INGESTION-V6.1.md](./proposals/CLIENT-HOSTED-INGESTION-V6.1.md)
|
|
130
132
|
- [LOCAL-TRACE-WORKSPACE.md](./proposals/LOCAL-TRACE-WORKSPACE.md)
|
package/docs/STANDARDS.md
CHANGED
|
@@ -21,6 +21,8 @@ Shape validation is **compatible**; semantic checks add field-level warnings for
|
|
|
21
21
|
|
|
22
22
|
Fixture: [fixtures/standards/openinference-basic.json](../fixtures/standards/openinference-basic.json)
|
|
23
23
|
|
|
24
|
+
The fixture's top-level `version` is an **AgentInspect reference fixture revision**, not an upstream OpenInference version.
|
|
25
|
+
|
|
24
26
|
## OTLP JSON (experimental)
|
|
25
27
|
|
|
26
28
|
```bash
|
|
@@ -31,10 +33,23 @@ GenAI attribute mapping follows `OTEL_GEN_AI_SEMCONV_PIN` (see exporters API). N
|
|
|
31
33
|
|
|
32
34
|
Fixture: [fixtures/standards/otlp-basic.json](../fixtures/standards/otlp-basic.json)
|
|
33
35
|
|
|
36
|
+
The fixture's `scope.version` is an **AgentInspect test-scope fixture revision**, not the `OTEL_GEN_AI_SEMCONV_PIN` and not an upstream OpenTelemetry version.
|
|
37
|
+
|
|
34
38
|
## Graduation guide
|
|
35
39
|
|
|
36
40
|
Full path from local export through review to optional customer-owned import: [STANDARDS-GRADUATION.md](./STANDARDS-GRADUATION.md).
|
|
37
41
|
|
|
42
|
+
That guide is the canonical source for standards known-loss boundaries, including kind degradation, bounded metadata, no chain-of-thought capture, and snapshot limitations.
|
|
43
|
+
|
|
44
|
+
## Maintaining tested provenance
|
|
45
|
+
|
|
46
|
+
- When the OTLP mapping changes, update `OTEL_GEN_AI_SEMCONV_PIN` in [`packages/core/src/exporters/semconv.ts`](../packages/core/src/exporters/semconv.ts) and any explicit tested-version claims together.
|
|
47
|
+
- When the OTLP reference shape changes, update its test-scope fixture revision in [`fixtures/standards/otlp-basic.json`](../fixtures/standards/otlp-basic.json) and any repeated fixture revision together.
|
|
48
|
+
- When the OpenInference reference shape changes, update the top-level fixture revision in [`fixtures/standards/openinference-basic.json`](../fixtures/standards/openinference-basic.json) and any repeated fixture revision together. The value remains an AgentInspect fixture revision, not an upstream version.
|
|
49
|
+
- Keep known-loss behavior canonical in [`STANDARDS-GRADUATION.md`](./STANDARDS-GRADUATION.md) and update its validation alongside any intentional exporter behavior change.
|
|
50
|
+
|
|
51
|
+
Run `pnpm public-truth:check` and `pnpm docs:check` after changing these sources.
|
|
52
|
+
|
|
38
53
|
## Import recipes
|
|
39
54
|
|
|
40
55
|
- [Phoenix / OpenInference](../examples/recipes/phoenix-openinference-import/)
|
package/docs/SUPPORT-LEVELS.md
CHANGED
|
@@ -37,6 +37,29 @@ Canonical maturity labels for AgentInspect public packages and major surfaces (6
|
|
|
37
37
|
|
|
38
38
|
Part of the fixed AgentInspect release line — see the npm badge for the current version.
|
|
39
39
|
|
|
40
|
+
> Every level in this matrix must be one of the Definitions levels above, and `docs/product/PUBLIC-PRODUCT-FACTS.json` `matchers.status` must match the matchers row here. `pnpm public-truth:check` enforces both; update the doc and the facts file together.
|
|
41
|
+
|
|
42
|
+
## Changing a level
|
|
43
|
+
|
|
44
|
+
A level is stated in two places, and they must not disagree: the row above, and
|
|
45
|
+
the `**Support level:**` line in the package's own README (which is what npm
|
|
46
|
+
shows). `pnpm package-readmes:check` enforces the agreement and runs as part of
|
|
47
|
+
`pnpm docs:check`.
|
|
48
|
+
|
|
49
|
+
To promote or demote a surface:
|
|
50
|
+
|
|
51
|
+
1. Edit the row in the matrix above.
|
|
52
|
+
2. Edit the `**Support level:**` line in each affected `packages/*/README.md`.
|
|
53
|
+
3. Run `pnpm package-readmes:check`.
|
|
54
|
+
|
|
55
|
+
The check also reports packages whose level is **unenforced** — those the matrix
|
|
56
|
+
above does not name, so the README is their only source. Adding a row that names
|
|
57
|
+
the package (in backticks) puts it under enforcement.
|
|
58
|
+
|
|
59
|
+
It rejects an absolute "no network" claim from any surface
|
|
60
|
+
[NETWORK-BEHAVIOR.md](./NETWORK-BEHAVIOR.md) records as making network calls, so
|
|
61
|
+
a promotion that changes network behavior cannot leave a stale guarantee on npm.
|
|
62
|
+
|
|
40
63
|
## Public package groups (presentation only)
|
|
41
64
|
|
|
42
65
|
Physical packages stay the fixed group of 18. Outreach/install kits group them as:
|
package/docs/TRACE-CONTRACTS.md
CHANGED
|
@@ -14,6 +14,36 @@ Contracts compile to deterministic check rules for common cases:
|
|
|
14
14
|
- evidence-bearing findings on failures
|
|
15
15
|
- evaluation over **logical** TraceFacts (raw events remain available)
|
|
16
16
|
|
|
17
|
+
## `tools.requiredOrder` semantics
|
|
18
|
+
|
|
19
|
+
`requiredOrder` is expanded into **adjacent pair** ordering rules with unique ids:
|
|
20
|
+
|
|
21
|
+
```text
|
|
22
|
+
[A, B, C]
|
|
23
|
+
→ contract.tool.order.0: A before B
|
|
24
|
+
→ contract.tool.order.1: B before C
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Each pair compares the **first occurrence** (start/encounter order in the evaluated event stream):
|
|
28
|
+
|
|
29
|
+
- unlisted intermediate tools are allowed;
|
|
30
|
+
- later repetitions do not invalidate an earlier valid first-occurrence order;
|
|
31
|
+
- TraceContract `requiredOrder` **implies presence** — every listed name is added to the effective required-tool set;
|
|
32
|
+
- this is **not** causal happens-before; overlapping intervals emit a non-failing `tool.order.overlap` warning;
|
|
33
|
+
- combine ordering with `maxCalls` or custom rules when repeated calls matter.
|
|
34
|
+
|
|
35
|
+
Examples for `requiredOrder: ["retrieve", "generate"]`:
|
|
36
|
+
|
|
37
|
+
| Trajectory | Result |
|
|
38
|
+
| --- | --- |
|
|
39
|
+
| `retrieve → generate` | PASS |
|
|
40
|
+
| `retrieve → rerank → generate` | PASS |
|
|
41
|
+
| `retrieve → generate → retrieve` | PASS (first-occurrence) |
|
|
42
|
+
| `generate → retrieve` | FAIL (order) |
|
|
43
|
+
| `cache_lookup → generate` | FAIL (missing `retrieve` via implied presence) |
|
|
44
|
+
|
|
45
|
+
Low-level `createToolOrderingRule({ before, after })` alone may still pass when an endpoint is missing (compositional). TraceContract `requiredOrder` does not.
|
|
46
|
+
|
|
17
47
|
### Experimental Vitest / Jest matchers (shipped)
|
|
18
48
|
|
|
19
49
|
| Package | Export | Matchers |
|
|
@@ -25,6 +55,66 @@ These are **Experimental** — API names may evolve. There is no `expectTrace(..
|
|
|
25
55
|
|
|
26
56
|
See [API.md](./API.md), [TRACE-FACTS.md](./TRACE-FACTS.md), and `packages/core/src/checks/contract.ts`.
|
|
27
57
|
|
|
58
|
+
## Rule kinds (shipped vs planned)
|
|
59
|
+
|
|
60
|
+
TraceContract rules fall into distinct categories. Mixing them incorrectly is a common source of false failures (see GitHub #308 and #309).
|
|
61
|
+
|
|
62
|
+
### `tools.required` (shipped)
|
|
63
|
+
|
|
64
|
+
Unconditional path invariant: every named tool must appear **at least once** in the trace.
|
|
65
|
+
|
|
66
|
+
- Use when the tool is always part of a valid execution path.
|
|
67
|
+
- **Do not** use for steps that legitimate shortcuts may skip (for example cache hits that bypass `retrieve`).
|
|
68
|
+
- When a shortcut is valid but you still need evidence of the outcome, prefer `observations.required` until `alternatives.anyOf` ships (6.20.0).
|
|
69
|
+
|
|
70
|
+
### `tools.requiredOrder` (shipped — first-occurrence / start-encounter)
|
|
71
|
+
|
|
72
|
+
Legacy first-start / encounter ordering. The evaluator walks the trace and checks that each listed tool's **first occurrence** appears after the previous tool's first occurrence.
|
|
73
|
+
|
|
74
|
+
- TraceContract `requiredOrder` **implies presence** of every listed tool (unioned into `tools.required`).
|
|
75
|
+
- Default mode is **first-occurrence** start/encounter order — **not** causal happens-before.
|
|
76
|
+
- Overlapping intervals that still satisfy start order emit a non-failing overlap warning.
|
|
77
|
+
- **Planned (6.20.0, GitHub #308):**
|
|
78
|
+
- `requiredOrderMode: "happens-before"` — first before must **end** before first after **starts**
|
|
79
|
+
- `requiredOrderMode: "all-occurrences"` — every before must end before every after starts
|
|
80
|
+
|
|
81
|
+
### `observations.required` (shipped)
|
|
82
|
+
|
|
83
|
+
Requires externally observed or effect evidence (for example HTTP status, file write, cache key) rather than a specific tool call. Prefer this when the invariant is about **outcome** rather than **which tool ran**.
|
|
84
|
+
|
|
85
|
+
### Planned (6.20.0 — not shipped)
|
|
86
|
+
|
|
87
|
+
Document only; **do not** use these fields in contracts today:
|
|
88
|
+
|
|
89
|
+
| Planned field | Purpose | GitHub |
|
|
90
|
+
|---------------|---------|--------|
|
|
91
|
+
| `alternatives.anyOf` | One of several deterministic valid paths (one level, no nested groups, no predicates) | #309 |
|
|
92
|
+
| `requiredOrderMode: "happens-before"` | Causal completion-before-start ordering | #308 |
|
|
93
|
+
| `requiredOrderMode: "all-occurrences"` | Strict ordering across all tool occurrences | #308 |
|
|
94
|
+
|
|
95
|
+
API shape for both requires maintainer approval before external PR lands. @HsienW volunteered on #308 for `requiredOrderMode` implementation.
|
|
96
|
+
|
|
97
|
+
## Workaround until 6.20.0
|
|
98
|
+
|
|
99
|
+
When a legitimate shortcut skips a tool you would otherwise require:
|
|
100
|
+
|
|
101
|
+
1. **Remove** unconditional `tools.required` for that step.
|
|
102
|
+
2. **Express** the verified outcome via `observations.required` when possible.
|
|
103
|
+
3. **Document** the cache-hit or alternate path in contract comments for reviewers.
|
|
104
|
+
|
|
105
|
+
Example matching GitHub #309 (cache hit skips second `retrieve`):
|
|
106
|
+
|
|
107
|
+
```yaml
|
|
108
|
+
contract:
|
|
109
|
+
tools:
|
|
110
|
+
required: [generate] # not retrieve — cache may skip it
|
|
111
|
+
requiredOrder: [generate] # ordering only among tools that ran
|
|
112
|
+
observations:
|
|
113
|
+
required: [cache_hit_or_retrieve_evidence]
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
With first-occurrence ordering, `retrieve → generate → retrieve` still **passes** when both retrieves are present (see worked example below).
|
|
117
|
+
|
|
28
118
|
## What is not shipped (yet)
|
|
29
119
|
|
|
30
120
|
Do **not** document these as available:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-inspect",
|
|
3
|
-
"version": "6.17.
|
|
3
|
+
"version": "6.17.5",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Local evidence debugger and trajectory-test toolkit for TypeScript AI agents — execution trees, TraceContract checks, Evidence v2, and read-only MCP",
|
|
@@ -235,13 +235,16 @@
|
|
|
235
235
|
"test:all": "pnpm run typecheck && pnpm run linked-versions:check && pnpm run build && pnpm run test && pnpm run size",
|
|
236
236
|
"prepublish:checks": "pnpm run typecheck && pnpm run test && pnpm run test:coverage && pnpm run build && pnpm run fixtures:check && pnpm run recipes:check && pnpm run size && pnpm run linked-versions:check && pnpm run repo:health && pnpm run pack:smoke",
|
|
237
237
|
"pack:dry-run": "pnpm run build && npm pack --dry-run",
|
|
238
|
-
"pack:smoke": "pnpm run build && node scripts/package-smoke.mjs && node scripts/packed-quickstart-e2e.mjs && node scripts/packed-semantic-loop-e2e.mjs && node scripts/packed-swarm-loop-e2e.mjs && node scripts/evidence-ci-golden-paths.mjs",
|
|
238
|
+
"pack:smoke": "pnpm run build && node scripts/package-smoke.mjs && node scripts/packed-openai-agents-e2e.mjs && node scripts/packed-quickstart-e2e.mjs && node scripts/packed-semantic-loop-e2e.mjs && node scripts/packed-swarm-loop-e2e.mjs && node scripts/evidence-ci-golden-paths.mjs",
|
|
239
239
|
"linked-versions:check": "node scripts/check-linked-versions.mjs",
|
|
240
240
|
"docs:commands": "node scripts/validate-doc-commands.mjs",
|
|
241
241
|
"docs:links": "node scripts/validate-doc-links.mjs",
|
|
242
|
+
"public-truth:sync": "node scripts/sync-public-truth.mjs",
|
|
242
243
|
"public-truth:check": "node scripts/validate-public-truth.mjs",
|
|
243
244
|
"ai-assets:check": "node scripts/validate-ai-assets.mjs",
|
|
244
|
-
"
|
|
245
|
+
"package-readmes:check": "node scripts/validate-package-readmes.mjs",
|
|
246
|
+
"docs:check": "pnpm run docs:commands && pnpm run docs:links && pnpm run public-truth:check && pnpm run ai-assets:check && pnpm run package-readmes:check && pnpm run repo:health && pnpm run build:cli && pnpm run demo:verify",
|
|
247
|
+
"build:cli": "pnpm exec tsup --config tsup.cli.config.ts",
|
|
245
248
|
"repo:health": "node scripts/validate-repo-health.mjs",
|
|
246
249
|
"demo:generate": "node scripts/demo-generate.mjs",
|
|
247
250
|
"demo:verify": "node scripts/demo-verify.mjs",
|
|
@@ -251,6 +254,7 @@
|
|
|
251
254
|
"perf:baseline": "node scripts/performance-baseline.mjs",
|
|
252
255
|
"examples:check": "pnpm install && pnpm --filter agent-inspect-example-01-basic run start",
|
|
253
256
|
"changeset": "changeset",
|
|
257
|
+
"version:packages": "changeset version && pnpm public-truth:sync && pnpm public-truth:check",
|
|
254
258
|
"release": "changeset publish",
|
|
255
259
|
"website:dev": "pnpm --filter @agent-inspect/website dev",
|
|
256
260
|
"website:build": "pnpm --filter @agent-inspect/website build",
|