@warlock.js/ai-panoptic 4.8.1 → 4.8.2

package/esm/evaluate/find-span-by-id.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-span-by-id.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai-panoptic/src/evaluate/find-span-by-id.ts"],"mappings":";;;;iBAGgB,YAAA,CAAa,IAAA,EAAM,SAAA,EAAW,MAAA,WAAiB,SAAS"}

package/esm/evaluate/find-span-by-id.mjs

@@ -0,0 +1,13 @@
+//#region ../@warlock.js/ai-panoptic/src/evaluate/find-span-by-id.ts
+/** Depth-first search for the span with `spanId` inside a trace's span tree. */
+function findSpanById(root, spanId) {
+	if (root.spanId === spanId) return root;
+	for (const child of root.children) {
+		const found = findSpanById(child, spanId);
+		if (found !== void 0) return found;
+	}
+}
+
+//#endregion
+export { findSpanById };
+//# sourceMappingURL=find-span-by-id.mjs.map

package/esm/evaluate/find-span-by-id.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-span-by-id.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai-panoptic/src/evaluate/find-span-by-id.ts"],"sourcesContent":["import type { TraceSpan } from \"../contracts/trace.type\";\n\n/** Depth-first search for the span with `spanId` inside a trace's span tree. */\nexport function findSpanById(root: TraceSpan, spanId: string): TraceSpan | undefined {\n  if (root.spanId === spanId) {\n    return root;\n  }\n\n  for (const child of root.children) {\n    const found = findSpanById(child, spanId);\n\n    if (found !== undefined) {\n      return found;\n    }\n  }\n\n  return undefined;\n}\n"],"mappings":";;AAGA,SAAgB,aAAa,MAAiB,QAAuC;CACnF,IAAI,KAAK,WAAW,QAClB,OAAO;CAGT,KAAK,MAAM,SAAS,KAAK,UAAU;EACjC,MAAM,QAAQ,aAAa,OAAO,MAAM;EAExC,IAAI,UAAU,QACZ,OAAO;CAEX;AAGF"}

package/esm/evaluate/index.d.mts

@@ -0,0 +1,4 @@
+import { EvaluateConfig, EvaluateVerdict } from "./evaluate.type.mjs";
+import { evaluateSystemPrompt } from "./evaluate-system-prompt.mjs";
+import { extractLastSystemPrompt } from "./extract-last-system-prompt.mjs";
+import { findSpanById } from "./find-span-by-id.mjs";

package/esm/evaluate/index.mjs

@@ -0,0 +1,5 @@
+import { evaluateSystemPrompt } from "./evaluate-system-prompt.mjs";
+import { extractLastSystemPrompt } from "./extract-last-system-prompt.mjs";
+import { findSpanById } from "./find-span-by-id.mjs";
+
+export {  };

package/esm/index.d.mts

@@ -31,10 +31,14 @@ import { CompletedEventPayload, PanopticTarget } from "./panoptic/panoptic-targe
 import { Panoptic, PanopticOptions } from "./panoptic/panoptic.type.mjs";
 import { panoptic } from "./panoptic/panoptic.mjs";
 import { createPanopticMiddleware } from "./panoptic/panoptic-middleware.mjs";
+import { EvaluateConfig, EvaluateVerdict } from "./evaluate/evaluate.type.mjs";
+import { evaluateSystemPrompt } from "./evaluate/evaluate-system-prompt.mjs";
+import { extractLastSystemPrompt } from "./evaluate/extract-last-system-prompt.mjs";
+import { findSpanById } from "./evaluate/find-span-by-id.mjs";
 import { DashboardHandle, DashboardOptions } from "./dashboard/dashboard.type.mjs";
 import { dashboard } from "./dashboard/dashboard.mjs";
 import { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, PromptGroup, SessionGroup, TraceFilter, TypeGroup, TypeStat, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey } from "./dashboard/trace-filter.mjs";
 import { PanopticConfig } from "./config/panoptic-config.type.mjs";
 import { applyPanopticConfig } from "./config/apply-panoptic-config.mjs";
-export { type AttributeValue, type CacheDriverInput, type CacheTraceStoreHandle, type CacheTraceStoreOptions, type CollectorContract, type CollectorOptions, type CompletedEventPayload, type ConsoleExporterOptions, type ConsoleLike, type ContentCaptureOptions, type ContentRedactor, type DashboardHandle, type DashboardOptions, type ExporterContract, type ExporterErrorHandler, type FileExporterOptions, GEN_AI_ATTRIBUTES, type InMemoryTraceStoreOptions, type LangfuseClientLike, type LangfuseExporterOptions, type LangfuseObservationBody, type LangfuseObservationEndBody, type LangfuseObservationLevel, type LangfuseObservationLike, type LangfuseTraceBody, type LangfuseTraceLike, type LangfuseUsageBody, NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, type OtelExporterOptions, type Panoptic, type PanopticConfig, type PanopticOptions, type PanopticTarget, type PromptGroup, type SessionGroup, type Trace, type TraceAggregate, type TraceFilter, type TraceQuery, type TraceRecord, type TraceSpan, type TraceSpanError, type TraceStoreContract, type TypeGroup, type TypeStat, WARLOCK_ATTRIBUTES, aggregateByType, applyPanopticConfig, consoleExporter, createCacheTraceStore, createCollector, createInMemoryTraceStore, createPanopticMiddleware, dashboard, emptyUsage, extractSpanAttributes, fileExporter, filterTraces, formatSpanIO, formatSpanLine, groupByPrompt, groupBySession, groupByType, heatIntensity, langfuseExporter, matchTrace, matchesFilter, maxNodeCost, normalizeError, otelExporter, panoptic, percentile, reportToSpan, reportToTrace, rollupCost, sumUsage, toGenAiAttributes, totalCostUsd, traceCost, tracePromptKey, walkSpans };
+export { type AttributeValue, type CacheDriverInput, type CacheTraceStoreHandle, type CacheTraceStoreOptions, type CollectorContract, type CollectorOptions, type CompletedEventPayload, type ConsoleExporterOptions, type ConsoleLike, type ContentCaptureOptions, type ContentRedactor, type DashboardHandle, type DashboardOptions, type EvaluateConfig, type EvaluateVerdict, type ExporterContract, type ExporterErrorHandler, type FileExporterOptions, GEN_AI_ATTRIBUTES, type InMemoryTraceStoreOptions, type LangfuseClientLike, type LangfuseExporterOptions, type LangfuseObservationBody, type LangfuseObservationEndBody, type LangfuseObservationLevel, type LangfuseObservationLike, type LangfuseTraceBody, type LangfuseTraceLike, type LangfuseUsageBody, NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, type OtelExporterOptions, type Panoptic, type PanopticConfig, type PanopticOptions, type PanopticTarget, type PromptGroup, type SessionGroup, type Trace, type TraceAggregate, type TraceFilter, type TraceQuery, type TraceRecord, type TraceSpan, type TraceSpanError, type TraceStoreContract, type TypeGroup, type TypeStat, WARLOCK_ATTRIBUTES, aggregateByType, applyPanopticConfig, consoleExporter, createCacheTraceStore, createCollector, createInMemoryTraceStore, createPanopticMiddleware, dashboard, emptyUsage, evaluateSystemPrompt, extractLastSystemPrompt, extractSpanAttributes, fileExporter, filterTraces, findSpanById, formatSpanIO, formatSpanLine, groupByPrompt, groupBySession, groupByType, heatIntensity, langfuseExporter, matchTrace, matchesFilter, maxNodeCost, normalizeError, otelExporter, panoptic, percentile, reportToSpan, reportToTrace, rollupCost, sumUsage, toGenAiAttributes, totalCostUsd, traceCost, tracePromptKey, walkSpans };
 import "./config/panoptic-config.type.mjs";

package/esm/index.mjs

@@ -22,10 +22,15 @@ import "./exporters/index.mjs";
 import { createPanopticMiddleware } from "./panoptic/panoptic-middleware.mjs";
 import { panoptic } from "./panoptic/panoptic.mjs";
 import "./panoptic/index.mjs";
+import { evaluateSystemPrompt } from "./evaluate/evaluate-system-prompt.mjs";
+import { extractLastSystemPrompt } from "./evaluate/extract-last-system-prompt.mjs";
+import { findSpanById } from "./evaluate/find-span-by-id.mjs";
+import "./evaluate/index.mjs";
 import { dashboard } from "./dashboard/dashboard.mjs";
 import { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey } from "./dashboard/trace-filter.mjs";
+import "./dashboard/index.mjs";
 import { applyPanopticConfig } from "./config/apply-panoptic-config.mjs";
 import "./config/index.mjs";
 import "./register.mjs";
 
-export { GEN_AI_ATTRIBUTES, NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, WARLOCK_ATTRIBUTES, aggregateByType, applyPanopticConfig, consoleExporter, createCacheTraceStore, createCollector, createInMemoryTraceStore, createPanopticMiddleware, dashboard, emptyUsage, extractSpanAttributes, fileExporter, filterTraces, formatSpanIO, formatSpanLine, groupByPrompt, groupBySession, groupByType, heatIntensity, langfuseExporter, matchTrace, matchesFilter, maxNodeCost, normalizeError, otelExporter, panoptic, percentile, reportToSpan, reportToTrace, rollupCost, sumUsage, toGenAiAttributes, totalCostUsd, traceCost, tracePromptKey, walkSpans };
+export { GEN_AI_ATTRIBUTES, NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, WARLOCK_ATTRIBUTES, aggregateByType, applyPanopticConfig, consoleExporter, createCacheTraceStore, createCollector, createInMemoryTraceStore, createPanopticMiddleware, dashboard, emptyUsage, evaluateSystemPrompt, extractLastSystemPrompt, extractSpanAttributes, fileExporter, filterTraces, findSpanById, formatSpanIO, formatSpanLine, groupByPrompt, groupBySession, groupByType, heatIntensity, langfuseExporter, matchTrace, matchesFilter, maxNodeCost, normalizeError, otelExporter, panoptic, percentile, reportToSpan, reportToTrace, rollupCost, sumUsage, toGenAiAttributes, totalCostUsd, traceCost, tracePromptKey, walkSpans };

package/llms-full.txt

@@ -4,6 +4,96 @@
 
 > Generated artifact. Concatenates every SKILL.md and reference file under `@warlock.js/ai-panoptic/skills/`. Re-run `node scripts/generate-llms.mjs` after any change.
 
+## evaluate-system-prompt  `@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md`
+
+---
+name: evaluate-system-prompt
+description: 'Grade a single trace''s LAST captured system prompt from the dashboard drawer — the dashboard''s only write-capable feature, config-gated via `DashboardOptions.evaluate` (`ai.config({ panoptic: { dashboard: { evaluate: { model, instructions } } } })`). Triggers: `evaluate`, `EvaluateConfig`, `EvaluateVerdict`, `evaluateSystemPrompt`, `extractLastSystemPrompt`, `findSpanById`, "Evaluate system prompt" button, `POST .../spans/:spanId/evaluate`, grading a prompt from a real trace, judging a captured system prompt, per-run instructions override, judge model factory; ''evaluate this trace''s prompt'', ''grade the system prompt from the dashboard'', ''judge a captured run'', ''score a live system prompt''. Skip: grading raw prompt TEXT you already have in code (no trace involved) — `@warlock.js/ai`''s `ai.prompts().validate({ criteria })`, the same judge machinery this wraps; a DATASET of test cases run through an agent — `@warlock.js/ai`''s `ai.dataset` + `agent.eval`; viewing/filtering/grouping traces without grading them — `@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md`; capturing the content this feature reads — `@warlock.js/ai-panoptic/export-traces/SKILL.md` (`captureContent`).'
+---
+
+# Evaluate a trace's system prompt from the dashboard
+
+A drawer button that grades the LAST `{role: "system"}` message captured on the selected span — not the whole conversation, not the tool code, just the system prompt that was actually in effect for that run. It's the dashboard's only write-capable feature: every other route is `GET`-only and read-only by design (see `use-local-dashboard/SKILL.md`'s security note); this one POSTs, and it's off unless you explicitly opt in.
+
+## Enable it
+
+```ts
+import { ai } from "@warlock.js/ai";
+import "@warlock.js/ai-panoptic";
+
+ai.config({
+  panoptic: {
+    dashboard: {
+      evaluate: {
+        model: openai.model("gpt-4o-mini"),
+        instructions: "Must address the user by name. Never invent prices.",
+      },
+    },
+    observeAll: true,
+    captureContent: true, // required — evaluate reads the captured span.input
+  },
+});
+```
+
+Absent `evaluate` ⇒ the drawer never renders the button and a `POST` to its route 405s like any other route would (the path is never even pattern-matched) — the dashboard stays fully read-only unless you opt in.
+
+### `EvaluateConfig`
+
+| Field | Required | Notes |
+|---|---|---|
+| `model` | yes | A `ModelContract`, or a `() => ModelContract \| Promise<ModelContract>` factory — mirrors `panoptic.cache`'s literal-or-factory ergonomics. Resolved fresh on every evaluate click (a deliberate, infrequent, human-triggered action — not the collector's hot path — so no memoization, unlike the cache store's driver). |
+| `instructions` | no | Default grading rubric/criteria, verbatim — the same shape `ai.prompts().validate({ criteria })` accepts. Seeds the dashboard's editable textarea (baked into the served page at boot, so the operator sees and can extend the actual default, not a blank box); the built-in prompt-quality rubric is used when neither this nor a per-run override is supplied. |
+
+## The UI
+
+Open a trace, select the span whose prompt you want graded (any span carrying a captured `[system, user]` pair or `fullHistory` message array — an agent span, not a tool span). An "Evaluate system prompt" button appears under the input/output block **only when that span actually has a system prompt to grade** (`extractLastSystemPrompt` finds one) — no dead button on tool spans or system-prompt-less runs.
+
+Click it to open an inline panel: a textarea pre-filled with the configured `instructions` (fully editable — type over it, append to it, or clear it to fall back to the built-in rubric) and a "Run" button. The result — `score` (0–1, shown as a percentage) and `issues` (the judge's reasoning, as a list) — renders inline once it lands. State (open/closed, typed text, last result) is kept per-span in client memory, so switching to a sibling span and back doesn't lose it; it does NOT persist to the trace store or survive a page reload.
+
+## The building blocks (for embedding elsewhere)
+
+```ts
+import { evaluateSystemPrompt, extractLastSystemPrompt, findSpanById } from "@warlock.js/ai-panoptic";
+
+const span = findSpanById(trace.root, spanId);          // walk the span tree
+const prompt = extractLastSystemPrompt(span);            // undefined if none captured
+if (prompt) {
+  const verdict = await evaluateSystemPrompt(prompt, evaluateConfig, "Per-run override, optional");
+  // verdict: { score?: number; issues: string[] }
+}
+```
+
+- **`extractLastSystemPrompt(span)`** — reads `span.input` (only present under `captureContent`). Handles both shapes `report-to-span.ts` produces: the `[system, user]` first-trip pair, and — under `fullHistory` — the full `CapturedMessage[]` conversation, where it takes the LAST system-role turn (a long-running agent can carry more than one; the most recent one is the one actually in effect). Returns `undefined` for a tool span, an agent with no system prompt, or when content capture is off.
+- **`findSpanById(root, spanId)`** — depth-first lookup inside a trace's span tree; the route uses it to resolve which node the drawer had selected.
+- **`evaluateSystemPrompt(text, config, instructionsOverride?)`** — the grading call. `instructionsOverride` (trimmed; blank-after-trim is treated as absent, never as "grade against nothing") wins over `config.instructions`. Reuses `@warlock.js/ai`'s `judgePromptBody` verbatim — the SAME LLM-as-judge machinery `ai.prompts().validate({ criteria })` runs — so there's exactly one judging implementation in the whole framework, not two. `judgePromptBody` itself never throws (a broken judge degrades to an issues-only outcome with no `score`); the only step that CAN throw is resolving `config.model` when it's a factory (e.g. constructing an SDK client) — the dashboard route catches that and returns `502`.
+
+## The route
+
+`POST {basePath}api/traces/:traceId/spans/:spanId/evaluate`, body `{ instructions?: string }` (all optional — an empty body is valid). Gated by the same `authToken` / `allowedHosts` checks as every other route (S4) — off-loopback still requires a token, exactly like the rest of the dashboard. Body reads are capped at 64 KB (`413` past that) so a hostile/oversized payload can't hold the connection open.
+
+| Status | Meaning |
+|---|---|
+| `200` | `EvaluateVerdict` — `{ score?: number; issues: string[] }` |
+| `404` | trace or span not found |
+| `422` | the span carried no captured system prompt to grade |
+| `413` | request body over 64 KB |
+| `400` | request body isn't valid JSON |
+| `502` | `config.model` (a factory) threw resolving the judge model |
+| `405` | POST hit this path but `evaluate` isn't configured |
+
+```sh
+curl -X POST 'http://127.0.0.1:4319/api/traces/tr_1/spans/sp_1/evaluate' \
+  -H 'content-type: application/json' \
+  -d '{"instructions": "Must never invent prices."}'
+```
+
+## See also
+
+- [`@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md`](@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md) — the dashboard this feature is a drawer action of; its security note now says "every route is GET-only and read-only EXCEPT the one opt-in write route `evaluate` enables."
+- [`@warlock.js/ai-panoptic/export-traces/SKILL.md`](@warlock.js/ai-panoptic/export-traces/SKILL.md) — `captureContent` (and `fullHistory`), the prerequisite for `span.input` to carry anything gradable.
+- `ai.prompts().validate({ criteria })` in `@warlock.js/ai` — the sibling capability this reuses: grading raw prompt TEXT you already have in code, no trace or dashboard involved.
+
+
 ## export-traces  `@warlock.js/ai-panoptic/export-traces/SKILL.md`
 
 ---
@@ -748,7 +838,7 @@ type DashboardHandle = {
 
 ## The JSON API
 
-All routes are **`GET`-only and read-only** (non-`GET` → `405`), mounted under `basePath`. The page polls them; you can curl them too. Store shapes are already JSON-safe, so responses are a plain `JSON.stringify`.
+Every route is **`GET`-only and read-only** (non-`GET` → `405`), mounted under `basePath` — EXCEPT the one opt-in `POST .../spans/:spanId/evaluate` route `DashboardOptions.evaluate` enables (see [`evaluate-system-prompt/SKILL.md`](@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md)); absent that config, it 405s like everything else. The page polls the `GET` routes; you can curl them too. Store shapes are already JSON-safe, so responses are a plain `JSON.stringify`.
 
 | Route | Returns |
 |---|---|
@@ -756,6 +846,7 @@ All routes are **`GET`-only and read-only** (non-`GET` → `405`), mounted under
 | `GET /api/traces/:id` | `store.get(id)` — one full trace, or `404 { error: "trace_not_found" }`. |
 | `GET /api/aggregate` | `store.aggregate(...)` — the usage + cost + status rollup over the same query filter. |
 | `GET /` (basePath) | the self-contained HTML page. |
+| `POST /api/traces/:traceId/spans/:spanId/evaluate` | grades the span's last captured system prompt — only when `evaluate` is configured. |
 
 Query-string filters map 1:1 onto `TraceQuery`:
 
@@ -782,16 +873,17 @@ const handler = createRequestHandler(store, {
 createServer(handler).listen(4319, "127.0.0.1");
 ```
 
-`dashboardHtml(basePath, title)` returns the served page as a string if you embed it elsewhere. `ServeConfig` is `{ basePath; title; allowedHosts; authToken? }`.
+`dashboardHtml(basePath, title, evaluateEnabled?, evaluateDefaultInstructions?)` returns the served page as a string if you embed it elsewhere. `ServeConfig` is `{ basePath; title; allowedHosts; authToken?; evaluate? }`.
 
 ## Security note
 
-The dashboard surfaces whatever the store holds — including captured prompt/response content when content capture is on (`captureContent`). It binds **loopback-only** by default precisely so that content never leaves the machine. Binding a non-loopback host is gated — it requires an `authToken` (and checks a `Host` allowlist, sending `nosniff` / CSP / `X-Frame-Options: DENY` on every response) — but still prefer loopback for dev, and for production ship to a real backend via an exporter rather than exposing the dashboard.
+The dashboard surfaces whatever the store holds — including captured prompt/response content when content capture is on (`captureContent`). It binds **loopback-only** by default precisely so that content never leaves the machine. Binding a non-loopback host is gated — it requires an `authToken` (and checks a `Host` allowlist, sending `nosniff` / CSP / `X-Frame-Options: DENY` on every response) — but still prefer loopback for dev, and for production ship to a real backend via an exporter rather than exposing the dashboard. The dashboard has exactly one write-capable route (`evaluate`, off by default) — everything else stays read-only regardless of configuration.
 
 ## See also
 
 - [`@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md`](@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md) — the `ai.config({ panoptic })` wiring and the underlying `panoptic(...)` subscriber that fills the store.
 - [`@warlock.js/ai-panoptic/query-traces/SKILL.md`](@warlock.js/ai-panoptic/query-traces/SKILL.md) — the `TraceStoreContract` (`query` / `get` / `aggregate`) the dashboard reads, the `TraceQuery` filter, and capacity eviction.
 - [`@warlock.js/ai-panoptic/export-traces/SKILL.md`](@warlock.js/ai-panoptic/export-traces/SKILL.md) — shipping traces to OTel / Langfuse / a file instead of (or alongside) the local dashboard.
+- [`@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md`](@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md) — the drawer's "Evaluate system prompt" action, the dashboard's one write route.
 
 

package/llms.txt

@@ -6,6 +6,7 @@
 
 ## Skills
 
+- [evaluate-system-prompt](@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md): Grade a single trace's LAST captured system prompt from the dashboard drawer — the dashboard's only write-capable feature, config-gated via `DashboardOptions.evaluate` (`ai.config({ panoptic: { dashboard: { evaluate: { model, instructions } } } })`). Triggers: `evaluate`, `EvaluateConfig`, `EvaluateVerdict`, `evaluateSystemPrompt`, `extractLastSystemPrompt`, `findSpanById`, "Evaluate system prompt" button, `POST .../spans/:spanId/evaluate`, grading a prompt from a real trace, judging a captured system prompt, per-run instructions override, judge model factory; 'evaluate this trace's prompt', 'grade the system prompt from the dashboard', 'judge a captured run', 'score a live system prompt'. Skip: grading raw prompt TEXT you already have in code (no trace involved) — `@warlock.js/ai`'s `ai.prompts().validate({ criteria })`, the same judge machinery this wraps; a DATASET of test cases run through an agent — `@warlock.js/ai`'s `ai.dataset` + `agent.eval`; viewing/filtering/grouping traces without grading them — `@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md`; capturing the content this feature reads — `@warlock.js/ai-panoptic/export-traces/SKILL.md` (`captureContent`).
 - [export-traces](@warlock.js/ai-panoptic/export-traces/SKILL.md): Send @warlock.js/ai-panoptic traces to an observability backend via pluggable exporters, including capturing prompt/response content onto spans. Triggers: `consoleExporter`, `fileExporter`, `otelExporter`, `langfuseExporter`, `ExporterContract`, `ContentCaptureOptions`, `captureContent`, `redactContent`, `fullHistory`, `ContentRedactor`, `toGenAiAttributes`, `walkSpans`, `totalCostUsd`, `GEN_AI_ATTRIBUTES`, `WARLOCK_ATTRIBUTES`, `reportToTrace`, `reportToSpan`, `extractSpanAttributes`, `normalizeError`, `formatSpanLine`, `createPanopticMiddleware`; 'export AI traces to OpenTelemetry', 'send traces to Langfuse', 'log traces to console / a file', 'capture the full prompt/conversation onto a span', 'gen_ai semantic conventions', 'write a custom exporter'; typical import `import { otelExporter, langfuseExporter, consoleExporter, fileExporter } from "@warlock.js/ai-panoptic"`. Skip: wiring the subscriber/collector into a run — `@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md`; the core report shape — `@warlock.js/ai/run-ai-agent/SKILL.md`.
 - [observe-with-panoptic](@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md): Wire @warlock.js/ai-panoptic into an agent/workflow/supervisor/orchestrator — declaratively via `ai.config({ panoptic })` (registers panoptic on core's Observer seam) or with the one-call `panoptic({ exporters })` subscriber. Triggers: `panoptic`, `Panoptic`, `PanopticOptions`, `PanopticConfig`, `ai.config({ panoptic })`, `observeAll`, per-flow `observe`, `FlowObserveOption`, `registerObserver`, `Observer`, `.attach`, `.middleware`, `.collect`, `.toTrace`, `observe.attach(agent)`, `panoptic().middleware()`, `completedEvents`, `middlewareName`; 'observe a warlock agent', 'trace an agent run', 'attach observability to a workflow/supervisor', 'observe every flow by default', 'add a tracing middleware', 'collect an orchestrator turn report', 'wire OTel/Langfuse into my agent'; typical import `import { ai } from "@warlock.js/ai"` + `import "@warlock.js/ai-panoptic"` (or `import { panoptic } from "@warlock.js/ai-panoptic"`). Skip: the zero-setup local dashboard — `@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md`; writing/choosing an exporter (the sink end) — `@warlock.js/ai-panoptic/export-traces/SKILL.md`; the core report shape / running the agent itself — `@warlock.js/ai/run-ai-agent/SKILL.md`.
 - [query-traces](@warlock.js/ai-panoptic/query-traces/SKILL.md): Retain @warlock.js/ai-panoptic traces in a queryable store — in-memory, or cache-backed so they survive a process restart — and slice them after the fact: get one run by id, list a session, filter failed runs in a window, roll up usage + cost. Triggers: `createInMemoryTraceStore`, `createCacheTraceStore`, `CacheTraceStoreHandle`, `CacheTraceStoreOptions`, `CacheDriverInput`, `store.ready`, `TraceStoreContract`, `TraceQuery`, `TraceAggregate`, `InMemoryTraceStoreOptions`, `store.query`, `store.aggregate`, `store.get`, `store.add`, `store.clear`, `store.size`, `capacity`, `prefix`, `onError`, `sumUsage`, `emptyUsage`, `matchTrace`; 'query collected traces', 'aggregate AI cost per session', 'how much did this session spend', 'list failed runs', 'retain traces in memory', 'persist traces across a restart', 'cache-backed trace store', 'Redis trace store', 'roll up token usage'; typical import `import { createInMemoryTraceStore, createCacheTraceStore } from "@warlock.js/ai-panoptic"`. Skip: sending traces to an external backend (OTel/Langfuse/console/file) — `@warlock.js/ai-panoptic/export-traces/SKILL.md`; wiring the subscriber into a run — `@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md`.

package/package.json

@@ -21,11 +21,12 @@
   "peerDependencies": {
     "@opentelemetry/api": "*",
     "@opentelemetry/sdk-trace-base": "*",
-    "@warlock.js/ai": "4.8.1",
-    "@warlock.js/cache": "4.8.1",
+    "@warlock.js/ai": "4.8.2",
+    "@warlock.js/cache": "4.8.2",
+    "@warlock.js/logger": "4.8.2",
     "langfuse": "*"
   },
-  "version": "4.8.1",
+  "version": "4.8.2",
   "main": "./cjs/index.cjs",
   "module": "./esm/index.mjs",
   "types": "./esm/index.d.mts",

package/skills/README.md

@@ -19,3 +19,7 @@ Retain traces in a queryable store — in-memory (`createInMemoryTraceStore`) or
 ### [`use-local-dashboard/`](./use-local-dashboard/SKILL.md)
 
 Run the zero-setup local Panoptic dashboard — a loopback `node:http` server over a trace store served from `ai.config({ panoptic: { dashboard } })`. Covers the redesigned UI (light/dark/system theme, two-pane nested call tree + metadata panel, colour-coded Title-case type labels including the first-class `Team` type, started/ended wall-clock timestamps, arrow-coded `↓in · ↑out · total` tokens, per-node rollup cost, hide-zero rows, Warlock logo), the new dashboard features (client-side search/filter, group-by-session + group-by-prompt + group-by-type, per-type stats panel, cost heatmap, Gantt timeline, errors-only toggle, hash deep-links) and their pure `trace-filter.ts` helpers (`filterTraces` / `groupBySession` / `groupByPrompt` / `rollupCost` / `maxNodeCost` / `heatIntensity` / `tracePromptKey`), the persistent `cache` store wiring, the declarative path (which store it reads, idempotent wiring), the low-level `dashboard(store, options)` building block, `DashboardOptions` (port `4319`, loopback host, `basePath`, `open`, `title`), the `DashboardHandle`, the read-only `/api/traces` / `/api/aggregate` / `/api/traces/:id` JSON API, `createRequestHandler` / `dashboardHtml` for embedding, prompt-version linkage, and the loopback-only security posture. Load when opening a local AI trace viewer in the browser, theming/filtering it, inspecting traces without a backend, persisting them across a restart, or spinning up a dev observability server.
+
+### [`evaluate-system-prompt/`](./evaluate-system-prompt/SKILL.md)
+
+Grade a trace's LAST captured system prompt from the drawer — the dashboard's only write-capable feature, config-gated via `DashboardOptions.evaluate` (`{ model, instructions }`). Covers `evaluateSystemPrompt` / `extractLastSystemPrompt` / `findSpanById`, the "Evaluate system prompt" button + editable-instructions panel, the `POST .../spans/:spanId/evaluate` route and its status codes, and how it reuses `@warlock.js/ai`'s `judgePromptBody` (the same machinery `ai.prompts().validate()` runs) rather than a second judging implementation. Load when grading a live/captured system prompt from the dashboard, wiring `evaluate` into `ai.config`, or embedding the evaluate building blocks elsewhere.

package/skills/evaluate-system-prompt/SKILL.md

@@ -0,0 +1,86 @@
+---
+name: evaluate-system-prompt
+description: 'Grade a single trace''s LAST captured system prompt from the dashboard drawer — the dashboard''s only write-capable feature, config-gated via `DashboardOptions.evaluate` (`ai.config({ panoptic: { dashboard: { evaluate: { model, instructions } } } })`). Triggers: `evaluate`, `EvaluateConfig`, `EvaluateVerdict`, `evaluateSystemPrompt`, `extractLastSystemPrompt`, `findSpanById`, "Evaluate system prompt" button, `POST .../spans/:spanId/evaluate`, grading a prompt from a real trace, judging a captured system prompt, per-run instructions override, judge model factory; ''evaluate this trace''s prompt'', ''grade the system prompt from the dashboard'', ''judge a captured run'', ''score a live system prompt''. Skip: grading raw prompt TEXT you already have in code (no trace involved) — `@warlock.js/ai`''s `ai.prompts().validate({ criteria })`, the same judge machinery this wraps; a DATASET of test cases run through an agent — `@warlock.js/ai`''s `ai.dataset` + `agent.eval`; viewing/filtering/grouping traces without grading them — `@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md`; capturing the content this feature reads — `@warlock.js/ai-panoptic/export-traces/SKILL.md` (`captureContent`).'
+---
+
+# Evaluate a trace's system prompt from the dashboard
+
+A drawer button that grades the LAST `{role: "system"}` message captured on the selected span — not the whole conversation, not the tool code, just the system prompt that was actually in effect for that run. It's the dashboard's only write-capable feature: every other route is `GET`-only and read-only by design (see `use-local-dashboard/SKILL.md`'s security note); this one POSTs, and it's off unless you explicitly opt in.
+
+## Enable it
+
+```ts
+import { ai } from "@warlock.js/ai";
+import "@warlock.js/ai-panoptic";
+
+ai.config({
+  panoptic: {
+    dashboard: {
+      evaluate: {
+        model: openai.model("gpt-4o-mini"),
+        instructions: "Must address the user by name. Never invent prices.",
+      },
+    },
+    observeAll: true,
+    captureContent: true, // required — evaluate reads the captured span.input
+  },
+});
+```
+
+Absent `evaluate` ⇒ the drawer never renders the button and a `POST` to its route 405s like any other route would (the path is never even pattern-matched) — the dashboard stays fully read-only unless you opt in.
+
+### `EvaluateConfig`
+
+| Field | Required | Notes |
+|---|---|---|
+| `model` | yes | A `ModelContract`, or a `() => ModelContract \| Promise<ModelContract>` factory — mirrors `panoptic.cache`'s literal-or-factory ergonomics. Resolved fresh on every evaluate click (a deliberate, infrequent, human-triggered action — not the collector's hot path — so no memoization, unlike the cache store's driver). |
+| `instructions` | no | Default grading rubric/criteria, verbatim — the same shape `ai.prompts().validate({ criteria })` accepts. Seeds the dashboard's editable textarea (baked into the served page at boot, so the operator sees and can extend the actual default, not a blank box); the built-in prompt-quality rubric is used when neither this nor a per-run override is supplied. |
+
+## The UI
+
+Open a trace, select the span whose prompt you want graded (any span carrying a captured `[system, user]` pair or `fullHistory` message array — an agent span, not a tool span). An "Evaluate system prompt" button appears under the input/output block **only when that span actually has a system prompt to grade** (`extractLastSystemPrompt` finds one) — no dead button on tool spans or system-prompt-less runs.
+
+Click it to open an inline panel: a textarea pre-filled with the configured `instructions` (fully editable — type over it, append to it, or clear it to fall back to the built-in rubric) and a "Run" button. The result — `score` (0–1, shown as a percentage) and `issues` (the judge's reasoning, as a list) — renders inline once it lands. State (open/closed, typed text, last result) is kept per-span in client memory, so switching to a sibling span and back doesn't lose it; it does NOT persist to the trace store or survive a page reload.
+
+## The building blocks (for embedding elsewhere)
+
+```ts
+import { evaluateSystemPrompt, extractLastSystemPrompt, findSpanById } from "@warlock.js/ai-panoptic";
+
+const span = findSpanById(trace.root, spanId);          // walk the span tree
+const prompt = extractLastSystemPrompt(span);            // undefined if none captured
+if (prompt) {
+  const verdict = await evaluateSystemPrompt(prompt, evaluateConfig, "Per-run override, optional");
+  // verdict: { score?: number; issues: string[] }
+}
+```
+
+- **`extractLastSystemPrompt(span)`** — reads `span.input` (only present under `captureContent`). Handles both shapes `report-to-span.ts` produces: the `[system, user]` first-trip pair, and — under `fullHistory` — the full `CapturedMessage[]` conversation, where it takes the LAST system-role turn (a long-running agent can carry more than one; the most recent one is the one actually in effect). Returns `undefined` for a tool span, an agent with no system prompt, or when content capture is off.
+- **`findSpanById(root, spanId)`** — depth-first lookup inside a trace's span tree; the route uses it to resolve which node the drawer had selected.
+- **`evaluateSystemPrompt(text, config, instructionsOverride?)`** — the grading call. `instructionsOverride` (trimmed; blank-after-trim is treated as absent, never as "grade against nothing") wins over `config.instructions`. Reuses `@warlock.js/ai`'s `judgePromptBody` verbatim — the SAME LLM-as-judge machinery `ai.prompts().validate({ criteria })` runs — so there's exactly one judging implementation in the whole framework, not two. `judgePromptBody` itself never throws (a broken judge degrades to an issues-only outcome with no `score`); the only step that CAN throw is resolving `config.model` when it's a factory (e.g. constructing an SDK client) — the dashboard route catches that and returns `502`.
+
+## The route
+
+`POST {basePath}api/traces/:traceId/spans/:spanId/evaluate`, body `{ instructions?: string }` (all optional — an empty body is valid). Gated by the same `authToken` / `allowedHosts` checks as every other route (S4) — off-loopback still requires a token, exactly like the rest of the dashboard. Body reads are capped at 64 KB (`413` past that) so a hostile/oversized payload can't hold the connection open.
+
+| Status | Meaning |
+|---|---|
+| `200` | `EvaluateVerdict` — `{ score?: number; issues: string[] }` |
+| `404` | trace or span not found |
+| `422` | the span carried no captured system prompt to grade |
+| `413` | request body over 64 KB |
+| `400` | request body isn't valid JSON |
+| `502` | `config.model` (a factory) threw resolving the judge model |
+| `405` | POST hit this path but `evaluate` isn't configured |
+
+```sh
+curl -X POST 'http://127.0.0.1:4319/api/traces/tr_1/spans/sp_1/evaluate' \
+  -H 'content-type: application/json' \
+  -d '{"instructions": "Must never invent prices."}'
+```
+
+## See also
+
+- [`@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md`](@warlock.js/ai-panoptic/use-local-dashboard/SKILL.md) — the dashboard this feature is a drawer action of; its security note now says "every route is GET-only and read-only EXCEPT the one opt-in write route `evaluate` enables."
+- [`@warlock.js/ai-panoptic/export-traces/SKILL.md`](@warlock.js/ai-panoptic/export-traces/SKILL.md) — `captureContent` (and `fullHistory`), the prerequisite for `span.input` to carry anything gradable.
+- `ai.prompts().validate({ criteria })` in `@warlock.js/ai` — the sibling capability this reuses: grading raw prompt TEXT you already have in code, no trace or dashboard involved.

package/skills/use-local-dashboard/SKILL.md

@@ -166,7 +166,7 @@ type DashboardHandle = {
 
 ## The JSON API
 
-All routes are **`GET`-only and read-only** (non-`GET` → `405`), mounted under `basePath`. The page polls them; you can curl them too. Store shapes are already JSON-safe, so responses are a plain `JSON.stringify`.
+Every route is **`GET`-only and read-only** (non-`GET` → `405`), mounted under `basePath` — EXCEPT the one opt-in `POST .../spans/:spanId/evaluate` route `DashboardOptions.evaluate` enables (see [`evaluate-system-prompt/SKILL.md`](@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md)); absent that config, it 405s like everything else. The page polls the `GET` routes; you can curl them too. Store shapes are already JSON-safe, so responses are a plain `JSON.stringify`.
 
 | Route | Returns |
 |---|---|
@@ -174,6 +174,7 @@ All routes are **`GET`-only and read-only** (non-`GET` → `405`), mounted under
 | `GET /api/traces/:id` | `store.get(id)` — one full trace, or `404 { error: "trace_not_found" }`. |
 | `GET /api/aggregate` | `store.aggregate(...)` — the usage + cost + status rollup over the same query filter. |
 | `GET /` (basePath) | the self-contained HTML page. |
+| `POST /api/traces/:traceId/spans/:spanId/evaluate` | grades the span's last captured system prompt — only when `evaluate` is configured. |
 
 Query-string filters map 1:1 onto `TraceQuery`:
 
@@ -200,14 +201,15 @@ const handler = createRequestHandler(store, {
 createServer(handler).listen(4319, "127.0.0.1");
 ```
 
-`dashboardHtml(basePath, title)` returns the served page as a string if you embed it elsewhere. `ServeConfig` is `{ basePath; title; allowedHosts; authToken? }`.
+`dashboardHtml(basePath, title, evaluateEnabled?, evaluateDefaultInstructions?)` returns the served page as a string if you embed it elsewhere. `ServeConfig` is `{ basePath; title; allowedHosts; authToken?; evaluate? }`.
 
 ## Security note
 
-The dashboard surfaces whatever the store holds — including captured prompt/response content when content capture is on (`captureContent`). It binds **loopback-only** by default precisely so that content never leaves the machine. Binding a non-loopback host is gated — it requires an `authToken` (and checks a `Host` allowlist, sending `nosniff` / CSP / `X-Frame-Options: DENY` on every response) — but still prefer loopback for dev, and for production ship to a real backend via an exporter rather than exposing the dashboard.
+The dashboard surfaces whatever the store holds — including captured prompt/response content when content capture is on (`captureContent`). It binds **loopback-only** by default precisely so that content never leaves the machine. Binding a non-loopback host is gated — it requires an `authToken` (and checks a `Host` allowlist, sending `nosniff` / CSP / `X-Frame-Options: DENY` on every response) — but still prefer loopback for dev, and for production ship to a real backend via an exporter rather than exposing the dashboard. The dashboard has exactly one write-capable route (`evaluate`, off by default) — everything else stays read-only regardless of configuration.
 
 ## See also
 
 - [`@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md`](@warlock.js/ai-panoptic/observe-with-panoptic/SKILL.md) — the `ai.config({ panoptic })` wiring and the underlying `panoptic(...)` subscriber that fills the store.
 - [`@warlock.js/ai-panoptic/query-traces/SKILL.md`](@warlock.js/ai-panoptic/query-traces/SKILL.md) — the `TraceStoreContract` (`query` / `get` / `aggregate`) the dashboard reads, the `TraceQuery` filter, and capacity eviction.
 - [`@warlock.js/ai-panoptic/export-traces/SKILL.md`](@warlock.js/ai-panoptic/export-traces/SKILL.md) — shipping traces to OTel / Langfuse / a file instead of (or alongside) the local dashboard.
+- [`@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md`](@warlock.js/ai-panoptic/evaluate-system-prompt/SKILL.md) — the drawer's "Evaluate system prompt" action, the dashboard's one write route.