@warlock.js/ai-panoptic 5.31.0 → 5.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/cjs/index.cjs +16 -65
- package/cjs/index.cjs.map +1 -1
- package/esm/collector/collector.d.mts +3 -5
- package/esm/collector/collector.d.mts.map +1 -1
- package/esm/collector/content-capture.type.d.mts +4 -5
- package/esm/collector/content-capture.type.d.mts.map +1 -1
- package/esm/collector/extract-span-attributes.d.mts +1 -3
- package/esm/collector/extract-span-attributes.d.mts.map +1 -1
- package/esm/collector/extract-span-attributes.mjs +1 -4
- package/esm/collector/extract-span-attributes.mjs.map +1 -1
- package/esm/collector/index.d.mts +2 -1
- package/esm/collector/index.mjs +1 -1
- package/esm/collector/normalize-error.d.mts +1 -3
- package/esm/collector/normalize-error.d.mts.map +1 -1
- package/esm/collector/report-to-span.d.mts +1 -3
- package/esm/collector/report-to-span.d.mts.map +1 -1
- package/esm/collector/report-to-trace.d.mts +1 -3
- package/esm/collector/report-to-trace.d.mts.map +1 -1
- package/esm/config/apply-panoptic-config.d.mts +2 -3
- package/esm/config/apply-panoptic-config.d.mts.map +1 -1
- package/esm/config/index.d.mts +2 -1
- package/esm/config/index.mjs +1 -1
- package/esm/config/panoptic-config.type.d.mts +2 -4
- package/esm/config/panoptic-config.type.d.mts.map +1 -1
- package/esm/contracts/collector.contract.d.mts +1 -3
- package/esm/contracts/collector.contract.d.mts.map +1 -1
- package/esm/contracts/exporter.contract.d.mts +1 -3
- package/esm/contracts/exporter.contract.d.mts.map +1 -1
- package/esm/contracts/index.d.mts +2 -1
- package/esm/contracts/trace.type.d.mts +35 -20
- package/esm/contracts/trace.type.d.mts.map +1 -1
- package/esm/dashboard/dashboard.d.mts +1 -3
- package/esm/dashboard/dashboard.d.mts.map +1 -1
- package/esm/dashboard/dashboard.mjs +3 -2
- package/esm/dashboard/dashboard.mjs.map +1 -1
- package/esm/dashboard/dashboard.type.d.mts +9 -7
- package/esm/dashboard/dashboard.type.d.mts.map +1 -1
- package/esm/dashboard/index.d.mts +4 -1
- package/esm/dashboard/index.mjs +1 -1
- package/esm/dashboard/parse-query.d.mts +1 -1
- package/esm/dashboard/serve.d.mts +2 -1
- package/esm/dashboard/serve.mjs +1 -1
- package/esm/dashboard/serve.mjs.map +1 -1
- package/esm/dashboard/trace-filter.d.mts +44 -32
- package/esm/dashboard/trace-filter.d.mts.map +1 -1
- package/esm/evaluate/evaluate-system-prompt.d.mts +1 -3
- package/esm/evaluate/evaluate-system-prompt.d.mts.map +1 -1
- package/esm/evaluate/evaluate-system-prompt.mjs +3 -1
- package/esm/evaluate/evaluate-system-prompt.mjs.map +1 -1
- package/esm/evaluate/evaluate.type.d.mts +2 -4
- package/esm/evaluate/evaluate.type.d.mts.map +1 -1
- package/esm/evaluate/extract-last-system-prompt.d.mts +1 -3
- package/esm/evaluate/extract-last-system-prompt.d.mts.map +1 -1
- package/esm/evaluate/find-span-by-id.d.mts +1 -3
- package/esm/evaluate/find-span-by-id.d.mts.map +1 -1
- package/esm/evaluate/index.d.mts +2 -1
- package/esm/evaluate/index.mjs +1 -1
- package/esm/exporters/console/console-exporter.d.mts +2 -3
- package/esm/exporters/console/console-exporter.d.mts.map +1 -1
- package/esm/exporters/console/console-exporter.type.d.mts +2 -3
- package/esm/exporters/console/console-exporter.type.d.mts.map +1 -1
- package/esm/exporters/console/format-span-io.d.mts +2 -2
- package/esm/exporters/console/format-span-io.d.mts.map +1 -1
- package/esm/exporters/console/format-span-line.d.mts +2 -2
- package/esm/exporters/console/format-span-line.d.mts.map +1 -1
- package/esm/exporters/console/index.d.mts +2 -1
- package/esm/exporters/console/index.mjs +1 -1
- package/esm/exporters/file/file-exporter.d.mts +2 -3
- package/esm/exporters/file/file-exporter.d.mts.map +1 -1
- package/esm/exporters/file/file-exporter.type.d.mts +8 -5
- package/esm/exporters/file/file-exporter.type.d.mts.map +1 -1
- package/esm/exporters/file/index.d.mts +2 -1
- package/esm/exporters/index.d.mts +7 -1
- package/esm/exporters/index.mjs +1 -1
- package/esm/exporters/langfuse/index.d.mts +2 -1
- package/esm/exporters/langfuse/index.mjs +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.d.mts +2 -3
- package/esm/exporters/langfuse/langfuse-exporter.d.mts.map +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.mjs +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.mjs.map +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.type.d.mts +23 -17
- package/esm/exporters/langfuse/langfuse-exporter.type.d.mts.map +1 -1
- package/esm/exporters/otel/index.d.mts +2 -1
- package/esm/exporters/otel/index.mjs +1 -1
- package/esm/exporters/otel/otel-api.shim.type.d.mts +6 -7
- package/esm/exporters/otel/otel-api.shim.type.d.mts.map +1 -1
- package/esm/exporters/otel/otel-exporter.d.mts +2 -3
- package/esm/exporters/otel/otel-exporter.d.mts.map +1 -1
- package/esm/exporters/otel/otel-exporter.type.d.mts +1 -3
- package/esm/exporters/otel/otel-exporter.type.d.mts.map +1 -1
- package/esm/exporters/utils/gen-ai-attributes.d.mts +9 -7
- package/esm/exporters/utils/gen-ai-attributes.d.mts.map +1 -1
- package/esm/exporters/utils/index.d.mts +2 -1
- package/esm/exporters/utils/index.mjs +1 -1
- package/esm/exporters/utils/total-cost.d.mts +1 -3
- package/esm/exporters/utils/total-cost.d.mts.map +1 -1
- package/esm/exporters/utils/walk-spans.d.mts +2 -2
- package/esm/exporters/utils/walk-spans.d.mts.map +1 -1
- package/esm/index.d.mts +8 -0
- package/esm/panoptic/index.d.mts +2 -1
- package/esm/panoptic/index.mjs +1 -1
- package/esm/panoptic/panoptic-middleware.d.mts +1 -3
- package/esm/panoptic/panoptic-middleware.d.mts.map +1 -1
- package/esm/panoptic/panoptic-target.type.d.mts +4 -5
- package/esm/panoptic/panoptic-target.type.d.mts.map +1 -1
- package/esm/panoptic/panoptic.d.mts +1 -3
- package/esm/panoptic/panoptic.d.mts.map +1 -1
- package/esm/panoptic/panoptic.type.d.mts +8 -7
- package/esm/panoptic/panoptic.type.d.mts.map +1 -1
- package/esm/store/cache-trace-store.d.mts +4 -6
- package/esm/store/cache-trace-store.d.mts.map +1 -1
- package/esm/store/cache-trace-store.mjs +1 -4
- package/esm/store/cache-trace-store.mjs.map +1 -1
- package/esm/store/in-memory-trace-store.d.mts +2 -4
- package/esm/store/in-memory-trace-store.d.mts.map +1 -1
- package/esm/store/in-memory-trace-store.mjs +1 -4
- package/esm/store/in-memory-trace-store.mjs.map +1 -1
- package/esm/store/index.d.mts +2 -1
- package/esm/store/index.mjs +1 -1
- package/esm/store/match-trace.d.mts +1 -3
- package/esm/store/match-trace.d.mts.map +1 -1
- package/esm/store/sum-usage.d.mts +4 -30
- package/esm/store/sum-usage.d.mts.map +1 -1
- package/esm/store/sum-usage.mjs +6 -49
- package/esm/store/sum-usage.mjs.map +1 -1
- package/esm/store/trace-aggregate.type.d.mts +12 -9
- package/esm/store/trace-aggregate.type.d.mts.map +1 -1
- package/esm/store/trace-query.type.d.mts +5 -5
- package/esm/store/trace-query.type.d.mts.map +1 -1
- package/esm/store/trace-store.contract.d.mts +1 -3
- package/esm/store/trace-store.contract.d.mts.map +1 -1
- package/llms-full.txt +14 -0
- package/package.json +4 -4
- package/skills/query-traces/SKILL.md +14 -0
package/esm/dashboard/index.mjs
CHANGED
|
@@ -4,4 +4,4 @@ import { createRequestHandler } from "./serve.mjs";
|
|
|
4
4
|
import { dashboard } from "./dashboard.mjs";
|
|
5
5
|
import { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey } from "./trace-filter.mjs";
|
|
6
6
|
|
|
7
|
-
export {
|
|
7
|
+
export { dashboard };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
import "../store/trace-query.type.mjs";
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
import "../store/trace-store.contract.mjs";
|
|
2
|
+
import "../evaluate/evaluate.type.mjs";
|
package/esm/dashboard/serve.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { timingSafeEqual } from "node:crypto";
|
|
|
7
7
|
|
|
8
8
|
//#region ../../packages/ai-panoptic/src/dashboard/serve.ts
|
|
9
9
|
/** Hard cap on the evaluate route's request body — well beyond a real instructions string. */
|
|
10
|
-
const MAX_EVALUATE_BODY_BYTES =
|
|
10
|
+
const MAX_EVALUATE_BODY_BYTES = 65536;
|
|
11
11
|
/**
|
|
12
12
|
* Security response headers added to every dashboard response (S4):
|
|
13
13
|
* block MIME-sniffing, framing, referrer leakage, and lock the page's
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serve.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/serve.ts"],"sourcesContent":["import { timingSafeEqual } from \"node:crypto\";\nimport type { IncomingMessage, ServerResponse } from \"node:http\";\nimport type { EvaluateConfig } from \"../evaluate/evaluate.type\";\nimport { evaluateSystemPrompt } from \"../evaluate/evaluate-system-prompt\";\nimport { extractLastSystemPrompt } from \"../evaluate/extract-last-system-prompt\";\nimport { findSpanById } from \"../evaluate/find-span-by-id\";\nimport type { TraceStoreContract } from \"../store/trace-store.contract\";\nimport { parseQuery } from \"./parse-query\";\nimport { dashboardHtml } from \"./ui.html\";\n\n/** Fully-resolved routing config the request handler closes over. */\nexport type ServeConfig = {\n /** Normalized mount path, always ending in `/` (e.g. `\"/\"`). */\n basePath: string;\n /** Header title baked into the served page. */\n title: string;\n /** Bearer token required on every request when set (S4). */\n authToken?: string;\n /** `Host` header allowlist — defends against DNS-rebinding (S4). */\n allowedHosts: string[];\n /**\n * Enables the one write route: `POST\n * {basePath}api/traces/:traceId/spans/:spanId/evaluate`. Absent ⇒ the\n * route 405s (POST isn't accepted by any route) and the served page never\n * renders the Evaluate button.\n */\n evaluate?: EvaluateConfig;\n};\n\n/** Request body accepted by the evaluate route — everything optional. */\ntype EvaluateRequestBody = {\n /** Per-run instructions override; falls back to `config.evaluate.instructions`. */\n instructions?: string;\n};\n\n/** Hard cap on the evaluate route's request body — well beyond a real instructions string. */\nconst MAX_EVALUATE_BODY_BYTES = 64 * 1024;\n\n/**\n * Security response headers added to every dashboard response (S4):\n * block MIME-sniffing, framing, referrer leakage, and lock the page's\n * content sources down to itself (the UI is fully self-contained — no CDN).\n */\nconst SECURITY_HEADERS: Record<string, string> = {\n \"x-content-type-options\": \"nosniff\",\n \"x-frame-options\": \"DENY\",\n \"referrer-policy\": \"no-referrer\",\n \"content-security-policy\":\n \"default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'\",\n};\n\n/** Extract the host (no port) from a `Host` header value. */\nfunction hostHeaderName(hostHeader: string | undefined): string | undefined {\n if (!hostHeader) return undefined;\n // IPv6 literal `[::1]:4319` → `[::1]`; otherwise strip `:port`.\n if (hostHeader.startsWith(\"[\")) {\n return hostHeader.slice(0, hostHeader.indexOf(\"]\") + 1);\n }\n const colon = hostHeader.indexOf(\":\");\n return colon === -1 ? hostHeader : hostHeader.slice(0, colon);\n}\n\n/**\n * Constant-time string equality (S4) — guards against a timing\n * side-channel that could otherwise let a network-adjacent attacker\n * recover the token byte-by-byte via repeated timed guesses. Plain `===`\n * short-circuits on the first differing byte, so response latency leaks\n * how many leading bytes matched; `timingSafeEqual` does not. Lengths are\n * compared first (a length mismatch is not secret and `timingSafeEqual`\n * requires equal-length buffers anyway).\n */\nfunction constantTimeEqual(a: string, b: string): boolean {\n const bufA = Buffer.from(a);\n const bufB = Buffer.from(b);\n\n if (bufA.length !== bufB.length) return false;\n\n return timingSafeEqual(bufA, bufB);\n}\n\n/**\n * Bearer-token check (S4): `Authorization: Bearer <token>` header, or\n * (only when `allowQueryToken`) a `?token=` query param.\n *\n * The query-string form only exists for the one request that structurally\n * cannot carry a custom header — the initial browser navigation that\n * loads the HTML shell (a typed/clicked/bookmarked URL). Every other\n * request the served page makes is same-origin `fetch()`, which can and\n * does set `Authorization` (see `ui.html.ts`'s `fetchAuthed`), so the API\n * routes never need to accept a query-string token. Restricting the\n * fallback to just the page route minimizes the token's exposure in\n * server access logs, proxy logs, and browser history to a single route\n * instead of every poll.\n */\nfunction isAuthorized(\n req: IncomingMessage,\n url: URL,\n token: string,\n allowQueryToken: boolean,\n): boolean {\n const header = req.headers.authorization;\n if (header && constantTimeEqual(header, `Bearer ${token}`)) return true;\n if (!allowQueryToken) return false;\n const queryToken = url.searchParams.get(\"token\");\n return queryToken !== null && constantTimeEqual(queryToken, token);\n}\n\n/**\n * Build the `node:http` request handler for the dashboard over a given\n * trace store. Kept separate from the server lifecycle so it can be unit\n * tested by feeding it a fake `req`/`res` without binding a port.\n *\n * Routes (all under `config.basePath`):\n *\n * - `GET api/traces` → `store.query(parseQuery(searchParams))`\n * - `GET api/traces/:id` → `store.get(id)` or `404`\n * - `GET api/aggregate` → `store.aggregate(parseQuery(searchParams))`\n * - `GET {basePath}` → the self-contained HTML page\n * - `POST api/traces/:traceId/spans/:spanId/evaluate` → ONLY when\n * `config.evaluate` is set; grades the span's last captured system\n * prompt and returns a {@link EvaluateVerdict}. The dashboard's one\n * write route — absent config, POST 405s like it would against any\n * other route (the path is never even pattern-matched).\n *\n * Anything else → `404`. A method the matched route doesn't accept → `405`.\n * Host allowlist + bearer-token auth (S4) are checked for EVERY request,\n * regardless of method, before any routing. The store shapes\n * ({@link Trace}/{@link TraceSpan}) are already JSON-safe, so GET responses\n * are a plain `JSON.stringify` with no serializer.\n *\n * @example\n * const handler = createRequestHandler(store, { basePath: \"/\", title: \"Panoptic\" });\n * http.createServer(handler).listen(4319, \"127.0.0.1\");\n */\nexport function createRequestHandler(\n store: TraceStoreContract,\n config: ServeConfig,\n): (req: IncomingMessage, res: ServerResponse) => void {\n const base = config.basePath;\n const apiPrefix = `${base}api`;\n\n return function handle(req: IncomingMessage, res: ServerResponse): void {\n // `req.url` is path + query only; a dummy origin lets URL parse it.\n const url = new URL(req.url ?? \"/\", \"http://localhost\");\n const pathname = url.pathname;\n\n // Host-header allowlist — blocks DNS-rebinding attacks that point a\n // hostile domain at this loopback port (S4). Checked for every method.\n const host = hostHeaderName(req.headers.host);\n if (!host || !config.allowedHosts.includes(host.toLowerCase())) {\n sendJson(res, 403, { error: \"host_not_allowed\" });\n\n return;\n }\n\n // Bearer-token auth when configured (always required off loopback, S4).\n // `?token=` is only honored on the HTML page route — see `isAuthorized`.\n const isPageRoute = pathname === base || pathname === base.replace(/\\/$/, \"\");\n if (config.authToken && !isAuthorized(req, url, config.authToken, isPageRoute)) {\n sendJson(res, 401, { error: \"unauthorized\" });\n\n return;\n }\n\n // The one write route — opt-in via `config.evaluate`, so it must be\n // checked before the blanket GET-only gate below.\n if (req.method === \"POST\" && config.evaluate) {\n const match = matchEvaluatePath(pathname, apiPrefix);\n\n if (match) {\n void handleEvaluate(store, config.evaluate, match.traceId, match.spanId, req, res);\n\n return;\n }\n }\n\n if (req.method !== \"GET\") {\n sendJson(res, 405, { error: \"method_not_allowed\" });\n\n return;\n }\n\n if (pathname === `${apiPrefix}/traces`) {\n sendJson(res, 200, store.query(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname.startsWith(`${apiPrefix}/traces/`)) {\n const traceId = decodeURIComponent(pathname.slice(`${apiPrefix}/traces/`.length));\n const trace = traceId.length > 0 ? store.get(traceId) : undefined;\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n sendJson(res, 200, trace);\n\n return;\n }\n\n if (pathname === `${apiPrefix}/aggregate`) {\n sendJson(res, 200, store.aggregate(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname === base || pathname === base.replace(/\\/$/, \"\")) {\n res.writeHead(200, {\n \"content-type\": \"text/html; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(\n dashboardHtml(\n base,\n config.title,\n Boolean(config.evaluate),\n config.evaluate?.instructions ?? \"\",\n ),\n );\n\n return;\n }\n\n sendJson(res, 404, { error: \"not_found\" });\n };\n}\n\n/** Write a JSON response with the given status code and security headers. */\nfunction sendJson(res: ServerResponse, status: number, body: unknown): void {\n res.writeHead(status, {\n \"content-type\": \"application/json; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(JSON.stringify(body));\n}\n\n/**\n * Match `{apiPrefix}/traces/:traceId/spans/:spanId/evaluate` — the one\n * write route. Returns `undefined` for anything else, including the plain\n * `{apiPrefix}/traces/:id` read route (no `/spans/.../evaluate` suffix),\n * so the two never collide.\n */\nfunction matchEvaluatePath(\n pathname: string,\n apiPrefix: string,\n): { traceId: string; spanId: string } | undefined {\n const prefix = `${apiPrefix}/traces/`;\n\n if (!pathname.startsWith(prefix)) {\n return undefined;\n }\n\n const match = /^([^/]+)\\/spans\\/([^/]+)\\/evaluate$/.exec(pathname.slice(prefix.length));\n\n if (!match) {\n return undefined;\n }\n\n return {\n traceId: decodeURIComponent(match[1]),\n spanId: decodeURIComponent(match[2]),\n };\n}\n\n/**\n * Collect and JSON-parse a request body, capped at\n * {@link MAX_EVALUATE_BODY_BYTES} so an oversized body can't hold the\n * connection open indefinitely. An empty body resolves to `undefined` —\n * the evaluate route treats that as \"no per-run override\".\n */\nfunction readJsonBody<T>(req: IncomingMessage): Promise<T | undefined> {\n return new Promise((resolve, reject) => {\n const chunks: Buffer[] = [];\n let size = 0;\n\n req.on(\"data\", (chunk: Buffer) => {\n size += chunk.length;\n\n if (size > MAX_EVALUATE_BODY_BYTES) {\n req.destroy();\n reject(new Error(\"payload_too_large\"));\n\n return;\n }\n\n chunks.push(chunk);\n });\n\n req.on(\"end\", () => {\n if (chunks.length === 0) {\n resolve(undefined);\n\n return;\n }\n\n try {\n resolve(JSON.parse(Buffer.concat(chunks).toString(\"utf-8\")) as T);\n } catch {\n reject(new Error(\"invalid_json\"));\n }\n });\n\n req.on(\"error\", reject);\n });\n}\n\n/**\n * Handle `POST {apiPrefix}/traces/:traceId/spans/:spanId/evaluate`. Looks\n * up the trace + span, extracts its last captured system prompt, and\n * grades it via {@link evaluateSystemPrompt}. `judgePromptBody` itself\n * never throws (a broken judge degrades to an issues-only outcome) — the\n * try/catch here only guards `config.evaluate.model` resolution, the one\n * step that CAN throw (e.g. a factory constructing an SDK client).\n */\nasync function handleEvaluate(\n store: TraceStoreContract,\n evaluate: EvaluateConfig,\n traceId: string,\n spanId: string,\n req: IncomingMessage,\n res: ServerResponse,\n): Promise<void> {\n const trace = store.get(traceId);\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n const span = findSpanById(trace.root, spanId);\n\n if (span === undefined) {\n sendJson(res, 404, { error: \"span_not_found\", spanId });\n\n return;\n }\n\n const systemPrompt = extractLastSystemPrompt(span);\n\n if (systemPrompt === undefined) {\n sendJson(res, 422, { error: \"no_system_prompt\" });\n\n return;\n }\n\n let body: EvaluateRequestBody | undefined;\n\n try {\n body = await readJsonBody<EvaluateRequestBody>(req);\n } catch (error) {\n const message = error instanceof Error ? error.message : \"invalid_json\";\n sendJson(res, message === \"payload_too_large\" ? 413 : 400, { error: message });\n\n return;\n }\n\n try {\n const verdict = await evaluateSystemPrompt(systemPrompt, evaluate, body?.instructions);\n sendJson(res, 200, verdict);\n } catch (error) {\n sendJson(res, 502, {\n error: \"evaluate_failed\",\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n"],"mappings":";;;;;;;;;AAoCA,MAAM,0BAA0B,KAAK;;;;;;AAOrC,MAAM,mBAA2C;CAC/C,0BAA0B;CAC1B,mBAAmB;CACnB,mBAAmB;CACnB,2BACE;AACJ;;AAGA,SAAS,eAAe,YAAoD;CAC1E,IAAI,CAAC,YAAY,OAAO;CAExB,IAAI,WAAW,WAAW,GAAG,GAC3B,OAAO,WAAW,MAAM,GAAG,WAAW,QAAQ,GAAG,IAAI,CAAC;CAExD,MAAM,QAAQ,WAAW,QAAQ,GAAG;CACpC,OAAO,UAAU,KAAK,aAAa,WAAW,MAAM,GAAG,KAAK;AAC9D;;;;;;;;;;AAWA,SAAS,kBAAkB,GAAW,GAAoB;CACxD,MAAM,OAAO,OAAO,KAAK,CAAC;CAC1B,MAAM,OAAO,OAAO,KAAK,CAAC;CAE1B,IAAI,KAAK,WAAW,KAAK,QAAQ,OAAO;CAExC,OAAO,gBAAgB,MAAM,IAAI;AACnC;;;;;;;;;;;;;;;AAgBA,SAAS,aACP,KACA,KACA,OACA,iBACS;CACT,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,UAAU,kBAAkB,QAAQ,UAAU,OAAO,GAAG,OAAO;CACnE,IAAI,CAAC,iBAAiB,OAAO;CAC7B,MAAM,aAAa,IAAI,aAAa,IAAI,OAAO;CAC/C,OAAO,eAAe,QAAQ,kBAAkB,YAAY,KAAK;AACnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,qBACd,OACA,QACqD;CACrD,MAAM,OAAO,OAAO;CACpB,MAAM,YAAY,GAAG,KAAK;CAE1B,OAAO,SAAS,OAAO,KAAsB,KAA2B;EAEtE,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,KAAK,kBAAkB;EACtD,MAAM,WAAW,IAAI;EAIrB,MAAM,OAAO,eAAe,IAAI,QAAQ,IAAI;EAC5C,IAAI,CAAC,QAAQ,CAAC,OAAO,aAAa,SAAS,KAAK,YAAY,CAAC,GAAG;GAC9D,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;GAEhD;EACF;EAIA,MAAM,cAAc,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE;EAC5E,IAAI,OAAO,aAAa,CAAC,aAAa,KAAK,KAAK,OAAO,WAAW,WAAW,GAAG;GAC9E,SAAS,KAAK,KAAK,EAAE,OAAO,eAAe,CAAC;GAE5C;EACF;EAIA,IAAI,IAAI,WAAW,UAAU,OAAO,UAAU;GAC5C,MAAM,QAAQ,kBAAkB,UAAU,SAAS;GAEnD,IAAI,OAAO;IACT,AAAK,eAAe,OAAO,OAAO,UAAU,MAAM,SAAS,MAAM,QAAQ,KAAK,GAAG;IAEjF;GACF;EACF;EAEA,IAAI,IAAI,WAAW,OAAO;GACxB,SAAS,KAAK,KAAK,EAAE,OAAO,qBAAqB,CAAC;GAElD;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,UAAU;GACtC,SAAS,KAAK,KAAK,MAAM,MAAM,WAAW,IAAI,YAAY,CAAC,CAAC;GAE5D;EACF;EAEA,IAAI,SAAS,WAAW,GAAG,UAAU,SAAS,GAAG;GAC/C,MAAM,UAAU,mBAAmB,SAAS,MAAM,GAAG,UAAU,UAAU,MAAM,CAAC;GAChF,MAAM,QAAQ,QAAQ,SAAS,IAAI,MAAM,IAAI,OAAO,IAAI;GAExD,IAAI,UAAU,QAAW;IACvB,SAAS,KAAK,KAAK;KAAE,OAAO;KAAmB;IAAQ,CAAC;IAExD;GACF;GAEA,SAAS,KAAK,KAAK,KAAK;GAExB;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,aAAa;GACzC,SAAS,KAAK,KAAK,MAAM,UAAU,WAAW,IAAI,YAAY,CAAC,CAAC;GAEhE;EACF;EAEA,IAAI,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE,GAAG;GAC7D,IAAI,UAAU,KAAK;IACjB,gBAAgB;IAChB,GAAG;GACL,CAAC;GACD,IAAI,IACF,cACE,MACA,OAAO,OACP,QAAQ,OAAO,QAAQ,GACvB,OAAO,UAAU,gBAAgB,EACnC,CACF;GAEA;EACF;EAEA,SAAS,KAAK,KAAK,EAAE,OAAO,YAAY,CAAC;CAC3C;AACF;;AAGA,SAAS,SAAS,KAAqB,QAAgB,MAAqB;CAC1E,IAAI,UAAU,QAAQ;EACpB,gBAAgB;EAChB,GAAG;CACL,CAAC;CACD,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC;AAC9B;;;;;;;AAQA,SAAS,kBACP,UACA,WACiD;CACjD,MAAM,SAAS,GAAG,UAAU;CAE5B,IAAI,CAAC,SAAS,WAAW,MAAM,GAC7B;CAGF,MAAM,QAAQ,sCAAsC,KAAK,SAAS,MAAM,OAAO,MAAM,CAAC;CAEtF,IAAI,CAAC,OACH;CAGF,OAAO;EACL,SAAS,mBAAmB,MAAM,EAAE;EACpC,QAAQ,mBAAmB,MAAM,EAAE;CACrC;AACF;;;;;;;AAQA,SAAS,aAAgB,KAA8C;CACrE,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,MAAM,SAAmB,CAAC;EAC1B,IAAI,OAAO;EAEX,IAAI,GAAG,SAAS,UAAkB;GAChC,QAAQ,MAAM;GAEd,IAAI,OAAO,yBAAyB;IAClC,IAAI,QAAQ;IACZ,uBAAO,IAAI,MAAM,mBAAmB,CAAC;IAErC;GACF;GAEA,OAAO,KAAK,KAAK;EACnB,CAAC;EAED,IAAI,GAAG,aAAa;GAClB,IAAI,OAAO,WAAW,GAAG;IACvB,QAAQ,MAAS;IAEjB;GACF;GAEA,IAAI;IACF,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM,CAAC,CAAC,SAAS,OAAO,CAAC,CAAM;GAClE,QAAQ;IACN,uBAAO,IAAI,MAAM,cAAc,CAAC;GAClC;EACF,CAAC;EAED,IAAI,GAAG,SAAS,MAAM;CACxB,CAAC;AACH;;;;;;;;;AAUA,eAAe,eACb,OACA,UACA,SACA,QACA,KACA,KACe;CACf,MAAM,QAAQ,MAAM,IAAI,OAAO;CAE/B,IAAI,UAAU,QAAW;EACvB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAmB;EAAQ,CAAC;EAExD;CACF;CAEA,MAAM,OAAO,aAAa,MAAM,MAAM,MAAM;CAE5C,IAAI,SAAS,QAAW;EACtB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAkB;EAAO,CAAC;EAEtD;CACF;CAEA,MAAM,eAAe,wBAAwB,IAAI;CAEjD,IAAI,iBAAiB,QAAW;EAC9B,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;EAEhD;CACF;CAEA,IAAI;CAEJ,IAAI;EACF,OAAO,MAAM,aAAkC,GAAG;CACpD,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;EACzD,SAAS,KAAK,YAAY,sBAAsB,MAAM,KAAK,EAAE,OAAO,QAAQ,CAAC;EAE7E;CACF;CAEA,IAAI;EAEF,SAAS,KAAK,KAAK,MADG,qBAAqB,cAAc,UAAU,MAAM,YAAY,CAC3D;CAC5B,SAAS,OAAO;EACd,SAAS,KAAK,KAAK;GACjB,OAAO;GACP,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAChE,CAAC;CACH;AACF"}
|
|
1
|
+
{"version":3,"file":"serve.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/serve.ts"],"sourcesContent":["import { timingSafeEqual } from \"node:crypto\";\nimport type { IncomingMessage, ServerResponse } from \"node:http\";\nimport type { EvaluateConfig } from \"../evaluate/evaluate.type\";\nimport { evaluateSystemPrompt } from \"../evaluate/evaluate-system-prompt\";\nimport { extractLastSystemPrompt } from \"../evaluate/extract-last-system-prompt\";\nimport { findSpanById } from \"../evaluate/find-span-by-id\";\nimport type { TraceStoreContract } from \"../store/trace-store.contract\";\nimport { parseQuery } from \"./parse-query\";\nimport { dashboardHtml } from \"./ui.html\";\n\n/** Fully-resolved routing config the request handler closes over. */\nexport type ServeConfig = {\n /** Normalized mount path, always ending in `/` (e.g. `\"/\"`). */\n basePath: string;\n /** Header title baked into the served page. */\n title: string;\n /** Bearer token required on every request when set (S4). */\n authToken?: string;\n /** `Host` header allowlist — defends against DNS-rebinding (S4). */\n allowedHosts: string[];\n /**\n * Enables the one write route: `POST\n * {basePath}api/traces/:traceId/spans/:spanId/evaluate`. Absent ⇒ the\n * route 405s (POST isn't accepted by any route) and the served page never\n * renders the Evaluate button.\n */\n evaluate?: EvaluateConfig;\n};\n\n/** Request body accepted by the evaluate route — everything optional. */\ntype EvaluateRequestBody = {\n /** Per-run instructions override; falls back to `config.evaluate.instructions`. */\n instructions?: string;\n};\n\n/** Hard cap on the evaluate route's request body — well beyond a real instructions string. */\nconst MAX_EVALUATE_BODY_BYTES = 64 * 1024;\n\n/**\n * Security response headers added to every dashboard response (S4):\n * block MIME-sniffing, framing, referrer leakage, and lock the page's\n * content sources down to itself (the UI is fully self-contained — no CDN).\n */\nconst SECURITY_HEADERS: Record<string, string> = {\n \"x-content-type-options\": \"nosniff\",\n \"x-frame-options\": \"DENY\",\n \"referrer-policy\": \"no-referrer\",\n \"content-security-policy\":\n \"default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'\",\n};\n\n/** Extract the host (no port) from a `Host` header value. */\nfunction hostHeaderName(hostHeader: string | undefined): string | undefined {\n if (!hostHeader) return undefined;\n // IPv6 literal `[::1]:4319` → `[::1]`; otherwise strip `:port`.\n if (hostHeader.startsWith(\"[\")) {\n return hostHeader.slice(0, hostHeader.indexOf(\"]\") + 1);\n }\n const colon = hostHeader.indexOf(\":\");\n return colon === -1 ? hostHeader : hostHeader.slice(0, colon);\n}\n\n/**\n * Constant-time string equality (S4) — guards against a timing\n * side-channel that could otherwise let a network-adjacent attacker\n * recover the token byte-by-byte via repeated timed guesses. Plain `===`\n * short-circuits on the first differing byte, so response latency leaks\n * how many leading bytes matched; `timingSafeEqual` does not. Lengths are\n * compared first (a length mismatch is not secret and `timingSafeEqual`\n * requires equal-length buffers anyway).\n */\nfunction constantTimeEqual(a: string, b: string): boolean {\n const bufA = Buffer.from(a);\n const bufB = Buffer.from(b);\n\n if (bufA.length !== bufB.length) return false;\n\n return timingSafeEqual(bufA, bufB);\n}\n\n/**\n * Bearer-token check (S4): `Authorization: Bearer <token>` header, or\n * (only when `allowQueryToken`) a `?token=` query param.\n *\n * The query-string form only exists for the one request that structurally\n * cannot carry a custom header — the initial browser navigation that\n * loads the HTML shell (a typed/clicked/bookmarked URL). Every other\n * request the served page makes is same-origin `fetch()`, which can and\n * does set `Authorization` (see `ui.html.ts`'s `fetchAuthed`), so the API\n * routes never need to accept a query-string token. Restricting the\n * fallback to just the page route minimizes the token's exposure in\n * server access logs, proxy logs, and browser history to a single route\n * instead of every poll.\n */\nfunction isAuthorized(\n req: IncomingMessage,\n url: URL,\n token: string,\n allowQueryToken: boolean,\n): boolean {\n const header = req.headers.authorization;\n if (header && constantTimeEqual(header, `Bearer ${token}`)) return true;\n if (!allowQueryToken) return false;\n const queryToken = url.searchParams.get(\"token\");\n return queryToken !== null && constantTimeEqual(queryToken, token);\n}\n\n/**\n * Build the `node:http` request handler for the dashboard over a given\n * trace store. Kept separate from the server lifecycle so it can be unit\n * tested by feeding it a fake `req`/`res` without binding a port.\n *\n * Routes (all under `config.basePath`):\n *\n * - `GET api/traces` → `store.query(parseQuery(searchParams))`\n * - `GET api/traces/:id` → `store.get(id)` or `404`\n * - `GET api/aggregate` → `store.aggregate(parseQuery(searchParams))`\n * - `GET {basePath}` → the self-contained HTML page\n * - `POST api/traces/:traceId/spans/:spanId/evaluate` → ONLY when\n * `config.evaluate` is set; grades the span's last captured system\n * prompt and returns a {@link EvaluateVerdict}. The dashboard's one\n * write route — absent config, POST 405s like it would against any\n * other route (the path is never even pattern-matched).\n *\n * Anything else → `404`. A method the matched route doesn't accept → `405`.\n * Host allowlist + bearer-token auth (S4) are checked for EVERY request,\n * regardless of method, before any routing. The store shapes\n * ({@link Trace}/{@link TraceSpan}) are already JSON-safe, so GET responses\n * are a plain `JSON.stringify` with no serializer.\n *\n * @example\n * const handler = createRequestHandler(store, { basePath: \"/\", title: \"Panoptic\" });\n * http.createServer(handler).listen(4319, \"127.0.0.1\");\n */\nexport function createRequestHandler(\n store: TraceStoreContract,\n config: ServeConfig,\n): (req: IncomingMessage, res: ServerResponse) => void {\n const base = config.basePath;\n const apiPrefix = `${base}api`;\n\n return function handle(req: IncomingMessage, res: ServerResponse): void {\n // `req.url` is path + query only; a dummy origin lets URL parse it.\n const url = new URL(req.url ?? \"/\", \"http://localhost\");\n const pathname = url.pathname;\n\n // Host-header allowlist — blocks DNS-rebinding attacks that point a\n // hostile domain at this loopback port (S4). Checked for every method.\n const host = hostHeaderName(req.headers.host);\n if (!host || !config.allowedHosts.includes(host.toLowerCase())) {\n sendJson(res, 403, { error: \"host_not_allowed\" });\n\n return;\n }\n\n // Bearer-token auth when configured (always required off loopback, S4).\n // `?token=` is only honored on the HTML page route — see `isAuthorized`.\n const isPageRoute = pathname === base || pathname === base.replace(/\\/$/, \"\");\n if (config.authToken && !isAuthorized(req, url, config.authToken, isPageRoute)) {\n sendJson(res, 401, { error: \"unauthorized\" });\n\n return;\n }\n\n // The one write route — opt-in via `config.evaluate`, so it must be\n // checked before the blanket GET-only gate below.\n if (req.method === \"POST\" && config.evaluate) {\n const match = matchEvaluatePath(pathname, apiPrefix);\n\n if (match) {\n void handleEvaluate(store, config.evaluate, match.traceId, match.spanId, req, res);\n\n return;\n }\n }\n\n if (req.method !== \"GET\") {\n sendJson(res, 405, { error: \"method_not_allowed\" });\n\n return;\n }\n\n if (pathname === `${apiPrefix}/traces`) {\n sendJson(res, 200, store.query(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname.startsWith(`${apiPrefix}/traces/`)) {\n const traceId = decodeURIComponent(pathname.slice(`${apiPrefix}/traces/`.length));\n const trace = traceId.length > 0 ? store.get(traceId) : undefined;\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n sendJson(res, 200, trace);\n\n return;\n }\n\n if (pathname === `${apiPrefix}/aggregate`) {\n sendJson(res, 200, store.aggregate(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname === base || pathname === base.replace(/\\/$/, \"\")) {\n res.writeHead(200, {\n \"content-type\": \"text/html; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(\n dashboardHtml(\n base,\n config.title,\n Boolean(config.evaluate),\n config.evaluate?.instructions ?? \"\",\n ),\n );\n\n return;\n }\n\n sendJson(res, 404, { error: \"not_found\" });\n };\n}\n\n/** Write a JSON response with the given status code and security headers. */\nfunction sendJson(res: ServerResponse, status: number, body: unknown): void {\n res.writeHead(status, {\n \"content-type\": \"application/json; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(JSON.stringify(body));\n}\n\n/**\n * Match `{apiPrefix}/traces/:traceId/spans/:spanId/evaluate` — the one\n * write route. Returns `undefined` for anything else, including the plain\n * `{apiPrefix}/traces/:id` read route (no `/spans/.../evaluate` suffix),\n * so the two never collide.\n */\nfunction matchEvaluatePath(\n pathname: string,\n apiPrefix: string,\n): { traceId: string; spanId: string } | undefined {\n const prefix = `${apiPrefix}/traces/`;\n\n if (!pathname.startsWith(prefix)) {\n return undefined;\n }\n\n const match = /^([^/]+)\\/spans\\/([^/]+)\\/evaluate$/.exec(pathname.slice(prefix.length));\n\n if (!match) {\n return undefined;\n }\n\n return {\n traceId: decodeURIComponent(match[1]),\n spanId: decodeURIComponent(match[2]),\n };\n}\n\n/**\n * Collect and JSON-parse a request body, capped at\n * {@link MAX_EVALUATE_BODY_BYTES} so an oversized body can't hold the\n * connection open indefinitely. An empty body resolves to `undefined` —\n * the evaluate route treats that as \"no per-run override\".\n */\nfunction readJsonBody<T>(req: IncomingMessage): Promise<T | undefined> {\n return new Promise((resolve, reject) => {\n const chunks: Buffer[] = [];\n let size = 0;\n\n req.on(\"data\", (chunk: Buffer) => {\n size += chunk.length;\n\n if (size > MAX_EVALUATE_BODY_BYTES) {\n req.destroy();\n reject(new Error(\"payload_too_large\"));\n\n return;\n }\n\n chunks.push(chunk);\n });\n\n req.on(\"end\", () => {\n if (chunks.length === 0) {\n resolve(undefined);\n\n return;\n }\n\n try {\n resolve(JSON.parse(Buffer.concat(chunks).toString(\"utf-8\")) as T);\n } catch {\n reject(new Error(\"invalid_json\"));\n }\n });\n\n req.on(\"error\", reject);\n });\n}\n\n/**\n * Handle `POST {apiPrefix}/traces/:traceId/spans/:spanId/evaluate`. Looks\n * up the trace + span, extracts its last captured system prompt, and\n * grades it via {@link evaluateSystemPrompt}. `judgePromptBody` itself\n * never throws (a broken judge degrades to an issues-only outcome) — the\n * try/catch here only guards `config.evaluate.model` resolution, the one\n * step that CAN throw (e.g. a factory constructing an SDK client).\n */\nasync function handleEvaluate(\n store: TraceStoreContract,\n evaluate: EvaluateConfig,\n traceId: string,\n spanId: string,\n req: IncomingMessage,\n res: ServerResponse,\n): Promise<void> {\n const trace = store.get(traceId);\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n const span = findSpanById(trace.root, spanId);\n\n if (span === undefined) {\n sendJson(res, 404, { error: \"span_not_found\", spanId });\n\n return;\n }\n\n const systemPrompt = extractLastSystemPrompt(span);\n\n if (systemPrompt === undefined) {\n sendJson(res, 422, { error: \"no_system_prompt\" });\n\n return;\n }\n\n let body: EvaluateRequestBody | undefined;\n\n try {\n body = await readJsonBody<EvaluateRequestBody>(req);\n } catch (error) {\n const message = error instanceof Error ? error.message : \"invalid_json\";\n sendJson(res, message === \"payload_too_large\" ? 413 : 400, { error: message });\n\n return;\n }\n\n try {\n const verdict = await evaluateSystemPrompt(systemPrompt, evaluate, body?.instructions);\n sendJson(res, 200, verdict);\n } catch (error) {\n sendJson(res, 502, {\n error: \"evaluate_failed\",\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n"],"mappings":";;;;;;;;;AAoCA,MAAM,0BAA0B;;;;;;AAOhC,MAAM,mBAA2C;CAC/C,0BAA0B;CAC1B,mBAAmB;CACnB,mBAAmB;CACnB,2BACE;AACJ;;AAGA,SAAS,eAAe,YAAoD;CAC1E,IAAI,CAAC,YAAY,OAAO;CAExB,IAAI,WAAW,WAAW,GAAG,GAC3B,OAAO,WAAW,MAAM,GAAG,WAAW,QAAQ,GAAG,IAAI,CAAC;CAExD,MAAM,QAAQ,WAAW,QAAQ,GAAG;CACpC,OAAO,UAAU,KAAK,aAAa,WAAW,MAAM,GAAG,KAAK;AAC9D;;;;;;;;;;AAWA,SAAS,kBAAkB,GAAW,GAAoB;CACxD,MAAM,OAAO,OAAO,KAAK,CAAC;CAC1B,MAAM,OAAO,OAAO,KAAK,CAAC;CAE1B,IAAI,KAAK,WAAW,KAAK,QAAQ,OAAO;CAExC,OAAO,gBAAgB,MAAM,IAAI;AACnC;;;;;;;;;;;;;;;AAgBA,SAAS,aACP,KACA,KACA,OACA,iBACS;CACT,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,UAAU,kBAAkB,QAAQ,UAAU,OAAO,GAAG,OAAO;CACnE,IAAI,CAAC,iBAAiB,OAAO;CAC7B,MAAM,aAAa,IAAI,aAAa,IAAI,OAAO;CAC/C,OAAO,eAAe,QAAQ,kBAAkB,YAAY,KAAK;AACnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,qBACd,OACA,QACqD;CACrD,MAAM,OAAO,OAAO;CACpB,MAAM,YAAY,GAAG,KAAK;CAE1B,OAAO,SAAS,OAAO,KAAsB,KAA2B;EAEtE,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,KAAK,kBAAkB;EACtD,MAAM,WAAW,IAAI;EAIrB,MAAM,OAAO,eAAe,IAAI,QAAQ,IAAI;EAC5C,IAAI,CAAC,QAAQ,CAAC,OAAO,aAAa,SAAS,KAAK,YAAY,CAAC,GAAG;GAC9D,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;GAEhD;EACF;EAIA,MAAM,cAAc,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE;EAC5E,IAAI,OAAO,aAAa,CAAC,aAAa,KAAK,KAAK,OAAO,WAAW,WAAW,GAAG;GAC9E,SAAS,KAAK,KAAK,EAAE,OAAO,eAAe,CAAC;GAE5C;EACF;EAIA,IAAI,IAAI,WAAW,UAAU,OAAO,UAAU;GAC5C,MAAM,QAAQ,kBAAkB,UAAU,SAAS;GAEnD,IAAI,OAAO;IACT,AAAK,eAAe,OAAO,OAAO,UAAU,MAAM,SAAS,MAAM,QAAQ,KAAK,GAAG;IAEjF;GACF;EACF;EAEA,IAAI,IAAI,WAAW,OAAO;GACxB,SAAS,KAAK,KAAK,EAAE,OAAO,qBAAqB,CAAC;GAElD;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,UAAU;GACtC,SAAS,KAAK,KAAK,MAAM,MAAM,WAAW,IAAI,YAAY,CAAC,CAAC;GAE5D;EACF;EAEA,IAAI,SAAS,WAAW,GAAG,UAAU,SAAS,GAAG;GAC/C,MAAM,UAAU,mBAAmB,SAAS,MAAM,GAAG,UAAU,UAAU,MAAM,CAAC;GAChF,MAAM,QAAQ,QAAQ,SAAS,IAAI,MAAM,IAAI,OAAO,IAAI;GAExD,IAAI,UAAU,QAAW;IACvB,SAAS,KAAK,KAAK;KAAE,OAAO;KAAmB;IAAQ,CAAC;IAExD;GACF;GAEA,SAAS,KAAK,KAAK,KAAK;GAExB;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,aAAa;GACzC,SAAS,KAAK,KAAK,MAAM,UAAU,WAAW,IAAI,YAAY,CAAC,CAAC;GAEhE;EACF;EAEA,IAAI,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE,GAAG;GAC7D,IAAI,UAAU,KAAK;IACjB,gBAAgB;IAChB,GAAG;GACL,CAAC;GACD,IAAI,IACF,cACE,MACA,OAAO,OACP,QAAQ,OAAO,QAAQ,GACvB,OAAO,UAAU,gBAAgB,EACnC,CACF;GAEA;EACF;EAEA,SAAS,KAAK,KAAK,EAAE,OAAO,YAAY,CAAC;CAC3C;AACF;;AAGA,SAAS,SAAS,KAAqB,QAAgB,MAAqB;CAC1E,IAAI,UAAU,QAAQ;EACpB,gBAAgB;EAChB,GAAG;CACL,CAAC;CACD,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC;AAC9B;;;;;;;AAQA,SAAS,kBACP,UACA,WACiD;CACjD,MAAM,SAAS,GAAG,UAAU;CAE5B,IAAI,CAAC,SAAS,WAAW,MAAM,GAC7B;CAGF,MAAM,QAAQ,sCAAsC,KAAK,SAAS,MAAM,OAAO,MAAM,CAAC;CAEtF,IAAI,CAAC,OACH;CAGF,OAAO;EACL,SAAS,mBAAmB,MAAM,EAAE;EACpC,QAAQ,mBAAmB,MAAM,EAAE;CACrC;AACF;;;;;;;AAQA,SAAS,aAAgB,KAA8C;CACrE,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,MAAM,SAAmB,CAAC;EAC1B,IAAI,OAAO;EAEX,IAAI,GAAG,SAAS,UAAkB;GAChC,QAAQ,MAAM;GAEd,IAAI,OAAO,yBAAyB;IAClC,IAAI,QAAQ;IACZ,uBAAO,IAAI,MAAM,mBAAmB,CAAC;IAErC;GACF;GAEA,OAAO,KAAK,KAAK;EACnB,CAAC;EAED,IAAI,GAAG,aAAa;GAClB,IAAI,OAAO,WAAW,GAAG;IACvB,QAAQ,MAAS;IAEjB;GACF;GAEA,IAAI;IACF,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM,CAAC,CAAC,SAAS,OAAO,CAAC,CAAM;GAClE,QAAQ;IACN,uBAAO,IAAI,MAAM,cAAc,CAAC;GAClC;EACF,CAAC;EAED,IAAI,GAAG,SAAS,MAAM;CACxB,CAAC;AACH;;;;;;;;;AAUA,eAAe,eACb,OACA,UACA,SACA,QACA,KACA,KACe;CACf,MAAM,QAAQ,MAAM,IAAI,OAAO;CAE/B,IAAI,UAAU,QAAW;EACvB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAmB;EAAQ,CAAC;EAExD;CACF;CAEA,MAAM,OAAO,aAAa,MAAM,MAAM,MAAM;CAE5C,IAAI,SAAS,QAAW;EACtB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAkB;EAAO,CAAC;EAEtD;CACF;CAEA,MAAM,eAAe,wBAAwB,IAAI;CAEjD,IAAI,iBAAiB,QAAW;EAC9B,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;EAEhD;CACF;CAEA,IAAI;CAEJ,IAAI;EACF,OAAO,MAAM,aAAkC,GAAG;CACpD,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;EACzD,SAAS,KAAK,YAAY,sBAAsB,MAAM,KAAK,EAAE,OAAO,QAAQ,CAAC;EAE7E;CACF;CAEA,IAAI;EAEF,SAAS,KAAK,KAAK,MADG,qBAAqB,cAAc,UAAU,MAAM,YAAY,CAC3D;CAC5B,SAAS,OAAO;EACd,SAAS,KAAK,KAAK;GACjB,OAAO;GACP,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAChE,CAAC;CACH;AACF"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { Trace, TraceSpan } from "../contracts/trace.type.mjs";
|
|
2
2
|
import { ReportStatus, ReportType } from "@warlock.js/ai";
|
|
3
|
-
|
|
4
3
|
//#region ../../packages/ai-panoptic/src/dashboard/trace-filter.d.ts
|
|
5
4
|
/**
|
|
6
5
|
* Client-side trace-list filter criteria. The dashboard polls the full
|
|
@@ -13,7 +12,7 @@ import { ReportStatus, ReportType } from "@warlock.js/ai";
|
|
|
13
12
|
* Every field is optional — an absent field is "don't care". An empty
|
|
14
13
|
* filter therefore matches every trace.
|
|
15
14
|
*/
|
|
16
|
-
type TraceFilter = {
|
|
15
|
+
export type TraceFilter = {
|
|
17
16
|
/**
|
|
18
17
|
* Free-text needle matched case-insensitively against a trace's root
|
|
19
18
|
* span name and session id. Whitespace-trimmed; an empty/blank string
|
|
@@ -60,24 +59,26 @@ type TraceFilter = {
|
|
|
60
59
|
* versions are distinct keys — that is the whole point of the linkage: to
|
|
61
60
|
* group / filter by the *exact* prompt revision that produced a run.
|
|
62
61
|
*/
|
|
63
|
-
declare function tracePromptKey(trace: Trace): string | undefined;
|
|
62
|
+
export declare function tracePromptKey(trace: Trace): string | undefined;
|
|
64
63
|
/**
|
|
65
64
|
* Does a single trace pass the given filter? Pure predicate — all
|
|
66
65
|
* conditions are ANDed; an unset field never excludes. Used by
|
|
67
66
|
* {@link filterTraces} and mirrored by the dashboard's inlined JS.
|
|
68
67
|
*/
|
|
69
|
-
declare function matchesFilter(trace: Trace, filter: TraceFilter): boolean;
|
|
68
|
+
export declare function matchesFilter(trace: Trace, filter: TraceFilter): boolean;
|
|
70
69
|
/** Narrow a trace list to those matching the filter, order preserved. */
|
|
71
|
-
declare function filterTraces(traces: Trace[], filter: TraceFilter): Trace[];
|
|
70
|
+
export declare function filterTraces(traces: Trace[], filter: TraceFilter): Trace[];
|
|
72
71
|
/**
|
|
73
72
|
* The grouping key a trace falls under when the list is grouped by
|
|
74
73
|
* session. Traces without a `sessionId` collapse into one synthetic
|
|
75
74
|
* "(no session)" bucket so they remain visible.
|
|
76
75
|
*/
|
|
77
|
-
declare const NO_SESSION_KEY = "(no session)";
|
|
76
|
+
export declare const NO_SESSION_KEY = "(no session)";
|
|
78
77
|
/** One session bucket — its key plus the traces under it, order preserved. */
|
|
79
|
-
type SessionGroup = {
|
|
80
|
-
/** The `sessionId`, or {@link NO_SESSION_KEY} for sessionless traces. */
|
|
78
|
+
export type SessionGroup = {
|
|
79
|
+
/** The `sessionId`, or {@link NO_SESSION_KEY} for sessionless traces. */
|
|
80
|
+
sessionId: string;
|
|
81
|
+
/** Traces in this group, in the order they appeared in the input. */
|
|
81
82
|
traces: Trace[];
|
|
82
83
|
};
|
|
83
84
|
/**
|
|
@@ -85,16 +86,18 @@ type SessionGroup = {
|
|
|
85
86
|
* preserving first-seen order of both the groups and the traces within
|
|
86
87
|
* each. Sessionless traces bucket under {@link NO_SESSION_KEY}.
|
|
87
88
|
*/
|
|
88
|
-
declare function groupBySession(traces: Trace[]): SessionGroup[];
|
|
89
|
+
export declare function groupBySession(traces: Trace[]): SessionGroup[];
|
|
89
90
|
/**
|
|
90
91
|
* The grouping key a trace falls under when the list is grouped by prompt
|
|
91
92
|
* version. Runs that resolved no named prompt collapse into one synthetic
|
|
92
93
|
* "(no prompt)" bucket so they remain visible.
|
|
93
94
|
*/
|
|
94
|
-
declare const NO_PROMPT_KEY = "(no prompt)";
|
|
95
|
+
export declare const NO_PROMPT_KEY = "(no prompt)";
|
|
95
96
|
/** One prompt-version bucket — its `name@version` key plus its traces. */
|
|
96
|
-
type PromptGroup = {
|
|
97
|
-
/** The `name@version` key, or {@link NO_PROMPT_KEY} for unlinked traces. */
|
|
97
|
+
export type PromptGroup = {
|
|
98
|
+
/** The `name@version` key, or {@link NO_PROMPT_KEY} for unlinked traces. */
|
|
99
|
+
promptKey: string;
|
|
100
|
+
/** Traces in this group, in the order they appeared in the input. */
|
|
98
101
|
traces: Trace[];
|
|
99
102
|
};
|
|
100
103
|
/**
|
|
@@ -108,16 +111,18 @@ type PromptGroup = {
|
|
|
108
111
|
* so a reviewer can see every run of "support@2" together and compare its
|
|
109
112
|
* cost / failure rate against "support@3".
|
|
110
113
|
*/
|
|
111
|
-
declare function groupByPrompt(traces: Trace[]): PromptGroup[];
|
|
114
|
+
export declare function groupByPrompt(traces: Trace[]): PromptGroup[];
|
|
112
115
|
/**
|
|
113
116
|
* The grouping key a trace falls under when the list is grouped by root
|
|
114
117
|
* type. `root.type` is always present, so this is only a defensive
|
|
115
118
|
* fallback for a malformed trace with no typed root.
|
|
116
119
|
*/
|
|
117
|
-
declare const NO_TYPE_KEY = "(no type)";
|
|
120
|
+
export declare const NO_TYPE_KEY = "(no type)";
|
|
118
121
|
/** One root-type bucket — its type discriminator plus its traces. */
|
|
119
|
-
type TypeGroup = {
|
|
120
|
-
/** The root span's `type` (e.g. `agent` / `workflow` / `planner`), or {@link NO_TYPE_KEY}. */
|
|
122
|
+
export type TypeGroup = {
|
|
123
|
+
/** The root span's `type` (e.g. `agent` / `workflow` / `planner`), or {@link NO_TYPE_KEY}. */
|
|
124
|
+
type: string;
|
|
125
|
+
/** Traces in this group, in the order they appeared in the input. */
|
|
121
126
|
traces: Trace[];
|
|
122
127
|
};
|
|
123
128
|
/**
|
|
@@ -132,29 +137,37 @@ type TypeGroup = {
|
|
|
132
137
|
* behind a per-type header so the structural primitives (the lone planner,
|
|
133
138
|
* the team, the supervisors) are one click away instead of a scroll past.
|
|
134
139
|
*/
|
|
135
|
-
declare function groupByType(traces: Trace[]): TypeGroup[];
|
|
140
|
+
export declare function groupByType(traces: Trace[]): TypeGroup[];
|
|
136
141
|
/** One row of the per-type aggregate-stats panel. */
|
|
137
|
-
type TypeStat = {
|
|
138
|
-
/** Root type discriminator, or {@link NO_TYPE_KEY}. */
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
142
|
+
export type TypeStat = {
|
|
143
|
+
/** Root type discriminator, or {@link NO_TYPE_KEY}. */
|
|
144
|
+
type: string;
|
|
145
|
+
/** Number of traces of this type. */
|
|
146
|
+
count: number;
|
|
147
|
+
/** Traces whose status is `failed` / `cancelled`. */
|
|
148
|
+
failed: number;
|
|
149
|
+
/** `failed / count` in `[0, 1]`. */
|
|
150
|
+
failRate: number;
|
|
151
|
+
/** Median (p50) root duration in ms. */
|
|
152
|
+
p50: number;
|
|
153
|
+
/** p95 root duration in ms. */
|
|
154
|
+
p95: number;
|
|
155
|
+
/** Summed `usage.total` tokens across the bucket. */
|
|
156
|
+
tokens: number;
|
|
157
|
+
/** Summed per-trace USD cost across the bucket. */
|
|
145
158
|
cost: number;
|
|
146
159
|
};
|
|
147
160
|
/**
|
|
148
161
|
* Nearest-rank percentile of `values` at `p` (0–100). Returns 0 for an
|
|
149
162
|
* empty input and sorts a copy so the caller's array is left untouched.
|
|
150
163
|
*/
|
|
151
|
-
declare function percentile(values: number[], p: number): number;
|
|
164
|
+
export declare function percentile(values: number[], p: number): number;
|
|
152
165
|
/**
|
|
153
166
|
* One trace's total USD cost: its explicit root-usage cost when priced,
|
|
154
167
|
* else the rollup of its subtree. Mirrors the dashboard's `traceCost` so the
|
|
155
168
|
* stats panel and the trace rows agree to the cent.
|
|
156
169
|
*/
|
|
157
|
-
declare function traceCost(trace: Trace): number;
|
|
170
|
+
export declare function traceCost(trace: Trace): number;
|
|
158
171
|
/**
|
|
159
172
|
* Aggregate an (already filtered) trace list into one {@link TypeStat} row
|
|
160
173
|
* per root type — count, failure rate, p50/p95 latency, total tokens, total
|
|
@@ -162,26 +175,25 @@ declare function traceCost(trace: Trace): number;
|
|
|
162
175
|
* stats panel; computed over the same filtered list the trace view shows, so
|
|
163
176
|
* the panel always reflects the active filters.
|
|
164
177
|
*/
|
|
165
|
-
declare function aggregateByType(traces: Trace[]): TypeStat[];
|
|
178
|
+
export declare function aggregateByType(traces: Trace[]): TypeStat[];
|
|
166
179
|
/**
|
|
167
180
|
* Rollup-aware subtree cost: a node's own cost when it carries one
|
|
168
181
|
* (it already rolls up its trips), otherwise the sum of its children.
|
|
169
182
|
* Avoids double-counting wrapper nodes (workflow/supervisor roots carry
|
|
170
183
|
* tokens but no cost). Mirrors the dashboard's `rollupCost`.
|
|
171
184
|
*/
|
|
172
|
-
declare function rollupCost(span: TraceSpan): number;
|
|
185
|
+
export declare function rollupCost(span: TraceSpan): number;
|
|
173
186
|
/**
|
|
174
187
|
* Heatmap intensity in `[0, 1]` for a node's `rollupCost` relative to the
|
|
175
188
|
* trace's most-expensive node. The dashboard tints each tree node's left
|
|
176
189
|
* accent by this. `max <= 0` (a free trace) yields `0` for every node so
|
|
177
190
|
* the heatmap simply stays cold rather than dividing by zero.
|
|
178
191
|
*/
|
|
179
|
-
declare function heatIntensity(nodeCost: number, maxCost: number): number;
|
|
192
|
+
export declare function heatIntensity(nodeCost: number, maxCost: number): number;
|
|
180
193
|
/**
|
|
181
194
|
* The largest single-node {@link rollupCost} anywhere in a span tree —
|
|
182
195
|
* the denominator for {@link heatIntensity}. Walks the whole subtree.
|
|
183
196
|
*/
|
|
184
|
-
declare function maxNodeCost(root: TraceSpan): number;
|
|
197
|
+
export declare function maxNodeCost(root: TraceSpan): number;
|
|
185
198
|
//#endregion
|
|
186
|
-
export { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, PromptGroup, SessionGroup, TraceFilter, TypeGroup, TypeStat, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey };
|
|
187
199
|
//# sourceMappingURL=trace-filter.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trace-filter.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/trace-filter.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"trace-filter.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/trace-filter.ts"],"mappings":";;;;;;;;;;;;;;YAcY;;;;;;EAMV;;;;;EAKA,WAAW;;;;;EAKX,QAAQ;;;;;;EAMR;;;;;;;EAOA;;;;;EAKA;;;;;;;;;;;;;wBAyBc,eAAe,OAAO;;;;;;wBAwBtB,cAAc,OAAO,OAAO,QAAQ;;wBA2CpC,aAAa,QAAQ,SAAS,QAAQ,cAAc;;;;;;qBASvD;;YAGD;;EAEV;;EAEA,QAAQ;;;;;;;wBAQM,eAAe,QAAQ,UAAU;;;;;;qBAyBpC;;YAGD;;EAEV;;EAEA,QAAQ;;;;;;;;;;;;;wBAcM,cAAc,QAAQ,UAAU;;;;;;qBAyBnC;;YAGD;;EAEV;;EAEA,QAAQ;;;;;;;;;;;;;;wBAeM,YAAY,QAAQ,UAAU;;YAqBlC;;EAEV;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;wBAOc,WAAW,kBAAkB;;;;;;wBAiB7B,UAAU,OAAO;;;;;;;;wBAYjB,gBAAgB,QAAQ,UAAU;;;;;;;wBAwElC,WAAW,MAAM;;;;;;;wBAoBjB,cAAc,kBAAkB;;;;;wBAchC,YAAY,MAAM"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { EvaluateConfig, EvaluateVerdict } from "./evaluate.type.mjs";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Grade `systemPrompt` with the configured judge model, reusing
|
|
@@ -13,7 +12,6 @@ import { EvaluateConfig, EvaluateVerdict } from "./evaluate.type.mjs";
|
|
|
13
12
|
* resolving `config.model` (a factory constructing an SDK client), which
|
|
14
13
|
* the caller (the dashboard route) is expected to catch.
|
|
15
14
|
*/
|
|
16
|
-
declare function evaluateSystemPrompt(systemPrompt: string, config: EvaluateConfig, instructionsOverride?: string): Promise<EvaluateVerdict>;
|
|
15
|
+
export declare function evaluateSystemPrompt(systemPrompt: string, config: EvaluateConfig, instructionsOverride?: string): Promise<EvaluateVerdict>;
|
|
17
16
|
//#endregion
|
|
18
|
-
export { evaluateSystemPrompt };
|
|
19
17
|
//# sourceMappingURL=evaluate-system-prompt.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate-system-prompt.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"evaluate-system-prompt.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"mappings":";;;;;;;;;;;;;;wBAesB,qBACpB,sBACA,QAAQ,gBACR,gCACC,QAAQ"}
|
|
@@ -14,7 +14,9 @@ import { judgePromptBody } from "@warlock.js/ai";
|
|
|
14
14
|
* the caller (the dashboard route) is expected to catch.
|
|
15
15
|
*/
|
|
16
16
|
async function evaluateSystemPrompt(systemPrompt, config, instructionsOverride) {
|
|
17
|
-
|
|
17
|
+
const model = typeof config.model === "function" ? await config.model() : config.model;
|
|
18
|
+
const instructions = instructionsOverride?.trim() || config.instructions;
|
|
19
|
+
return judgePromptBody(systemPrompt, model, instructions);
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate-system-prompt.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"sourcesContent":["import { judgePromptBody } from \"@warlock.js/ai\";\nimport type { EvaluateConfig, EvaluateVerdict } from \"./evaluate.type\";\n\n/**\n * Grade `systemPrompt` with the configured judge model, reusing\n * `ai.prompts().validate()`'s own `judgePromptBody` — never a second\n * judging implementation. `instructionsOverride` (the dashboard's\n * per-run textarea) wins over `config.instructions`; with neither, the\n * judge falls back to `judgePromptBody`'s built-in prompt-quality rubric.\n *\n * The judge itself never throws (`judgePromptBody` degrades to an\n * issues-only outcome on failure) — the only thing that CAN throw here is\n * resolving `config.model` (a factory constructing an SDK client), which\n * the caller (the dashboard route) is expected to catch.\n */\nexport async function evaluateSystemPrompt(\n systemPrompt: string,\n config: EvaluateConfig,\n instructionsOverride?: string,\n): Promise<EvaluateVerdict> {\n const model = typeof config.model === \"function\" ? await config.model() : config.model;\n const instructions = instructionsOverride?.trim() || config.instructions;\n\n return judgePromptBody(systemPrompt, model, instructions);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAeA,eAAsB,qBACpB,cACA,QACA,sBAC0B;
|
|
1
|
+
{"version":3,"file":"evaluate-system-prompt.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"sourcesContent":["import { judgePromptBody } from \"@warlock.js/ai\";\nimport type { EvaluateConfig, EvaluateVerdict } from \"./evaluate.type\";\n\n/**\n * Grade `systemPrompt` with the configured judge model, reusing\n * `ai.prompts().validate()`'s own `judgePromptBody` — never a second\n * judging implementation. `instructionsOverride` (the dashboard's\n * per-run textarea) wins over `config.instructions`; with neither, the\n * judge falls back to `judgePromptBody`'s built-in prompt-quality rubric.\n *\n * The judge itself never throws (`judgePromptBody` degrades to an\n * issues-only outcome on failure) — the only thing that CAN throw here is\n * resolving `config.model` (a factory constructing an SDK client), which\n * the caller (the dashboard route) is expected to catch.\n */\nexport async function evaluateSystemPrompt(\n systemPrompt: string,\n config: EvaluateConfig,\n instructionsOverride?: string,\n): Promise<EvaluateVerdict> {\n const model = typeof config.model === \"function\" ? await config.model() : config.model;\n const instructions = instructionsOverride?.trim() || config.instructions;\n\n return judgePromptBody(systemPrompt, model, instructions);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAeA,eAAsB,qBACpB,cACA,QACA,sBAC0B;CAC1B,MAAM,QAAQ,OAAO,OAAO,UAAU,aAAa,MAAM,OAAO,MAAM,IAAI,OAAO;CACjF,MAAM,eAAe,sBAAsB,KAAK,KAAK,OAAO;CAE5D,OAAO,gBAAgB,cAAc,OAAO,YAAY;AAC1D"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { JudgeOutcome, ModelContract } from "@warlock.js/ai";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/evaluate/evaluate.type.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Enables the dashboard's per-trace "Evaluate system prompt" action — the
|
|
@@ -22,7 +21,7 @@ import { JudgeOutcome, ModelContract } from "@warlock.js/ai";
|
|
|
22
21
|
* },
|
|
23
22
|
* });
|
|
24
23
|
*/
|
|
25
|
-
type EvaluateConfig = {
|
|
24
|
+
export type EvaluateConfig = {
|
|
26
25
|
/**
|
|
27
26
|
* The judge model, or a (possibly async) factory resolved on first use —
|
|
28
27
|
* the factory form defers construction (e.g. an SDK client) until the
|
|
@@ -41,7 +40,6 @@ type EvaluateConfig = {
|
|
|
41
40
|
instructions?: string;
|
|
42
41
|
};
|
|
43
42
|
/** The judge verdict returned by the evaluate endpoint — `{@link JudgeOutcome}` verbatim. */
|
|
44
|
-
type EvaluateVerdict = JudgeOutcome;
|
|
43
|
+
export type EvaluateVerdict = JudgeOutcome;
|
|
45
44
|
//#endregion
|
|
46
|
-
export { EvaluateConfig, EvaluateVerdict };
|
|
47
45
|
//# sourceMappingURL=evaluate.type.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.type.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate.type.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"evaluate.type.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate.type.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;YAuBY;;;;;;;;EAQV,OAAO,uBAAuB,gBAAgB,QAAQ;;;;;;;;EAQtD;;;YAIU,kBAAkB"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { TraceSpan } from "../contracts/trace.type.mjs";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/evaluate/extract-last-system-prompt.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Pull the LAST `{role: "system"}` message off a span's captured `input`.
|
|
@@ -13,7 +12,6 @@ import { TraceSpan } from "../contracts/trace.type.mjs";
|
|
|
13
12
|
* `undefined` for a tool span, an agent with no system prompt, or when
|
|
14
13
|
* content capture is off — the caller treats that as "nothing to evaluate."
|
|
15
14
|
*/
|
|
16
|
-
declare function extractLastSystemPrompt(span: TraceSpan): string | undefined;
|
|
15
|
+
export declare function extractLastSystemPrompt(span: TraceSpan): string | undefined;
|
|
17
16
|
//#endregion
|
|
18
|
-
export { extractLastSystemPrompt };
|
|
19
17
|
//# sourceMappingURL=extract-last-system-prompt.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-last-system-prompt.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/extract-last-system-prompt.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-last-system-prompt.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/extract-last-system-prompt.ts"],"mappings":";;;;;;;;;;;;;;wBAcgB,wBAAwB,MAAM"}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { TraceSpan } from "../contracts/trace.type.mjs";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/evaluate/find-span-by-id.d.ts
|
|
4
3
|
/** Depth-first search for the span with `spanId` inside a trace's span tree. */
|
|
5
|
-
declare function findSpanById(root: TraceSpan, spanId: string): TraceSpan | undefined;
|
|
4
|
+
export declare function findSpanById(root: TraceSpan, spanId: string): TraceSpan | undefined;
|
|
6
5
|
//#endregion
|
|
7
|
-
export { findSpanById };
|
|
8
6
|
//# sourceMappingURL=find-span-by-id.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"find-span-by-id.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/find-span-by-id.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"find-span-by-id.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/find-span-by-id.ts"],"mappings":";;;wBAGgB,aAAa,MAAM,WAAW,iBAAiB"}
|
package/esm/evaluate/index.d.mts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { EvaluateConfig, EvaluateVerdict } from "./evaluate.type.mjs";
|
|
2
2
|
import { evaluateSystemPrompt } from "./evaluate-system-prompt.mjs";
|
|
3
3
|
import { extractLastSystemPrompt } from "./extract-last-system-prompt.mjs";
|
|
4
|
-
import { findSpanById } from "./find-span-by-id.mjs";
|
|
4
|
+
import { findSpanById } from "./find-span-by-id.mjs";
|
|
5
|
+
export { type EvaluateConfig, type EvaluateVerdict, evaluateSystemPrompt, extractLastSystemPrompt, findSpanById };
|
package/esm/evaluate/index.mjs
CHANGED
|
@@ -2,4 +2,4 @@ import { evaluateSystemPrompt } from "./evaluate-system-prompt.mjs";
|
|
|
2
2
|
import { extractLastSystemPrompt } from "./extract-last-system-prompt.mjs";
|
|
3
3
|
import { findSpanById } from "./find-span-by-id.mjs";
|
|
4
4
|
|
|
5
|
-
export {
|
|
5
|
+
export { evaluateSystemPrompt, extractLastSystemPrompt, findSpanById };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ExporterContract } from "../../contracts/exporter.contract.mjs";
|
|
2
|
+
import "../../contracts/index.mjs";
|
|
2
3
|
import { ConsoleExporterOptions } from "./console-exporter.type.mjs";
|
|
3
|
-
|
|
4
4
|
//#region ../../packages/ai-panoptic/src/exporters/console/console-exporter.d.ts
|
|
5
5
|
/**
|
|
6
6
|
* Zero-dependency {@link ExporterContract} that prints traces to a
|
|
@@ -27,7 +27,6 @@ import { ConsoleExporterOptions } from "./console-exporter.type.mjs";
|
|
|
27
27
|
* exporters: [consoleExporter({ tree: true, io: true })],
|
|
28
28
|
* });
|
|
29
29
|
*/
|
|
30
|
-
declare function consoleExporter(options?: ConsoleExporterOptions): ExporterContract;
|
|
30
|
+
export declare function consoleExporter(options?: ConsoleExporterOptions): ExporterContract;
|
|
31
31
|
//#endregion
|
|
32
|
-
export { consoleExporter };
|
|
33
32
|
//# sourceMappingURL=console-exporter.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"console-exporter.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/console-exporter.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"console-exporter.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/console-exporter.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAiCgB,gBAAgB,UAAS,yBAA8B"}
|
|
@@ -5,14 +5,14 @@
|
|
|
5
5
|
* monkey-patching the global. Only the two levels the exporter uses are
|
|
6
6
|
* required.
|
|
7
7
|
*/
|
|
8
|
-
type ConsoleLike = {
|
|
8
|
+
export type ConsoleLike = {
|
|
9
9
|
log(...args: unknown[]): void;
|
|
10
10
|
error(...args: unknown[]): void;
|
|
11
11
|
};
|
|
12
12
|
/**
|
|
13
13
|
* Options for {@link consoleExporter}.
|
|
14
14
|
*/
|
|
15
|
-
type ConsoleExporterOptions = {
|
|
15
|
+
export type ConsoleExporterOptions = {
|
|
16
16
|
/**
|
|
17
17
|
* Where to write. Defaults to the global `console`. Inject a custom
|
|
18
18
|
* sink in tests or to redirect to a structured logger.
|
|
@@ -47,5 +47,4 @@ type ConsoleExporterOptions = {
|
|
|
47
47
|
ioMaxChars?: number;
|
|
48
48
|
};
|
|
49
49
|
//#endregion
|
|
50
|
-
export { ConsoleExporterOptions, ConsoleLike };
|
|
51
50
|
//# sourceMappingURL=console-exporter.type.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"console-exporter.type.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/console-exporter.type.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"console-exporter.type.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/console-exporter.type.ts"],"mappings":";;;;;;;YAMY;EACV,OAAO;EACP,SAAS;;;;;YAMC;;;;;EAKV,UAAU;;;;;;EAMV;;;;;;EAMA;;;;;;;;EAQA;;;;;;;EAOA"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { TraceSpan } from "../../contracts/trace.type.mjs";
|
|
2
|
+
import "../../contracts/index.mjs";
|
|
2
3
|
//#region ../../packages/ai-panoptic/src/exporters/console/format-span-io.d.ts
|
|
3
4
|
/**
|
|
4
5
|
* Render a span's captured content ({@link TraceSpan.input} /
|
|
@@ -21,7 +22,6 @@ import { TraceSpan } from "../../contracts/trace.type.mjs";
|
|
|
21
22
|
* formatSpanIO(toolSpan, 1);
|
|
22
23
|
* // [' in: {"query":"specialty coffee demand"}', ' out: {"results":[…]}']
|
|
23
24
|
*/
|
|
24
|
-
declare function formatSpanIO(span: TraceSpan, depth?: number, maxChars?: number): string[];
|
|
25
|
+
export declare function formatSpanIO(span: TraceSpan, depth?: number, maxChars?: number): string[];
|
|
25
26
|
//#endregion
|
|
26
|
-
export { formatSpanIO };
|
|
27
27
|
//# sourceMappingURL=format-span-io.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format-span-io.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/format-span-io.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"format-span-io.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/format-span-io.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;wBA0BgB,aACd,MAAM,WACN,gBACA"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { TraceSpan } from "../../contracts/trace.type.mjs";
|
|
2
|
+
import "../../contracts/index.mjs";
|
|
2
3
|
//#region ../../packages/ai-panoptic/src/exporters/console/format-span-line.d.ts
|
|
3
4
|
/**
|
|
4
5
|
* Render a single {@link TraceSpan} as one scannable console line:
|
|
@@ -11,7 +12,6 @@ import { TraceSpan } from "../../contracts/trace.type.mjs";
|
|
|
11
12
|
* formatSpanLine(span, 1);
|
|
12
13
|
* // ' ok agent "router" — 1240ms, 470 tok, $0.0021'
|
|
13
14
|
*/
|
|
14
|
-
declare function formatSpanLine(span: TraceSpan, depth?: number): string;
|
|
15
|
+
export declare function formatSpanLine(span: TraceSpan, depth?: number): string;
|
|
15
16
|
//#endregion
|
|
16
|
-
export { formatSpanLine };
|
|
17
17
|
//# sourceMappingURL=format-span-line.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format-span-line.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/format-span-line.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"format-span-line.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/console/format-span-line.ts"],"mappings":";;;;;;;;;;;;;;wBAcgB,eAAe,MAAM,WAAW"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ConsoleExporterOptions, ConsoleLike } from "./console-exporter.type.mjs";
|
|
2
2
|
import { consoleExporter } from "./console-exporter.mjs";
|
|
3
3
|
import { formatSpanLine } from "./format-span-line.mjs";
|
|
4
|
-
import { formatSpanIO } from "./format-span-io.mjs";
|
|
4
|
+
import { formatSpanIO } from "./format-span-io.mjs";
|
|
5
|
+
export { type ConsoleExporterOptions, type ConsoleLike, consoleExporter, formatSpanIO, formatSpanLine };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ExporterContract } from "../../contracts/exporter.contract.mjs";
|
|
2
|
+
import "../../contracts/index.mjs";
|
|
2
3
|
import { FileExporterOptions } from "./file-exporter.type.mjs";
|
|
3
|
-
|
|
4
4
|
//#region ../../packages/ai-panoptic/src/exporters/file/file-exporter.d.ts
|
|
5
5
|
/**
|
|
6
6
|
* Zero-dependency {@link ExporterContract} that appends completed traces
|
|
@@ -17,7 +17,6 @@ import { FileExporterOptions } from "./file-exporter.type.mjs";
|
|
|
17
17
|
* // on shutdown:
|
|
18
18
|
* await collector.shutdown(); // drains the buffer
|
|
19
19
|
*/
|
|
20
|
-
declare function fileExporter(options: FileExporterOptions): ExporterContract;
|
|
20
|
+
export declare function fileExporter(options: FileExporterOptions): ExporterContract;
|
|
21
21
|
//#endregion
|
|
22
|
-
export { fileExporter };
|
|
23
22
|
//# sourceMappingURL=file-exporter.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-exporter.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/file/file-exporter.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"file-exporter.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/file/file-exporter.ts"],"mappings":";;;;;;;;;;;;;;;;;;;wBAsBgB,aAAa,SAAS,sBAAsB"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Trace } from "../../contracts/trace.type.mjs";
|
|
2
|
+
import "../../contracts/index.mjs";
|
|
2
3
|
//#region ../../packages/ai-panoptic/src/exporters/file/file-exporter.type.d.ts
|
|
3
4
|
/**
|
|
4
5
|
* One JSON-Lines record the {@link fileExporter} appends per trace. A
|
|
@@ -6,15 +7,18 @@ import { Trace } from "../../contracts/trace.type.mjs";
|
|
|
6
7
|
* the time the line was written, so a downstream reader can tell records
|
|
7
8
|
* apart and order them without re-deriving anything from the payload.
|
|
8
9
|
*/
|
|
9
|
-
type TraceRecord = {
|
|
10
|
-
/** Discriminator for the record format — always `"trace"` today. */
|
|
11
|
-
|
|
10
|
+
export type TraceRecord = {
|
|
11
|
+
/** Discriminator for the record format — always `"trace"` today. */
|
|
12
|
+
type: "trace";
|
|
13
|
+
/** ISO-8601 timestamp the exporter wrote this record. */
|
|
14
|
+
exportedAt: string;
|
|
15
|
+
/** The completed trace, verbatim. */
|
|
12
16
|
trace: Trace;
|
|
13
17
|
};
|
|
14
18
|
/**
|
|
15
19
|
* Options for {@link fileExporter}.
|
|
16
20
|
*/
|
|
17
|
-
type FileExporterOptions = {
|
|
21
|
+
export type FileExporterOptions = {
|
|
18
22
|
/**
|
|
19
23
|
* Absolute or cwd-relative path of the JSON-Lines file traces are
|
|
20
24
|
* appended to. The parent directory is created on first write if
|
|
@@ -35,5 +39,4 @@ type FileExporterOptions = {
|
|
|
35
39
|
pretty?: boolean;
|
|
36
40
|
};
|
|
37
41
|
//#endregion
|
|
38
|
-
export { FileExporterOptions, TraceRecord };
|
|
39
42
|
//# sourceMappingURL=file-exporter.type.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-exporter.type.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/file/file-exporter.type.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"file-exporter.type.d.mts","names":[],"sources":["../../../../../../../../../packages/ai-panoptic/src/exporters/file/file-exporter.type.ts"],"mappings":";;;;;;;;;YAQY;;EAEV;;EAEA;;EAEA,OAAO;;;;;YAMG;;;;;;EAMV;;;;;;EAMA;;;;;;EAMA"}
|