@warlock.js/ai-panoptic 5.31.0 → 5.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/cjs/index.cjs +72 -67
- package/cjs/index.cjs.map +1 -1
- package/esm/collector/collector.d.mts +3 -5
- package/esm/collector/collector.d.mts.map +1 -1
- package/esm/collector/collector.mjs +2 -0
- package/esm/collector/collector.mjs.map +1 -1
- package/esm/collector/content-capture.type.d.mts +4 -5
- package/esm/collector/content-capture.type.d.mts.map +1 -1
- package/esm/collector/extract-span-attributes.d.mts +1 -3
- package/esm/collector/extract-span-attributes.d.mts.map +1 -1
- package/esm/collector/extract-span-attributes.mjs +30 -6
- package/esm/collector/extract-span-attributes.mjs.map +1 -1
- package/esm/collector/index.d.mts +3 -1
- package/esm/collector/index.mjs +2 -1
- package/esm/collector/normalize-error.d.mts +1 -3
- package/esm/collector/normalize-error.d.mts.map +1 -1
- package/esm/collector/report-to-span.d.mts +1 -3
- package/esm/collector/report-to-span.d.mts.map +1 -1
- package/esm/collector/report-to-trace.d.mts +1 -3
- package/esm/collector/report-to-trace.d.mts.map +1 -1
- package/esm/collector/trace-subscriptions.d.mts +14 -0
- package/esm/collector/trace-subscriptions.d.mts.map +1 -0
- package/esm/collector/trace-subscriptions.mjs +26 -0
- package/esm/collector/trace-subscriptions.mjs.map +1 -0
- package/esm/config/apply-panoptic-config.d.mts +2 -3
- package/esm/config/apply-panoptic-config.d.mts.map +1 -1
- package/esm/config/index.d.mts +2 -1
- package/esm/config/index.mjs +1 -1
- package/esm/config/panoptic-config.type.d.mts +2 -4
- package/esm/config/panoptic-config.type.d.mts.map +1 -1
- package/esm/contracts/collector.contract.d.mts +1 -3
- package/esm/contracts/collector.contract.d.mts.map +1 -1
- package/esm/contracts/exporter.contract.d.mts +1 -3
- package/esm/contracts/exporter.contract.d.mts.map +1 -1
- package/esm/contracts/index.d.mts +2 -1
- package/esm/contracts/trace.type.d.mts +37 -20
- package/esm/contracts/trace.type.d.mts.map +1 -1
- package/esm/dashboard/dashboard.d.mts +1 -3
- package/esm/dashboard/dashboard.d.mts.map +1 -1
- package/esm/dashboard/dashboard.mjs +3 -2
- package/esm/dashboard/dashboard.mjs.map +1 -1
- package/esm/dashboard/dashboard.type.d.mts +9 -7
- package/esm/dashboard/dashboard.type.d.mts.map +1 -1
- package/esm/dashboard/index.d.mts +4 -1
- package/esm/dashboard/index.mjs +1 -1
- package/esm/dashboard/parse-query.d.mts +1 -1
- package/esm/dashboard/serve.d.mts +2 -1
- package/esm/dashboard/serve.mjs +1 -1
- package/esm/dashboard/serve.mjs.map +1 -1
- package/esm/dashboard/trace-filter.d.mts +44 -32
- package/esm/dashboard/trace-filter.d.mts.map +1 -1
- package/esm/evaluate/evaluate-system-prompt.d.mts +1 -3
- package/esm/evaluate/evaluate-system-prompt.d.mts.map +1 -1
- package/esm/evaluate/evaluate-system-prompt.mjs +3 -1
- package/esm/evaluate/evaluate-system-prompt.mjs.map +1 -1
- package/esm/evaluate/evaluate.type.d.mts +2 -4
- package/esm/evaluate/evaluate.type.d.mts.map +1 -1
- package/esm/evaluate/extract-last-system-prompt.d.mts +1 -3
- package/esm/evaluate/extract-last-system-prompt.d.mts.map +1 -1
- package/esm/evaluate/find-span-by-id.d.mts +1 -3
- package/esm/evaluate/find-span-by-id.d.mts.map +1 -1
- package/esm/evaluate/index.d.mts +2 -1
- package/esm/evaluate/index.mjs +1 -1
- package/esm/exporters/console/console-exporter.d.mts +2 -3
- package/esm/exporters/console/console-exporter.d.mts.map +1 -1
- package/esm/exporters/console/console-exporter.type.d.mts +2 -3
- package/esm/exporters/console/console-exporter.type.d.mts.map +1 -1
- package/esm/exporters/console/format-span-io.d.mts +2 -2
- package/esm/exporters/console/format-span-io.d.mts.map +1 -1
- package/esm/exporters/console/format-span-line.d.mts +2 -2
- package/esm/exporters/console/format-span-line.d.mts.map +1 -1
- package/esm/exporters/console/index.d.mts +2 -1
- package/esm/exporters/console/index.mjs +1 -1
- package/esm/exporters/file/file-exporter.d.mts +2 -3
- package/esm/exporters/file/file-exporter.d.mts.map +1 -1
- package/esm/exporters/file/file-exporter.type.d.mts +8 -5
- package/esm/exporters/file/file-exporter.type.d.mts.map +1 -1
- package/esm/exporters/file/index.d.mts +2 -1
- package/esm/exporters/index.d.mts +7 -1
- package/esm/exporters/index.mjs +1 -1
- package/esm/exporters/langfuse/index.d.mts +2 -1
- package/esm/exporters/langfuse/index.mjs +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.d.mts +2 -3
- package/esm/exporters/langfuse/langfuse-exporter.d.mts.map +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.mjs +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.mjs.map +1 -1
- package/esm/exporters/langfuse/langfuse-exporter.type.d.mts +23 -17
- package/esm/exporters/langfuse/langfuse-exporter.type.d.mts.map +1 -1
- package/esm/exporters/otel/index.d.mts +2 -1
- package/esm/exporters/otel/index.mjs +1 -1
- package/esm/exporters/otel/otel-api.shim.type.d.mts +6 -7
- package/esm/exporters/otel/otel-api.shim.type.d.mts.map +1 -1
- package/esm/exporters/otel/otel-exporter.d.mts +2 -3
- package/esm/exporters/otel/otel-exporter.d.mts.map +1 -1
- package/esm/exporters/otel/otel-exporter.type.d.mts +1 -3
- package/esm/exporters/otel/otel-exporter.type.d.mts.map +1 -1
- package/esm/exporters/utils/gen-ai-attributes.d.mts +9 -7
- package/esm/exporters/utils/gen-ai-attributes.d.mts.map +1 -1
- package/esm/exporters/utils/index.d.mts +2 -1
- package/esm/exporters/utils/index.mjs +1 -1
- package/esm/exporters/utils/total-cost.d.mts +1 -3
- package/esm/exporters/utils/total-cost.d.mts.map +1 -1
- package/esm/exporters/utils/walk-spans.d.mts +2 -2
- package/esm/exporters/utils/walk-spans.d.mts.map +1 -1
- package/esm/index.d.mts +10 -1
- package/esm/index.mjs +2 -1
- package/esm/panoptic/index.d.mts +2 -1
- package/esm/panoptic/index.mjs +1 -1
- package/esm/panoptic/panoptic-middleware.d.mts +1 -3
- package/esm/panoptic/panoptic-middleware.d.mts.map +1 -1
- package/esm/panoptic/panoptic-target.type.d.mts +4 -5
- package/esm/panoptic/panoptic-target.type.d.mts.map +1 -1
- package/esm/panoptic/panoptic.d.mts +1 -3
- package/esm/panoptic/panoptic.d.mts.map +1 -1
- package/esm/panoptic/panoptic.type.d.mts +8 -7
- package/esm/panoptic/panoptic.type.d.mts.map +1 -1
- package/esm/store/cache-trace-store.d.mts +4 -6
- package/esm/store/cache-trace-store.d.mts.map +1 -1
- package/esm/store/cache-trace-store.mjs +1 -4
- package/esm/store/cache-trace-store.mjs.map +1 -1
- package/esm/store/in-memory-trace-store.d.mts +2 -4
- package/esm/store/in-memory-trace-store.d.mts.map +1 -1
- package/esm/store/in-memory-trace-store.mjs +1 -4
- package/esm/store/in-memory-trace-store.mjs.map +1 -1
- package/esm/store/index.d.mts +2 -1
- package/esm/store/index.mjs +1 -1
- package/esm/store/match-trace.d.mts +1 -3
- package/esm/store/match-trace.d.mts.map +1 -1
- package/esm/store/match-trace.mjs +1 -0
- package/esm/store/match-trace.mjs.map +1 -1
- package/esm/store/sum-usage.d.mts +4 -30
- package/esm/store/sum-usage.d.mts.map +1 -1
- package/esm/store/sum-usage.mjs +6 -49
- package/esm/store/sum-usage.mjs.map +1 -1
- package/esm/store/trace-aggregate.type.d.mts +12 -9
- package/esm/store/trace-aggregate.type.d.mts.map +1 -1
- package/esm/store/trace-query.type.d.mts +7 -5
- package/esm/store/trace-query.type.d.mts.map +1 -1
- package/esm/store/trace-store.contract.d.mts +1 -3
- package/esm/store/trace-store.contract.d.mts.map +1 -1
- package/llms-full.txt +105 -0
- package/package.json +4 -4
- package/skills/observe-with-panoptic/SKILL.md +35 -0
- package/skills/query-traces/SKILL.md +49 -0
- package/skills/use-local-dashboard/SKILL.md +21 -0
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { ReportStatus, ReportType, Usage } from "@warlock.js/ai";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/contracts/trace.type.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* A single node in a Panoptic trace tree, derived 1:1 from a core
|
|
@@ -31,7 +30,7 @@ import { ReportStatus, ReportType, Usage } from "@warlock.js/ai";
|
|
|
31
30
|
* children: [],
|
|
32
31
|
* };
|
|
33
32
|
*/
|
|
34
|
-
type TraceSpan = {
|
|
33
|
+
export type TraceSpan = {
|
|
35
34
|
/**
|
|
36
35
|
* Stable id for this span. Mirrors `BaseReport.runId` — one span per
|
|
37
36
|
* execution node.
|
|
@@ -54,19 +53,26 @@ type TraceSpan = {
|
|
|
54
53
|
* Surfaces as a session/thread dimension in backends that model
|
|
55
54
|
* conversations. Absent when the caller didn't supply one.
|
|
56
55
|
*/
|
|
57
|
-
sessionId?: string;
|
|
56
|
+
sessionId?: string;
|
|
57
|
+
/** Executable identity — the tool/agent/workflow/supervisor name. */
|
|
58
58
|
name: string;
|
|
59
59
|
/**
|
|
60
60
|
* Dev-curated version string mirrored from `BaseReport.version`.
|
|
61
61
|
* Free-form; surfaced as a span attribute so dashboards can slice
|
|
62
62
|
* runs of "agent X v2.1" from "v2.2". Absent when undeclared.
|
|
63
63
|
*/
|
|
64
|
-
version?: string;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
64
|
+
version?: string;
|
|
65
|
+
/** Discriminator for the kind of executable that produced this span. */
|
|
66
|
+
type: ReportType;
|
|
67
|
+
/** Terminal status of this execution, mirrored from `BaseReport.status`. */
|
|
68
|
+
status: ReportStatus;
|
|
69
|
+
/** ISO-8601 wall-clock timestamp when execution began. */
|
|
70
|
+
startedAt: string;
|
|
71
|
+
/** ISO-8601 wall-clock timestamp when execution finished. */
|
|
72
|
+
endedAt: string;
|
|
73
|
+
/** Monotonic duration in milliseconds. */
|
|
74
|
+
duration: number;
|
|
75
|
+
/** Rolled-up usage for this node (own cost + sum of children). */
|
|
70
76
|
usage: Usage;
|
|
71
77
|
/**
|
|
72
78
|
* Normalized error description when `status` is `"failed"` /
|
|
@@ -110,9 +116,12 @@ type TraceSpan = {
|
|
|
110
116
|
* Kept structural (not an `Error` instance) so it survives JSON
|
|
111
117
|
* serialization to a backend collector unchanged.
|
|
112
118
|
*/
|
|
113
|
-
type TraceSpanError = {
|
|
114
|
-
/** Stable error identifier (typed framework error name when available). */
|
|
115
|
-
|
|
119
|
+
export type TraceSpanError = {
|
|
120
|
+
/** Stable error identifier (typed framework error name when available). */
|
|
121
|
+
type: string;
|
|
122
|
+
/** Human-readable message — secret-scrubbed before storage (S4). */
|
|
123
|
+
message: string;
|
|
124
|
+
/** Optional captured stack trace, secret-scrubbed when present (S4). */
|
|
116
125
|
stack?: string;
|
|
117
126
|
/**
|
|
118
127
|
* Optional redacted view of the error's `cause` (S4) — a raw provider
|
|
@@ -128,13 +137,22 @@ type TraceSpanError = {
|
|
|
128
137
|
* `Trace` corresponds to one outermost `BaseReport` (a single
|
|
129
138
|
* `.execute()` / `.invoke()` call).
|
|
130
139
|
*/
|
|
131
|
-
type Trace = {
|
|
132
|
-
/**
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
140
|
+
export type Trace = {
|
|
141
|
+
/** HTTP request correlation supplied by an integration; independent of sessionId. */
|
|
142
|
+
requestId?: string;
|
|
143
|
+
/** Trace id — equals the root span's `traceId` / `rootRunId`. */
|
|
144
|
+
traceId: string;
|
|
145
|
+
/** Session grouping id, when the originating run carried one. */
|
|
146
|
+
sessionId?: string;
|
|
147
|
+
/** Root span of this trace. */
|
|
148
|
+
root: TraceSpan;
|
|
149
|
+
/** Wall-clock start of the trace (root span `startedAt`). */
|
|
150
|
+
startedAt: string;
|
|
151
|
+
/** Wall-clock end of the trace (root span `endedAt`). */
|
|
152
|
+
endedAt: string;
|
|
153
|
+
/** Total trace duration in milliseconds (root span `duration`). */
|
|
154
|
+
duration: number;
|
|
155
|
+
/** Trace-wide usage rollup (root span `usage`). */
|
|
138
156
|
usage: Usage;
|
|
139
157
|
/**
|
|
140
158
|
* Wire-format version of the source `BaseReport` shape, mirrored from
|
|
@@ -144,5 +162,4 @@ type Trace = {
|
|
|
144
162
|
reportSchemaVersion?: number;
|
|
145
163
|
};
|
|
146
164
|
//#endregion
|
|
147
|
-
export { Trace, TraceSpan, TraceSpanError };
|
|
148
165
|
//# sourceMappingURL=trace.type.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trace.type.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/contracts/trace.type.ts"],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"trace.type.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/contracts/trace.type.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAgCY;;;;;EAKV;;;;;EAKA;;;;;;;EAOA;;;;;;EAMA;;EAEA;;;;;;EAMA;;EAEA,MAAM;;EAEN,QAAQ;;EAER;;EAEA;;EAEA;;EAEA,OAAO;;;;;;EAMP,QAAQ;;;;;;;;;;EAUR,aAAa;;;;;;;;EAQb;;;;;;EAMA;;;;;;EAMA,UAAU;;;;;;;YAQA;;EAEV;;EAEA;;EAEA;;;;;;;EAOA;;;;;;;;YASU;;EAEV;;EAEA;;EAEA;;EAEA,MAAM;;EAEN;;EAEA;;EAEA;;EAEA,OAAO;;;;;;EAMP"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { TraceStoreContract } from "../store/trace-store.contract.mjs";
|
|
2
2
|
import { DashboardHandle, DashboardOptions } from "./dashboard.type.mjs";
|
|
3
|
-
|
|
4
3
|
//#region ../../packages/ai-panoptic/src/dashboard/dashboard.d.ts
|
|
5
4
|
/**
|
|
6
5
|
* Start the local Panoptic dashboard over a {@link TraceStoreContract} —
|
|
@@ -26,7 +25,6 @@ import { DashboardHandle, DashboardOptions } from "./dashboard.type.mjs";
|
|
|
26
25
|
* // ...later:
|
|
27
26
|
* await handle.close();
|
|
28
27
|
*/
|
|
29
|
-
declare function dashboard(store: TraceStoreContract, options?: DashboardOptions): Promise<DashboardHandle>;
|
|
28
|
+
export declare function dashboard(store: TraceStoreContract, options?: DashboardOptions): Promise<DashboardHandle>;
|
|
30
29
|
//#endregion
|
|
31
|
-
export { dashboard };
|
|
32
30
|
//# sourceMappingURL=dashboard.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dashboard.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/dashboard.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"dashboard.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/dashboard.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;wBAiCgB,UACd,OAAO,oBACP,UAAS,mBACR,QAAQ"}
|
|
@@ -36,13 +36,14 @@ function dashboard(store, options = {}) {
|
|
|
36
36
|
const basePath = normalizeBasePath(options.basePath);
|
|
37
37
|
if (!isLoopbackHost(host) && !options.authToken) return Promise.reject(/* @__PURE__ */ new Error(`Panoptic dashboard: binding to a non-loopback host ("${host}") requires an \`authToken\` (the dashboard exposes raw prompt content). Pass \`authToken\`, or bind to 127.0.0.1.`));
|
|
38
38
|
const allowedHosts = (options.allowedHosts ?? defaultAllowedHosts(host)).map((h) => h.toLowerCase());
|
|
39
|
-
const
|
|
39
|
+
const handler = createRequestHandler(store, {
|
|
40
40
|
basePath,
|
|
41
41
|
title,
|
|
42
42
|
authToken: options.authToken,
|
|
43
43
|
allowedHosts,
|
|
44
44
|
evaluate: options.evaluate
|
|
45
|
-
})
|
|
45
|
+
});
|
|
46
|
+
const server = createServer(handler);
|
|
46
47
|
return new Promise((resolve, reject) => {
|
|
47
48
|
const onError = (error) => {
|
|
48
49
|
server.off("error", onError);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dashboard.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/dashboard.ts"],"sourcesContent":["import { createServer } from \"node:http\";\nimport type { TraceStoreContract } from \"../store/trace-store.contract\";\nimport type { DashboardHandle, DashboardOptions } from \"./dashboard.type\";\nimport { createRequestHandler } from \"./serve\";\n\nconst DEFAULT_PORT = 4319;\nconst DEFAULT_HOST = \"127.0.0.1\";\nconst DEFAULT_TITLE = \"Panoptic\";\n\n/**\n * Start the local Panoptic dashboard over a {@link TraceStoreContract} —\n * a zero-dependency `node:http` server that serves a read-only JSON API\n * and one self-contained HTML page polling it. The store is the live\n * object the collector fills, so each poll reflects the latest completed\n * traces with no extra wiring.\n *\n * This is a low-level building block. The documented path is\n * `ai.config({ panoptic: { dashboard } })`, which constructs (or reuses)\n * the store and calls this for you. Use it directly only when you manage\n * the store yourself.\n *\n * Binds loopback-only by default (`127.0.0.1`) so prompt content is never\n * exposed to the LAN. Pass `port: 0` for an ephemeral port — the resolved\n * port comes back on the handle. A port already in use rejects with a\n * clear, actionable `Error` rather than the raw `EADDRINUSE`.\n *\n * @example\n * const store = createInMemoryTraceStore();\n * const handle = await dashboard(store, { port: 4319, open: true });\n * console.log(handle.url); // http://127.0.0.1:4319/\n * // ...later:\n * await handle.close();\n */\nexport function dashboard(\n store: TraceStoreContract,\n options: DashboardOptions = {},\n): Promise<DashboardHandle> {\n const port = options.port ?? DEFAULT_PORT;\n const host = options.host ?? DEFAULT_HOST;\n const title = options.title ?? DEFAULT_TITLE;\n const basePath = normalizeBasePath(options.basePath);\n\n // Secure-by-default off loopback (S4): a non-loopback bind without an\n // auth token would expose raw prompt content to the network, so refuse\n // to start rather than silently exposing it.\n if (!isLoopbackHost(host) && !options.authToken) {\n return Promise.reject(\n new Error(\n `Panoptic dashboard: binding to a non-loopback host (\"${host}\") requires an \\`authToken\\` ` +\n \"(the dashboard exposes raw prompt content). Pass `authToken`, or bind to 127.0.0.1.\",\n ),\n );\n }\n\n const allowedHosts = (options.allowedHosts ?? defaultAllowedHosts(host)).map((h) =>\n h.toLowerCase(),\n );\n\n const handler = createRequestHandler(store, {\n basePath,\n title,\n authToken: options.authToken,\n allowedHosts,\n evaluate: options.evaluate,\n });\n const server = createServer(handler);\n\n return new Promise<DashboardHandle>((resolve, reject) => {\n const onError = (error: NodeJS.ErrnoException): void => {\n server.off(\"error\", onError);\n\n if (error.code === \"EADDRINUSE\") {\n reject(\n new Error(\n `Panoptic dashboard: port ${port} in use; pass { port: 0 } for an ephemeral port`,\n ),\n );\n\n return;\n }\n\n reject(error);\n };\n\n server.on(\"error\", onError);\n\n server.listen(port, host, () => {\n server.off(\"error\", onError);\n\n const address = server.address();\n const resolvedPort = typeof address === \"object\" && address !== null ? address.port : port;\n const url = `http://${host}:${resolvedPort}${basePath}`;\n\n if (options.open) {\n openBrowser(url);\n }\n\n resolve({\n url,\n port: resolvedPort,\n close(): Promise<void> {\n return new Promise<void>((closeResolve, closeReject) => {\n server.close((closeError) => {\n if (closeError) {\n closeReject(closeError);\n\n return;\n }\n\n closeResolve();\n });\n });\n },\n });\n });\n });\n}\n\n/** Loopback hosts the dashboard may bind without an auth token. */\nfunction isLoopbackHost(host: string): boolean {\n const h = host.toLowerCase();\n return h === \"127.0.0.1\" || h === \"::1\" || h === \"[::1]\" || h === \"localhost\";\n}\n\n/**\n * Default `Host` allowlist for a given bind host. A loopback bind accepts\n * the loopback names a browser would send; a non-loopback bind defaults to\n * just the bound host (callers can widen via `allowedHosts`).\n */\nfunction defaultAllowedHosts(host: string): string[] {\n if (isLoopbackHost(host)) {\n return [\"localhost\", \"127.0.0.1\", \"[::1]\", \"::1\"];\n }\n return [host];\n}\n\n/**\n * Normalize a caller `basePath` to a leading-and-trailing-slash form the\n * router can prefix routes with. `undefined` / `\"\"` → `\"/\"`.\n */\nfunction normalizeBasePath(basePath?: string): string {\n if (basePath === undefined || basePath.length === 0 || basePath === \"/\") {\n return \"/\";\n }\n\n const withLeading = basePath.startsWith(\"/\") ? basePath : `/${basePath}`;\n\n return withLeading.endsWith(\"/\") ? withLeading : `${withLeading}/`;\n}\n\n/**\n * Best-effort open of the default browser at `url`. Fire-and-forget and\n * fully swallowed — failing to open a browser must never reject the\n * dashboard start. Uses the platform's native opener via a lazy\n * `node:child_process` import so the dependency is paid only when\n * `open: true`.\n */\nfunction openBrowser(url: string): void {\n void import(\"node:child_process\")\n .then(({ spawn }) => {\n const command =\n process.platform === \"win32\" ? \"cmd\" : process.platform === \"darwin\" ? \"open\" : \"xdg-open\";\n const args = process.platform === \"win32\" ? [\"/c\", \"start\", \"\", url] : [url];\n\n const child = spawn(command, args, { stdio: \"ignore\", detached: true });\n child.on(\"error\", () => {\n // Swallow — opening a browser is best-effort.\n });\n child.unref();\n })\n .catch(() => {\n // Swallow — opening a browser is best-effort.\n });\n}\n"],"mappings":";;;;AAKA,MAAM,eAAe;AACrB,MAAM,eAAe;AACrB,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;AA0BtB,SAAgB,UACd,OACA,UAA4B,CAAC,GACH;CAC1B,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,WAAW,kBAAkB,QAAQ,QAAQ;CAKnD,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,QAAQ,WACpC,OAAO,QAAQ,uBACb,IAAI,MACF,wDAAwD,KAAK,mHAE/D,CACF;CAGF,MAAM,gBAAgB,QAAQ,gBAAgB,oBAAoB,IAAI,EAAC,CAAE,KAAK,MAC5E,EAAE,YAAY,CAChB;
|
|
1
|
+
{"version":3,"file":"dashboard.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/dashboard.ts"],"sourcesContent":["import { createServer } from \"node:http\";\nimport type { TraceStoreContract } from \"../store/trace-store.contract\";\nimport type { DashboardHandle, DashboardOptions } from \"./dashboard.type\";\nimport { createRequestHandler } from \"./serve\";\n\nconst DEFAULT_PORT = 4319;\nconst DEFAULT_HOST = \"127.0.0.1\";\nconst DEFAULT_TITLE = \"Panoptic\";\n\n/**\n * Start the local Panoptic dashboard over a {@link TraceStoreContract} —\n * a zero-dependency `node:http` server that serves a read-only JSON API\n * and one self-contained HTML page polling it. The store is the live\n * object the collector fills, so each poll reflects the latest completed\n * traces with no extra wiring.\n *\n * This is a low-level building block. The documented path is\n * `ai.config({ panoptic: { dashboard } })`, which constructs (or reuses)\n * the store and calls this for you. Use it directly only when you manage\n * the store yourself.\n *\n * Binds loopback-only by default (`127.0.0.1`) so prompt content is never\n * exposed to the LAN. Pass `port: 0` for an ephemeral port — the resolved\n * port comes back on the handle. A port already in use rejects with a\n * clear, actionable `Error` rather than the raw `EADDRINUSE`.\n *\n * @example\n * const store = createInMemoryTraceStore();\n * const handle = await dashboard(store, { port: 4319, open: true });\n * console.log(handle.url); // http://127.0.0.1:4319/\n * // ...later:\n * await handle.close();\n */\nexport function dashboard(\n store: TraceStoreContract,\n options: DashboardOptions = {},\n): Promise<DashboardHandle> {\n const port = options.port ?? DEFAULT_PORT;\n const host = options.host ?? DEFAULT_HOST;\n const title = options.title ?? DEFAULT_TITLE;\n const basePath = normalizeBasePath(options.basePath);\n\n // Secure-by-default off loopback (S4): a non-loopback bind without an\n // auth token would expose raw prompt content to the network, so refuse\n // to start rather than silently exposing it.\n if (!isLoopbackHost(host) && !options.authToken) {\n return Promise.reject(\n new Error(\n `Panoptic dashboard: binding to a non-loopback host (\"${host}\") requires an \\`authToken\\` ` +\n \"(the dashboard exposes raw prompt content). Pass `authToken`, or bind to 127.0.0.1.\",\n ),\n );\n }\n\n const allowedHosts = (options.allowedHosts ?? defaultAllowedHosts(host)).map((h) =>\n h.toLowerCase(),\n );\n\n const handler = createRequestHandler(store, {\n basePath,\n title,\n authToken: options.authToken,\n allowedHosts,\n evaluate: options.evaluate,\n });\n const server = createServer(handler);\n\n return new Promise<DashboardHandle>((resolve, reject) => {\n const onError = (error: NodeJS.ErrnoException): void => {\n server.off(\"error\", onError);\n\n if (error.code === \"EADDRINUSE\") {\n reject(\n new Error(\n `Panoptic dashboard: port ${port} in use; pass { port: 0 } for an ephemeral port`,\n ),\n );\n\n return;\n }\n\n reject(error);\n };\n\n server.on(\"error\", onError);\n\n server.listen(port, host, () => {\n server.off(\"error\", onError);\n\n const address = server.address();\n const resolvedPort = typeof address === \"object\" && address !== null ? address.port : port;\n const url = `http://${host}:${resolvedPort}${basePath}`;\n\n if (options.open) {\n openBrowser(url);\n }\n\n resolve({\n url,\n port: resolvedPort,\n close(): Promise<void> {\n return new Promise<void>((closeResolve, closeReject) => {\n server.close((closeError) => {\n if (closeError) {\n closeReject(closeError);\n\n return;\n }\n\n closeResolve();\n });\n });\n },\n });\n });\n });\n}\n\n/** Loopback hosts the dashboard may bind without an auth token. */\nfunction isLoopbackHost(host: string): boolean {\n const h = host.toLowerCase();\n return h === \"127.0.0.1\" || h === \"::1\" || h === \"[::1]\" || h === \"localhost\";\n}\n\n/**\n * Default `Host` allowlist for a given bind host. A loopback bind accepts\n * the loopback names a browser would send; a non-loopback bind defaults to\n * just the bound host (callers can widen via `allowedHosts`).\n */\nfunction defaultAllowedHosts(host: string): string[] {\n if (isLoopbackHost(host)) {\n return [\"localhost\", \"127.0.0.1\", \"[::1]\", \"::1\"];\n }\n return [host];\n}\n\n/**\n * Normalize a caller `basePath` to a leading-and-trailing-slash form the\n * router can prefix routes with. `undefined` / `\"\"` → `\"/\"`.\n */\nfunction normalizeBasePath(basePath?: string): string {\n if (basePath === undefined || basePath.length === 0 || basePath === \"/\") {\n return \"/\";\n }\n\n const withLeading = basePath.startsWith(\"/\") ? basePath : `/${basePath}`;\n\n return withLeading.endsWith(\"/\") ? withLeading : `${withLeading}/`;\n}\n\n/**\n * Best-effort open of the default browser at `url`. Fire-and-forget and\n * fully swallowed — failing to open a browser must never reject the\n * dashboard start. Uses the platform's native opener via a lazy\n * `node:child_process` import so the dependency is paid only when\n * `open: true`.\n */\nfunction openBrowser(url: string): void {\n void import(\"node:child_process\")\n .then(({ spawn }) => {\n const command =\n process.platform === \"win32\" ? \"cmd\" : process.platform === \"darwin\" ? \"open\" : \"xdg-open\";\n const args = process.platform === \"win32\" ? [\"/c\", \"start\", \"\", url] : [url];\n\n const child = spawn(command, args, { stdio: \"ignore\", detached: true });\n child.on(\"error\", () => {\n // Swallow — opening a browser is best-effort.\n });\n child.unref();\n })\n .catch(() => {\n // Swallow — opening a browser is best-effort.\n });\n}\n"],"mappings":";;;;AAKA,MAAM,eAAe;AACrB,MAAM,eAAe;AACrB,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;AA0BtB,SAAgB,UACd,OACA,UAA4B,CAAC,GACH;CAC1B,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,WAAW,kBAAkB,QAAQ,QAAQ;CAKnD,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,QAAQ,WACpC,OAAO,QAAQ,uBACb,IAAI,MACF,wDAAwD,KAAK,mHAE/D,CACF;CAGF,MAAM,gBAAgB,QAAQ,gBAAgB,oBAAoB,IAAI,EAAC,CAAE,KAAK,MAC5E,EAAE,YAAY,CAChB;CAEA,MAAM,UAAU,qBAAqB,OAAO;EAC1C;EACA;EACA,WAAW,QAAQ;EACnB;EACA,UAAU,QAAQ;CACpB,CAAC;CACD,MAAM,SAAS,aAAa,OAAO;CAEnC,OAAO,IAAI,SAA0B,SAAS,WAAW;EACvD,MAAM,WAAW,UAAuC;GACtD,OAAO,IAAI,SAAS,OAAO;GAE3B,IAAI,MAAM,SAAS,cAAc;IAC/B,uBACE,IAAI,MACF,4BAA4B,KAAK,gDACnC,CACF;IAEA;GACF;GAEA,OAAO,KAAK;EACd;EAEA,OAAO,GAAG,SAAS,OAAO;EAE1B,OAAO,OAAO,MAAM,YAAY;GAC9B,OAAO,IAAI,SAAS,OAAO;GAE3B,MAAM,UAAU,OAAO,QAAQ;GAC/B,MAAM,eAAe,OAAO,YAAY,YAAY,YAAY,OAAO,QAAQ,OAAO;GACtF,MAAM,MAAM,UAAU,KAAK,GAAG,eAAe;GAE7C,IAAI,QAAQ,MACV,YAAY,GAAG;GAGjB,QAAQ;IACN;IACA,MAAM;IACN,QAAuB;KACrB,OAAO,IAAI,SAAe,cAAc,gBAAgB;MACtD,OAAO,OAAO,eAAe;OAC3B,IAAI,YAAY;QACd,YAAY,UAAU;QAEtB;OACF;OAEA,aAAa;MACf,CAAC;KACH,CAAC;IACH;GACF,CAAC;EACH,CAAC;CACH,CAAC;AACH;;AAGA,SAAS,eAAe,MAAuB;CAC7C,MAAM,IAAI,KAAK,YAAY;CAC3B,OAAO,MAAM,eAAe,MAAM,SAAS,MAAM,WAAW,MAAM;AACpE;;;;;;AAOA,SAAS,oBAAoB,MAAwB;CACnD,IAAI,eAAe,IAAI,GACrB,OAAO;EAAC;EAAa;EAAa;EAAS;CAAK;CAElD,OAAO,CAAC,IAAI;AACd;;;;;AAMA,SAAS,kBAAkB,UAA2B;CACpD,IAAI,aAAa,UAAa,SAAS,WAAW,KAAK,aAAa,KAClE,OAAO;CAGT,MAAM,cAAc,SAAS,WAAW,GAAG,IAAI,WAAW,IAAI;CAE9D,OAAO,YAAY,SAAS,GAAG,IAAI,cAAc,GAAG,YAAY;AAClE;;;;;;;;AASA,SAAS,YAAY,KAAmB;CACtC,AAAK,OAAO,qBAAqB,CAC9B,MAAM,EAAE,YAAY;EAKnB,MAAM,QAAQ,MAHZ,QAAQ,aAAa,UAAU,QAAQ,QAAQ,aAAa,WAAW,SAAS,YACrE,QAAQ,aAAa,UAAU;GAAC;GAAM;GAAS;GAAI;EAAG,IAAI,CAAC,GAAG,GAExC;GAAE,OAAO;GAAU,UAAU;EAAK,CAAC;EACtE,MAAM,GAAG,eAAe,CAExB,CAAC;EACD,MAAM,MAAM;CACd,CAAC,CAAC,CACD,YAAY,CAEb,CAAC;AACL"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { EvaluateConfig } from "../evaluate/evaluate.type.mjs";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/dashboard/dashboard.type.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Options for the local Panoptic {@link dashboard} server.
|
|
@@ -12,7 +11,7 @@ import { EvaluateConfig } from "../evaluate/evaluate.type.mjs";
|
|
|
12
11
|
* {@link DashboardOptions.evaluate} enables. There is no auth or persistence
|
|
13
12
|
* by default either.
|
|
14
13
|
*/
|
|
15
|
-
type DashboardOptions = {
|
|
14
|
+
export type DashboardOptions = {
|
|
16
15
|
/**
|
|
17
16
|
* TCP port to listen on. Default `4319`. Pass `0` for an ephemeral
|
|
18
17
|
* port — the OS-assigned port is then returned on
|
|
@@ -35,7 +34,8 @@ type DashboardOptions = {
|
|
|
35
34
|
* Open the default browser at the resolved URL once the server is
|
|
36
35
|
* listening. Default `false`.
|
|
37
36
|
*/
|
|
38
|
-
open?: boolean;
|
|
37
|
+
open?: boolean;
|
|
38
|
+
/** Title shown in the dashboard header. Default `"Panoptic"`. */
|
|
39
39
|
title?: string;
|
|
40
40
|
/**
|
|
41
41
|
* Bearer token required on every request (S4). When set, a request must
|
|
@@ -66,11 +66,13 @@ type DashboardOptions = {
|
|
|
66
66
|
* Carries the resolved URL + port (useful when `port: 0` requested an
|
|
67
67
|
* ephemeral one) and a `close()` to release the port.
|
|
68
68
|
*/
|
|
69
|
-
type DashboardHandle = {
|
|
70
|
-
/** Resolved URL the UI is served at (`http://host:port{basePath}`). */
|
|
71
|
-
readonly
|
|
69
|
+
export type DashboardHandle = {
|
|
70
|
+
/** Resolved URL the UI is served at (`http://host:port{basePath}`). */
|
|
71
|
+
readonly url: string;
|
|
72
|
+
/** Resolved port — the real OS-assigned port when `port: 0` was passed. */
|
|
73
|
+
readonly port: number;
|
|
74
|
+
/** Stop the server and release the port. Resolves once fully closed. */
|
|
72
75
|
close(): Promise<void>;
|
|
73
76
|
};
|
|
74
77
|
//#endregion
|
|
75
|
-
export { DashboardHandle, DashboardOptions };
|
|
76
78
|
//# sourceMappingURL=dashboard.type.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dashboard.type.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/dashboard.type.ts"],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"dashboard.type.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/dashboard.type.ts"],"mappings":";;;;;;;;;;;;;YAaY;;;;;;EAMV;;;;;;EAMA;;;;;;EAMA;;;;;EAKA;;EAEA;;;;;;;EAOA;;;;;;;EAOA;;;;;;;;;EASA,WAAW;;;;;;;YAQD;;WAED;;WAEA;;EAET,SAAS"}
|
|
@@ -1,3 +1,6 @@
|
|
|
1
1
|
import { DashboardHandle, DashboardOptions } from "./dashboard.type.mjs";
|
|
2
2
|
import { dashboard } from "./dashboard.mjs";
|
|
3
|
-
import
|
|
3
|
+
import "./serve.mjs";
|
|
4
|
+
import "./parse-query.mjs";
|
|
5
|
+
import { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, PromptGroup, SessionGroup, TraceFilter, TypeGroup, TypeStat, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey } from "./trace-filter.mjs";
|
|
6
|
+
export { type DashboardHandle, type DashboardOptions, dashboard };
|
package/esm/dashboard/index.mjs
CHANGED
|
@@ -4,4 +4,4 @@ import { createRequestHandler } from "./serve.mjs";
|
|
|
4
4
|
import { dashboard } from "./dashboard.mjs";
|
|
5
5
|
import { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey } from "./trace-filter.mjs";
|
|
6
6
|
|
|
7
|
-
export {
|
|
7
|
+
export { dashboard };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
import "../store/trace-query.type.mjs";
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
import "../store/trace-store.contract.mjs";
|
|
2
|
+
import "../evaluate/evaluate.type.mjs";
|
package/esm/dashboard/serve.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { timingSafeEqual } from "node:crypto";
|
|
|
7
7
|
|
|
8
8
|
//#region ../../packages/ai-panoptic/src/dashboard/serve.ts
|
|
9
9
|
/** Hard cap on the evaluate route's request body — well beyond a real instructions string. */
|
|
10
|
-
const MAX_EVALUATE_BODY_BYTES =
|
|
10
|
+
const MAX_EVALUATE_BODY_BYTES = 65536;
|
|
11
11
|
/**
|
|
12
12
|
* Security response headers added to every dashboard response (S4):
|
|
13
13
|
* block MIME-sniffing, framing, referrer leakage, and lock the page's
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serve.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/serve.ts"],"sourcesContent":["import { timingSafeEqual } from \"node:crypto\";\nimport type { IncomingMessage, ServerResponse } from \"node:http\";\nimport type { EvaluateConfig } from \"../evaluate/evaluate.type\";\nimport { evaluateSystemPrompt } from \"../evaluate/evaluate-system-prompt\";\nimport { extractLastSystemPrompt } from \"../evaluate/extract-last-system-prompt\";\nimport { findSpanById } from \"../evaluate/find-span-by-id\";\nimport type { TraceStoreContract } from \"../store/trace-store.contract\";\nimport { parseQuery } from \"./parse-query\";\nimport { dashboardHtml } from \"./ui.html\";\n\n/** Fully-resolved routing config the request handler closes over. */\nexport type ServeConfig = {\n /** Normalized mount path, always ending in `/` (e.g. `\"/\"`). */\n basePath: string;\n /** Header title baked into the served page. */\n title: string;\n /** Bearer token required on every request when set (S4). */\n authToken?: string;\n /** `Host` header allowlist — defends against DNS-rebinding (S4). */\n allowedHosts: string[];\n /**\n * Enables the one write route: `POST\n * {basePath}api/traces/:traceId/spans/:spanId/evaluate`. Absent ⇒ the\n * route 405s (POST isn't accepted by any route) and the served page never\n * renders the Evaluate button.\n */\n evaluate?: EvaluateConfig;\n};\n\n/** Request body accepted by the evaluate route — everything optional. */\ntype EvaluateRequestBody = {\n /** Per-run instructions override; falls back to `config.evaluate.instructions`. */\n instructions?: string;\n};\n\n/** Hard cap on the evaluate route's request body — well beyond a real instructions string. */\nconst MAX_EVALUATE_BODY_BYTES = 64 * 1024;\n\n/**\n * Security response headers added to every dashboard response (S4):\n * block MIME-sniffing, framing, referrer leakage, and lock the page's\n * content sources down to itself (the UI is fully self-contained — no CDN).\n */\nconst SECURITY_HEADERS: Record<string, string> = {\n \"x-content-type-options\": \"nosniff\",\n \"x-frame-options\": \"DENY\",\n \"referrer-policy\": \"no-referrer\",\n \"content-security-policy\":\n \"default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'\",\n};\n\n/** Extract the host (no port) from a `Host` header value. */\nfunction hostHeaderName(hostHeader: string | undefined): string | undefined {\n if (!hostHeader) return undefined;\n // IPv6 literal `[::1]:4319` → `[::1]`; otherwise strip `:port`.\n if (hostHeader.startsWith(\"[\")) {\n return hostHeader.slice(0, hostHeader.indexOf(\"]\") + 1);\n }\n const colon = hostHeader.indexOf(\":\");\n return colon === -1 ? hostHeader : hostHeader.slice(0, colon);\n}\n\n/**\n * Constant-time string equality (S4) — guards against a timing\n * side-channel that could otherwise let a network-adjacent attacker\n * recover the token byte-by-byte via repeated timed guesses. Plain `===`\n * short-circuits on the first differing byte, so response latency leaks\n * how many leading bytes matched; `timingSafeEqual` does not. Lengths are\n * compared first (a length mismatch is not secret and `timingSafeEqual`\n * requires equal-length buffers anyway).\n */\nfunction constantTimeEqual(a: string, b: string): boolean {\n const bufA = Buffer.from(a);\n const bufB = Buffer.from(b);\n\n if (bufA.length !== bufB.length) return false;\n\n return timingSafeEqual(bufA, bufB);\n}\n\n/**\n * Bearer-token check (S4): `Authorization: Bearer <token>` header, or\n * (only when `allowQueryToken`) a `?token=` query param.\n *\n * The query-string form only exists for the one request that structurally\n * cannot carry a custom header — the initial browser navigation that\n * loads the HTML shell (a typed/clicked/bookmarked URL). Every other\n * request the served page makes is same-origin `fetch()`, which can and\n * does set `Authorization` (see `ui.html.ts`'s `fetchAuthed`), so the API\n * routes never need to accept a query-string token. Restricting the\n * fallback to just the page route minimizes the token's exposure in\n * server access logs, proxy logs, and browser history to a single route\n * instead of every poll.\n */\nfunction isAuthorized(\n req: IncomingMessage,\n url: URL,\n token: string,\n allowQueryToken: boolean,\n): boolean {\n const header = req.headers.authorization;\n if (header && constantTimeEqual(header, `Bearer ${token}`)) return true;\n if (!allowQueryToken) return false;\n const queryToken = url.searchParams.get(\"token\");\n return queryToken !== null && constantTimeEqual(queryToken, token);\n}\n\n/**\n * Build the `node:http` request handler for the dashboard over a given\n * trace store. Kept separate from the server lifecycle so it can be unit\n * tested by feeding it a fake `req`/`res` without binding a port.\n *\n * Routes (all under `config.basePath`):\n *\n * - `GET api/traces` → `store.query(parseQuery(searchParams))`\n * - `GET api/traces/:id` → `store.get(id)` or `404`\n * - `GET api/aggregate` → `store.aggregate(parseQuery(searchParams))`\n * - `GET {basePath}` → the self-contained HTML page\n * - `POST api/traces/:traceId/spans/:spanId/evaluate` → ONLY when\n * `config.evaluate` is set; grades the span's last captured system\n * prompt and returns a {@link EvaluateVerdict}. The dashboard's one\n * write route — absent config, POST 405s like it would against any\n * other route (the path is never even pattern-matched).\n *\n * Anything else → `404`. A method the matched route doesn't accept → `405`.\n * Host allowlist + bearer-token auth (S4) are checked for EVERY request,\n * regardless of method, before any routing. The store shapes\n * ({@link Trace}/{@link TraceSpan}) are already JSON-safe, so GET responses\n * are a plain `JSON.stringify` with no serializer.\n *\n * @example\n * const handler = createRequestHandler(store, { basePath: \"/\", title: \"Panoptic\" });\n * http.createServer(handler).listen(4319, \"127.0.0.1\");\n */\nexport function createRequestHandler(\n store: TraceStoreContract,\n config: ServeConfig,\n): (req: IncomingMessage, res: ServerResponse) => void {\n const base = config.basePath;\n const apiPrefix = `${base}api`;\n\n return function handle(req: IncomingMessage, res: ServerResponse): void {\n // `req.url` is path + query only; a dummy origin lets URL parse it.\n const url = new URL(req.url ?? \"/\", \"http://localhost\");\n const pathname = url.pathname;\n\n // Host-header allowlist — blocks DNS-rebinding attacks that point a\n // hostile domain at this loopback port (S4). Checked for every method.\n const host = hostHeaderName(req.headers.host);\n if (!host || !config.allowedHosts.includes(host.toLowerCase())) {\n sendJson(res, 403, { error: \"host_not_allowed\" });\n\n return;\n }\n\n // Bearer-token auth when configured (always required off loopback, S4).\n // `?token=` is only honored on the HTML page route — see `isAuthorized`.\n const isPageRoute = pathname === base || pathname === base.replace(/\\/$/, \"\");\n if (config.authToken && !isAuthorized(req, url, config.authToken, isPageRoute)) {\n sendJson(res, 401, { error: \"unauthorized\" });\n\n return;\n }\n\n // The one write route — opt-in via `config.evaluate`, so it must be\n // checked before the blanket GET-only gate below.\n if (req.method === \"POST\" && config.evaluate) {\n const match = matchEvaluatePath(pathname, apiPrefix);\n\n if (match) {\n void handleEvaluate(store, config.evaluate, match.traceId, match.spanId, req, res);\n\n return;\n }\n }\n\n if (req.method !== \"GET\") {\n sendJson(res, 405, { error: \"method_not_allowed\" });\n\n return;\n }\n\n if (pathname === `${apiPrefix}/traces`) {\n sendJson(res, 200, store.query(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname.startsWith(`${apiPrefix}/traces/`)) {\n const traceId = decodeURIComponent(pathname.slice(`${apiPrefix}/traces/`.length));\n const trace = traceId.length > 0 ? store.get(traceId) : undefined;\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n sendJson(res, 200, trace);\n\n return;\n }\n\n if (pathname === `${apiPrefix}/aggregate`) {\n sendJson(res, 200, store.aggregate(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname === base || pathname === base.replace(/\\/$/, \"\")) {\n res.writeHead(200, {\n \"content-type\": \"text/html; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(\n dashboardHtml(\n base,\n config.title,\n Boolean(config.evaluate),\n config.evaluate?.instructions ?? \"\",\n ),\n );\n\n return;\n }\n\n sendJson(res, 404, { error: \"not_found\" });\n };\n}\n\n/** Write a JSON response with the given status code and security headers. */\nfunction sendJson(res: ServerResponse, status: number, body: unknown): void {\n res.writeHead(status, {\n \"content-type\": \"application/json; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(JSON.stringify(body));\n}\n\n/**\n * Match `{apiPrefix}/traces/:traceId/spans/:spanId/evaluate` — the one\n * write route. Returns `undefined` for anything else, including the plain\n * `{apiPrefix}/traces/:id` read route (no `/spans/.../evaluate` suffix),\n * so the two never collide.\n */\nfunction matchEvaluatePath(\n pathname: string,\n apiPrefix: string,\n): { traceId: string; spanId: string } | undefined {\n const prefix = `${apiPrefix}/traces/`;\n\n if (!pathname.startsWith(prefix)) {\n return undefined;\n }\n\n const match = /^([^/]+)\\/spans\\/([^/]+)\\/evaluate$/.exec(pathname.slice(prefix.length));\n\n if (!match) {\n return undefined;\n }\n\n return {\n traceId: decodeURIComponent(match[1]),\n spanId: decodeURIComponent(match[2]),\n };\n}\n\n/**\n * Collect and JSON-parse a request body, capped at\n * {@link MAX_EVALUATE_BODY_BYTES} so an oversized body can't hold the\n * connection open indefinitely. An empty body resolves to `undefined` —\n * the evaluate route treats that as \"no per-run override\".\n */\nfunction readJsonBody<T>(req: IncomingMessage): Promise<T | undefined> {\n return new Promise((resolve, reject) => {\n const chunks: Buffer[] = [];\n let size = 0;\n\n req.on(\"data\", (chunk: Buffer) => {\n size += chunk.length;\n\n if (size > MAX_EVALUATE_BODY_BYTES) {\n req.destroy();\n reject(new Error(\"payload_too_large\"));\n\n return;\n }\n\n chunks.push(chunk);\n });\n\n req.on(\"end\", () => {\n if (chunks.length === 0) {\n resolve(undefined);\n\n return;\n }\n\n try {\n resolve(JSON.parse(Buffer.concat(chunks).toString(\"utf-8\")) as T);\n } catch {\n reject(new Error(\"invalid_json\"));\n }\n });\n\n req.on(\"error\", reject);\n });\n}\n\n/**\n * Handle `POST {apiPrefix}/traces/:traceId/spans/:spanId/evaluate`. Looks\n * up the trace + span, extracts its last captured system prompt, and\n * grades it via {@link evaluateSystemPrompt}. `judgePromptBody` itself\n * never throws (a broken judge degrades to an issues-only outcome) — the\n * try/catch here only guards `config.evaluate.model` resolution, the one\n * step that CAN throw (e.g. a factory constructing an SDK client).\n */\nasync function handleEvaluate(\n store: TraceStoreContract,\n evaluate: EvaluateConfig,\n traceId: string,\n spanId: string,\n req: IncomingMessage,\n res: ServerResponse,\n): Promise<void> {\n const trace = store.get(traceId);\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n const span = findSpanById(trace.root, spanId);\n\n if (span === undefined) {\n sendJson(res, 404, { error: \"span_not_found\", spanId });\n\n return;\n }\n\n const systemPrompt = extractLastSystemPrompt(span);\n\n if (systemPrompt === undefined) {\n sendJson(res, 422, { error: \"no_system_prompt\" });\n\n return;\n }\n\n let body: EvaluateRequestBody | undefined;\n\n try {\n body = await readJsonBody<EvaluateRequestBody>(req);\n } catch (error) {\n const message = error instanceof Error ? error.message : \"invalid_json\";\n sendJson(res, message === \"payload_too_large\" ? 413 : 400, { error: message });\n\n return;\n }\n\n try {\n const verdict = await evaluateSystemPrompt(systemPrompt, evaluate, body?.instructions);\n sendJson(res, 200, verdict);\n } catch (error) {\n sendJson(res, 502, {\n error: \"evaluate_failed\",\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n"],"mappings":";;;;;;;;;AAoCA,MAAM,0BAA0B,KAAK;;;;;;AAOrC,MAAM,mBAA2C;CAC/C,0BAA0B;CAC1B,mBAAmB;CACnB,mBAAmB;CACnB,2BACE;AACJ;;AAGA,SAAS,eAAe,YAAoD;CAC1E,IAAI,CAAC,YAAY,OAAO;CAExB,IAAI,WAAW,WAAW,GAAG,GAC3B,OAAO,WAAW,MAAM,GAAG,WAAW,QAAQ,GAAG,IAAI,CAAC;CAExD,MAAM,QAAQ,WAAW,QAAQ,GAAG;CACpC,OAAO,UAAU,KAAK,aAAa,WAAW,MAAM,GAAG,KAAK;AAC9D;;;;;;;;;;AAWA,SAAS,kBAAkB,GAAW,GAAoB;CACxD,MAAM,OAAO,OAAO,KAAK,CAAC;CAC1B,MAAM,OAAO,OAAO,KAAK,CAAC;CAE1B,IAAI,KAAK,WAAW,KAAK,QAAQ,OAAO;CAExC,OAAO,gBAAgB,MAAM,IAAI;AACnC;;;;;;;;;;;;;;;AAgBA,SAAS,aACP,KACA,KACA,OACA,iBACS;CACT,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,UAAU,kBAAkB,QAAQ,UAAU,OAAO,GAAG,OAAO;CACnE,IAAI,CAAC,iBAAiB,OAAO;CAC7B,MAAM,aAAa,IAAI,aAAa,IAAI,OAAO;CAC/C,OAAO,eAAe,QAAQ,kBAAkB,YAAY,KAAK;AACnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,qBACd,OACA,QACqD;CACrD,MAAM,OAAO,OAAO;CACpB,MAAM,YAAY,GAAG,KAAK;CAE1B,OAAO,SAAS,OAAO,KAAsB,KAA2B;EAEtE,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,KAAK,kBAAkB;EACtD,MAAM,WAAW,IAAI;EAIrB,MAAM,OAAO,eAAe,IAAI,QAAQ,IAAI;EAC5C,IAAI,CAAC,QAAQ,CAAC,OAAO,aAAa,SAAS,KAAK,YAAY,CAAC,GAAG;GAC9D,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;GAEhD;EACF;EAIA,MAAM,cAAc,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE;EAC5E,IAAI,OAAO,aAAa,CAAC,aAAa,KAAK,KAAK,OAAO,WAAW,WAAW,GAAG;GAC9E,SAAS,KAAK,KAAK,EAAE,OAAO,eAAe,CAAC;GAE5C;EACF;EAIA,IAAI,IAAI,WAAW,UAAU,OAAO,UAAU;GAC5C,MAAM,QAAQ,kBAAkB,UAAU,SAAS;GAEnD,IAAI,OAAO;IACT,AAAK,eAAe,OAAO,OAAO,UAAU,MAAM,SAAS,MAAM,QAAQ,KAAK,GAAG;IAEjF;GACF;EACF;EAEA,IAAI,IAAI,WAAW,OAAO;GACxB,SAAS,KAAK,KAAK,EAAE,OAAO,qBAAqB,CAAC;GAElD;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,UAAU;GACtC,SAAS,KAAK,KAAK,MAAM,MAAM,WAAW,IAAI,YAAY,CAAC,CAAC;GAE5D;EACF;EAEA,IAAI,SAAS,WAAW,GAAG,UAAU,SAAS,GAAG;GAC/C,MAAM,UAAU,mBAAmB,SAAS,MAAM,GAAG,UAAU,UAAU,MAAM,CAAC;GAChF,MAAM,QAAQ,QAAQ,SAAS,IAAI,MAAM,IAAI,OAAO,IAAI;GAExD,IAAI,UAAU,QAAW;IACvB,SAAS,KAAK,KAAK;KAAE,OAAO;KAAmB;IAAQ,CAAC;IAExD;GACF;GAEA,SAAS,KAAK,KAAK,KAAK;GAExB;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,aAAa;GACzC,SAAS,KAAK,KAAK,MAAM,UAAU,WAAW,IAAI,YAAY,CAAC,CAAC;GAEhE;EACF;EAEA,IAAI,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE,GAAG;GAC7D,IAAI,UAAU,KAAK;IACjB,gBAAgB;IAChB,GAAG;GACL,CAAC;GACD,IAAI,IACF,cACE,MACA,OAAO,OACP,QAAQ,OAAO,QAAQ,GACvB,OAAO,UAAU,gBAAgB,EACnC,CACF;GAEA;EACF;EAEA,SAAS,KAAK,KAAK,EAAE,OAAO,YAAY,CAAC;CAC3C;AACF;;AAGA,SAAS,SAAS,KAAqB,QAAgB,MAAqB;CAC1E,IAAI,UAAU,QAAQ;EACpB,gBAAgB;EAChB,GAAG;CACL,CAAC;CACD,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC;AAC9B;;;;;;;AAQA,SAAS,kBACP,UACA,WACiD;CACjD,MAAM,SAAS,GAAG,UAAU;CAE5B,IAAI,CAAC,SAAS,WAAW,MAAM,GAC7B;CAGF,MAAM,QAAQ,sCAAsC,KAAK,SAAS,MAAM,OAAO,MAAM,CAAC;CAEtF,IAAI,CAAC,OACH;CAGF,OAAO;EACL,SAAS,mBAAmB,MAAM,EAAE;EACpC,QAAQ,mBAAmB,MAAM,EAAE;CACrC;AACF;;;;;;;AAQA,SAAS,aAAgB,KAA8C;CACrE,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,MAAM,SAAmB,CAAC;EAC1B,IAAI,OAAO;EAEX,IAAI,GAAG,SAAS,UAAkB;GAChC,QAAQ,MAAM;GAEd,IAAI,OAAO,yBAAyB;IAClC,IAAI,QAAQ;IACZ,uBAAO,IAAI,MAAM,mBAAmB,CAAC;IAErC;GACF;GAEA,OAAO,KAAK,KAAK;EACnB,CAAC;EAED,IAAI,GAAG,aAAa;GAClB,IAAI,OAAO,WAAW,GAAG;IACvB,QAAQ,MAAS;IAEjB;GACF;GAEA,IAAI;IACF,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM,CAAC,CAAC,SAAS,OAAO,CAAC,CAAM;GAClE,QAAQ;IACN,uBAAO,IAAI,MAAM,cAAc,CAAC;GAClC;EACF,CAAC;EAED,IAAI,GAAG,SAAS,MAAM;CACxB,CAAC;AACH;;;;;;;;;AAUA,eAAe,eACb,OACA,UACA,SACA,QACA,KACA,KACe;CACf,MAAM,QAAQ,MAAM,IAAI,OAAO;CAE/B,IAAI,UAAU,QAAW;EACvB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAmB;EAAQ,CAAC;EAExD;CACF;CAEA,MAAM,OAAO,aAAa,MAAM,MAAM,MAAM;CAE5C,IAAI,SAAS,QAAW;EACtB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAkB;EAAO,CAAC;EAEtD;CACF;CAEA,MAAM,eAAe,wBAAwB,IAAI;CAEjD,IAAI,iBAAiB,QAAW;EAC9B,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;EAEhD;CACF;CAEA,IAAI;CAEJ,IAAI;EACF,OAAO,MAAM,aAAkC,GAAG;CACpD,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;EACzD,SAAS,KAAK,YAAY,sBAAsB,MAAM,KAAK,EAAE,OAAO,QAAQ,CAAC;EAE7E;CACF;CAEA,IAAI;EAEF,SAAS,KAAK,KAAK,MADG,qBAAqB,cAAc,UAAU,MAAM,YAAY,CAC3D;CAC5B,SAAS,OAAO;EACd,SAAS,KAAK,KAAK;GACjB,OAAO;GACP,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAChE,CAAC;CACH;AACF"}
|
|
1
|
+
{"version":3,"file":"serve.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/serve.ts"],"sourcesContent":["import { timingSafeEqual } from \"node:crypto\";\nimport type { IncomingMessage, ServerResponse } from \"node:http\";\nimport type { EvaluateConfig } from \"../evaluate/evaluate.type\";\nimport { evaluateSystemPrompt } from \"../evaluate/evaluate-system-prompt\";\nimport { extractLastSystemPrompt } from \"../evaluate/extract-last-system-prompt\";\nimport { findSpanById } from \"../evaluate/find-span-by-id\";\nimport type { TraceStoreContract } from \"../store/trace-store.contract\";\nimport { parseQuery } from \"./parse-query\";\nimport { dashboardHtml } from \"./ui.html\";\n\n/** Fully-resolved routing config the request handler closes over. */\nexport type ServeConfig = {\n /** Normalized mount path, always ending in `/` (e.g. `\"/\"`). */\n basePath: string;\n /** Header title baked into the served page. */\n title: string;\n /** Bearer token required on every request when set (S4). */\n authToken?: string;\n /** `Host` header allowlist — defends against DNS-rebinding (S4). */\n allowedHosts: string[];\n /**\n * Enables the one write route: `POST\n * {basePath}api/traces/:traceId/spans/:spanId/evaluate`. Absent ⇒ the\n * route 405s (POST isn't accepted by any route) and the served page never\n * renders the Evaluate button.\n */\n evaluate?: EvaluateConfig;\n};\n\n/** Request body accepted by the evaluate route — everything optional. */\ntype EvaluateRequestBody = {\n /** Per-run instructions override; falls back to `config.evaluate.instructions`. */\n instructions?: string;\n};\n\n/** Hard cap on the evaluate route's request body — well beyond a real instructions string. */\nconst MAX_EVALUATE_BODY_BYTES = 64 * 1024;\n\n/**\n * Security response headers added to every dashboard response (S4):\n * block MIME-sniffing, framing, referrer leakage, and lock the page's\n * content sources down to itself (the UI is fully self-contained — no CDN).\n */\nconst SECURITY_HEADERS: Record<string, string> = {\n \"x-content-type-options\": \"nosniff\",\n \"x-frame-options\": \"DENY\",\n \"referrer-policy\": \"no-referrer\",\n \"content-security-policy\":\n \"default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'\",\n};\n\n/** Extract the host (no port) from a `Host` header value. */\nfunction hostHeaderName(hostHeader: string | undefined): string | undefined {\n if (!hostHeader) return undefined;\n // IPv6 literal `[::1]:4319` → `[::1]`; otherwise strip `:port`.\n if (hostHeader.startsWith(\"[\")) {\n return hostHeader.slice(0, hostHeader.indexOf(\"]\") + 1);\n }\n const colon = hostHeader.indexOf(\":\");\n return colon === -1 ? hostHeader : hostHeader.slice(0, colon);\n}\n\n/**\n * Constant-time string equality (S4) — guards against a timing\n * side-channel that could otherwise let a network-adjacent attacker\n * recover the token byte-by-byte via repeated timed guesses. Plain `===`\n * short-circuits on the first differing byte, so response latency leaks\n * how many leading bytes matched; `timingSafeEqual` does not. Lengths are\n * compared first (a length mismatch is not secret and `timingSafeEqual`\n * requires equal-length buffers anyway).\n */\nfunction constantTimeEqual(a: string, b: string): boolean {\n const bufA = Buffer.from(a);\n const bufB = Buffer.from(b);\n\n if (bufA.length !== bufB.length) return false;\n\n return timingSafeEqual(bufA, bufB);\n}\n\n/**\n * Bearer-token check (S4): `Authorization: Bearer <token>` header, or\n * (only when `allowQueryToken`) a `?token=` query param.\n *\n * The query-string form only exists for the one request that structurally\n * cannot carry a custom header — the initial browser navigation that\n * loads the HTML shell (a typed/clicked/bookmarked URL). Every other\n * request the served page makes is same-origin `fetch()`, which can and\n * does set `Authorization` (see `ui.html.ts`'s `fetchAuthed`), so the API\n * routes never need to accept a query-string token. Restricting the\n * fallback to just the page route minimizes the token's exposure in\n * server access logs, proxy logs, and browser history to a single route\n * instead of every poll.\n */\nfunction isAuthorized(\n req: IncomingMessage,\n url: URL,\n token: string,\n allowQueryToken: boolean,\n): boolean {\n const header = req.headers.authorization;\n if (header && constantTimeEqual(header, `Bearer ${token}`)) return true;\n if (!allowQueryToken) return false;\n const queryToken = url.searchParams.get(\"token\");\n return queryToken !== null && constantTimeEqual(queryToken, token);\n}\n\n/**\n * Build the `node:http` request handler for the dashboard over a given\n * trace store. Kept separate from the server lifecycle so it can be unit\n * tested by feeding it a fake `req`/`res` without binding a port.\n *\n * Routes (all under `config.basePath`):\n *\n * - `GET api/traces` → `store.query(parseQuery(searchParams))`\n * - `GET api/traces/:id` → `store.get(id)` or `404`\n * - `GET api/aggregate` → `store.aggregate(parseQuery(searchParams))`\n * - `GET {basePath}` → the self-contained HTML page\n * - `POST api/traces/:traceId/spans/:spanId/evaluate` → ONLY when\n * `config.evaluate` is set; grades the span's last captured system\n * prompt and returns a {@link EvaluateVerdict}. The dashboard's one\n * write route — absent config, POST 405s like it would against any\n * other route (the path is never even pattern-matched).\n *\n * Anything else → `404`. A method the matched route doesn't accept → `405`.\n * Host allowlist + bearer-token auth (S4) are checked for EVERY request,\n * regardless of method, before any routing. The store shapes\n * ({@link Trace}/{@link TraceSpan}) are already JSON-safe, so GET responses\n * are a plain `JSON.stringify` with no serializer.\n *\n * @example\n * const handler = createRequestHandler(store, { basePath: \"/\", title: \"Panoptic\" });\n * http.createServer(handler).listen(4319, \"127.0.0.1\");\n */\nexport function createRequestHandler(\n store: TraceStoreContract,\n config: ServeConfig,\n): (req: IncomingMessage, res: ServerResponse) => void {\n const base = config.basePath;\n const apiPrefix = `${base}api`;\n\n return function handle(req: IncomingMessage, res: ServerResponse): void {\n // `req.url` is path + query only; a dummy origin lets URL parse it.\n const url = new URL(req.url ?? \"/\", \"http://localhost\");\n const pathname = url.pathname;\n\n // Host-header allowlist — blocks DNS-rebinding attacks that point a\n // hostile domain at this loopback port (S4). Checked for every method.\n const host = hostHeaderName(req.headers.host);\n if (!host || !config.allowedHosts.includes(host.toLowerCase())) {\n sendJson(res, 403, { error: \"host_not_allowed\" });\n\n return;\n }\n\n // Bearer-token auth when configured (always required off loopback, S4).\n // `?token=` is only honored on the HTML page route — see `isAuthorized`.\n const isPageRoute = pathname === base || pathname === base.replace(/\\/$/, \"\");\n if (config.authToken && !isAuthorized(req, url, config.authToken, isPageRoute)) {\n sendJson(res, 401, { error: \"unauthorized\" });\n\n return;\n }\n\n // The one write route — opt-in via `config.evaluate`, so it must be\n // checked before the blanket GET-only gate below.\n if (req.method === \"POST\" && config.evaluate) {\n const match = matchEvaluatePath(pathname, apiPrefix);\n\n if (match) {\n void handleEvaluate(store, config.evaluate, match.traceId, match.spanId, req, res);\n\n return;\n }\n }\n\n if (req.method !== \"GET\") {\n sendJson(res, 405, { error: \"method_not_allowed\" });\n\n return;\n }\n\n if (pathname === `${apiPrefix}/traces`) {\n sendJson(res, 200, store.query(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname.startsWith(`${apiPrefix}/traces/`)) {\n const traceId = decodeURIComponent(pathname.slice(`${apiPrefix}/traces/`.length));\n const trace = traceId.length > 0 ? store.get(traceId) : undefined;\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n sendJson(res, 200, trace);\n\n return;\n }\n\n if (pathname === `${apiPrefix}/aggregate`) {\n sendJson(res, 200, store.aggregate(parseQuery(url.searchParams)));\n\n return;\n }\n\n if (pathname === base || pathname === base.replace(/\\/$/, \"\")) {\n res.writeHead(200, {\n \"content-type\": \"text/html; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(\n dashboardHtml(\n base,\n config.title,\n Boolean(config.evaluate),\n config.evaluate?.instructions ?? \"\",\n ),\n );\n\n return;\n }\n\n sendJson(res, 404, { error: \"not_found\" });\n };\n}\n\n/** Write a JSON response with the given status code and security headers. */\nfunction sendJson(res: ServerResponse, status: number, body: unknown): void {\n res.writeHead(status, {\n \"content-type\": \"application/json; charset=utf-8\",\n ...SECURITY_HEADERS,\n });\n res.end(JSON.stringify(body));\n}\n\n/**\n * Match `{apiPrefix}/traces/:traceId/spans/:spanId/evaluate` — the one\n * write route. Returns `undefined` for anything else, including the plain\n * `{apiPrefix}/traces/:id` read route (no `/spans/.../evaluate` suffix),\n * so the two never collide.\n */\nfunction matchEvaluatePath(\n pathname: string,\n apiPrefix: string,\n): { traceId: string; spanId: string } | undefined {\n const prefix = `${apiPrefix}/traces/`;\n\n if (!pathname.startsWith(prefix)) {\n return undefined;\n }\n\n const match = /^([^/]+)\\/spans\\/([^/]+)\\/evaluate$/.exec(pathname.slice(prefix.length));\n\n if (!match) {\n return undefined;\n }\n\n return {\n traceId: decodeURIComponent(match[1]),\n spanId: decodeURIComponent(match[2]),\n };\n}\n\n/**\n * Collect and JSON-parse a request body, capped at\n * {@link MAX_EVALUATE_BODY_BYTES} so an oversized body can't hold the\n * connection open indefinitely. An empty body resolves to `undefined` —\n * the evaluate route treats that as \"no per-run override\".\n */\nfunction readJsonBody<T>(req: IncomingMessage): Promise<T | undefined> {\n return new Promise((resolve, reject) => {\n const chunks: Buffer[] = [];\n let size = 0;\n\n req.on(\"data\", (chunk: Buffer) => {\n size += chunk.length;\n\n if (size > MAX_EVALUATE_BODY_BYTES) {\n req.destroy();\n reject(new Error(\"payload_too_large\"));\n\n return;\n }\n\n chunks.push(chunk);\n });\n\n req.on(\"end\", () => {\n if (chunks.length === 0) {\n resolve(undefined);\n\n return;\n }\n\n try {\n resolve(JSON.parse(Buffer.concat(chunks).toString(\"utf-8\")) as T);\n } catch {\n reject(new Error(\"invalid_json\"));\n }\n });\n\n req.on(\"error\", reject);\n });\n}\n\n/**\n * Handle `POST {apiPrefix}/traces/:traceId/spans/:spanId/evaluate`. Looks\n * up the trace + span, extracts its last captured system prompt, and\n * grades it via {@link evaluateSystemPrompt}. `judgePromptBody` itself\n * never throws (a broken judge degrades to an issues-only outcome) — the\n * try/catch here only guards `config.evaluate.model` resolution, the one\n * step that CAN throw (e.g. a factory constructing an SDK client).\n */\nasync function handleEvaluate(\n store: TraceStoreContract,\n evaluate: EvaluateConfig,\n traceId: string,\n spanId: string,\n req: IncomingMessage,\n res: ServerResponse,\n): Promise<void> {\n const trace = store.get(traceId);\n\n if (trace === undefined) {\n sendJson(res, 404, { error: \"trace_not_found\", traceId });\n\n return;\n }\n\n const span = findSpanById(trace.root, spanId);\n\n if (span === undefined) {\n sendJson(res, 404, { error: \"span_not_found\", spanId });\n\n return;\n }\n\n const systemPrompt = extractLastSystemPrompt(span);\n\n if (systemPrompt === undefined) {\n sendJson(res, 422, { error: \"no_system_prompt\" });\n\n return;\n }\n\n let body: EvaluateRequestBody | undefined;\n\n try {\n body = await readJsonBody<EvaluateRequestBody>(req);\n } catch (error) {\n const message = error instanceof Error ? error.message : \"invalid_json\";\n sendJson(res, message === \"payload_too_large\" ? 413 : 400, { error: message });\n\n return;\n }\n\n try {\n const verdict = await evaluateSystemPrompt(systemPrompt, evaluate, body?.instructions);\n sendJson(res, 200, verdict);\n } catch (error) {\n sendJson(res, 502, {\n error: \"evaluate_failed\",\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n"],"mappings":";;;;;;;;;AAoCA,MAAM,0BAA0B;;;;;;AAOhC,MAAM,mBAA2C;CAC/C,0BAA0B;CAC1B,mBAAmB;CACnB,mBAAmB;CACnB,2BACE;AACJ;;AAGA,SAAS,eAAe,YAAoD;CAC1E,IAAI,CAAC,YAAY,OAAO;CAExB,IAAI,WAAW,WAAW,GAAG,GAC3B,OAAO,WAAW,MAAM,GAAG,WAAW,QAAQ,GAAG,IAAI,CAAC;CAExD,MAAM,QAAQ,WAAW,QAAQ,GAAG;CACpC,OAAO,UAAU,KAAK,aAAa,WAAW,MAAM,GAAG,KAAK;AAC9D;;;;;;;;;;AAWA,SAAS,kBAAkB,GAAW,GAAoB;CACxD,MAAM,OAAO,OAAO,KAAK,CAAC;CAC1B,MAAM,OAAO,OAAO,KAAK,CAAC;CAE1B,IAAI,KAAK,WAAW,KAAK,QAAQ,OAAO;CAExC,OAAO,gBAAgB,MAAM,IAAI;AACnC;;;;;;;;;;;;;;;AAgBA,SAAS,aACP,KACA,KACA,OACA,iBACS;CACT,MAAM,SAAS,IAAI,QAAQ;CAC3B,IAAI,UAAU,kBAAkB,QAAQ,UAAU,OAAO,GAAG,OAAO;CACnE,IAAI,CAAC,iBAAiB,OAAO;CAC7B,MAAM,aAAa,IAAI,aAAa,IAAI,OAAO;CAC/C,OAAO,eAAe,QAAQ,kBAAkB,YAAY,KAAK;AACnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,qBACd,OACA,QACqD;CACrD,MAAM,OAAO,OAAO;CACpB,MAAM,YAAY,GAAG,KAAK;CAE1B,OAAO,SAAS,OAAO,KAAsB,KAA2B;EAEtE,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,KAAK,kBAAkB;EACtD,MAAM,WAAW,IAAI;EAIrB,MAAM,OAAO,eAAe,IAAI,QAAQ,IAAI;EAC5C,IAAI,CAAC,QAAQ,CAAC,OAAO,aAAa,SAAS,KAAK,YAAY,CAAC,GAAG;GAC9D,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;GAEhD;EACF;EAIA,MAAM,cAAc,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE;EAC5E,IAAI,OAAO,aAAa,CAAC,aAAa,KAAK,KAAK,OAAO,WAAW,WAAW,GAAG;GAC9E,SAAS,KAAK,KAAK,EAAE,OAAO,eAAe,CAAC;GAE5C;EACF;EAIA,IAAI,IAAI,WAAW,UAAU,OAAO,UAAU;GAC5C,MAAM,QAAQ,kBAAkB,UAAU,SAAS;GAEnD,IAAI,OAAO;IACT,AAAK,eAAe,OAAO,OAAO,UAAU,MAAM,SAAS,MAAM,QAAQ,KAAK,GAAG;IAEjF;GACF;EACF;EAEA,IAAI,IAAI,WAAW,OAAO;GACxB,SAAS,KAAK,KAAK,EAAE,OAAO,qBAAqB,CAAC;GAElD;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,UAAU;GACtC,SAAS,KAAK,KAAK,MAAM,MAAM,WAAW,IAAI,YAAY,CAAC,CAAC;GAE5D;EACF;EAEA,IAAI,SAAS,WAAW,GAAG,UAAU,SAAS,GAAG;GAC/C,MAAM,UAAU,mBAAmB,SAAS,MAAM,GAAG,UAAU,UAAU,MAAM,CAAC;GAChF,MAAM,QAAQ,QAAQ,SAAS,IAAI,MAAM,IAAI,OAAO,IAAI;GAExD,IAAI,UAAU,QAAW;IACvB,SAAS,KAAK,KAAK;KAAE,OAAO;KAAmB;IAAQ,CAAC;IAExD;GACF;GAEA,SAAS,KAAK,KAAK,KAAK;GAExB;EACF;EAEA,IAAI,aAAa,GAAG,UAAU,aAAa;GACzC,SAAS,KAAK,KAAK,MAAM,UAAU,WAAW,IAAI,YAAY,CAAC,CAAC;GAEhE;EACF;EAEA,IAAI,aAAa,QAAQ,aAAa,KAAK,QAAQ,OAAO,EAAE,GAAG;GAC7D,IAAI,UAAU,KAAK;IACjB,gBAAgB;IAChB,GAAG;GACL,CAAC;GACD,IAAI,IACF,cACE,MACA,OAAO,OACP,QAAQ,OAAO,QAAQ,GACvB,OAAO,UAAU,gBAAgB,EACnC,CACF;GAEA;EACF;EAEA,SAAS,KAAK,KAAK,EAAE,OAAO,YAAY,CAAC;CAC3C;AACF;;AAGA,SAAS,SAAS,KAAqB,QAAgB,MAAqB;CAC1E,IAAI,UAAU,QAAQ;EACpB,gBAAgB;EAChB,GAAG;CACL,CAAC;CACD,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC;AAC9B;;;;;;;AAQA,SAAS,kBACP,UACA,WACiD;CACjD,MAAM,SAAS,GAAG,UAAU;CAE5B,IAAI,CAAC,SAAS,WAAW,MAAM,GAC7B;CAGF,MAAM,QAAQ,sCAAsC,KAAK,SAAS,MAAM,OAAO,MAAM,CAAC;CAEtF,IAAI,CAAC,OACH;CAGF,OAAO;EACL,SAAS,mBAAmB,MAAM,EAAE;EACpC,QAAQ,mBAAmB,MAAM,EAAE;CACrC;AACF;;;;;;;AAQA,SAAS,aAAgB,KAA8C;CACrE,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,MAAM,SAAmB,CAAC;EAC1B,IAAI,OAAO;EAEX,IAAI,GAAG,SAAS,UAAkB;GAChC,QAAQ,MAAM;GAEd,IAAI,OAAO,yBAAyB;IAClC,IAAI,QAAQ;IACZ,uBAAO,IAAI,MAAM,mBAAmB,CAAC;IAErC;GACF;GAEA,OAAO,KAAK,KAAK;EACnB,CAAC;EAED,IAAI,GAAG,aAAa;GAClB,IAAI,OAAO,WAAW,GAAG;IACvB,QAAQ,MAAS;IAEjB;GACF;GAEA,IAAI;IACF,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM,CAAC,CAAC,SAAS,OAAO,CAAC,CAAM;GAClE,QAAQ;IACN,uBAAO,IAAI,MAAM,cAAc,CAAC;GAClC;EACF,CAAC;EAED,IAAI,GAAG,SAAS,MAAM;CACxB,CAAC;AACH;;;;;;;;;AAUA,eAAe,eACb,OACA,UACA,SACA,QACA,KACA,KACe;CACf,MAAM,QAAQ,MAAM,IAAI,OAAO;CAE/B,IAAI,UAAU,QAAW;EACvB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAmB;EAAQ,CAAC;EAExD;CACF;CAEA,MAAM,OAAO,aAAa,MAAM,MAAM,MAAM;CAE5C,IAAI,SAAS,QAAW;EACtB,SAAS,KAAK,KAAK;GAAE,OAAO;GAAkB;EAAO,CAAC;EAEtD;CACF;CAEA,MAAM,eAAe,wBAAwB,IAAI;CAEjD,IAAI,iBAAiB,QAAW;EAC9B,SAAS,KAAK,KAAK,EAAE,OAAO,mBAAmB,CAAC;EAEhD;CACF;CAEA,IAAI;CAEJ,IAAI;EACF,OAAO,MAAM,aAAkC,GAAG;CACpD,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;EACzD,SAAS,KAAK,YAAY,sBAAsB,MAAM,KAAK,EAAE,OAAO,QAAQ,CAAC;EAE7E;CACF;CAEA,IAAI;EAEF,SAAS,KAAK,KAAK,MADG,qBAAqB,cAAc,UAAU,MAAM,YAAY,CAC3D;CAC5B,SAAS,OAAO;EACd,SAAS,KAAK,KAAK;GACjB,OAAO;GACP,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAChE,CAAC;CACH;AACF"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { Trace, TraceSpan } from "../contracts/trace.type.mjs";
|
|
2
2
|
import { ReportStatus, ReportType } from "@warlock.js/ai";
|
|
3
|
-
|
|
4
3
|
//#region ../../packages/ai-panoptic/src/dashboard/trace-filter.d.ts
|
|
5
4
|
/**
|
|
6
5
|
* Client-side trace-list filter criteria. The dashboard polls the full
|
|
@@ -13,7 +12,7 @@ import { ReportStatus, ReportType } from "@warlock.js/ai";
|
|
|
13
12
|
* Every field is optional — an absent field is "don't care". An empty
|
|
14
13
|
* filter therefore matches every trace.
|
|
15
14
|
*/
|
|
16
|
-
type TraceFilter = {
|
|
15
|
+
export type TraceFilter = {
|
|
17
16
|
/**
|
|
18
17
|
* Free-text needle matched case-insensitively against a trace's root
|
|
19
18
|
* span name and session id. Whitespace-trimmed; an empty/blank string
|
|
@@ -60,24 +59,26 @@ type TraceFilter = {
|
|
|
60
59
|
* versions are distinct keys — that is the whole point of the linkage: to
|
|
61
60
|
* group / filter by the *exact* prompt revision that produced a run.
|
|
62
61
|
*/
|
|
63
|
-
declare function tracePromptKey(trace: Trace): string | undefined;
|
|
62
|
+
export declare function tracePromptKey(trace: Trace): string | undefined;
|
|
64
63
|
/**
|
|
65
64
|
* Does a single trace pass the given filter? Pure predicate — all
|
|
66
65
|
* conditions are ANDed; an unset field never excludes. Used by
|
|
67
66
|
* {@link filterTraces} and mirrored by the dashboard's inlined JS.
|
|
68
67
|
*/
|
|
69
|
-
declare function matchesFilter(trace: Trace, filter: TraceFilter): boolean;
|
|
68
|
+
export declare function matchesFilter(trace: Trace, filter: TraceFilter): boolean;
|
|
70
69
|
/** Narrow a trace list to those matching the filter, order preserved. */
|
|
71
|
-
declare function filterTraces(traces: Trace[], filter: TraceFilter): Trace[];
|
|
70
|
+
export declare function filterTraces(traces: Trace[], filter: TraceFilter): Trace[];
|
|
72
71
|
/**
|
|
73
72
|
* The grouping key a trace falls under when the list is grouped by
|
|
74
73
|
* session. Traces without a `sessionId` collapse into one synthetic
|
|
75
74
|
* "(no session)" bucket so they remain visible.
|
|
76
75
|
*/
|
|
77
|
-
declare const NO_SESSION_KEY = "(no session)";
|
|
76
|
+
export declare const NO_SESSION_KEY = "(no session)";
|
|
78
77
|
/** One session bucket — its key plus the traces under it, order preserved. */
|
|
79
|
-
type SessionGroup = {
|
|
80
|
-
/** The `sessionId`, or {@link NO_SESSION_KEY} for sessionless traces. */
|
|
78
|
+
export type SessionGroup = {
|
|
79
|
+
/** The `sessionId`, or {@link NO_SESSION_KEY} for sessionless traces. */
|
|
80
|
+
sessionId: string;
|
|
81
|
+
/** Traces in this group, in the order they appeared in the input. */
|
|
81
82
|
traces: Trace[];
|
|
82
83
|
};
|
|
83
84
|
/**
|
|
@@ -85,16 +86,18 @@ type SessionGroup = {
|
|
|
85
86
|
* preserving first-seen order of both the groups and the traces within
|
|
86
87
|
* each. Sessionless traces bucket under {@link NO_SESSION_KEY}.
|
|
87
88
|
*/
|
|
88
|
-
declare function groupBySession(traces: Trace[]): SessionGroup[];
|
|
89
|
+
export declare function groupBySession(traces: Trace[]): SessionGroup[];
|
|
89
90
|
/**
|
|
90
91
|
* The grouping key a trace falls under when the list is grouped by prompt
|
|
91
92
|
* version. Runs that resolved no named prompt collapse into one synthetic
|
|
92
93
|
* "(no prompt)" bucket so they remain visible.
|
|
93
94
|
*/
|
|
94
|
-
declare const NO_PROMPT_KEY = "(no prompt)";
|
|
95
|
+
export declare const NO_PROMPT_KEY = "(no prompt)";
|
|
95
96
|
/** One prompt-version bucket — its `name@version` key plus its traces. */
|
|
96
|
-
type PromptGroup = {
|
|
97
|
-
/** The `name@version` key, or {@link NO_PROMPT_KEY} for unlinked traces. */
|
|
97
|
+
export type PromptGroup = {
|
|
98
|
+
/** The `name@version` key, or {@link NO_PROMPT_KEY} for unlinked traces. */
|
|
99
|
+
promptKey: string;
|
|
100
|
+
/** Traces in this group, in the order they appeared in the input. */
|
|
98
101
|
traces: Trace[];
|
|
99
102
|
};
|
|
100
103
|
/**
|
|
@@ -108,16 +111,18 @@ type PromptGroup = {
|
|
|
108
111
|
* so a reviewer can see every run of "support@2" together and compare its
|
|
109
112
|
* cost / failure rate against "support@3".
|
|
110
113
|
*/
|
|
111
|
-
declare function groupByPrompt(traces: Trace[]): PromptGroup[];
|
|
114
|
+
export declare function groupByPrompt(traces: Trace[]): PromptGroup[];
|
|
112
115
|
/**
|
|
113
116
|
* The grouping key a trace falls under when the list is grouped by root
|
|
114
117
|
* type. `root.type` is always present, so this is only a defensive
|
|
115
118
|
* fallback for a malformed trace with no typed root.
|
|
116
119
|
*/
|
|
117
|
-
declare const NO_TYPE_KEY = "(no type)";
|
|
120
|
+
export declare const NO_TYPE_KEY = "(no type)";
|
|
118
121
|
/** One root-type bucket — its type discriminator plus its traces. */
|
|
119
|
-
type TypeGroup = {
|
|
120
|
-
/** The root span's `type` (e.g. `agent` / `workflow` / `planner`), or {@link NO_TYPE_KEY}. */
|
|
122
|
+
export type TypeGroup = {
|
|
123
|
+
/** The root span's `type` (e.g. `agent` / `workflow` / `planner`), or {@link NO_TYPE_KEY}. */
|
|
124
|
+
type: string;
|
|
125
|
+
/** Traces in this group, in the order they appeared in the input. */
|
|
121
126
|
traces: Trace[];
|
|
122
127
|
};
|
|
123
128
|
/**
|
|
@@ -132,29 +137,37 @@ type TypeGroup = {
|
|
|
132
137
|
* behind a per-type header so the structural primitives (the lone planner,
|
|
133
138
|
* the team, the supervisors) are one click away instead of a scroll past.
|
|
134
139
|
*/
|
|
135
|
-
declare function groupByType(traces: Trace[]): TypeGroup[];
|
|
140
|
+
export declare function groupByType(traces: Trace[]): TypeGroup[];
|
|
136
141
|
/** One row of the per-type aggregate-stats panel. */
|
|
137
|
-
type TypeStat = {
|
|
138
|
-
/** Root type discriminator, or {@link NO_TYPE_KEY}. */
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
142
|
+
export type TypeStat = {
|
|
143
|
+
/** Root type discriminator, or {@link NO_TYPE_KEY}. */
|
|
144
|
+
type: string;
|
|
145
|
+
/** Number of traces of this type. */
|
|
146
|
+
count: number;
|
|
147
|
+
/** Traces whose status is `failed` / `cancelled`. */
|
|
148
|
+
failed: number;
|
|
149
|
+
/** `failed / count` in `[0, 1]`. */
|
|
150
|
+
failRate: number;
|
|
151
|
+
/** Median (p50) root duration in ms. */
|
|
152
|
+
p50: number;
|
|
153
|
+
/** p95 root duration in ms. */
|
|
154
|
+
p95: number;
|
|
155
|
+
/** Summed `usage.total` tokens across the bucket. */
|
|
156
|
+
tokens: number;
|
|
157
|
+
/** Summed per-trace USD cost across the bucket. */
|
|
145
158
|
cost: number;
|
|
146
159
|
};
|
|
147
160
|
/**
|
|
148
161
|
* Nearest-rank percentile of `values` at `p` (0–100). Returns 0 for an
|
|
149
162
|
* empty input and sorts a copy so the caller's array is left untouched.
|
|
150
163
|
*/
|
|
151
|
-
declare function percentile(values: number[], p: number): number;
|
|
164
|
+
export declare function percentile(values: number[], p: number): number;
|
|
152
165
|
/**
|
|
153
166
|
* One trace's total USD cost: its explicit root-usage cost when priced,
|
|
154
167
|
* else the rollup of its subtree. Mirrors the dashboard's `traceCost` so the
|
|
155
168
|
* stats panel and the trace rows agree to the cent.
|
|
156
169
|
*/
|
|
157
|
-
declare function traceCost(trace: Trace): number;
|
|
170
|
+
export declare function traceCost(trace: Trace): number;
|
|
158
171
|
/**
|
|
159
172
|
* Aggregate an (already filtered) trace list into one {@link TypeStat} row
|
|
160
173
|
* per root type — count, failure rate, p50/p95 latency, total tokens, total
|
|
@@ -162,26 +175,25 @@ declare function traceCost(trace: Trace): number;
|
|
|
162
175
|
* stats panel; computed over the same filtered list the trace view shows, so
|
|
163
176
|
* the panel always reflects the active filters.
|
|
164
177
|
*/
|
|
165
|
-
declare function aggregateByType(traces: Trace[]): TypeStat[];
|
|
178
|
+
export declare function aggregateByType(traces: Trace[]): TypeStat[];
|
|
166
179
|
/**
|
|
167
180
|
* Rollup-aware subtree cost: a node's own cost when it carries one
|
|
168
181
|
* (it already rolls up its trips), otherwise the sum of its children.
|
|
169
182
|
* Avoids double-counting wrapper nodes (workflow/supervisor roots carry
|
|
170
183
|
* tokens but no cost). Mirrors the dashboard's `rollupCost`.
|
|
171
184
|
*/
|
|
172
|
-
declare function rollupCost(span: TraceSpan): number;
|
|
185
|
+
export declare function rollupCost(span: TraceSpan): number;
|
|
173
186
|
/**
|
|
174
187
|
* Heatmap intensity in `[0, 1]` for a node's `rollupCost` relative to the
|
|
175
188
|
* trace's most-expensive node. The dashboard tints each tree node's left
|
|
176
189
|
* accent by this. `max <= 0` (a free trace) yields `0` for every node so
|
|
177
190
|
* the heatmap simply stays cold rather than dividing by zero.
|
|
178
191
|
*/
|
|
179
|
-
declare function heatIntensity(nodeCost: number, maxCost: number): number;
|
|
192
|
+
export declare function heatIntensity(nodeCost: number, maxCost: number): number;
|
|
180
193
|
/**
|
|
181
194
|
* The largest single-node {@link rollupCost} anywhere in a span tree —
|
|
182
195
|
* the denominator for {@link heatIntensity}. Walks the whole subtree.
|
|
183
196
|
*/
|
|
184
|
-
declare function maxNodeCost(root: TraceSpan): number;
|
|
197
|
+
export declare function maxNodeCost(root: TraceSpan): number;
|
|
185
198
|
//#endregion
|
|
186
|
-
export { NO_PROMPT_KEY, NO_SESSION_KEY, NO_TYPE_KEY, PromptGroup, SessionGroup, TraceFilter, TypeGroup, TypeStat, aggregateByType, filterTraces, groupByPrompt, groupBySession, groupByType, heatIntensity, matchesFilter, maxNodeCost, percentile, rollupCost, traceCost, tracePromptKey };
|
|
187
199
|
//# sourceMappingURL=trace-filter.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trace-filter.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/trace-filter.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"trace-filter.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/dashboard/trace-filter.ts"],"mappings":";;;;;;;;;;;;;;YAcY;;;;;;EAMV;;;;;EAKA,WAAW;;;;;EAKX,QAAQ;;;;;;EAMR;;;;;;;EAOA;;;;;EAKA;;;;;;;;;;;;;wBAyBc,eAAe,OAAO;;;;;;wBAwBtB,cAAc,OAAO,OAAO,QAAQ;;wBA2CpC,aAAa,QAAQ,SAAS,QAAQ,cAAc;;;;;;qBASvD;;YAGD;;EAEV;;EAEA,QAAQ;;;;;;;wBAQM,eAAe,QAAQ,UAAU;;;;;;qBAyBpC;;YAGD;;EAEV;;EAEA,QAAQ;;;;;;;;;;;;;wBAcM,cAAc,QAAQ,UAAU;;;;;;qBAyBnC;;YAGD;;EAEV;;EAEA,QAAQ;;;;;;;;;;;;;;wBAeM,YAAY,QAAQ,UAAU;;YAqBlC;;EAEV;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;wBAOc,WAAW,kBAAkB;;;;;;wBAiB7B,UAAU,OAAO;;;;;;;;wBAYjB,gBAAgB,QAAQ,UAAU;;;;;;;wBAwElC,WAAW,MAAM;;;;;;;wBAoBjB,cAAc,kBAAkB;;;;;wBAchC,YAAY,MAAM"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { EvaluateConfig, EvaluateVerdict } from "./evaluate.type.mjs";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Grade `systemPrompt` with the configured judge model, reusing
|
|
@@ -13,7 +12,6 @@ import { EvaluateConfig, EvaluateVerdict } from "./evaluate.type.mjs";
|
|
|
13
12
|
* resolving `config.model` (a factory constructing an SDK client), which
|
|
14
13
|
* the caller (the dashboard route) is expected to catch.
|
|
15
14
|
*/
|
|
16
|
-
declare function evaluateSystemPrompt(systemPrompt: string, config: EvaluateConfig, instructionsOverride?: string): Promise<EvaluateVerdict>;
|
|
15
|
+
export declare function evaluateSystemPrompt(systemPrompt: string, config: EvaluateConfig, instructionsOverride?: string): Promise<EvaluateVerdict>;
|
|
17
16
|
//#endregion
|
|
18
|
-
export { evaluateSystemPrompt };
|
|
19
17
|
//# sourceMappingURL=evaluate-system-prompt.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate-system-prompt.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"evaluate-system-prompt.d.mts","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"mappings":";;;;;;;;;;;;;;wBAesB,qBACpB,sBACA,QAAQ,gBACR,gCACC,QAAQ"}
|
|
@@ -14,7 +14,9 @@ import { judgePromptBody } from "@warlock.js/ai";
|
|
|
14
14
|
* the caller (the dashboard route) is expected to catch.
|
|
15
15
|
*/
|
|
16
16
|
async function evaluateSystemPrompt(systemPrompt, config, instructionsOverride) {
|
|
17
|
-
|
|
17
|
+
const model = typeof config.model === "function" ? await config.model() : config.model;
|
|
18
|
+
const instructions = instructionsOverride?.trim() || config.instructions;
|
|
19
|
+
return judgePromptBody(systemPrompt, model, instructions);
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate-system-prompt.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"sourcesContent":["import { judgePromptBody } from \"@warlock.js/ai\";\nimport type { EvaluateConfig, EvaluateVerdict } from \"./evaluate.type\";\n\n/**\n * Grade `systemPrompt` with the configured judge model, reusing\n * `ai.prompts().validate()`'s own `judgePromptBody` — never a second\n * judging implementation. `instructionsOverride` (the dashboard's\n * per-run textarea) wins over `config.instructions`; with neither, the\n * judge falls back to `judgePromptBody`'s built-in prompt-quality rubric.\n *\n * The judge itself never throws (`judgePromptBody` degrades to an\n * issues-only outcome on failure) — the only thing that CAN throw here is\n * resolving `config.model` (a factory constructing an SDK client), which\n * the caller (the dashboard route) is expected to catch.\n */\nexport async function evaluateSystemPrompt(\n systemPrompt: string,\n config: EvaluateConfig,\n instructionsOverride?: string,\n): Promise<EvaluateVerdict> {\n const model = typeof config.model === \"function\" ? await config.model() : config.model;\n const instructions = instructionsOverride?.trim() || config.instructions;\n\n return judgePromptBody(systemPrompt, model, instructions);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAeA,eAAsB,qBACpB,cACA,QACA,sBAC0B;
|
|
1
|
+
{"version":3,"file":"evaluate-system-prompt.mjs","names":[],"sources":["../../../../../../../../packages/ai-panoptic/src/evaluate/evaluate-system-prompt.ts"],"sourcesContent":["import { judgePromptBody } from \"@warlock.js/ai\";\nimport type { EvaluateConfig, EvaluateVerdict } from \"./evaluate.type\";\n\n/**\n * Grade `systemPrompt` with the configured judge model, reusing\n * `ai.prompts().validate()`'s own `judgePromptBody` — never a second\n * judging implementation. `instructionsOverride` (the dashboard's\n * per-run textarea) wins over `config.instructions`; with neither, the\n * judge falls back to `judgePromptBody`'s built-in prompt-quality rubric.\n *\n * The judge itself never throws (`judgePromptBody` degrades to an\n * issues-only outcome on failure) — the only thing that CAN throw here is\n * resolving `config.model` (a factory constructing an SDK client), which\n * the caller (the dashboard route) is expected to catch.\n */\nexport async function evaluateSystemPrompt(\n systemPrompt: string,\n config: EvaluateConfig,\n instructionsOverride?: string,\n): Promise<EvaluateVerdict> {\n const model = typeof config.model === \"function\" ? await config.model() : config.model;\n const instructions = instructionsOverride?.trim() || config.instructions;\n\n return judgePromptBody(systemPrompt, model, instructions);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAeA,eAAsB,qBACpB,cACA,QACA,sBAC0B;CAC1B,MAAM,QAAQ,OAAO,OAAO,UAAU,aAAa,MAAM,OAAO,MAAM,IAAI,OAAO;CACjF,MAAM,eAAe,sBAAsB,KAAK,KAAK,OAAO;CAE5D,OAAO,gBAAgB,cAAc,OAAO,YAAY;AAC1D"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { JudgeOutcome, ModelContract } from "@warlock.js/ai";
|
|
2
|
-
|
|
3
2
|
//#region ../../packages/ai-panoptic/src/evaluate/evaluate.type.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Enables the dashboard's per-trace "Evaluate system prompt" action — the
|
|
@@ -22,7 +21,7 @@ import { JudgeOutcome, ModelContract } from "@warlock.js/ai";
|
|
|
22
21
|
* },
|
|
23
22
|
* });
|
|
24
23
|
*/
|
|
25
|
-
type EvaluateConfig = {
|
|
24
|
+
export type EvaluateConfig = {
|
|
26
25
|
/**
|
|
27
26
|
* The judge model, or a (possibly async) factory resolved on first use —
|
|
28
27
|
* the factory form defers construction (e.g. an SDK client) until the
|
|
@@ -41,7 +40,6 @@ type EvaluateConfig = {
|
|
|
41
40
|
instructions?: string;
|
|
42
41
|
};
|
|
43
42
|
/** The judge verdict returned by the evaluate endpoint — `{@link JudgeOutcome}` verbatim. */
|
|
44
|
-
type EvaluateVerdict = JudgeOutcome;
|
|
43
|
+
export type EvaluateVerdict = JudgeOutcome;
|
|
45
44
|
//#endregion
|
|
46
|
-
export { EvaluateConfig, EvaluateVerdict };
|
|
47
45
|
//# sourceMappingURL=evaluate.type.d.mts.map
|