@terreno/ai 57.21.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +2 -2
  4. package/dist/harness/commit.d.ts +4 -3
  5. package/dist/harness/commit.js +23 -4
  6. package/dist/index.d.ts +6 -9
  7. package/dist/index.js +14 -25
  8. package/dist/observability/adminScreens.js +5 -1
  9. package/dist/observability/evaluate.d.ts +8 -1
  10. package/dist/observability/evaluate.js +10 -2
  11. package/dist/observability/judgeAgreement.d.ts +53 -0
  12. package/dist/observability/judgeAgreement.js +123 -0
  13. package/dist/observability/local/datasetStore.d.ts +5 -0
  14. package/dist/observability/local/datasetStore.js +7 -2
  15. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  16. package/dist/observability/local/evaluatorStore.js +73 -2
  17. package/dist/observability/local/experimentRunner.d.ts +40 -3
  18. package/dist/observability/local/experimentRunner.js +390 -11
  19. package/dist/observability/local/humanAnswers.d.ts +57 -0
  20. package/dist/observability/local/humanAnswers.js +174 -0
  21. package/dist/observability/local/judgeRun.d.ts +56 -0
  22. package/dist/observability/local/judgeRun.js +111 -0
  23. package/dist/observability/local/localPlugin.js +4 -0
  24. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  25. package/dist/observability/local/models/obsEvaluator.js +4 -0
  26. package/dist/observability/local/models/obsExperiment.js +28 -0
  27. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  28. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  29. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  30. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  31. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  32. package/dist/observability/local/models/obsReviewItem.js +10 -2
  33. package/dist/observability/local/models/obsTrace.js +4 -0
  34. package/dist/observability/local/promptReferences.d.ts +31 -0
  35. package/dist/observability/local/promptReferences.js +114 -0
  36. package/dist/observability/local/promptStore.d.ts +55 -19
  37. package/dist/observability/local/promptStore.js +171 -7
  38. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  39. package/dist/observability/local/reviewBatchStore.js +517 -0
  40. package/dist/observability/local/reviewStore.d.ts +24 -3
  41. package/dist/observability/local/reviewStore.js +98 -22
  42. package/dist/observability/local/traceStore.d.ts +14 -1
  43. package/dist/observability/local/traceStore.js +144 -11
  44. package/dist/observability/observabilityApp.js +10 -1
  45. package/dist/observability/questionFingerprint.d.ts +7 -0
  46. package/dist/observability/questionFingerprint.js +23 -0
  47. package/dist/observability/routes/experiments.js +12 -1
  48. package/dist/observability/routes/review.js +3 -1
  49. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  50. package/dist/observability/routes/reviewBatches.js +78 -0
  51. package/dist/observability/routes/traces.d.ts +1 -1
  52. package/dist/observability/routes/traces.js +34 -2
  53. package/dist/observability/types.d.ts +10 -1
  54. package/dist/service/aiService.js +4 -1
  55. package/dist/service/chatTurn.js +38 -31
  56. package/dist/types/harness.d.ts +5 -2
  57. package/dist/types/index.d.ts +8 -3
  58. package/dist/types/observability.d.ts +74 -2
  59. package/package.json +8 -24
  60. package/src/harness/agentLoop.test.ts +0 -3
  61. package/src/harness/agentRecovery.test.ts +0 -3
  62. package/src/harness/commit.ts +24 -4
  63. package/src/harness/extensions.test.ts +0 -3
  64. package/src/harness/subagent.test.ts +27 -3
  65. package/src/index.test.ts +0 -17
  66. package/src/index.ts +23 -26
  67. package/src/isolated/gptStream.isolated.ts +0 -8
  68. package/src/observability/adminScreens.test.ts +7 -0
  69. package/src/observability/adminScreens.ts +6 -1
  70. package/src/observability/evaluate.ts +12 -2
  71. package/src/observability/judgeAgreement.test.ts +111 -0
  72. package/src/observability/judgeAgreement.ts +184 -0
  73. package/src/observability/local/datasetStore.ts +30 -11
  74. package/src/observability/local/evaluatorStore.ts +96 -2
  75. package/src/observability/local/experimentRunner.ts +495 -15
  76. package/src/observability/local/humanAnswers.ts +233 -0
  77. package/src/observability/local/judgePipeline.test.ts +444 -0
  78. package/src/observability/local/judgeRun.test.ts +173 -0
  79. package/src/observability/local/judgeRun.ts +184 -0
  80. package/src/observability/local/localPlugin.ts +4 -0
  81. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  82. package/src/observability/local/models/obsEvaluator.ts +4 -0
  83. package/src/observability/local/models/obsExperiment.ts +29 -0
  84. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  85. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  86. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  87. package/src/observability/local/models/obsReviewItem.ts +11 -2
  88. package/src/observability/local/models/obsTrace.ts +5 -0
  89. package/src/observability/local/obsModels.test.ts +13 -0
  90. package/src/observability/local/promptReferences.test.ts +128 -0
  91. package/src/observability/local/promptReferences.ts +149 -0
  92. package/src/observability/local/promptStore.test.ts +291 -0
  93. package/src/observability/local/promptStore.ts +243 -32
  94. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  95. package/src/observability/local/reviewBatchStore.ts +656 -0
  96. package/src/observability/local/reviewStore.ts +122 -25
  97. package/src/observability/local/traceStore.test.ts +139 -1
  98. package/src/observability/local/traceStore.ts +162 -11
  99. package/src/observability/observabilityApp.test.ts +13 -10
  100. package/src/observability/observabilityApp.ts +10 -1
  101. package/src/observability/questionFingerprint.test.ts +33 -0
  102. package/src/observability/questionFingerprint.ts +24 -0
  103. package/src/observability/routes/experiments.test.ts +37 -0
  104. package/src/observability/routes/experiments.ts +13 -1
  105. package/src/observability/routes/review.ts +3 -1
  106. package/src/observability/routes/reviewBatches.test.ts +209 -0
  107. package/src/observability/routes/reviewBatches.ts +138 -0
  108. package/src/observability/routes/status.test.ts +4 -4
  109. package/src/observability/routes/traces.test.ts +61 -0
  110. package/src/observability/routes/traces.ts +46 -4
  111. package/src/observability/types.ts +5 -2
  112. package/src/routes/routes.test.ts +101 -76
  113. package/src/service/aiService.test.ts +21 -0
  114. package/src/service/aiService.ts +4 -1
  115. package/src/service/chatTurn.ts +42 -33
  116. package/src/tests/bunSetup.ts +0 -54
  117. package/src/types/harness.ts +5 -2
  118. package/src/types/index.ts +8 -3
  119. package/src/types/observability.ts +86 -2
  120. package/dist/langfuseApp.d.ts +0 -9
  121. package/dist/langfuseApp.js +0 -53
  122. package/dist/langfuseCache.d.ts +0 -15
  123. package/dist/langfuseCache.js +0 -54
  124. package/dist/langfuseClient.d.ts +0 -6
  125. package/dist/langfuseClient.js +0 -34
  126. package/dist/langfusePrompts.d.ts +0 -12
  127. package/dist/langfusePrompts.js +0 -89
  128. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  129. package/dist/langfuseRoutesEvaluations.js +0 -32
  130. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  131. package/dist/langfuseRoutesMiddleware.js +0 -13
  132. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  133. package/dist/langfuseRoutesPlayground.js +0 -36
  134. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  135. package/dist/langfuseRoutesPrompts.js +0 -56
  136. package/dist/langfuseRoutesTraces.d.ts +0 -2
  137. package/dist/langfuseRoutesTraces.js +0 -30
  138. package/dist/langfuseTracing.d.ts +0 -4
  139. package/dist/langfuseTracing.js +0 -32
  140. package/dist/langfuseTypes.d.ts +0 -96
  141. package/dist/langfuseTypes.js +0 -2
  142. package/dist/langfuseVercelAi.d.ts +0 -12
  143. package/dist/langfuseVercelAi.js +0 -49
  144. package/src/isolated/langfuseClient.isolated.ts +0 -66
  145. package/src/langfuseApp.test.ts +0 -119
  146. package/src/langfuseApp.ts +0 -62
  147. package/src/langfuseCache.test.ts +0 -75
  148. package/src/langfuseCache.ts +0 -72
  149. package/src/langfuseClient.test.ts +0 -69
  150. package/src/langfuseClient.ts +0 -34
  151. package/src/langfusePrompts.test.ts +0 -188
  152. package/src/langfusePrompts.ts +0 -121
  153. package/src/langfuseRoutes.test.ts +0 -261
  154. package/src/langfuseRoutesEvaluations.ts +0 -48
  155. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  156. package/src/langfuseRoutesMiddleware.ts +0 -12
  157. package/src/langfuseRoutesPlayground.ts +0 -50
  158. package/src/langfuseRoutesPrompts.ts +0 -87
  159. package/src/langfuseRoutesTraces.ts +0 -40
  160. package/src/langfuseTracing.test.ts +0 -72
  161. package/src/langfuseTracing.ts +0 -33
  162. package/src/langfuseTypes.ts +0 -103
  163. package/src/langfuseVercelAi.test.ts +0 -82
  164. package/src/langfuseVercelAi.ts +0 -72
  165. package/src/tests/fakeLangfusePrompt.ts +0 -10
package/CHANGELOG.md CHANGED
@@ -20,6 +20,37 @@ Upgrade notes for consumer action live in [`mcp-server/src/docs/upgrades/`](mcp-
20
20
 
21
21
  Unreleased changes live in [`changelog/unreleased/`](changelog/unreleased/). Add one Markdown file per feature (see that directory's README) instead of editing this section.
22
22
 
23
+ ## [57.22.0] - 2026-11-03
24
+
25
+ Upgrade note: [`mcp-server/src/docs/upgrades/57.22.0.md`](mcp-server/src/docs/upgrades/57.22.0.md).
26
+
27
+ ### Breaking
28
+
29
+ - `@terreno/ai` no longer integrates with Langfuse. The built-in observability prompt registry, traces, and review queue (`ObservabilityApp` with `createLocalObservabilityPlugin()`) replace it. Removed exports: `LangfuseApp`, `LangfuseCache`, `getCached`, `setCached`, `invalidateCache`, `initLangfuseClient`, `getLangfuseClient`, `isLangfuseInitialized`, `shutdownLangfuseClient`, `compilePrompt`, `createPrompt`, `getPrompt`, `invalidatePromptCache`, `initTracing`, `shutdownTracing`, `createTelemetryConfig`, `preparePromptForAI`, and the types `ChatMessage`, `GetPromptOptions`, `LangfuseAppOptions`, `LangfuseCachedPrompt`, `PaginatedResult`, `PreparePromptResult`, `PromptListItem`, `ScoreSubmission`, `ScoringFunction`, `TelemetrySettings`, `TraceListItem`. The `@terreno/ai/langfuseApp` and `@terreno/ai/langfuseClient` subpath exports are gone, along with the `@langfuse/client`, `@langfuse/otel`, and `@opentelemetry/*` dependencies of `@terreno/ai`.
30
+ - `addGptRoutes` drops `langfuseSystemPromptName`; use `systemPromptName` (a registry prompt) instead. Chat turns no longer set `experimental_telemetry`.
31
+ - `ControlPrimary` is now only `"local"`.
32
+ - The example backend no longer reads `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_BASE_URL`, `LANGFUSE_ORGANIZATION`, `LANGFUSE_PROJECT`, or `LANGFUSE_PROJECT_ID`. Deploys (`.github/workflows/cd.yml`, `scripts/ci/gcp-deploy.sh`) no longer mount the Langfuse secrets, and `infra/flourish/main.tf` drops the `backend_secret_langfuse_secret_key` and `backend_secret_langfuse_public_key` modules, so the next `terraform apply` deletes those two Secret Manager secrets. Deploy a revision without the mounts before applying.
33
+
34
+ ### Added
35
+
36
+ - `addGptRoutes` accepts `systemPromptName` and `systemPromptLabel`. A chat turn prepends that observability registry prompt to the system prompt and records its version on the trace.
37
+ - The example chat reads `chat-assistant` and the `generate_joke` tool reads `joke-generator` from the observability prompt registry.
38
+ - `bun run backend:seed` fills every AI Observability screen: labelled prompt versions for `example-summarize`, `joke-generator`, and `chat-assistant`, golden datasets, finished experiments, traces with scores, and review items in every status. Run it on an existing database with the `seedAiObservabilityDemo` admin script.
39
+ - Review batches: pick a human question set and press **Grab 10 recent traces** to give reviewers a fixed set of inputs and outputs (`POST /ai/observability/review/batches`).
40
+ - The review item screen shows one input, one output, and every question as Yes / No rows. On web, `Y` / `N` answer the highlighted question, `Enter` submits and moves to the next batch item, and `S` skips. Evaluator dimensions accept a reviewer `question`.
41
+ - Run any LLM judge over a batch and see overall and per-question agreement with the human answers, plus a list of disagreements (`POST /ai/observability/review/batches/:id/judgeRuns`).
42
+ - The AI observability seed adds the `summary-review-questions` question set, two judges for it, and a 10-item batch with 8 answers where one judge agrees 97% and the other 78%.
43
+ - Observability registry prompts can reference other prompts with `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}` (default label `production`). Shared instructions live in one prompt; `PromptRegistry.get`, the playground, experiments, and judges use the resolved text.
44
+ - Every reference must resolve when it is saved. Creating a prompt, saving a version, and moving a label return 400 `Invalid prompt reference` for a malformed tag, an unknown prompt or version, or a label the referenced prompt does not have yet, 400 `Prompt reference cycle` when the change would make prompts include each other, and 400 `Prompt references nested too deeply` when it would nest the prompt, or any prompt that includes it, more than 5 levels.
45
+ - Traces stay named after the called prompt. `ObsTrace.prompts` lists it first, then each shared prompt version it pulled in, marked with `includedBy`. `PromptVersionRef.includes` and harness prompt refs carry the same list.
46
+
47
+ ### Fixed
48
+
49
+ - Review queue: `Y` and `N` answer the highlighted question when a Yes/No segment is focused. **What the AI was given** and **What the AI wrote** are headings above the field cards.
50
+ - Review batches: **Save batch** sits on the New batch card. Search and **More filters** share a baseline.
51
+ - Prompt hub: description, folder, and tags sit on the Versions screen with the editor. System and template fields explain what to write.
52
+ - Evaluator detail loads the judge prompt `outputSchema` (so helpfulness no longer reports a missing dimension) and experiment rows open that experiment.
53
+
23
54
  ## [57.21.0] - 2026-11-01
24
55
 
25
56
  Upgrade note: [`mcp-server/src/docs/upgrades/57.21.0.md`](mcp-server/src/docs/upgrades/57.21.0.md).
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  AI service layer for Terreno apps - GPT chat, request logging, and admin tools.
4
4
 
5
- AI service layer for Terreno backends — provider-agnostic chat, request logging, GPT history, projects, file uploads, MCP tools, and optional Langfuse integration.
5
+ AI service layer for Terreno backends — provider-agnostic chat, request logging, GPT history, projects, file uploads, MCP tools, and built-in observability (prompt registry, traces, evaluators).
6
6
 
7
7
  ## Install
8
8
 
@@ -29,7 +29,7 @@ new TerrenoApp({userModel: User})
29
29
  .start();
30
30
  ```
31
31
 
32
- This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), projects, and the admin AI request explorer. Add `LangfuseApp`, `FileStorageService`, and `MCPService` when you need those features.
32
+ This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), projects, and the admin AI request explorer. Add `ObservabilityApp`, `FileStorageService`, and `MCPService` when you need those features.
33
33
 
34
34
  ## What's included
35
35
 
@@ -38,7 +38,7 @@ This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), proj
38
38
  - Mongoose models: `AIRequest`, `GptHistory`, `Project`, `FileAttachment`
39
39
  - Route registrars: `addGptRoutes`, `addGptHistoryRoutes`, `addProjectRoutes`, `addFileRoutes`, `addMcpRoutes`, `addAiRequestsExplorerRoutes`
40
40
  - Agent UI Asks (opt-in `asks` option on `AiApp` / `addGptRoutes`) — the agent pauses a turn to ask a typed question (`choice`, `confirm`, `markdown`, `form`, `files`), server-enforced approval for tools with `needsApproval`, and headless `pendingAsks` / `turn` endpoints for watches and other small clients. See [Agent UI Asks](https://github.com/TerrenoLabs/terreno/blob/master/docs/reference/agent-ui-asks.md)
41
- - `LangfuseApp` — optional Langfuse tracing and admin UI routes
41
+ - `ObservabilityApp` — prompt registry, traces, evaluators, datasets, experiments, and review queue
42
42
  - `FileStorageService` — GCS uploads with signed URLs
43
43
  - `MCPService` — SSE MCP client for tool calling
44
44
  - `getMCPTools` — wrap registered `modelRouter` MCP tools as Vercel AI SDK tools for in-process `streamText` / `generateText`
package/biome.jsonc CHANGED
@@ -1,5 +1,4 @@
1
1
  {
2
- "root": true,
3
2
  "$schema": "./node_modules/@biomejs/biome/configuration_schema.json",
4
3
  "extends": ["../biome.jsonc"],
5
4
  "files": {
@@ -17,5 +16,6 @@
17
16
  "!!**/coverage",
18
17
  "!!coverage"
19
18
  ]
20
- }
19
+ },
20
+ "root": true
21
21
  }
@@ -42,9 +42,10 @@ export declare const newTaskLease: (lease: HarnessLeaseSettings) => {
42
42
  token: string;
43
43
  };
44
44
  /**
45
- * Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a registry
46
- * `body` is dropped), once each. Throws `invalidRequest` for a ref without a name, with a
47
- * non-string label, or without a positive integer version.
45
+ * Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
46
+ * registry `body` is dropped and its `includes` follow it), once each. Throws
47
+ * `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
48
+ * non-array includes, or without a positive integer version.
48
49
  */
49
50
  export declare const tracePromptRefs: (refs: HarnessPromptRef[] | undefined) => HarnessPromptRef[];
50
51
  /**
@@ -76,9 +76,10 @@ const findByRequestId = async ({ definition, models, requestId, userId, }) => {
76
76
  return existing;
77
77
  };
78
78
  /**
79
- * Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a registry
80
- * `body` is dropped), once each. Throws `invalidRequest` for a ref without a name, with a
81
- * non-string label, or without a positive integer version.
79
+ * Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
80
+ * registry `body` is dropped and its `includes` follow it), once each. Throws
81
+ * `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
82
+ * non-array includes, or without a positive integer version.
82
83
  */
83
84
  const tracePromptRefs = (refs) => {
84
85
  if (refs === undefined) {
@@ -88,7 +89,18 @@ const tracePromptRefs = (refs) => {
88
89
  throw (0, errors_1.harnessError)({ detail: "prompts must be an array", kind: "invalidRequest" });
89
90
  }
90
91
  const unique = new Map();
91
- for (const ref of refs) {
92
+ const expanded = refs.flatMap((ref) => {
93
+ if (ref?.includes !== undefined && !Array.isArray(ref.includes)) {
94
+ throw (0, errors_1.harnessError)({
95
+ detail: typeof ref.name === "string" && ref.name.trim()
96
+ ? `Prompt ref "${ref.name}" includes must be an array`
97
+ : "Prompt ref includes must be an array",
98
+ kind: "invalidRequest",
99
+ });
100
+ }
101
+ return [ref, ...(ref?.includes ?? [])];
102
+ });
103
+ for (const ref of expanded) {
92
104
  if (typeof ref?.name !== "string" || !ref.name.trim()) {
93
105
  throw (0, errors_1.harnessError)({ detail: "Each prompt ref needs a name", kind: "invalidRequest" });
94
106
  }
@@ -104,7 +116,14 @@ const tracePromptRefs = (refs) => {
104
116
  kind: "invalidRequest",
105
117
  });
106
118
  }
119
+ if (ref.includedBy !== undefined && typeof ref.includedBy !== "string") {
120
+ throw (0, errors_1.harnessError)({
121
+ detail: `Prompt ref "${ref.name}" includedBy must be a string`,
122
+ kind: "invalidRequest",
123
+ });
124
+ }
107
125
  const normalized = {
126
+ ...(ref.includedBy !== undefined ? { includedBy: ref.includedBy } : {}),
108
127
  ...(ref.label !== undefined ? { label: ref.label } : {}),
109
128
  name: ref.name,
110
129
  version: ref.version,
package/dist/index.d.ts CHANGED
@@ -5,27 +5,24 @@ export type { AIAdminAppOptions } from "./aiAdminApp";
5
5
  export { AIAdminApp } from "./aiAdminApp";
6
6
  export type { AiAppOptions } from "./aiApp";
7
7
  export { AiApp } from "./aiApp";
8
- export { LangfuseApp } from "./langfuseApp";
9
- export { getCached, invalidateCache, LangfuseCache, setCached } from "./langfuseCache";
10
- export { getLangfuseClient, initLangfuseClient, isLangfuseInitialized, shutdownLangfuseClient, } from "./langfuseClient";
11
- export { compilePrompt, createPrompt, getPrompt, invalidatePromptCache } from "./langfusePrompts";
12
- export { initTracing, shutdownTracing } from "./langfuseTracing";
13
- export type { ChatMessage, GetPromptOptions, LangfuseAppOptions, LangfuseCachedPrompt, PaginatedResult, PreparePromptResult, PromptListItem, ScoreSubmission, ScoringFunction, TelemetrySettings, TraceListItem, } from "./langfuseTypes";
14
- export { createTelemetryConfig, preparePromptForAI } from "./langfuseVercelAi";
15
8
  export { AIDataset } from "./models/aiDataset";
16
9
  export { AIRequest } from "./models/aiRequest";
17
10
  export { FileAttachment } from "./models/fileAttachment";
18
11
  export { GptHistory } from "./models/gptHistory";
19
12
  export { getProjectModel, Project } from "./models/project";
20
13
  export { AI_OBSERVABILITY_GROUP, observabilityAdminScreens } from "./observability/adminScreens";
14
+ export { answersAgree, computeJudgeAgreement, type JudgeAgreement, } from "./observability/judgeAgreement";
21
15
  export { LocalDatasetStore } from "./observability/local/datasetStore";
22
- export { LocalEvaluatorStore } from "./observability/local/evaluatorStore";
23
- export { LocalExperimentRunner } from "./observability/local/experimentRunner";
16
+ export { type EvaluatorCalibration, LocalEvaluatorStore } from "./observability/local/evaluatorStore";
17
+ export { DEFAULT_TARGET_AGREEMENT, type JudgeExperimentCreateInput, LocalExperimentRunner, } from "./observability/local/experimentRunner";
18
+ export { type DatasetJudgmentSummary, loadCurrentHumanAnswers, } from "./observability/local/humanAnswers";
24
19
  export { createLocalObservabilityBundle, createLocalObservabilityPlugin, } from "./observability/local/localPlugin";
25
20
  export { LocalPromptStore } from "./observability/local/promptStore";
21
+ export { DEFAULT_REVIEW_BATCH_SIZE, LocalReviewBatchStore, MAX_REVIEW_BATCH_SIZE, type ReviewBatchDetail, type ReviewBatchListItem, } from "./observability/local/reviewBatchStore";
26
22
  export { LocalReviewStore } from "./observability/local/reviewStore";
27
23
  export { LocalScoreSink, LocalTraceSink, LocalTraceStore, MemoryScoreSink, MemoryTraceSink, } from "./observability/local/traceStore";
28
24
  export { getObservabilityApp, ObservabilityApp, resetObservabilityApp, } from "./observability/observabilityApp";
25
+ export { questionSetFingerprint } from "./observability/questionFingerprint";
29
26
  export type { ObservabilityPluginStatus, ObservabilityStatus, PlaygroundAiSource, PlaygroundAiStatus, } from "./observability/status";
30
27
  export { buildObservabilityStatus, buildPlaygroundAiStatus, isLocalObservabilityPluginOn, } from "./observability/status";
31
28
  export type { ControlPrimary, ObservabilityAiServiceFactory, ObservabilityAppOptions, ObservabilityCapability, ObservabilityControlConfig, ObservabilityGenerateClient, ObservabilityPlugin, ObservabilityRequestAiServiceFactory, PromptRegistry, ReviewQueue, ScoreRecord, ScoreSink, SpanRecord, TraceExportResult, TraceRecord, TraceSink, } from "./observability/types";
package/dist/index.js CHANGED
@@ -14,8 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.validateObservabilityConfig = exports.resolveObservabilityControl = exports.DEFAULT_OBSERVABILITY_CONTROL = exports.isLocalObservabilityPluginOn = exports.buildPlaygroundAiStatus = exports.buildObservabilityStatus = exports.resetObservabilityApp = exports.ObservabilityApp = exports.getObservabilityApp = exports.MemoryTraceSink = exports.MemoryScoreSink = exports.LocalTraceStore = exports.LocalTraceSink = exports.LocalScoreSink = exports.LocalReviewStore = exports.LocalPromptStore = exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = exports.LocalExperimentRunner = exports.LocalEvaluatorStore = exports.LocalDatasetStore = exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = exports.Project = exports.getProjectModel = exports.GptHistory = exports.FileAttachment = exports.AIRequest = exports.AIDataset = exports.preparePromptForAI = exports.createTelemetryConfig = exports.shutdownTracing = exports.initTracing = exports.invalidatePromptCache = exports.getPrompt = exports.createPrompt = exports.compilePrompt = exports.shutdownLangfuseClient = exports.isLangfuseInitialized = exports.initLangfuseClient = exports.getLangfuseClient = exports.setCached = exports.LangfuseCache = exports.invalidateCache = exports.getCached = exports.LangfuseApp = exports.AiApp = exports.AIAdminApp = exports.Output = exports.jsonSchema = void 0;
18
- exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService = exports.registerAiDataset = exports.configureAiDatasets = exports.findAgentBlock = exports.addProjectRoutes = exports.addMcpRoutes = exports.addGptHistoryRoutes = exports.addGptRoutes = exports.addFileRoutes = exports.addAiRequestsExplorerRoutes = void 0;
17
+ exports.registerAiDataset = exports.configureAiDatasets = exports.findAgentBlock = exports.addProjectRoutes = exports.addMcpRoutes = exports.addGptHistoryRoutes = exports.addGptRoutes = exports.addFileRoutes = exports.addAiRequestsExplorerRoutes = exports.validateObservabilityConfig = exports.resolveObservabilityControl = exports.DEFAULT_OBSERVABILITY_CONTROL = exports.isLocalObservabilityPluginOn = exports.buildPlaygroundAiStatus = exports.buildObservabilityStatus = exports.questionSetFingerprint = exports.resetObservabilityApp = exports.ObservabilityApp = exports.getObservabilityApp = exports.MemoryTraceSink = exports.MemoryScoreSink = exports.LocalTraceStore = exports.LocalTraceSink = exports.LocalScoreSink = exports.LocalReviewStore = exports.MAX_REVIEW_BATCH_SIZE = exports.LocalReviewBatchStore = exports.DEFAULT_REVIEW_BATCH_SIZE = exports.LocalPromptStore = exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = exports.loadCurrentHumanAnswers = exports.LocalExperimentRunner = exports.DEFAULT_TARGET_AGREEMENT = exports.LocalEvaluatorStore = exports.LocalDatasetStore = exports.computeJudgeAgreement = exports.answersAgree = exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = exports.Project = exports.getProjectModel = exports.GptHistory = exports.FileAttachment = exports.AIRequest = exports.AIDataset = exports.AiApp = exports.AIAdminApp = exports.Output = exports.jsonSchema = void 0;
18
+ exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService = void 0;
19
19
  var ai_1 = require("ai");
20
20
  Object.defineProperty(exports, "jsonSchema", { enumerable: true, get: function () { return ai_1.jsonSchema; } });
21
21
  Object.defineProperty(exports, "Output", { enumerable: true, get: function () { return ai_1.Output; } });
@@ -23,29 +23,6 @@ var aiAdminApp_1 = require("./aiAdminApp");
23
23
  Object.defineProperty(exports, "AIAdminApp", { enumerable: true, get: function () { return aiAdminApp_1.AIAdminApp; } });
24
24
  var aiApp_1 = require("./aiApp");
25
25
  Object.defineProperty(exports, "AiApp", { enumerable: true, get: function () { return aiApp_1.AiApp; } });
26
- var langfuseApp_1 = require("./langfuseApp");
27
- Object.defineProperty(exports, "LangfuseApp", { enumerable: true, get: function () { return langfuseApp_1.LangfuseApp; } });
28
- var langfuseCache_1 = require("./langfuseCache");
29
- Object.defineProperty(exports, "getCached", { enumerable: true, get: function () { return langfuseCache_1.getCached; } });
30
- Object.defineProperty(exports, "invalidateCache", { enumerable: true, get: function () { return langfuseCache_1.invalidateCache; } });
31
- Object.defineProperty(exports, "LangfuseCache", { enumerable: true, get: function () { return langfuseCache_1.LangfuseCache; } });
32
- Object.defineProperty(exports, "setCached", { enumerable: true, get: function () { return langfuseCache_1.setCached; } });
33
- var langfuseClient_1 = require("./langfuseClient");
34
- Object.defineProperty(exports, "getLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.getLangfuseClient; } });
35
- Object.defineProperty(exports, "initLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.initLangfuseClient; } });
36
- Object.defineProperty(exports, "isLangfuseInitialized", { enumerable: true, get: function () { return langfuseClient_1.isLangfuseInitialized; } });
37
- Object.defineProperty(exports, "shutdownLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.shutdownLangfuseClient; } });
38
- var langfusePrompts_1 = require("./langfusePrompts");
39
- Object.defineProperty(exports, "compilePrompt", { enumerable: true, get: function () { return langfusePrompts_1.compilePrompt; } });
40
- Object.defineProperty(exports, "createPrompt", { enumerable: true, get: function () { return langfusePrompts_1.createPrompt; } });
41
- Object.defineProperty(exports, "getPrompt", { enumerable: true, get: function () { return langfusePrompts_1.getPrompt; } });
42
- Object.defineProperty(exports, "invalidatePromptCache", { enumerable: true, get: function () { return langfusePrompts_1.invalidatePromptCache; } });
43
- var langfuseTracing_1 = require("./langfuseTracing");
44
- Object.defineProperty(exports, "initTracing", { enumerable: true, get: function () { return langfuseTracing_1.initTracing; } });
45
- Object.defineProperty(exports, "shutdownTracing", { enumerable: true, get: function () { return langfuseTracing_1.shutdownTracing; } });
46
- var langfuseVercelAi_1 = require("./langfuseVercelAi");
47
- Object.defineProperty(exports, "createTelemetryConfig", { enumerable: true, get: function () { return langfuseVercelAi_1.createTelemetryConfig; } });
48
- Object.defineProperty(exports, "preparePromptForAI", { enumerable: true, get: function () { return langfuseVercelAi_1.preparePromptForAI; } });
49
26
  var aiDataset_1 = require("./models/aiDataset");
50
27
  Object.defineProperty(exports, "AIDataset", { enumerable: true, get: function () { return aiDataset_1.AIDataset; } });
51
28
  var aiRequest_1 = require("./models/aiRequest");
@@ -60,17 +37,27 @@ Object.defineProperty(exports, "Project", { enumerable: true, get: function () {
60
37
  var adminScreens_1 = require("./observability/adminScreens");
61
38
  Object.defineProperty(exports, "AI_OBSERVABILITY_GROUP", { enumerable: true, get: function () { return adminScreens_1.AI_OBSERVABILITY_GROUP; } });
62
39
  Object.defineProperty(exports, "observabilityAdminScreens", { enumerable: true, get: function () { return adminScreens_1.observabilityAdminScreens; } });
40
+ var judgeAgreement_1 = require("./observability/judgeAgreement");
41
+ Object.defineProperty(exports, "answersAgree", { enumerable: true, get: function () { return judgeAgreement_1.answersAgree; } });
42
+ Object.defineProperty(exports, "computeJudgeAgreement", { enumerable: true, get: function () { return judgeAgreement_1.computeJudgeAgreement; } });
63
43
  var datasetStore_1 = require("./observability/local/datasetStore");
64
44
  Object.defineProperty(exports, "LocalDatasetStore", { enumerable: true, get: function () { return datasetStore_1.LocalDatasetStore; } });
65
45
  var evaluatorStore_1 = require("./observability/local/evaluatorStore");
66
46
  Object.defineProperty(exports, "LocalEvaluatorStore", { enumerable: true, get: function () { return evaluatorStore_1.LocalEvaluatorStore; } });
67
47
  var experimentRunner_1 = require("./observability/local/experimentRunner");
48
+ Object.defineProperty(exports, "DEFAULT_TARGET_AGREEMENT", { enumerable: true, get: function () { return experimentRunner_1.DEFAULT_TARGET_AGREEMENT; } });
68
49
  Object.defineProperty(exports, "LocalExperimentRunner", { enumerable: true, get: function () { return experimentRunner_1.LocalExperimentRunner; } });
50
+ var humanAnswers_1 = require("./observability/local/humanAnswers");
51
+ Object.defineProperty(exports, "loadCurrentHumanAnswers", { enumerable: true, get: function () { return humanAnswers_1.loadCurrentHumanAnswers; } });
69
52
  var localPlugin_1 = require("./observability/local/localPlugin");
70
53
  Object.defineProperty(exports, "createLocalObservabilityBundle", { enumerable: true, get: function () { return localPlugin_1.createLocalObservabilityBundle; } });
71
54
  Object.defineProperty(exports, "createLocalObservabilityPlugin", { enumerable: true, get: function () { return localPlugin_1.createLocalObservabilityPlugin; } });
72
55
  var promptStore_1 = require("./observability/local/promptStore");
73
56
  Object.defineProperty(exports, "LocalPromptStore", { enumerable: true, get: function () { return promptStore_1.LocalPromptStore; } });
57
+ var reviewBatchStore_1 = require("./observability/local/reviewBatchStore");
58
+ Object.defineProperty(exports, "DEFAULT_REVIEW_BATCH_SIZE", { enumerable: true, get: function () { return reviewBatchStore_1.DEFAULT_REVIEW_BATCH_SIZE; } });
59
+ Object.defineProperty(exports, "LocalReviewBatchStore", { enumerable: true, get: function () { return reviewBatchStore_1.LocalReviewBatchStore; } });
60
+ Object.defineProperty(exports, "MAX_REVIEW_BATCH_SIZE", { enumerable: true, get: function () { return reviewBatchStore_1.MAX_REVIEW_BATCH_SIZE; } });
74
61
  var reviewStore_1 = require("./observability/local/reviewStore");
75
62
  Object.defineProperty(exports, "LocalReviewStore", { enumerable: true, get: function () { return reviewStore_1.LocalReviewStore; } });
76
63
  var traceStore_1 = require("./observability/local/traceStore");
@@ -83,6 +70,8 @@ var observabilityApp_1 = require("./observability/observabilityApp");
83
70
  Object.defineProperty(exports, "getObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.getObservabilityApp; } });
84
71
  Object.defineProperty(exports, "ObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.ObservabilityApp; } });
85
72
  Object.defineProperty(exports, "resetObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.resetObservabilityApp; } });
73
+ var questionFingerprint_1 = require("./observability/questionFingerprint");
74
+ Object.defineProperty(exports, "questionSetFingerprint", { enumerable: true, get: function () { return questionFingerprint_1.questionSetFingerprint; } });
86
75
  var status_1 = require("./observability/status");
87
76
  Object.defineProperty(exports, "buildObservabilityStatus", { enumerable: true, get: function () { return status_1.buildObservabilityStatus; } });
88
77
  Object.defineProperty(exports, "buildPlaygroundAiStatus", { enumerable: true, get: function () { return status_1.buildPlaygroundAiStatus; } });
@@ -25,7 +25,11 @@ const observabilityAdminScreens = ({ localOn }) => {
25
25
  }),
26
26
  ];
27
27
  if (localOn) {
28
- screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" }));
28
+ screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" }), observabilityScreen({
29
+ displayName: "Review batches",
30
+ name: "ai-review-batches",
31
+ resource: "aiReview",
32
+ }));
29
33
  }
30
34
  return screens;
31
35
  };
@@ -23,7 +23,14 @@ export interface EvaluatorExecutionResult {
23
23
  scores?: Record<string, boolean | number | string>;
24
24
  }
25
25
  export declare const runJsonAssertEvaluator: (context: EvaluatorExecutionContext) => EvaluatorExecutionResult;
26
- export declare const runLlmJudgeEvaluator: (context: EvaluatorExecutionContext, ai: EvaluatorAiClient, judgeOutputSchema: Record<string, unknown>) => Promise<EvaluatorExecutionResult>;
26
+ /**
27
+ * Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
28
+ * version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
29
+ * version instead, as judge calibration experiments do.
30
+ */
31
+ export declare const runLlmJudgeEvaluator: (context: EvaluatorExecutionContext, ai: EvaluatorAiClient, judgeOutputSchema: Record<string, unknown>, options?: {
32
+ judgeSystemPrompt?: string;
33
+ }) => Promise<EvaluatorExecutionResult>;
27
34
  export declare const runEvaluator: (context: EvaluatorExecutionContext, deps: {
28
35
  ai?: EvaluatorAiClient;
29
36
  judgeOutputSchema?: Record<string, unknown>;
@@ -130,15 +130,23 @@ const runJsonAssertEvaluator = (context) => {
130
130
  };
131
131
  };
132
132
  exports.runJsonAssertEvaluator = runJsonAssertEvaluator;
133
- const runLlmJudgeEvaluator = async (context, ai, judgeOutputSchema) => {
133
+ /**
134
+ * Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
135
+ * version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
136
+ * version instead, as judge calibration experiments do.
137
+ */
138
+ const runLlmJudgeEvaluator = async (context, ai, judgeOutputSchema, options = {}) => {
134
139
  const judgePromptName = context.evaluator.judgePromptName;
135
140
  if (!judgePromptName) {
136
141
  return { error: "llm-judge evaluator is missing judgePromptName" };
137
142
  }
138
143
  try {
144
+ const promptSource = options.judgeSystemPrompt === undefined
145
+ ? { promptName: judgePromptName }
146
+ : { systemPrompt: options.judgeSystemPrompt };
139
147
  const payload = (await ai.generateJsonObject({
140
148
  prompt: buildJudgePrompt(context),
141
- promptName: judgePromptName,
149
+ ...promptSource,
142
150
  schema: (0, ai_1.jsonSchema)(judgeOutputSchema),
143
151
  schemaName: `${context.evaluator.name}-judge`,
144
152
  skipTrace: true,
@@ -0,0 +1,53 @@
1
+ import type { EvaluatorDimension } from "../types/observability";
2
+ export type ScoreValue = boolean | number | string;
3
+ /**
4
+ * Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
5
+ * otherwise by `traceId`.
6
+ */
7
+ export interface AgreementAnswerSet {
8
+ datasetItemId?: string;
9
+ scores?: Record<string, ScoreValue>;
10
+ traceId?: string;
11
+ }
12
+ export interface JudgeAnswerSet extends AgreementAnswerSet {
13
+ error?: string;
14
+ }
15
+ export interface JudgeQuestionAgreement {
16
+ compared: number;
17
+ /** False when the judge does not answer this question, so it is left out of the totals. */
18
+ covered: boolean;
19
+ key: string;
20
+ matches: number;
21
+ question?: string;
22
+ rate?: number;
23
+ }
24
+ export interface JudgeDisagreement {
25
+ datasetItemId?: string;
26
+ human: ScoreValue;
27
+ judge?: ScoreValue;
28
+ key: string;
29
+ traceId?: string;
30
+ }
31
+ export interface JudgeAgreement {
32
+ compared: number;
33
+ disagreements: JudgeDisagreement[];
34
+ errors: number;
35
+ itemsCompared: number;
36
+ matches: number;
37
+ questions: JudgeQuestionAgreement[];
38
+ rate?: number;
39
+ }
40
+ /** True when a judge answer counts as the same answer a human gave for this dimension. */
41
+ export declare const answersAgree: (dimension: EvaluatorDimension, human: ScoreValue, judge: ScoreValue | undefined) => boolean;
42
+ /** Key that pairs human and judge answers for the same item. */
43
+ export declare const agreementAnswerKey: (answer: AgreementAnswerSet) => string;
44
+ /**
45
+ * Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
46
+ * answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
47
+ */
48
+ export declare const computeJudgeAgreement: (params: {
49
+ dimensions: EvaluatorDimension[];
50
+ humanAnswers: AgreementAnswerSet[];
51
+ judgeAnswers: JudgeAnswerSet[];
52
+ judgeKeys: string[];
53
+ }) => JudgeAgreement;
@@ -0,0 +1,123 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.computeJudgeAgreement = exports.agreementAnswerKey = exports.answersAgree = void 0;
4
+ // Numeric answers agree when they land within this share of the dimension's range.
5
+ const NUMERIC_TOLERANCE = 0.1;
6
+ const numericSpan = (range) => {
7
+ const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
8
+ if (!matches) {
9
+ return 1;
10
+ }
11
+ const span = Math.abs(Number(matches[2]) - Number(matches[1]));
12
+ return span > 0 ? span : 1;
13
+ };
14
+ const toBoolean = (value) => {
15
+ if (typeof value === "boolean") {
16
+ return value;
17
+ }
18
+ if (value === "true" || value === "yes") {
19
+ return true;
20
+ }
21
+ if (value === "false" || value === "no") {
22
+ return false;
23
+ }
24
+ return undefined;
25
+ };
26
+ /** True when a judge answer counts as the same answer a human gave for this dimension. */
27
+ const answersAgree = (dimension, human, judge) => {
28
+ if (judge === undefined) {
29
+ return false;
30
+ }
31
+ if (dimension.dataType === "boolean") {
32
+ const humanBool = toBoolean(human);
33
+ return humanBool !== undefined && humanBool === toBoolean(judge);
34
+ }
35
+ if (dimension.dataType === "numeric") {
36
+ const humanNumber = Number(human);
37
+ const judgeNumber = Number(judge);
38
+ if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
39
+ return false;
40
+ }
41
+ return (Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9);
42
+ }
43
+ return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
44
+ };
45
+ exports.answersAgree = answersAgree;
46
+ /** Key that pairs human and judge answers for the same item. */
47
+ const agreementAnswerKey = (answer) => {
48
+ return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
49
+ };
50
+ exports.agreementAnswerKey = agreementAnswerKey;
51
+ const rateOf = (matches, compared) => {
52
+ return compared > 0 ? matches / compared : undefined;
53
+ };
54
+ /**
55
+ * Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
56
+ * answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
57
+ */
58
+ const computeJudgeAgreement = (params) => {
59
+ const judgeKeys = new Set(params.judgeKeys);
60
+ const judgeByKey = new Map(params.judgeAnswers.map((answer) => {
61
+ return [(0, exports.agreementAnswerKey)(answer), answer];
62
+ }));
63
+ const questions = params.dimensions.map((dimension) => {
64
+ return {
65
+ compared: 0,
66
+ covered: judgeKeys.has(dimension.key),
67
+ key: dimension.key,
68
+ matches: 0,
69
+ question: dimension.question,
70
+ };
71
+ });
72
+ const disagreements = [];
73
+ let itemsCompared = 0;
74
+ let errors = 0;
75
+ for (const human of params.humanAnswers) {
76
+ const judge = judgeByKey.get((0, exports.agreementAnswerKey)(human));
77
+ if (!judge || !human.scores) {
78
+ continue;
79
+ }
80
+ let comparedThisItem = false;
81
+ if (judge.error) {
82
+ errors += 1;
83
+ }
84
+ params.dimensions.forEach((dimension, index) => {
85
+ const humanValue = human.scores?.[dimension.key];
86
+ const question = questions[index];
87
+ if (!question?.covered || humanValue === undefined) {
88
+ return;
89
+ }
90
+ comparedThisItem = true;
91
+ question.compared += 1;
92
+ const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
93
+ if ((0, exports.answersAgree)(dimension, humanValue, judgeValue)) {
94
+ question.matches += 1;
95
+ return;
96
+ }
97
+ disagreements.push({
98
+ ...(human.datasetItemId ? { datasetItemId: human.datasetItemId } : {}),
99
+ human: humanValue,
100
+ judge: judgeValue,
101
+ key: dimension.key,
102
+ ...(human.traceId ? { traceId: human.traceId } : {}),
103
+ });
104
+ });
105
+ if (comparedThisItem) {
106
+ itemsCompared += 1;
107
+ }
108
+ }
109
+ const compared = questions.reduce((total, question) => total + question.compared, 0);
110
+ const matches = questions.reduce((total, question) => total + question.matches, 0);
111
+ return {
112
+ compared,
113
+ disagreements,
114
+ errors,
115
+ itemsCompared,
116
+ matches,
117
+ questions: questions.map((question) => {
118
+ return { ...question, rate: rateOf(question.matches, question.compared) };
119
+ }),
120
+ rate: rateOf(matches, compared),
121
+ };
122
+ };
123
+ exports.computeJudgeAgreement = computeJudgeAgreement;
@@ -1,5 +1,6 @@
1
1
  import type { ObsDatasetAnnotatedBy } from "../../types/observability";
2
2
  import { type ParsedDatasetImportRow } from "../datasetImport";
3
+ import { type DatasetHumanAnswerView, type DatasetJudgmentSummary } from "./humanAnswers";
3
4
  import { LocalPromptStore } from "./promptStore";
4
5
  export interface DatasetWriteInput {
5
6
  description?: string | null;
@@ -31,6 +32,8 @@ export interface DatasetView {
31
32
  expectedOutputSchema?: Record<string, unknown>;
32
33
  id: string;
33
34
  inputSchemaPromptName?: string;
35
+ /** Human judging progress per question set (see `summarizeDatasetJudgments`). */
36
+ judgments: DatasetJudgmentSummary[];
34
37
  name: string;
35
38
  tags: string[];
36
39
  updated: string;
@@ -40,6 +43,8 @@ export interface DatasetItemView {
40
43
  created: string;
41
44
  datasetId: string;
42
45
  expectedOutput?: unknown;
46
+ /** Human answers keyed by question-set evaluator id. */
47
+ humanAnswers?: Record<string, DatasetHumanAnswerView>;
43
48
  id: string;
44
49
  input: unknown;
45
50
  metadata?: Record<string, unknown>;
@@ -9,6 +9,7 @@ const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
10
10
  const datasetImport_1 = require("../datasetImport");
11
11
  const schemaValidation_1 = require("../schemaValidation");
12
+ const humanAnswers_1 = require("./humanAnswers");
12
13
  const obsDataset_1 = require("./models/obsDataset");
13
14
  const obsDatasetItem_1 = require("./models/obsDatasetItem");
14
15
  const obsTrace_1 = require("./models/obsTrace");
@@ -24,8 +25,9 @@ const throwOnDatasetImportErrors = (result) => {
24
25
  });
25
26
  };
26
27
  exports.throwOnDatasetImportErrors = throwOnDatasetImportErrors;
27
- const toDatasetView = async (doc) => {
28
+ const toDatasetView = async (doc, judgments) => {
28
29
  const counts = await computeDatasetCounts(String(doc._id));
30
+ const resolvedJudgments = judgments ?? (await (0, humanAnswers_1.summarizeDatasetJudgments)([doc._id])).get(String(doc._id)) ?? [];
29
31
  return {
30
32
  counts,
31
33
  created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
@@ -33,6 +35,7 @@ const toDatasetView = async (doc) => {
33
35
  expectedOutputSchema: doc.expectedOutputSchema,
34
36
  id: String(doc._id),
35
37
  inputSchemaPromptName: doc.inputSchemaPromptName,
38
+ judgments: resolvedJudgments,
36
39
  name: doc.name,
37
40
  tags: doc.tags,
38
41
  updated: luxon_1.DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
@@ -44,6 +47,7 @@ const toItemView = (doc) => {
44
47
  created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
45
48
  datasetId: String(doc.datasetId),
46
49
  expectedOutput: doc.expectedOutput,
50
+ humanAnswers: (0, humanAnswers_1.toHumanAnswerViews)(doc.humanAnswers),
47
51
  id: String(doc._id),
48
52
  input: doc.input,
49
53
  metadata: doc.metadata,
@@ -106,8 +110,9 @@ class LocalDatasetStore {
106
110
  }
107
111
  async list() {
108
112
  const rows = await (0, obsDataset_1.registerObsDataset)().find({}).sort({ name: 1 });
113
+ const judgments = await (0, humanAnswers_1.summarizeDatasetJudgments)(rows.map((row) => row._id));
109
114
  return Promise.all(rows.map((row) => {
110
- return toDatasetView(row);
115
+ return toDatasetView(row, judgments.get(String(row._id)) ?? []);
111
116
  }));
112
117
  }
113
118
  async get(id) {
@@ -15,11 +15,27 @@ export interface EvaluatorWriteInput {
15
15
  target: "dataset item" | "full trace" | "generation span";
16
16
  type: "human" | "json-assert" | "llm-judge";
17
17
  }
18
+ /** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
19
+ export interface EvaluatorCalibration {
20
+ datasetName?: string;
21
+ experimentId: string;
22
+ experimentName: string;
23
+ passed: boolean;
24
+ questionSetName?: string;
25
+ /** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
26
+ rate?: number;
27
+ /** True when the question set's questions changed since the experiment ran. */
28
+ stale: boolean;
29
+ targetAgreement?: number;
30
+ /** The judge prompt's production version when it was compared, else the best version. */
31
+ version: number;
32
+ }
18
33
  export interface EvaluatorView {
19
34
  assertion?: {
20
35
  constraint: string;
21
36
  path: string;
22
37
  };
38
+ calibration?: EvaluatorCalibration;
23
39
  confidenceAlertBelow: number;
24
40
  description?: string;
25
41
  dimensions: EvaluatorDimension[];
@@ -37,6 +53,7 @@ export declare class LocalEvaluatorStore {
37
53
  create(input: EvaluatorWriteInput): Promise<EvaluatorView>;
38
54
  list(): Promise<EvaluatorView[]>;
39
55
  get(id: string): Promise<EvaluatorView>;
56
+ private loadCalibrations;
40
57
  getByName(name: string): Promise<EvaluatorView | undefined>;
41
58
  update(id: string, input: Partial<EvaluatorWriteInput>): Promise<EvaluatorView>;
42
59
  remove(id: string): Promise<void>;