@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
package/CHANGELOG.md CHANGED
@@ -20,6 +20,87 @@ Upgrade notes for consumer action live in [`mcp-server/src/docs/upgrades/`](mcp-
20
20
 
21
21
  Unreleased changes live in [`changelog/unreleased/`](changelog/unreleased/). Add one Markdown file per feature (see that directory's README) instead of editing this section.
22
22
 
23
+ ## [57.22.0] - 2026-11-03
24
+
25
+ Upgrade note: [`mcp-server/src/docs/upgrades/57.22.0.md`](mcp-server/src/docs/upgrades/57.22.0.md).
26
+
27
+ ### Breaking
28
+
29
+ - `@terreno/ai` no longer integrates with Langfuse. The built-in observability prompt registry, traces, and review queue (`ObservabilityApp` with `createLocalObservabilityPlugin()`) replace it. Removed exports: `LangfuseApp`, `LangfuseCache`, `getCached`, `setCached`, `invalidateCache`, `initLangfuseClient`, `getLangfuseClient`, `isLangfuseInitialized`, `shutdownLangfuseClient`, `compilePrompt`, `createPrompt`, `getPrompt`, `invalidatePromptCache`, `initTracing`, `shutdownTracing`, `createTelemetryConfig`, `preparePromptForAI`, and the types `ChatMessage`, `GetPromptOptions`, `LangfuseAppOptions`, `LangfuseCachedPrompt`, `PaginatedResult`, `PreparePromptResult`, `PromptListItem`, `ScoreSubmission`, `ScoringFunction`, `TelemetrySettings`, `TraceListItem`. The `@terreno/ai/langfuseApp` and `@terreno/ai/langfuseClient` subpath exports are gone, along with the `@langfuse/client`, `@langfuse/otel`, and `@opentelemetry/*` dependencies of `@terreno/ai`.
30
+ - `addGptRoutes` drops `langfuseSystemPromptName`; use `systemPromptName` (a registry prompt) instead. Chat turns no longer set `experimental_telemetry`.
31
+ - `ControlPrimary` is now only `"local"`.
32
+ - The example backend no longer reads `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_BASE_URL`, `LANGFUSE_ORGANIZATION`, `LANGFUSE_PROJECT`, or `LANGFUSE_PROJECT_ID`. Deploys (`.github/workflows/cd.yml`, `scripts/ci/gcp-deploy.sh`) no longer mount the Langfuse secrets, and `infra/flourish/main.tf` drops the `backend_secret_langfuse_secret_key` and `backend_secret_langfuse_public_key` modules, so the next `terraform apply` deletes those two Secret Manager secrets. Deploy a revision without the mounts before applying.
33
+
34
+ ### Added
35
+
36
+ - `addGptRoutes` accepts `systemPromptName` and `systemPromptLabel`. A chat turn prepends that observability registry prompt to the system prompt and records its version on the trace.
37
+ - The example chat reads `chat-assistant` and the `generate_joke` tool reads `joke-generator` from the observability prompt registry.
38
+ - `bun run backend:seed` fills every AI Observability screen: labelled prompt versions for `example-summarize`, `joke-generator`, and `chat-assistant`, golden datasets, finished experiments, traces with scores, and review items in every status. Run it on an existing database with the `seedAiObservabilityDemo` admin script.
39
+ - Review batches: pick a human question set and press **Grab 10 recent traces** to give reviewers a fixed set of inputs and outputs (`POST /ai/observability/review/batches`).
40
+ - The review item screen shows one input, one output, and every question as Yes / No rows. On web, `Y` / `N` answer the highlighted question, `Enter` submits and moves to the next batch item, and `S` skips. Evaluator dimensions accept a reviewer `question`.
41
+ - Run any LLM judge over a batch and see overall and per-question agreement with the human answers, plus a list of disagreements (`POST /ai/observability/review/batches/:id/judgeRuns`).
42
+ - The AI observability seed adds the `summary-review-questions` question set, two judges for it, and a 10-item batch with 8 answers where one judge agrees 97% and the other 78%.
43
+ - Observability registry prompts can reference other prompts with `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}` (default label `production`). Shared instructions live in one prompt; `PromptRegistry.get`, the playground, experiments, and judges use the resolved text.
44
+ - Every reference must resolve when it is saved. Creating a prompt, saving a version, and moving a label return 400 `Invalid prompt reference` for a malformed tag, an unknown prompt or version, or a label the referenced prompt does not have yet, 400 `Prompt reference cycle` when the change would make prompts include each other, and 400 `Prompt references nested too deeply` when it would nest the prompt, or any prompt that includes it, more than 5 levels.
45
+ - Traces stay named after the called prompt. `ObsTrace.prompts` lists it first, then each shared prompt version it pulled in, marked with `includedBy`. `PromptVersionRef.includes` and harness prompt refs carry the same list.
46
+
47
+ ### Fixed
48
+
49
+ - Review queue: `Y` and `N` answer the highlighted question when a Yes/No segment is focused. **What the AI was given** and **What the AI wrote** are headings above the field cards.
50
+ - Review batches: **Save batch** sits on the New batch card. Search and **More filters** share a baseline.
51
+ - Prompt hub: description, folder, and tags sit on the Versions screen with the editor. System and template fields explain what to write.
52
+ - Evaluator detail loads the judge prompt `outputSchema` (so helpfulness no longer reports a missing dimension) and experiment rows open that experiment.
53
+
54
+ ## [57.21.0] - 2026-11-01
55
+
56
+ Upgrade note: [`mcp-server/src/docs/upgrades/57.21.0.md`](mcp-server/src/docs/upgrades/57.21.0.md).
57
+
58
+ ### Breaking
59
+
60
+ - The `terreno-scan` plugin is now `terreno-loop`, and its skills are renamed from `terreno-scan-*` to `terreno-loop-*`. The resident loop `terreno-scan-loop` is now `terreno-loop-run`: `/terreno-loop:run` in Claude Code, `/terreno-loop-run` in Cursor, `$terreno-loop-run` in Codex. Claude Code moves existing installs over through the marketplace `renames` entry. In Cursor and Codex, uninstall `terreno-scan` and install `terreno-loop`. Campaign state stays at `.terreno/scan/<goal>.json`, so campaigns already in progress resume.
61
+
62
+ ### Added
63
+
64
+ - `@terreno/admin-spa` serves an MCP OAuth consent screen at `/console/oauth/consent`, so a backend with no product frontend can set `mcpOAuth.consentUrl` to `${issuer}/console/oauth/consent`. Any signed-in user can approve or deny a request with their Better Auth session; signed-out users sign in and return through the login screen's new `returnTo` parameter, which accepts only in-app paths.
65
+ - Claude Code plugins now publish to npm with every release, as `@terreno/agent-plugin` and `@terreno/agent-plugin-loop`. A new `terreno-releases` marketplace installs them at the latest stable release. To pin one release, add `TerrenoLabs/terreno#<version>` instead. See [Install agent skills](docs/how-to/install-agent-skills.md#pick-a-channel).
66
+ - The generated Claude plugin manifest no longer pins `version`. Before this change, plugins installed from the `TerrenoLabs/terreno` marketplace stayed on the `2.12.0` copy, because their version never changed. They now update on every commit. Run `/plugin update terreno@terreno-plugins` once to leave `2.12.0`.
67
+ - The `terreno-planning` lifecycle plugin (2.13.0) plans e2e tests and manual regression cases in every IP:
68
+
69
+ - Grow requires `## E2E tests` and `## Regression cases` sections (either may be `None — <reason>`), read from the e2e guide at `docs/testing/e2e.md` or a `## Testing conventions` override in AGENTS.md or CLAUDE.md.
70
+ - Regression cases stay minimal: most features add none. A case is only for what a developer or QA must check by hand, such as push notifications, and Roast fails one that repeats an existing case.
71
+ - Pick writes each planned spec in the task that delivers it, and Roast runs it; a missing or failing row is `FAIL`.
72
+ - Brew gates on every planned spec and merges regression cases into `docs/regression/<area>.md` with the next free ID.
73
+ - Taste re-checks both sections after behaviour-changing fixes and renumbers regression ID collisions with `master`.
74
+ - `bun run check:lifecycle-skills` enforces these phrases in every stage.
75
+ - New `backfill-regression` skill proposes manual regression cases one area at a time and writes only the sets the human approves.
76
+ - A consumer opts out of manual regression suites with `Regression suites | none` in its `## Testing conventions` table.
77
+ - The `terreno-planning` lifecycle plugin (2.14.0) grows an end-user guide with every feature:
78
+
79
+ - Every IP has a `## User guide` section listing the guide pages it adds, updates, or retires, or `None — <reason>`. Pages live in `docs/user-guide/<area>/<feature>.md`, one feature per page, with `title`, `audience`, `platforms`, and `questions` frontmatter that chat agents use to answer "how do I…" questions.
80
+ - Pick writes each page in the task that ships the feature, Roast checks it against the shipped screens, Brew gates on it and the index, and Taste keeps it current after fixes.
81
+ - A repo opts out with `| User guide | none — <reason> |` in a `## User guide conventions` table in AGENTS.md or CLAUDE.md.
82
+ - New `backfill-user-guide` skill drafts pages for features that shipped without one, one area at a time, and writes only the pages the human approves.
83
+ - `bun run check:user-guide` validates the pages and index (CI `repo-policies`); `--uncovered <app dir>` lists screens no page names.
84
+ - Terreno's own guide covers the example app's main features.
85
+ - `@terreno/api`: opt-in `mcpRequestLog` on `TerrenoApp` records one metadata-only `McpRequestLog` row per MCP tool call (tool, user, credential, service token or OAuth client, duration, outcome, error class; never arguments, results, or error text), expiring after `retentionDays` (default 30).
86
+ - `@terreno/api`: opt-in `mcpToolSettings` on `TerrenoApp` registers every MCP tool in `McpToolSetting` at startup; tools an admin switches off are hidden from `tools/list` and rejected by `tools/call`.
87
+ - `@terreno/api`: deleting an `McpOAuthToken` row revokes its whole connection (token family) and keeps the rows. New `McpOAuthToken.revokeAllForUser` and `McpServiceToken.revokeAllForUser`.
88
+ - `@terreno/admin-backend`: `mcpAdminModels()` adds MCP admin screens (service tokens, OAuth clients, OAuth connections, request log, tool switches), `mcpAdminStatements` for RBAC, and `mcpAdminScripts()` adds `mcp-sign-out-everywhere`. See [Manage MCP from the admin panel](docs/how-to/manage-mcp-in-admin.md).
89
+ - Docs: [Expose an MCP server safely](docs/how-to/expose-mcp-server-safely.md) (OAuth, service tokens, Cloudflare Tunnel, Access) and the `bootstrap-mcp-server` skill for a single-binary MCP server with the admin console embedded.
90
+ - Every workspace package has a real-process smoke test and an e2e page:
91
+
92
+ - Each package's `e2e/smoke.ts` (or `smoke.spec.ts` for browser packages) runs its built output through its public entry: a server on a real port, the compiled binary, or a page in Chromium. `bun run --cwd <package> smoke` runs one; `bun run smoke` runs them all and prints each package's time. The CircleCI `smoke` job runs them on every package change.
93
+ - `@terreno/test` adds the smoke helpers `smokeMongoUri()`, `spawnUntilReady()`, `spawnExampleBackend()`, `runBin()`, and `freePort()`.
94
+ - The [E2E testing guide](docs/testing/e2e.md) explains the test tiers and links one page per package under `docs/testing/e2e/`.
95
+ - Manual regression cases live in `docs/regression/<area>.md`, one file per area, and `bun run check:regression-suites` validates them.
96
+ - example-frontend's smoke serves a web export that `bun run --cwd example-frontend export:smoke` builds once with a placeholder API URL; the smoke swaps in a `freePort()` API URL as it serves the bundle, so the 30 s smoke no longer rebuilds the export. `spawnUntilReady()` takes `stopGraceMs`, and `spawnExampleBackend()` stops example-backend after 0.5 s instead of 5 s.
97
+ - Every workspace package has a `typecheck` script that runs `tsc` over source and test files through a new `tsconfig.typecheck.json`. Root `bun run lint` ends with `bun run typecheck`, and CircleCI runs it in a new `typecheck` job. Test files that had type errors were fixed.
98
+
99
+ ### Changed
100
+
101
+ - `@terreno/api`: deleting an `McpOAuthToken` row with document or query `deleteOne` now revokes that row's whole token family (the connection) and keeps the rows instead of removing one pair. `deleteMany` is unchanged.
102
+ - `@terreno/ui`: new `theme.field` tokens (`background`, `backgroundDisabled`, `border`, `borderFocus`, `borderError`, `borderDisabled`) style form inputs. `TextField`, `SelectField`, `DateTimeField`, `HeightField`, and `OtpField` read them instead of `surface.base` and `border.dark`. Defaults equal the old colors, but overriding `surface` or `border` no longer recolors those fields; use `setTheme({field: ...})`.
103
+
23
104
  ## [57.20.0] - 2026-10-30
24
105
 
25
106
  Upgrade note: [`mcp-server/src/docs/upgrades/57.20.0.md`](mcp-server/src/docs/upgrades/57.20.0.md).
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  AI service layer for Terreno apps - GPT chat, request logging, and admin tools.
4
4
 
5
- AI service layer for Terreno backends — provider-agnostic chat, request logging, GPT history, projects, file uploads, MCP tools, and optional Langfuse integration.
5
+ AI service layer for Terreno backends — provider-agnostic chat, request logging, GPT history, projects, file uploads, MCP tools, and built-in observability (prompt registry, traces, evaluators).
6
6
 
7
7
  ## Install
8
8
 
@@ -29,7 +29,7 @@ new TerrenoApp({userModel: User})
29
29
  .start();
30
30
  ```
31
31
 
32
- This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), projects, and the admin AI request explorer. Add `LangfuseApp`, `FileStorageService`, and `MCPService` when you need those features.
32
+ This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), projects, and the admin AI request explorer. Add `ObservabilityApp`, `FileStorageService`, and `MCPService` when you need those features.
33
33
 
34
34
  ## What's included
35
35
 
@@ -38,7 +38,7 @@ This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), proj
38
38
  - Mongoose models: `AIRequest`, `GptHistory`, `Project`, `FileAttachment`
39
39
  - Route registrars: `addGptRoutes`, `addGptHistoryRoutes`, `addProjectRoutes`, `addFileRoutes`, `addMcpRoutes`, `addAiRequestsExplorerRoutes`
40
40
  - Agent UI Asks (opt-in `asks` option on `AiApp` / `addGptRoutes`) — the agent pauses a turn to ask a typed question (`choice`, `confirm`, `markdown`, `form`, `files`), server-enforced approval for tools with `needsApproval`, and headless `pendingAsks` / `turn` endpoints for watches and other small clients. See [Agent UI Asks](https://github.com/TerrenoLabs/terreno/blob/master/docs/reference/agent-ui-asks.md)
41
- - `LangfuseApp` — optional Langfuse tracing and admin UI routes
41
+ - `ObservabilityApp` — prompt registry, traces, evaluators, datasets, experiments, and review queue
42
42
  - `FileStorageService` — GCS uploads with signed URLs
43
43
  - `MCPService` — SSE MCP client for tool calling
44
44
  - `getMCPTools` — wrap registered `modelRouter` MCP tools as Vercel AI SDK tools for in-process `streamText` / `generateText`
package/biome.jsonc CHANGED
@@ -1,5 +1,4 @@
1
1
  {
2
- "root": true,
3
2
  "$schema": "./node_modules/@biomejs/biome/configuration_schema.json",
4
3
  "extends": ["../biome.jsonc"],
5
4
  "files": {
@@ -7,6 +6,7 @@
7
6
  "package.json",
8
7
  "src/**/*.ts",
9
8
  "src/**/*.tsx",
9
+ "e2e/**/*.ts",
10
10
  "!!**/node_modules",
11
11
  "!!node_modules",
12
12
  "!!**/dist",
@@ -16,5 +16,6 @@
16
16
  "!!**/coverage",
17
17
  "!!coverage"
18
18
  ]
19
- }
19
+ },
20
+ "root": true
20
21
  }
@@ -403,6 +403,7 @@ const createAgentTasks = (context) => {
403
403
  }
404
404
  let returned;
405
405
  try {
406
+ // `args.data` was parsed by this tool's own `parameters`, which match its `execute` args.
406
407
  returned = await tool.execute(args.data, {
407
408
  conversationId,
408
409
  env: rt.env,
@@ -42,9 +42,10 @@ export declare const newTaskLease: (lease: HarnessLeaseSettings) => {
42
42
  token: string;
43
43
  };
44
44
  /**
45
- * Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a registry
46
- * `body` is dropped), once each. Throws `invalidRequest` for a ref without a name, with a
47
- * non-string label, or without a positive integer version.
45
+ * Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
46
+ * registry `body` is dropped and its `includes` follow it), once each. Throws
47
+ * `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
48
+ * non-array includes, or without a positive integer version.
48
49
  */
49
50
  export declare const tracePromptRefs: (refs: HarnessPromptRef[] | undefined) => HarnessPromptRef[];
50
51
  /**
@@ -76,9 +76,10 @@ const findByRequestId = async ({ definition, models, requestId, userId, }) => {
76
76
  return existing;
77
77
  };
78
78
  /**
79
- * Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a registry
80
- * `body` is dropped), once each. Throws `invalidRequest` for a ref without a name, with a
81
- * non-string label, or without a positive integer version.
79
+ * Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
80
+ * registry `body` is dropped and its `includes` follow it), once each. Throws
81
+ * `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
82
+ * non-array includes, or without a positive integer version.
82
83
  */
83
84
  const tracePromptRefs = (refs) => {
84
85
  if (refs === undefined) {
@@ -88,7 +89,18 @@ const tracePromptRefs = (refs) => {
88
89
  throw (0, errors_1.harnessError)({ detail: "prompts must be an array", kind: "invalidRequest" });
89
90
  }
90
91
  const unique = new Map();
91
- for (const ref of refs) {
92
+ const expanded = refs.flatMap((ref) => {
93
+ if (ref?.includes !== undefined && !Array.isArray(ref.includes)) {
94
+ throw (0, errors_1.harnessError)({
95
+ detail: typeof ref.name === "string" && ref.name.trim()
96
+ ? `Prompt ref "${ref.name}" includes must be an array`
97
+ : "Prompt ref includes must be an array",
98
+ kind: "invalidRequest",
99
+ });
100
+ }
101
+ return [ref, ...(ref?.includes ?? [])];
102
+ });
103
+ for (const ref of expanded) {
92
104
  if (typeof ref?.name !== "string" || !ref.name.trim()) {
93
105
  throw (0, errors_1.harnessError)({ detail: "Each prompt ref needs a name", kind: "invalidRequest" });
94
106
  }
@@ -104,7 +116,14 @@ const tracePromptRefs = (refs) => {
104
116
  kind: "invalidRequest",
105
117
  });
106
118
  }
119
+ if (ref.includedBy !== undefined && typeof ref.includedBy !== "string") {
120
+ throw (0, errors_1.harnessError)({
121
+ detail: `Prompt ref "${ref.name}" includedBy must be a string`,
122
+ kind: "invalidRequest",
123
+ });
124
+ }
107
125
  const normalized = {
126
+ ...(ref.includedBy !== undefined ? { includedBy: ref.includedBy } : {}),
108
127
  ...(ref.label !== undefined ? { label: ref.label } : {}),
109
128
  name: ref.name,
110
129
  version: ref.version,
package/dist/index.d.ts CHANGED
@@ -5,27 +5,24 @@ export type { AIAdminAppOptions } from "./aiAdminApp";
5
5
  export { AIAdminApp } from "./aiAdminApp";
6
6
  export type { AiAppOptions } from "./aiApp";
7
7
  export { AiApp } from "./aiApp";
8
- export { LangfuseApp } from "./langfuseApp";
9
- export { getCached, invalidateCache, LangfuseCache, setCached } from "./langfuseCache";
10
- export { getLangfuseClient, initLangfuseClient, isLangfuseInitialized, shutdownLangfuseClient, } from "./langfuseClient";
11
- export { compilePrompt, createPrompt, getPrompt, invalidatePromptCache } from "./langfusePrompts";
12
- export { initTracing, shutdownTracing } from "./langfuseTracing";
13
- export type { ChatMessage, GetPromptOptions, LangfuseAppOptions, LangfuseCachedPrompt, PaginatedResult, PreparePromptResult, PromptListItem, ScoreSubmission, ScoringFunction, TelemetrySettings, TraceListItem, } from "./langfuseTypes";
14
- export { createTelemetryConfig, preparePromptForAI } from "./langfuseVercelAi";
15
8
  export { AIDataset } from "./models/aiDataset";
16
9
  export { AIRequest } from "./models/aiRequest";
17
10
  export { FileAttachment } from "./models/fileAttachment";
18
11
  export { GptHistory } from "./models/gptHistory";
19
12
  export { getProjectModel, Project } from "./models/project";
20
13
  export { AI_OBSERVABILITY_GROUP, observabilityAdminScreens } from "./observability/adminScreens";
14
+ export { answersAgree, computeJudgeAgreement, type JudgeAgreement, } from "./observability/judgeAgreement";
21
15
  export { LocalDatasetStore } from "./observability/local/datasetStore";
22
- export { LocalEvaluatorStore } from "./observability/local/evaluatorStore";
23
- export { LocalExperimentRunner } from "./observability/local/experimentRunner";
16
+ export { type EvaluatorCalibration, LocalEvaluatorStore } from "./observability/local/evaluatorStore";
17
+ export { DEFAULT_TARGET_AGREEMENT, type JudgeExperimentCreateInput, LocalExperimentRunner, } from "./observability/local/experimentRunner";
18
+ export { type DatasetJudgmentSummary, loadCurrentHumanAnswers, } from "./observability/local/humanAnswers";
24
19
  export { createLocalObservabilityBundle, createLocalObservabilityPlugin, } from "./observability/local/localPlugin";
25
20
  export { LocalPromptStore } from "./observability/local/promptStore";
21
+ export { DEFAULT_REVIEW_BATCH_SIZE, LocalReviewBatchStore, MAX_REVIEW_BATCH_SIZE, type ReviewBatchDetail, type ReviewBatchListItem, } from "./observability/local/reviewBatchStore";
26
22
  export { LocalReviewStore } from "./observability/local/reviewStore";
27
23
  export { LocalScoreSink, LocalTraceSink, LocalTraceStore, MemoryScoreSink, MemoryTraceSink, } from "./observability/local/traceStore";
28
24
  export { getObservabilityApp, ObservabilityApp, resetObservabilityApp, } from "./observability/observabilityApp";
25
+ export { questionSetFingerprint } from "./observability/questionFingerprint";
29
26
  export type { ObservabilityPluginStatus, ObservabilityStatus, PlaygroundAiSource, PlaygroundAiStatus, } from "./observability/status";
30
27
  export { buildObservabilityStatus, buildPlaygroundAiStatus, isLocalObservabilityPluginOn, } from "./observability/status";
31
28
  export type { ControlPrimary, ObservabilityAiServiceFactory, ObservabilityAppOptions, ObservabilityCapability, ObservabilityControlConfig, ObservabilityGenerateClient, ObservabilityPlugin, ObservabilityRequestAiServiceFactory, PromptRegistry, ReviewQueue, ScoreRecord, ScoreSink, SpanRecord, TraceExportResult, TraceRecord, TraceSink, } from "./observability/types";
package/dist/index.js CHANGED
@@ -14,8 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.validateObservabilityConfig = exports.resolveObservabilityControl = exports.DEFAULT_OBSERVABILITY_CONTROL = exports.isLocalObservabilityPluginOn = exports.buildPlaygroundAiStatus = exports.buildObservabilityStatus = exports.resetObservabilityApp = exports.ObservabilityApp = exports.getObservabilityApp = exports.MemoryTraceSink = exports.MemoryScoreSink = exports.LocalTraceStore = exports.LocalTraceSink = exports.LocalScoreSink = exports.LocalReviewStore = exports.LocalPromptStore = exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = exports.LocalExperimentRunner = exports.LocalEvaluatorStore = exports.LocalDatasetStore = exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = exports.Project = exports.getProjectModel = exports.GptHistory = exports.FileAttachment = exports.AIRequest = exports.AIDataset = exports.preparePromptForAI = exports.createTelemetryConfig = exports.shutdownTracing = exports.initTracing = exports.invalidatePromptCache = exports.getPrompt = exports.createPrompt = exports.compilePrompt = exports.shutdownLangfuseClient = exports.isLangfuseInitialized = exports.initLangfuseClient = exports.getLangfuseClient = exports.setCached = exports.LangfuseCache = exports.invalidateCache = exports.getCached = exports.LangfuseApp = exports.AiApp = exports.AIAdminApp = exports.Output = exports.jsonSchema = void 0;
18
- exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService = exports.registerAiDataset = exports.configureAiDatasets = exports.findAgentBlock = exports.addProjectRoutes = exports.addMcpRoutes = exports.addGptHistoryRoutes = exports.addGptRoutes = exports.addFileRoutes = exports.addAiRequestsExplorerRoutes = void 0;
17
+ exports.registerAiDataset = exports.configureAiDatasets = exports.findAgentBlock = exports.addProjectRoutes = exports.addMcpRoutes = exports.addGptHistoryRoutes = exports.addGptRoutes = exports.addFileRoutes = exports.addAiRequestsExplorerRoutes = exports.validateObservabilityConfig = exports.resolveObservabilityControl = exports.DEFAULT_OBSERVABILITY_CONTROL = exports.isLocalObservabilityPluginOn = exports.buildPlaygroundAiStatus = exports.buildObservabilityStatus = exports.questionSetFingerprint = exports.resetObservabilityApp = exports.ObservabilityApp = exports.getObservabilityApp = exports.MemoryTraceSink = exports.MemoryScoreSink = exports.LocalTraceStore = exports.LocalTraceSink = exports.LocalScoreSink = exports.LocalReviewStore = exports.MAX_REVIEW_BATCH_SIZE = exports.LocalReviewBatchStore = exports.DEFAULT_REVIEW_BATCH_SIZE = exports.LocalPromptStore = exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = exports.loadCurrentHumanAnswers = exports.LocalExperimentRunner = exports.DEFAULT_TARGET_AGREEMENT = exports.LocalEvaluatorStore = exports.LocalDatasetStore = exports.computeJudgeAgreement = exports.answersAgree = exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = exports.Project = exports.getProjectModel = exports.GptHistory = exports.FileAttachment = exports.AIRequest = exports.AIDataset = exports.AiApp = exports.AIAdminApp = exports.Output = exports.jsonSchema = void 0;
18
+ exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService = void 0;
19
19
  var ai_1 = require("ai");
20
20
  Object.defineProperty(exports, "jsonSchema", { enumerable: true, get: function () { return ai_1.jsonSchema; } });
21
21
  Object.defineProperty(exports, "Output", { enumerable: true, get: function () { return ai_1.Output; } });
@@ -23,29 +23,6 @@ var aiAdminApp_1 = require("./aiAdminApp");
23
23
  Object.defineProperty(exports, "AIAdminApp", { enumerable: true, get: function () { return aiAdminApp_1.AIAdminApp; } });
24
24
  var aiApp_1 = require("./aiApp");
25
25
  Object.defineProperty(exports, "AiApp", { enumerable: true, get: function () { return aiApp_1.AiApp; } });
26
- var langfuseApp_1 = require("./langfuseApp");
27
- Object.defineProperty(exports, "LangfuseApp", { enumerable: true, get: function () { return langfuseApp_1.LangfuseApp; } });
28
- var langfuseCache_1 = require("./langfuseCache");
29
- Object.defineProperty(exports, "getCached", { enumerable: true, get: function () { return langfuseCache_1.getCached; } });
30
- Object.defineProperty(exports, "invalidateCache", { enumerable: true, get: function () { return langfuseCache_1.invalidateCache; } });
31
- Object.defineProperty(exports, "LangfuseCache", { enumerable: true, get: function () { return langfuseCache_1.LangfuseCache; } });
32
- Object.defineProperty(exports, "setCached", { enumerable: true, get: function () { return langfuseCache_1.setCached; } });
33
- var langfuseClient_1 = require("./langfuseClient");
34
- Object.defineProperty(exports, "getLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.getLangfuseClient; } });
35
- Object.defineProperty(exports, "initLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.initLangfuseClient; } });
36
- Object.defineProperty(exports, "isLangfuseInitialized", { enumerable: true, get: function () { return langfuseClient_1.isLangfuseInitialized; } });
37
- Object.defineProperty(exports, "shutdownLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.shutdownLangfuseClient; } });
38
- var langfusePrompts_1 = require("./langfusePrompts");
39
- Object.defineProperty(exports, "compilePrompt", { enumerable: true, get: function () { return langfusePrompts_1.compilePrompt; } });
40
- Object.defineProperty(exports, "createPrompt", { enumerable: true, get: function () { return langfusePrompts_1.createPrompt; } });
41
- Object.defineProperty(exports, "getPrompt", { enumerable: true, get: function () { return langfusePrompts_1.getPrompt; } });
42
- Object.defineProperty(exports, "invalidatePromptCache", { enumerable: true, get: function () { return langfusePrompts_1.invalidatePromptCache; } });
43
- var langfuseTracing_1 = require("./langfuseTracing");
44
- Object.defineProperty(exports, "initTracing", { enumerable: true, get: function () { return langfuseTracing_1.initTracing; } });
45
- Object.defineProperty(exports, "shutdownTracing", { enumerable: true, get: function () { return langfuseTracing_1.shutdownTracing; } });
46
- var langfuseVercelAi_1 = require("./langfuseVercelAi");
47
- Object.defineProperty(exports, "createTelemetryConfig", { enumerable: true, get: function () { return langfuseVercelAi_1.createTelemetryConfig; } });
48
- Object.defineProperty(exports, "preparePromptForAI", { enumerable: true, get: function () { return langfuseVercelAi_1.preparePromptForAI; } });
49
26
  var aiDataset_1 = require("./models/aiDataset");
50
27
  Object.defineProperty(exports, "AIDataset", { enumerable: true, get: function () { return aiDataset_1.AIDataset; } });
51
28
  var aiRequest_1 = require("./models/aiRequest");
@@ -60,17 +37,27 @@ Object.defineProperty(exports, "Project", { enumerable: true, get: function () {
60
37
  var adminScreens_1 = require("./observability/adminScreens");
61
38
  Object.defineProperty(exports, "AI_OBSERVABILITY_GROUP", { enumerable: true, get: function () { return adminScreens_1.AI_OBSERVABILITY_GROUP; } });
62
39
  Object.defineProperty(exports, "observabilityAdminScreens", { enumerable: true, get: function () { return adminScreens_1.observabilityAdminScreens; } });
40
+ var judgeAgreement_1 = require("./observability/judgeAgreement");
41
+ Object.defineProperty(exports, "answersAgree", { enumerable: true, get: function () { return judgeAgreement_1.answersAgree; } });
42
+ Object.defineProperty(exports, "computeJudgeAgreement", { enumerable: true, get: function () { return judgeAgreement_1.computeJudgeAgreement; } });
63
43
  var datasetStore_1 = require("./observability/local/datasetStore");
64
44
  Object.defineProperty(exports, "LocalDatasetStore", { enumerable: true, get: function () { return datasetStore_1.LocalDatasetStore; } });
65
45
  var evaluatorStore_1 = require("./observability/local/evaluatorStore");
66
46
  Object.defineProperty(exports, "LocalEvaluatorStore", { enumerable: true, get: function () { return evaluatorStore_1.LocalEvaluatorStore; } });
67
47
  var experimentRunner_1 = require("./observability/local/experimentRunner");
48
+ Object.defineProperty(exports, "DEFAULT_TARGET_AGREEMENT", { enumerable: true, get: function () { return experimentRunner_1.DEFAULT_TARGET_AGREEMENT; } });
68
49
  Object.defineProperty(exports, "LocalExperimentRunner", { enumerable: true, get: function () { return experimentRunner_1.LocalExperimentRunner; } });
50
+ var humanAnswers_1 = require("./observability/local/humanAnswers");
51
+ Object.defineProperty(exports, "loadCurrentHumanAnswers", { enumerable: true, get: function () { return humanAnswers_1.loadCurrentHumanAnswers; } });
69
52
  var localPlugin_1 = require("./observability/local/localPlugin");
70
53
  Object.defineProperty(exports, "createLocalObservabilityBundle", { enumerable: true, get: function () { return localPlugin_1.createLocalObservabilityBundle; } });
71
54
  Object.defineProperty(exports, "createLocalObservabilityPlugin", { enumerable: true, get: function () { return localPlugin_1.createLocalObservabilityPlugin; } });
72
55
  var promptStore_1 = require("./observability/local/promptStore");
73
56
  Object.defineProperty(exports, "LocalPromptStore", { enumerable: true, get: function () { return promptStore_1.LocalPromptStore; } });
57
+ var reviewBatchStore_1 = require("./observability/local/reviewBatchStore");
58
+ Object.defineProperty(exports, "DEFAULT_REVIEW_BATCH_SIZE", { enumerable: true, get: function () { return reviewBatchStore_1.DEFAULT_REVIEW_BATCH_SIZE; } });
59
+ Object.defineProperty(exports, "LocalReviewBatchStore", { enumerable: true, get: function () { return reviewBatchStore_1.LocalReviewBatchStore; } });
60
+ Object.defineProperty(exports, "MAX_REVIEW_BATCH_SIZE", { enumerable: true, get: function () { return reviewBatchStore_1.MAX_REVIEW_BATCH_SIZE; } });
74
61
  var reviewStore_1 = require("./observability/local/reviewStore");
75
62
  Object.defineProperty(exports, "LocalReviewStore", { enumerable: true, get: function () { return reviewStore_1.LocalReviewStore; } });
76
63
  var traceStore_1 = require("./observability/local/traceStore");
@@ -83,6 +70,8 @@ var observabilityApp_1 = require("./observability/observabilityApp");
83
70
  Object.defineProperty(exports, "getObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.getObservabilityApp; } });
84
71
  Object.defineProperty(exports, "ObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.ObservabilityApp; } });
85
72
  Object.defineProperty(exports, "resetObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.resetObservabilityApp; } });
73
+ var questionFingerprint_1 = require("./observability/questionFingerprint");
74
+ Object.defineProperty(exports, "questionSetFingerprint", { enumerable: true, get: function () { return questionFingerprint_1.questionSetFingerprint; } });
86
75
  var status_1 = require("./observability/status");
87
76
  Object.defineProperty(exports, "buildObservabilityStatus", { enumerable: true, get: function () { return status_1.buildObservabilityStatus; } });
88
77
  Object.defineProperty(exports, "buildPlaygroundAiStatus", { enumerable: true, get: function () { return status_1.buildPlaygroundAiStatus; } });
@@ -25,7 +25,11 @@ const observabilityAdminScreens = ({ localOn }) => {
25
25
  }),
26
26
  ];
27
27
  if (localOn) {
28
- screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" }));
28
+ screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" }), observabilityScreen({
29
+ displayName: "Review batches",
30
+ name: "ai-review-batches",
31
+ resource: "aiReview",
32
+ }));
29
33
  }
30
34
  return screens;
31
35
  };
@@ -23,7 +23,14 @@ export interface EvaluatorExecutionResult {
23
23
  scores?: Record<string, boolean | number | string>;
24
24
  }
25
25
  export declare const runJsonAssertEvaluator: (context: EvaluatorExecutionContext) => EvaluatorExecutionResult;
26
- export declare const runLlmJudgeEvaluator: (context: EvaluatorExecutionContext, ai: EvaluatorAiClient, judgeOutputSchema: Record<string, unknown>) => Promise<EvaluatorExecutionResult>;
26
+ /**
27
+ * Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
28
+ * version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
29
+ * version instead, as judge calibration experiments do.
30
+ */
31
+ export declare const runLlmJudgeEvaluator: (context: EvaluatorExecutionContext, ai: EvaluatorAiClient, judgeOutputSchema: Record<string, unknown>, options?: {
32
+ judgeSystemPrompt?: string;
33
+ }) => Promise<EvaluatorExecutionResult>;
27
34
  export declare const runEvaluator: (context: EvaluatorExecutionContext, deps: {
28
35
  ai?: EvaluatorAiClient;
29
36
  judgeOutputSchema?: Record<string, unknown>;
@@ -130,15 +130,23 @@ const runJsonAssertEvaluator = (context) => {
130
130
  };
131
131
  };
132
132
  exports.runJsonAssertEvaluator = runJsonAssertEvaluator;
133
- const runLlmJudgeEvaluator = async (context, ai, judgeOutputSchema) => {
133
+ /**
134
+ * Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
135
+ * version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
136
+ * version instead, as judge calibration experiments do.
137
+ */
138
+ const runLlmJudgeEvaluator = async (context, ai, judgeOutputSchema, options = {}) => {
134
139
  const judgePromptName = context.evaluator.judgePromptName;
135
140
  if (!judgePromptName) {
136
141
  return { error: "llm-judge evaluator is missing judgePromptName" };
137
142
  }
138
143
  try {
144
+ const promptSource = options.judgeSystemPrompt === undefined
145
+ ? { promptName: judgePromptName }
146
+ : { systemPrompt: options.judgeSystemPrompt };
139
147
  const payload = (await ai.generateJsonObject({
140
148
  prompt: buildJudgePrompt(context),
141
- promptName: judgePromptName,
149
+ ...promptSource,
142
150
  schema: (0, ai_1.jsonSchema)(judgeOutputSchema),
143
151
  schemaName: `${context.evaluator.name}-judge`,
144
152
  skipTrace: true,
@@ -0,0 +1,53 @@
1
+ import type { EvaluatorDimension } from "../types/observability";
2
+ export type ScoreValue = boolean | number | string;
3
+ /**
4
+ * Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
5
+ * otherwise by `traceId`.
6
+ */
7
+ export interface AgreementAnswerSet {
8
+ datasetItemId?: string;
9
+ scores?: Record<string, ScoreValue>;
10
+ traceId?: string;
11
+ }
12
+ export interface JudgeAnswerSet extends AgreementAnswerSet {
13
+ error?: string;
14
+ }
15
+ export interface JudgeQuestionAgreement {
16
+ compared: number;
17
+ /** False when the judge does not answer this question, so it is left out of the totals. */
18
+ covered: boolean;
19
+ key: string;
20
+ matches: number;
21
+ question?: string;
22
+ rate?: number;
23
+ }
24
+ export interface JudgeDisagreement {
25
+ datasetItemId?: string;
26
+ human: ScoreValue;
27
+ judge?: ScoreValue;
28
+ key: string;
29
+ traceId?: string;
30
+ }
31
+ export interface JudgeAgreement {
32
+ compared: number;
33
+ disagreements: JudgeDisagreement[];
34
+ errors: number;
35
+ itemsCompared: number;
36
+ matches: number;
37
+ questions: JudgeQuestionAgreement[];
38
+ rate?: number;
39
+ }
40
+ /** True when a judge answer counts as the same answer a human gave for this dimension. */
41
+ export declare const answersAgree: (dimension: EvaluatorDimension, human: ScoreValue, judge: ScoreValue | undefined) => boolean;
42
+ /** Key that pairs human and judge answers for the same item. */
43
+ export declare const agreementAnswerKey: (answer: AgreementAnswerSet) => string;
44
+ /**
45
+ * Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
46
+ * answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
47
+ */
48
+ export declare const computeJudgeAgreement: (params: {
49
+ dimensions: EvaluatorDimension[];
50
+ humanAnswers: AgreementAnswerSet[];
51
+ judgeAnswers: JudgeAnswerSet[];
52
+ judgeKeys: string[];
53
+ }) => JudgeAgreement;
@@ -0,0 +1,123 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.computeJudgeAgreement = exports.agreementAnswerKey = exports.answersAgree = void 0;
4
+ // Numeric answers agree when they land within this share of the dimension's range.
5
+ const NUMERIC_TOLERANCE = 0.1;
6
+ const numericSpan = (range) => {
7
+ const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
8
+ if (!matches) {
9
+ return 1;
10
+ }
11
+ const span = Math.abs(Number(matches[2]) - Number(matches[1]));
12
+ return span > 0 ? span : 1;
13
+ };
14
+ const toBoolean = (value) => {
15
+ if (typeof value === "boolean") {
16
+ return value;
17
+ }
18
+ if (value === "true" || value === "yes") {
19
+ return true;
20
+ }
21
+ if (value === "false" || value === "no") {
22
+ return false;
23
+ }
24
+ return undefined;
25
+ };
26
+ /** True when a judge answer counts as the same answer a human gave for this dimension. */
27
+ const answersAgree = (dimension, human, judge) => {
28
+ if (judge === undefined) {
29
+ return false;
30
+ }
31
+ if (dimension.dataType === "boolean") {
32
+ const humanBool = toBoolean(human);
33
+ return humanBool !== undefined && humanBool === toBoolean(judge);
34
+ }
35
+ if (dimension.dataType === "numeric") {
36
+ const humanNumber = Number(human);
37
+ const judgeNumber = Number(judge);
38
+ if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
39
+ return false;
40
+ }
41
+ return (Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9);
42
+ }
43
+ return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
44
+ };
45
+ exports.answersAgree = answersAgree;
46
+ /** Key that pairs human and judge answers for the same item. */
47
+ const agreementAnswerKey = (answer) => {
48
+ return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
49
+ };
50
+ exports.agreementAnswerKey = agreementAnswerKey;
51
+ const rateOf = (matches, compared) => {
52
+ return compared > 0 ? matches / compared : undefined;
53
+ };
54
+ /**
55
+ * Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
56
+ * answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
57
+ */
58
+ const computeJudgeAgreement = (params) => {
59
+ const judgeKeys = new Set(params.judgeKeys);
60
+ const judgeByKey = new Map(params.judgeAnswers.map((answer) => {
61
+ return [(0, exports.agreementAnswerKey)(answer), answer];
62
+ }));
63
+ const questions = params.dimensions.map((dimension) => {
64
+ return {
65
+ compared: 0,
66
+ covered: judgeKeys.has(dimension.key),
67
+ key: dimension.key,
68
+ matches: 0,
69
+ question: dimension.question,
70
+ };
71
+ });
72
+ const disagreements = [];
73
+ let itemsCompared = 0;
74
+ let errors = 0;
75
+ for (const human of params.humanAnswers) {
76
+ const judge = judgeByKey.get((0, exports.agreementAnswerKey)(human));
77
+ if (!judge || !human.scores) {
78
+ continue;
79
+ }
80
+ let comparedThisItem = false;
81
+ if (judge.error) {
82
+ errors += 1;
83
+ }
84
+ params.dimensions.forEach((dimension, index) => {
85
+ const humanValue = human.scores?.[dimension.key];
86
+ const question = questions[index];
87
+ if (!question?.covered || humanValue === undefined) {
88
+ return;
89
+ }
90
+ comparedThisItem = true;
91
+ question.compared += 1;
92
+ const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
93
+ if ((0, exports.answersAgree)(dimension, humanValue, judgeValue)) {
94
+ question.matches += 1;
95
+ return;
96
+ }
97
+ disagreements.push({
98
+ ...(human.datasetItemId ? { datasetItemId: human.datasetItemId } : {}),
99
+ human: humanValue,
100
+ judge: judgeValue,
101
+ key: dimension.key,
102
+ ...(human.traceId ? { traceId: human.traceId } : {}),
103
+ });
104
+ });
105
+ if (comparedThisItem) {
106
+ itemsCompared += 1;
107
+ }
108
+ }
109
+ const compared = questions.reduce((total, question) => total + question.compared, 0);
110
+ const matches = questions.reduce((total, question) => total + question.matches, 0);
111
+ return {
112
+ compared,
113
+ disagreements,
114
+ errors,
115
+ itemsCompared,
116
+ matches,
117
+ questions: questions.map((question) => {
118
+ return { ...question, rate: rateOf(question.matches, question.compared) };
119
+ }),
120
+ rate: rateOf(matches, compared),
121
+ };
122
+ };
123
+ exports.computeJudgeAgreement = computeJudgeAgreement;
@@ -1,5 +1,6 @@
1
1
  import type { ObsDatasetAnnotatedBy } from "../../types/observability";
2
2
  import { type ParsedDatasetImportRow } from "../datasetImport";
3
+ import { type DatasetHumanAnswerView, type DatasetJudgmentSummary } from "./humanAnswers";
3
4
  import { LocalPromptStore } from "./promptStore";
4
5
  export interface DatasetWriteInput {
5
6
  description?: string | null;
@@ -31,6 +32,8 @@ export interface DatasetView {
31
32
  expectedOutputSchema?: Record<string, unknown>;
32
33
  id: string;
33
34
  inputSchemaPromptName?: string;
35
+ /** Human judging progress per question set (see `summarizeDatasetJudgments`). */
36
+ judgments: DatasetJudgmentSummary[];
34
37
  name: string;
35
38
  tags: string[];
36
39
  updated: string;
@@ -40,6 +43,8 @@ export interface DatasetItemView {
40
43
  created: string;
41
44
  datasetId: string;
42
45
  expectedOutput?: unknown;
46
+ /** Human answers keyed by question-set evaluator id. */
47
+ humanAnswers?: Record<string, DatasetHumanAnswerView>;
43
48
  id: string;
44
49
  input: unknown;
45
50
  metadata?: Record<string, unknown>;