@terreno/ai 57.21.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -3
- package/biome.jsonc +2 -2
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
package/CHANGELOG.md
CHANGED
|
@@ -20,6 +20,37 @@ Upgrade notes for consumer action live in [`mcp-server/src/docs/upgrades/`](mcp-
|
|
|
20
20
|
|
|
21
21
|
Unreleased changes live in [`changelog/unreleased/`](changelog/unreleased/). Add one Markdown file per feature (see that directory's README) instead of editing this section.
|
|
22
22
|
|
|
23
|
+
## [57.22.0] - 2026-11-03
|
|
24
|
+
|
|
25
|
+
Upgrade note: [`mcp-server/src/docs/upgrades/57.22.0.md`](mcp-server/src/docs/upgrades/57.22.0.md).
|
|
26
|
+
|
|
27
|
+
### Breaking
|
|
28
|
+
|
|
29
|
+
- `@terreno/ai` no longer integrates with Langfuse. The built-in observability prompt registry, traces, and review queue (`ObservabilityApp` with `createLocalObservabilityPlugin()`) replace it. Removed exports: `LangfuseApp`, `LangfuseCache`, `getCached`, `setCached`, `invalidateCache`, `initLangfuseClient`, `getLangfuseClient`, `isLangfuseInitialized`, `shutdownLangfuseClient`, `compilePrompt`, `createPrompt`, `getPrompt`, `invalidatePromptCache`, `initTracing`, `shutdownTracing`, `createTelemetryConfig`, `preparePromptForAI`, and the types `ChatMessage`, `GetPromptOptions`, `LangfuseAppOptions`, `LangfuseCachedPrompt`, `PaginatedResult`, `PreparePromptResult`, `PromptListItem`, `ScoreSubmission`, `ScoringFunction`, `TelemetrySettings`, `TraceListItem`. The `@terreno/ai/langfuseApp` and `@terreno/ai/langfuseClient` subpath exports are gone, along with the `@langfuse/client`, `@langfuse/otel`, and `@opentelemetry/*` dependencies of `@terreno/ai`.
|
|
30
|
+
- `addGptRoutes` drops `langfuseSystemPromptName`; use `systemPromptName` (a registry prompt) instead. Chat turns no longer set `experimental_telemetry`.
|
|
31
|
+
- `ControlPrimary` is now only `"local"`.
|
|
32
|
+
- The example backend no longer reads `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_BASE_URL`, `LANGFUSE_ORGANIZATION`, `LANGFUSE_PROJECT`, or `LANGFUSE_PROJECT_ID`. Deploys (`.github/workflows/cd.yml`, `scripts/ci/gcp-deploy.sh`) no longer mount the Langfuse secrets, and `infra/flourish/main.tf` drops the `backend_secret_langfuse_secret_key` and `backend_secret_langfuse_public_key` modules, so the next `terraform apply` deletes those two Secret Manager secrets. Deploy a revision without the mounts before applying.
|
|
33
|
+
|
|
34
|
+
### Added
|
|
35
|
+
|
|
36
|
+
- `addGptRoutes` accepts `systemPromptName` and `systemPromptLabel`. A chat turn prepends that observability registry prompt to the system prompt and records its version on the trace.
|
|
37
|
+
- The example chat reads `chat-assistant` and the `generate_joke` tool reads `joke-generator` from the observability prompt registry.
|
|
38
|
+
- `bun run backend:seed` fills every AI Observability screen: labelled prompt versions for `example-summarize`, `joke-generator`, and `chat-assistant`, golden datasets, finished experiments, traces with scores, and review items in every status. Run it on an existing database with the `seedAiObservabilityDemo` admin script.
|
|
39
|
+
- Review batches: pick a human question set and press **Grab 10 recent traces** to give reviewers a fixed set of inputs and outputs (`POST /ai/observability/review/batches`).
|
|
40
|
+
- The review item screen shows one input, one output, and every question as Yes / No rows. On web, `Y` / `N` answer the highlighted question, `Enter` submits and moves to the next batch item, and `S` skips. Evaluator dimensions accept a reviewer `question`.
|
|
41
|
+
- Run any LLM judge over a batch and see overall and per-question agreement with the human answers, plus a list of disagreements (`POST /ai/observability/review/batches/:id/judgeRuns`).
|
|
42
|
+
- The AI observability seed adds the `summary-review-questions` question set, two judges for it, and a 10-item batch with 8 answers where one judge agrees 97% and the other 78%.
|
|
43
|
+
- Observability registry prompts can reference other prompts with `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}` (default label `production`). Shared instructions live in one prompt; `PromptRegistry.get`, the playground, experiments, and judges use the resolved text.
|
|
44
|
+
- Every reference must resolve when it is saved. Creating a prompt, saving a version, and moving a label return 400 `Invalid prompt reference` for a malformed tag, an unknown prompt or version, or a label the referenced prompt does not have yet, 400 `Prompt reference cycle` when the change would make prompts include each other, and 400 `Prompt references nested too deeply` when it would nest the prompt, or any prompt that includes it, more than 5 levels.
|
|
45
|
+
- Traces stay named after the called prompt. `ObsTrace.prompts` lists it first, then each shared prompt version it pulled in, marked with `includedBy`. `PromptVersionRef.includes` and harness prompt refs carry the same list.
|
|
46
|
+
|
|
47
|
+
### Fixed
|
|
48
|
+
|
|
49
|
+
- Review queue: `Y` and `N` answer the highlighted question when a Yes/No segment is focused. **What the AI was given** and **What the AI wrote** are headings above the field cards.
|
|
50
|
+
- Review batches: **Save batch** sits on the New batch card. Search and **More filters** share a baseline.
|
|
51
|
+
- Prompt hub: description, folder, and tags sit on the Versions screen with the editor. System and template fields explain what to write.
|
|
52
|
+
- Evaluator detail loads the judge prompt `outputSchema` (so helpfulness no longer reports a missing dimension) and experiment rows open that experiment.
|
|
53
|
+
|
|
23
54
|
## [57.21.0] - 2026-11-01
|
|
24
55
|
|
|
25
56
|
Upgrade note: [`mcp-server/src/docs/upgrades/57.21.0.md`](mcp-server/src/docs/upgrades/57.21.0.md).
|
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
AI service layer for Terreno apps - GPT chat, request logging, and admin tools.
|
|
4
4
|
|
|
5
|
-
AI service layer for Terreno backends — provider-agnostic chat, request logging, GPT history, projects, file uploads, MCP tools, and
|
|
5
|
+
AI service layer for Terreno backends — provider-agnostic chat, request logging, GPT history, projects, file uploads, MCP tools, and built-in observability (prompt registry, traces, evaluators).
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
@@ -29,7 +29,7 @@ new TerrenoApp({userModel: User})
|
|
|
29
29
|
.start();
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
-
This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), projects, and the admin AI request explorer. Add `
|
|
32
|
+
This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), projects, and the admin AI request explorer. Add `ObservabilityApp`, `FileStorageService`, and `MCPService` when you need those features.
|
|
33
33
|
|
|
34
34
|
## What's included
|
|
35
35
|
|
|
@@ -38,7 +38,7 @@ This mounts GPT chat (`POST /gpt/prompt`), history CRUD (`/gpt/histories`), proj
|
|
|
38
38
|
- Mongoose models: `AIRequest`, `GptHistory`, `Project`, `FileAttachment`
|
|
39
39
|
- Route registrars: `addGptRoutes`, `addGptHistoryRoutes`, `addProjectRoutes`, `addFileRoutes`, `addMcpRoutes`, `addAiRequestsExplorerRoutes`
|
|
40
40
|
- Agent UI Asks (opt-in `asks` option on `AiApp` / `addGptRoutes`) — the agent pauses a turn to ask a typed question (`choice`, `confirm`, `markdown`, `form`, `files`), server-enforced approval for tools with `needsApproval`, and headless `pendingAsks` / `turn` endpoints for watches and other small clients. See [Agent UI Asks](https://github.com/TerrenoLabs/terreno/blob/master/docs/reference/agent-ui-asks.md)
|
|
41
|
-
- `
|
|
41
|
+
- `ObservabilityApp` — prompt registry, traces, evaluators, datasets, experiments, and review queue
|
|
42
42
|
- `FileStorageService` — GCS uploads with signed URLs
|
|
43
43
|
- `MCPService` — SSE MCP client for tool calling
|
|
44
44
|
- `getMCPTools` — wrap registered `modelRouter` MCP tools as Vercel AI SDK tools for in-process `streamText` / `generateText`
|
package/biome.jsonc
CHANGED
package/dist/harness/commit.d.ts
CHANGED
|
@@ -42,9 +42,10 @@ export declare const newTaskLease: (lease: HarnessLeaseSettings) => {
|
|
|
42
42
|
token: string;
|
|
43
43
|
};
|
|
44
44
|
/**
|
|
45
|
-
* Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a
|
|
46
|
-
* `body` is dropped
|
|
47
|
-
* non-string label
|
|
45
|
+
* Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
|
|
46
|
+
* registry `body` is dropped and its `includes` follow it), once each. Throws
|
|
47
|
+
* `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
|
|
48
|
+
* non-array includes, or without a positive integer version.
|
|
48
49
|
*/
|
|
49
50
|
export declare const tracePromptRefs: (refs: HarnessPromptRef[] | undefined) => HarnessPromptRef[];
|
|
50
51
|
/**
|
package/dist/harness/commit.js
CHANGED
|
@@ -76,9 +76,10 @@ const findByRequestId = async ({ definition, models, requestId, userId, }) => {
|
|
|
76
76
|
return existing;
|
|
77
77
|
};
|
|
78
78
|
/**
|
|
79
|
-
* Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a
|
|
80
|
-
* `body` is dropped
|
|
81
|
-
* non-string label
|
|
79
|
+
* Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
|
|
80
|
+
* registry `body` is dropped and its `includes` follow it), once each. Throws
|
|
81
|
+
* `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
|
|
82
|
+
* non-array includes, or without a positive integer version.
|
|
82
83
|
*/
|
|
83
84
|
const tracePromptRefs = (refs) => {
|
|
84
85
|
if (refs === undefined) {
|
|
@@ -88,7 +89,18 @@ const tracePromptRefs = (refs) => {
|
|
|
88
89
|
throw (0, errors_1.harnessError)({ detail: "prompts must be an array", kind: "invalidRequest" });
|
|
89
90
|
}
|
|
90
91
|
const unique = new Map();
|
|
91
|
-
|
|
92
|
+
const expanded = refs.flatMap((ref) => {
|
|
93
|
+
if (ref?.includes !== undefined && !Array.isArray(ref.includes)) {
|
|
94
|
+
throw (0, errors_1.harnessError)({
|
|
95
|
+
detail: typeof ref.name === "string" && ref.name.trim()
|
|
96
|
+
? `Prompt ref "${ref.name}" includes must be an array`
|
|
97
|
+
: "Prompt ref includes must be an array",
|
|
98
|
+
kind: "invalidRequest",
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return [ref, ...(ref?.includes ?? [])];
|
|
102
|
+
});
|
|
103
|
+
for (const ref of expanded) {
|
|
92
104
|
if (typeof ref?.name !== "string" || !ref.name.trim()) {
|
|
93
105
|
throw (0, errors_1.harnessError)({ detail: "Each prompt ref needs a name", kind: "invalidRequest" });
|
|
94
106
|
}
|
|
@@ -104,7 +116,14 @@ const tracePromptRefs = (refs) => {
|
|
|
104
116
|
kind: "invalidRequest",
|
|
105
117
|
});
|
|
106
118
|
}
|
|
119
|
+
if (ref.includedBy !== undefined && typeof ref.includedBy !== "string") {
|
|
120
|
+
throw (0, errors_1.harnessError)({
|
|
121
|
+
detail: `Prompt ref "${ref.name}" includedBy must be a string`,
|
|
122
|
+
kind: "invalidRequest",
|
|
123
|
+
});
|
|
124
|
+
}
|
|
107
125
|
const normalized = {
|
|
126
|
+
...(ref.includedBy !== undefined ? { includedBy: ref.includedBy } : {}),
|
|
108
127
|
...(ref.label !== undefined ? { label: ref.label } : {}),
|
|
109
128
|
name: ref.name,
|
|
110
129
|
version: ref.version,
|
package/dist/index.d.ts
CHANGED
|
@@ -5,27 +5,24 @@ export type { AIAdminAppOptions } from "./aiAdminApp";
|
|
|
5
5
|
export { AIAdminApp } from "./aiAdminApp";
|
|
6
6
|
export type { AiAppOptions } from "./aiApp";
|
|
7
7
|
export { AiApp } from "./aiApp";
|
|
8
|
-
export { LangfuseApp } from "./langfuseApp";
|
|
9
|
-
export { getCached, invalidateCache, LangfuseCache, setCached } from "./langfuseCache";
|
|
10
|
-
export { getLangfuseClient, initLangfuseClient, isLangfuseInitialized, shutdownLangfuseClient, } from "./langfuseClient";
|
|
11
|
-
export { compilePrompt, createPrompt, getPrompt, invalidatePromptCache } from "./langfusePrompts";
|
|
12
|
-
export { initTracing, shutdownTracing } from "./langfuseTracing";
|
|
13
|
-
export type { ChatMessage, GetPromptOptions, LangfuseAppOptions, LangfuseCachedPrompt, PaginatedResult, PreparePromptResult, PromptListItem, ScoreSubmission, ScoringFunction, TelemetrySettings, TraceListItem, } from "./langfuseTypes";
|
|
14
|
-
export { createTelemetryConfig, preparePromptForAI } from "./langfuseVercelAi";
|
|
15
8
|
export { AIDataset } from "./models/aiDataset";
|
|
16
9
|
export { AIRequest } from "./models/aiRequest";
|
|
17
10
|
export { FileAttachment } from "./models/fileAttachment";
|
|
18
11
|
export { GptHistory } from "./models/gptHistory";
|
|
19
12
|
export { getProjectModel, Project } from "./models/project";
|
|
20
13
|
export { AI_OBSERVABILITY_GROUP, observabilityAdminScreens } from "./observability/adminScreens";
|
|
14
|
+
export { answersAgree, computeJudgeAgreement, type JudgeAgreement, } from "./observability/judgeAgreement";
|
|
21
15
|
export { LocalDatasetStore } from "./observability/local/datasetStore";
|
|
22
|
-
export { LocalEvaluatorStore } from "./observability/local/evaluatorStore";
|
|
23
|
-
export { LocalExperimentRunner } from "./observability/local/experimentRunner";
|
|
16
|
+
export { type EvaluatorCalibration, LocalEvaluatorStore } from "./observability/local/evaluatorStore";
|
|
17
|
+
export { DEFAULT_TARGET_AGREEMENT, type JudgeExperimentCreateInput, LocalExperimentRunner, } from "./observability/local/experimentRunner";
|
|
18
|
+
export { type DatasetJudgmentSummary, loadCurrentHumanAnswers, } from "./observability/local/humanAnswers";
|
|
24
19
|
export { createLocalObservabilityBundle, createLocalObservabilityPlugin, } from "./observability/local/localPlugin";
|
|
25
20
|
export { LocalPromptStore } from "./observability/local/promptStore";
|
|
21
|
+
export { DEFAULT_REVIEW_BATCH_SIZE, LocalReviewBatchStore, MAX_REVIEW_BATCH_SIZE, type ReviewBatchDetail, type ReviewBatchListItem, } from "./observability/local/reviewBatchStore";
|
|
26
22
|
export { LocalReviewStore } from "./observability/local/reviewStore";
|
|
27
23
|
export { LocalScoreSink, LocalTraceSink, LocalTraceStore, MemoryScoreSink, MemoryTraceSink, } from "./observability/local/traceStore";
|
|
28
24
|
export { getObservabilityApp, ObservabilityApp, resetObservabilityApp, } from "./observability/observabilityApp";
|
|
25
|
+
export { questionSetFingerprint } from "./observability/questionFingerprint";
|
|
29
26
|
export type { ObservabilityPluginStatus, ObservabilityStatus, PlaygroundAiSource, PlaygroundAiStatus, } from "./observability/status";
|
|
30
27
|
export { buildObservabilityStatus, buildPlaygroundAiStatus, isLocalObservabilityPluginOn, } from "./observability/status";
|
|
31
28
|
export type { ControlPrimary, ObservabilityAiServiceFactory, ObservabilityAppOptions, ObservabilityCapability, ObservabilityControlConfig, ObservabilityGenerateClient, ObservabilityPlugin, ObservabilityRequestAiServiceFactory, PromptRegistry, ReviewQueue, ScoreRecord, ScoreSink, SpanRecord, TraceExportResult, TraceRecord, TraceSink, } from "./observability/types";
|
package/dist/index.js
CHANGED
|
@@ -14,8 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.
|
|
18
|
-
exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService =
|
|
17
|
+
exports.registerAiDataset = exports.configureAiDatasets = exports.findAgentBlock = exports.addProjectRoutes = exports.addMcpRoutes = exports.addGptHistoryRoutes = exports.addGptRoutes = exports.addFileRoutes = exports.addAiRequestsExplorerRoutes = exports.validateObservabilityConfig = exports.resolveObservabilityControl = exports.DEFAULT_OBSERVABILITY_CONTROL = exports.isLocalObservabilityPluginOn = exports.buildPlaygroundAiStatus = exports.buildObservabilityStatus = exports.questionSetFingerprint = exports.resetObservabilityApp = exports.ObservabilityApp = exports.getObservabilityApp = exports.MemoryTraceSink = exports.MemoryScoreSink = exports.LocalTraceStore = exports.LocalTraceSink = exports.LocalScoreSink = exports.LocalReviewStore = exports.MAX_REVIEW_BATCH_SIZE = exports.LocalReviewBatchStore = exports.DEFAULT_REVIEW_BATCH_SIZE = exports.LocalPromptStore = exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = exports.loadCurrentHumanAnswers = exports.LocalExperimentRunner = exports.DEFAULT_TARGET_AGREEMENT = exports.LocalEvaluatorStore = exports.LocalDatasetStore = exports.computeJudgeAgreement = exports.answersAgree = exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = exports.Project = exports.getProjectModel = exports.GptHistory = exports.FileAttachment = exports.AIRequest = exports.AIDataset = exports.AiApp = exports.AIAdminApp = exports.Output = exports.jsonSchema = void 0;
|
|
18
|
+
exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService = void 0;
|
|
19
19
|
var ai_1 = require("ai");
|
|
20
20
|
Object.defineProperty(exports, "jsonSchema", { enumerable: true, get: function () { return ai_1.jsonSchema; } });
|
|
21
21
|
Object.defineProperty(exports, "Output", { enumerable: true, get: function () { return ai_1.Output; } });
|
|
@@ -23,29 +23,6 @@ var aiAdminApp_1 = require("./aiAdminApp");
|
|
|
23
23
|
Object.defineProperty(exports, "AIAdminApp", { enumerable: true, get: function () { return aiAdminApp_1.AIAdminApp; } });
|
|
24
24
|
var aiApp_1 = require("./aiApp");
|
|
25
25
|
Object.defineProperty(exports, "AiApp", { enumerable: true, get: function () { return aiApp_1.AiApp; } });
|
|
26
|
-
var langfuseApp_1 = require("./langfuseApp");
|
|
27
|
-
Object.defineProperty(exports, "LangfuseApp", { enumerable: true, get: function () { return langfuseApp_1.LangfuseApp; } });
|
|
28
|
-
var langfuseCache_1 = require("./langfuseCache");
|
|
29
|
-
Object.defineProperty(exports, "getCached", { enumerable: true, get: function () { return langfuseCache_1.getCached; } });
|
|
30
|
-
Object.defineProperty(exports, "invalidateCache", { enumerable: true, get: function () { return langfuseCache_1.invalidateCache; } });
|
|
31
|
-
Object.defineProperty(exports, "LangfuseCache", { enumerable: true, get: function () { return langfuseCache_1.LangfuseCache; } });
|
|
32
|
-
Object.defineProperty(exports, "setCached", { enumerable: true, get: function () { return langfuseCache_1.setCached; } });
|
|
33
|
-
var langfuseClient_1 = require("./langfuseClient");
|
|
34
|
-
Object.defineProperty(exports, "getLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.getLangfuseClient; } });
|
|
35
|
-
Object.defineProperty(exports, "initLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.initLangfuseClient; } });
|
|
36
|
-
Object.defineProperty(exports, "isLangfuseInitialized", { enumerable: true, get: function () { return langfuseClient_1.isLangfuseInitialized; } });
|
|
37
|
-
Object.defineProperty(exports, "shutdownLangfuseClient", { enumerable: true, get: function () { return langfuseClient_1.shutdownLangfuseClient; } });
|
|
38
|
-
var langfusePrompts_1 = require("./langfusePrompts");
|
|
39
|
-
Object.defineProperty(exports, "compilePrompt", { enumerable: true, get: function () { return langfusePrompts_1.compilePrompt; } });
|
|
40
|
-
Object.defineProperty(exports, "createPrompt", { enumerable: true, get: function () { return langfusePrompts_1.createPrompt; } });
|
|
41
|
-
Object.defineProperty(exports, "getPrompt", { enumerable: true, get: function () { return langfusePrompts_1.getPrompt; } });
|
|
42
|
-
Object.defineProperty(exports, "invalidatePromptCache", { enumerable: true, get: function () { return langfusePrompts_1.invalidatePromptCache; } });
|
|
43
|
-
var langfuseTracing_1 = require("./langfuseTracing");
|
|
44
|
-
Object.defineProperty(exports, "initTracing", { enumerable: true, get: function () { return langfuseTracing_1.initTracing; } });
|
|
45
|
-
Object.defineProperty(exports, "shutdownTracing", { enumerable: true, get: function () { return langfuseTracing_1.shutdownTracing; } });
|
|
46
|
-
var langfuseVercelAi_1 = require("./langfuseVercelAi");
|
|
47
|
-
Object.defineProperty(exports, "createTelemetryConfig", { enumerable: true, get: function () { return langfuseVercelAi_1.createTelemetryConfig; } });
|
|
48
|
-
Object.defineProperty(exports, "preparePromptForAI", { enumerable: true, get: function () { return langfuseVercelAi_1.preparePromptForAI; } });
|
|
49
26
|
var aiDataset_1 = require("./models/aiDataset");
|
|
50
27
|
Object.defineProperty(exports, "AIDataset", { enumerable: true, get: function () { return aiDataset_1.AIDataset; } });
|
|
51
28
|
var aiRequest_1 = require("./models/aiRequest");
|
|
@@ -60,17 +37,27 @@ Object.defineProperty(exports, "Project", { enumerable: true, get: function () {
|
|
|
60
37
|
var adminScreens_1 = require("./observability/adminScreens");
|
|
61
38
|
Object.defineProperty(exports, "AI_OBSERVABILITY_GROUP", { enumerable: true, get: function () { return adminScreens_1.AI_OBSERVABILITY_GROUP; } });
|
|
62
39
|
Object.defineProperty(exports, "observabilityAdminScreens", { enumerable: true, get: function () { return adminScreens_1.observabilityAdminScreens; } });
|
|
40
|
+
var judgeAgreement_1 = require("./observability/judgeAgreement");
|
|
41
|
+
Object.defineProperty(exports, "answersAgree", { enumerable: true, get: function () { return judgeAgreement_1.answersAgree; } });
|
|
42
|
+
Object.defineProperty(exports, "computeJudgeAgreement", { enumerable: true, get: function () { return judgeAgreement_1.computeJudgeAgreement; } });
|
|
63
43
|
var datasetStore_1 = require("./observability/local/datasetStore");
|
|
64
44
|
Object.defineProperty(exports, "LocalDatasetStore", { enumerable: true, get: function () { return datasetStore_1.LocalDatasetStore; } });
|
|
65
45
|
var evaluatorStore_1 = require("./observability/local/evaluatorStore");
|
|
66
46
|
Object.defineProperty(exports, "LocalEvaluatorStore", { enumerable: true, get: function () { return evaluatorStore_1.LocalEvaluatorStore; } });
|
|
67
47
|
var experimentRunner_1 = require("./observability/local/experimentRunner");
|
|
48
|
+
Object.defineProperty(exports, "DEFAULT_TARGET_AGREEMENT", { enumerable: true, get: function () { return experimentRunner_1.DEFAULT_TARGET_AGREEMENT; } });
|
|
68
49
|
Object.defineProperty(exports, "LocalExperimentRunner", { enumerable: true, get: function () { return experimentRunner_1.LocalExperimentRunner; } });
|
|
50
|
+
var humanAnswers_1 = require("./observability/local/humanAnswers");
|
|
51
|
+
Object.defineProperty(exports, "loadCurrentHumanAnswers", { enumerable: true, get: function () { return humanAnswers_1.loadCurrentHumanAnswers; } });
|
|
69
52
|
var localPlugin_1 = require("./observability/local/localPlugin");
|
|
70
53
|
Object.defineProperty(exports, "createLocalObservabilityBundle", { enumerable: true, get: function () { return localPlugin_1.createLocalObservabilityBundle; } });
|
|
71
54
|
Object.defineProperty(exports, "createLocalObservabilityPlugin", { enumerable: true, get: function () { return localPlugin_1.createLocalObservabilityPlugin; } });
|
|
72
55
|
var promptStore_1 = require("./observability/local/promptStore");
|
|
73
56
|
Object.defineProperty(exports, "LocalPromptStore", { enumerable: true, get: function () { return promptStore_1.LocalPromptStore; } });
|
|
57
|
+
var reviewBatchStore_1 = require("./observability/local/reviewBatchStore");
|
|
58
|
+
Object.defineProperty(exports, "DEFAULT_REVIEW_BATCH_SIZE", { enumerable: true, get: function () { return reviewBatchStore_1.DEFAULT_REVIEW_BATCH_SIZE; } });
|
|
59
|
+
Object.defineProperty(exports, "LocalReviewBatchStore", { enumerable: true, get: function () { return reviewBatchStore_1.LocalReviewBatchStore; } });
|
|
60
|
+
Object.defineProperty(exports, "MAX_REVIEW_BATCH_SIZE", { enumerable: true, get: function () { return reviewBatchStore_1.MAX_REVIEW_BATCH_SIZE; } });
|
|
74
61
|
var reviewStore_1 = require("./observability/local/reviewStore");
|
|
75
62
|
Object.defineProperty(exports, "LocalReviewStore", { enumerable: true, get: function () { return reviewStore_1.LocalReviewStore; } });
|
|
76
63
|
var traceStore_1 = require("./observability/local/traceStore");
|
|
@@ -83,6 +70,8 @@ var observabilityApp_1 = require("./observability/observabilityApp");
|
|
|
83
70
|
Object.defineProperty(exports, "getObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.getObservabilityApp; } });
|
|
84
71
|
Object.defineProperty(exports, "ObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.ObservabilityApp; } });
|
|
85
72
|
Object.defineProperty(exports, "resetObservabilityApp", { enumerable: true, get: function () { return observabilityApp_1.resetObservabilityApp; } });
|
|
73
|
+
var questionFingerprint_1 = require("./observability/questionFingerprint");
|
|
74
|
+
Object.defineProperty(exports, "questionSetFingerprint", { enumerable: true, get: function () { return questionFingerprint_1.questionSetFingerprint; } });
|
|
86
75
|
var status_1 = require("./observability/status");
|
|
87
76
|
Object.defineProperty(exports, "buildObservabilityStatus", { enumerable: true, get: function () { return status_1.buildObservabilityStatus; } });
|
|
88
77
|
Object.defineProperty(exports, "buildPlaygroundAiStatus", { enumerable: true, get: function () { return status_1.buildPlaygroundAiStatus; } });
|
|
@@ -25,7 +25,11 @@ const observabilityAdminScreens = ({ localOn }) => {
|
|
|
25
25
|
}),
|
|
26
26
|
];
|
|
27
27
|
if (localOn) {
|
|
28
|
-
screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" })
|
|
28
|
+
screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" }), observabilityScreen({
|
|
29
|
+
displayName: "Review batches",
|
|
30
|
+
name: "ai-review-batches",
|
|
31
|
+
resource: "aiReview",
|
|
32
|
+
}));
|
|
29
33
|
}
|
|
30
34
|
return screens;
|
|
31
35
|
};
|
|
@@ -23,7 +23,14 @@ export interface EvaluatorExecutionResult {
|
|
|
23
23
|
scores?: Record<string, boolean | number | string>;
|
|
24
24
|
}
|
|
25
25
|
export declare const runJsonAssertEvaluator: (context: EvaluatorExecutionContext) => EvaluatorExecutionResult;
|
|
26
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
|
|
28
|
+
* version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
|
|
29
|
+
* version instead, as judge calibration experiments do.
|
|
30
|
+
*/
|
|
31
|
+
export declare const runLlmJudgeEvaluator: (context: EvaluatorExecutionContext, ai: EvaluatorAiClient, judgeOutputSchema: Record<string, unknown>, options?: {
|
|
32
|
+
judgeSystemPrompt?: string;
|
|
33
|
+
}) => Promise<EvaluatorExecutionResult>;
|
|
27
34
|
export declare const runEvaluator: (context: EvaluatorExecutionContext, deps: {
|
|
28
35
|
ai?: EvaluatorAiClient;
|
|
29
36
|
judgeOutputSchema?: Record<string, unknown>;
|
|
@@ -130,15 +130,23 @@ const runJsonAssertEvaluator = (context) => {
|
|
|
130
130
|
};
|
|
131
131
|
};
|
|
132
132
|
exports.runJsonAssertEvaluator = runJsonAssertEvaluator;
|
|
133
|
-
|
|
133
|
+
/**
|
|
134
|
+
* Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
|
|
135
|
+
* version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
|
|
136
|
+
* version instead, as judge calibration experiments do.
|
|
137
|
+
*/
|
|
138
|
+
const runLlmJudgeEvaluator = async (context, ai, judgeOutputSchema, options = {}) => {
|
|
134
139
|
const judgePromptName = context.evaluator.judgePromptName;
|
|
135
140
|
if (!judgePromptName) {
|
|
136
141
|
return { error: "llm-judge evaluator is missing judgePromptName" };
|
|
137
142
|
}
|
|
138
143
|
try {
|
|
144
|
+
const promptSource = options.judgeSystemPrompt === undefined
|
|
145
|
+
? { promptName: judgePromptName }
|
|
146
|
+
: { systemPrompt: options.judgeSystemPrompt };
|
|
139
147
|
const payload = (await ai.generateJsonObject({
|
|
140
148
|
prompt: buildJudgePrompt(context),
|
|
141
|
-
|
|
149
|
+
...promptSource,
|
|
142
150
|
schema: (0, ai_1.jsonSchema)(judgeOutputSchema),
|
|
143
151
|
schemaName: `${context.evaluator.name}-judge`,
|
|
144
152
|
skipTrace: true,
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { EvaluatorDimension } from "../types/observability";
|
|
2
|
+
export type ScoreValue = boolean | number | string;
|
|
3
|
+
/**
|
|
4
|
+
* Answers for one reviewed item. Human and judge answers pair up by `datasetItemId` when set,
|
|
5
|
+
* otherwise by `traceId`.
|
|
6
|
+
*/
|
|
7
|
+
export interface AgreementAnswerSet {
|
|
8
|
+
datasetItemId?: string;
|
|
9
|
+
scores?: Record<string, ScoreValue>;
|
|
10
|
+
traceId?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface JudgeAnswerSet extends AgreementAnswerSet {
|
|
13
|
+
error?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface JudgeQuestionAgreement {
|
|
16
|
+
compared: number;
|
|
17
|
+
/** False when the judge does not answer this question, so it is left out of the totals. */
|
|
18
|
+
covered: boolean;
|
|
19
|
+
key: string;
|
|
20
|
+
matches: number;
|
|
21
|
+
question?: string;
|
|
22
|
+
rate?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface JudgeDisagreement {
|
|
25
|
+
datasetItemId?: string;
|
|
26
|
+
human: ScoreValue;
|
|
27
|
+
judge?: ScoreValue;
|
|
28
|
+
key: string;
|
|
29
|
+
traceId?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface JudgeAgreement {
|
|
32
|
+
compared: number;
|
|
33
|
+
disagreements: JudgeDisagreement[];
|
|
34
|
+
errors: number;
|
|
35
|
+
itemsCompared: number;
|
|
36
|
+
matches: number;
|
|
37
|
+
questions: JudgeQuestionAgreement[];
|
|
38
|
+
rate?: number;
|
|
39
|
+
}
|
|
40
|
+
/** True when a judge answer counts as the same answer a human gave for this dimension. */
|
|
41
|
+
export declare const answersAgree: (dimension: EvaluatorDimension, human: ScoreValue, judge: ScoreValue | undefined) => boolean;
|
|
42
|
+
/** Key that pairs human and judge answers for the same item. */
|
|
43
|
+
export declare const agreementAnswerKey: (answer: AgreementAnswerSet) => string;
|
|
44
|
+
/**
|
|
45
|
+
* Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
|
|
46
|
+
* answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
|
|
47
|
+
*/
|
|
48
|
+
export declare const computeJudgeAgreement: (params: {
|
|
49
|
+
dimensions: EvaluatorDimension[];
|
|
50
|
+
humanAnswers: AgreementAnswerSet[];
|
|
51
|
+
judgeAnswers: JudgeAnswerSet[];
|
|
52
|
+
judgeKeys: string[];
|
|
53
|
+
}) => JudgeAgreement;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.computeJudgeAgreement = exports.agreementAnswerKey = exports.answersAgree = void 0;
|
|
4
|
+
// Numeric answers agree when they land within this share of the dimension's range.
|
|
5
|
+
const NUMERIC_TOLERANCE = 0.1;
|
|
6
|
+
const numericSpan = (range) => {
|
|
7
|
+
const matches = range?.match(/(-?\d+(?:\.\d+)?)\s*(?:-|\.\.)\s*(-?\d+(?:\.\d+)?)/);
|
|
8
|
+
if (!matches) {
|
|
9
|
+
return 1;
|
|
10
|
+
}
|
|
11
|
+
const span = Math.abs(Number(matches[2]) - Number(matches[1]));
|
|
12
|
+
return span > 0 ? span : 1;
|
|
13
|
+
};
|
|
14
|
+
const toBoolean = (value) => {
|
|
15
|
+
if (typeof value === "boolean") {
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
if (value === "true" || value === "yes") {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
if (value === "false" || value === "no") {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
return undefined;
|
|
25
|
+
};
|
|
26
|
+
/** True when a judge answer counts as the same answer a human gave for this dimension. */
|
|
27
|
+
const answersAgree = (dimension, human, judge) => {
|
|
28
|
+
if (judge === undefined) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (dimension.dataType === "boolean") {
|
|
32
|
+
const humanBool = toBoolean(human);
|
|
33
|
+
return humanBool !== undefined && humanBool === toBoolean(judge);
|
|
34
|
+
}
|
|
35
|
+
if (dimension.dataType === "numeric") {
|
|
36
|
+
const humanNumber = Number(human);
|
|
37
|
+
const judgeNumber = Number(judge);
|
|
38
|
+
if (Number.isNaN(humanNumber) || Number.isNaN(judgeNumber)) {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
return (Math.abs(humanNumber - judgeNumber) <= numericSpan(dimension.range) * NUMERIC_TOLERANCE + 1e-9);
|
|
42
|
+
}
|
|
43
|
+
return String(human).trim().toLowerCase() === String(judge).trim().toLowerCase();
|
|
44
|
+
};
|
|
45
|
+
exports.answersAgree = answersAgree;
|
|
46
|
+
/** Key that pairs human and judge answers for the same item. */
|
|
47
|
+
const agreementAnswerKey = (answer) => {
|
|
48
|
+
return answer.datasetItemId ? `item:${answer.datasetItemId}` : `trace:${answer.traceId ?? ""}`;
|
|
49
|
+
};
|
|
50
|
+
exports.agreementAnswerKey = agreementAnswerKey;
|
|
51
|
+
const rateOf = (matches, compared) => {
|
|
52
|
+
return compared > 0 ? matches / compared : undefined;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Compare one judge's answers with human answers for the same traces or dataset items. Only questions the judge
|
|
56
|
+
* answers (judgeKeys) count; a judge error or a missing answer counts as a disagreement.
|
|
57
|
+
*/
|
|
58
|
+
const computeJudgeAgreement = (params) => {
|
|
59
|
+
const judgeKeys = new Set(params.judgeKeys);
|
|
60
|
+
const judgeByKey = new Map(params.judgeAnswers.map((answer) => {
|
|
61
|
+
return [(0, exports.agreementAnswerKey)(answer), answer];
|
|
62
|
+
}));
|
|
63
|
+
const questions = params.dimensions.map((dimension) => {
|
|
64
|
+
return {
|
|
65
|
+
compared: 0,
|
|
66
|
+
covered: judgeKeys.has(dimension.key),
|
|
67
|
+
key: dimension.key,
|
|
68
|
+
matches: 0,
|
|
69
|
+
question: dimension.question,
|
|
70
|
+
};
|
|
71
|
+
});
|
|
72
|
+
const disagreements = [];
|
|
73
|
+
let itemsCompared = 0;
|
|
74
|
+
let errors = 0;
|
|
75
|
+
for (const human of params.humanAnswers) {
|
|
76
|
+
const judge = judgeByKey.get((0, exports.agreementAnswerKey)(human));
|
|
77
|
+
if (!judge || !human.scores) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
let comparedThisItem = false;
|
|
81
|
+
if (judge.error) {
|
|
82
|
+
errors += 1;
|
|
83
|
+
}
|
|
84
|
+
params.dimensions.forEach((dimension, index) => {
|
|
85
|
+
const humanValue = human.scores?.[dimension.key];
|
|
86
|
+
const question = questions[index];
|
|
87
|
+
if (!question?.covered || humanValue === undefined) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
comparedThisItem = true;
|
|
91
|
+
question.compared += 1;
|
|
92
|
+
const judgeValue = judge.error ? undefined : judge.scores?.[dimension.key];
|
|
93
|
+
if ((0, exports.answersAgree)(dimension, humanValue, judgeValue)) {
|
|
94
|
+
question.matches += 1;
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
disagreements.push({
|
|
98
|
+
...(human.datasetItemId ? { datasetItemId: human.datasetItemId } : {}),
|
|
99
|
+
human: humanValue,
|
|
100
|
+
judge: judgeValue,
|
|
101
|
+
key: dimension.key,
|
|
102
|
+
...(human.traceId ? { traceId: human.traceId } : {}),
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
if (comparedThisItem) {
|
|
106
|
+
itemsCompared += 1;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const compared = questions.reduce((total, question) => total + question.compared, 0);
|
|
110
|
+
const matches = questions.reduce((total, question) => total + question.matches, 0);
|
|
111
|
+
return {
|
|
112
|
+
compared,
|
|
113
|
+
disagreements,
|
|
114
|
+
errors,
|
|
115
|
+
itemsCompared,
|
|
116
|
+
matches,
|
|
117
|
+
questions: questions.map((question) => {
|
|
118
|
+
return { ...question, rate: rateOf(question.matches, question.compared) };
|
|
119
|
+
}),
|
|
120
|
+
rate: rateOf(matches, compared),
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
exports.computeJudgeAgreement = computeJudgeAgreement;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ObsDatasetAnnotatedBy } from "../../types/observability";
|
|
2
2
|
import { type ParsedDatasetImportRow } from "../datasetImport";
|
|
3
|
+
import { type DatasetHumanAnswerView, type DatasetJudgmentSummary } from "./humanAnswers";
|
|
3
4
|
import { LocalPromptStore } from "./promptStore";
|
|
4
5
|
export interface DatasetWriteInput {
|
|
5
6
|
description?: string | null;
|
|
@@ -31,6 +32,8 @@ export interface DatasetView {
|
|
|
31
32
|
expectedOutputSchema?: Record<string, unknown>;
|
|
32
33
|
id: string;
|
|
33
34
|
inputSchemaPromptName?: string;
|
|
35
|
+
/** Human judging progress per question set (see `summarizeDatasetJudgments`). */
|
|
36
|
+
judgments: DatasetJudgmentSummary[];
|
|
34
37
|
name: string;
|
|
35
38
|
tags: string[];
|
|
36
39
|
updated: string;
|
|
@@ -40,6 +43,8 @@ export interface DatasetItemView {
|
|
|
40
43
|
created: string;
|
|
41
44
|
datasetId: string;
|
|
42
45
|
expectedOutput?: unknown;
|
|
46
|
+
/** Human answers keyed by question-set evaluator id. */
|
|
47
|
+
humanAnswers?: Record<string, DatasetHumanAnswerView>;
|
|
43
48
|
id: string;
|
|
44
49
|
input: unknown;
|
|
45
50
|
metadata?: Record<string, unknown>;
|
|
@@ -9,6 +9,7 @@ const luxon_1 = require("luxon");
|
|
|
9
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
10
|
const datasetImport_1 = require("../datasetImport");
|
|
11
11
|
const schemaValidation_1 = require("../schemaValidation");
|
|
12
|
+
const humanAnswers_1 = require("./humanAnswers");
|
|
12
13
|
const obsDataset_1 = require("./models/obsDataset");
|
|
13
14
|
const obsDatasetItem_1 = require("./models/obsDatasetItem");
|
|
14
15
|
const obsTrace_1 = require("./models/obsTrace");
|
|
@@ -24,8 +25,9 @@ const throwOnDatasetImportErrors = (result) => {
|
|
|
24
25
|
});
|
|
25
26
|
};
|
|
26
27
|
exports.throwOnDatasetImportErrors = throwOnDatasetImportErrors;
|
|
27
|
-
const toDatasetView = async (doc) => {
|
|
28
|
+
const toDatasetView = async (doc, judgments) => {
|
|
28
29
|
const counts = await computeDatasetCounts(String(doc._id));
|
|
30
|
+
const resolvedJudgments = judgments ?? (await (0, humanAnswers_1.summarizeDatasetJudgments)([doc._id])).get(String(doc._id)) ?? [];
|
|
29
31
|
return {
|
|
30
32
|
counts,
|
|
31
33
|
created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
@@ -33,6 +35,7 @@ const toDatasetView = async (doc) => {
|
|
|
33
35
|
expectedOutputSchema: doc.expectedOutputSchema,
|
|
34
36
|
id: String(doc._id),
|
|
35
37
|
inputSchemaPromptName: doc.inputSchemaPromptName,
|
|
38
|
+
judgments: resolvedJudgments,
|
|
36
39
|
name: doc.name,
|
|
37
40
|
tags: doc.tags,
|
|
38
41
|
updated: luxon_1.DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
|
|
@@ -44,6 +47,7 @@ const toItemView = (doc) => {
|
|
|
44
47
|
created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
|
|
45
48
|
datasetId: String(doc.datasetId),
|
|
46
49
|
expectedOutput: doc.expectedOutput,
|
|
50
|
+
humanAnswers: (0, humanAnswers_1.toHumanAnswerViews)(doc.humanAnswers),
|
|
47
51
|
id: String(doc._id),
|
|
48
52
|
input: doc.input,
|
|
49
53
|
metadata: doc.metadata,
|
|
@@ -106,8 +110,9 @@ class LocalDatasetStore {
|
|
|
106
110
|
}
|
|
107
111
|
async list() {
|
|
108
112
|
const rows = await (0, obsDataset_1.registerObsDataset)().find({}).sort({ name: 1 });
|
|
113
|
+
const judgments = await (0, humanAnswers_1.summarizeDatasetJudgments)(rows.map((row) => row._id));
|
|
109
114
|
return Promise.all(rows.map((row) => {
|
|
110
|
-
return toDatasetView(row);
|
|
115
|
+
return toDatasetView(row, judgments.get(String(row._id)) ?? []);
|
|
111
116
|
}));
|
|
112
117
|
}
|
|
113
118
|
async get(id) {
|
|
@@ -15,11 +15,27 @@ export interface EvaluatorWriteInput {
|
|
|
15
15
|
target: "dataset item" | "full trace" | "generation span";
|
|
16
16
|
type: "human" | "json-assert" | "llm-judge";
|
|
17
17
|
}
|
|
18
|
+
/** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
|
|
19
|
+
export interface EvaluatorCalibration {
|
|
20
|
+
datasetName?: string;
|
|
21
|
+
experimentId: string;
|
|
22
|
+
experimentName: string;
|
|
23
|
+
passed: boolean;
|
|
24
|
+
questionSetName?: string;
|
|
25
|
+
/** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
|
|
26
|
+
rate?: number;
|
|
27
|
+
/** True when the question set's questions changed since the experiment ran. */
|
|
28
|
+
stale: boolean;
|
|
29
|
+
targetAgreement?: number;
|
|
30
|
+
/** The judge prompt's production version when it was compared, else the best version. */
|
|
31
|
+
version: number;
|
|
32
|
+
}
|
|
18
33
|
export interface EvaluatorView {
|
|
19
34
|
assertion?: {
|
|
20
35
|
constraint: string;
|
|
21
36
|
path: string;
|
|
22
37
|
};
|
|
38
|
+
calibration?: EvaluatorCalibration;
|
|
23
39
|
confidenceAlertBelow: number;
|
|
24
40
|
description?: string;
|
|
25
41
|
dimensions: EvaluatorDimension[];
|
|
@@ -37,6 +53,7 @@ export declare class LocalEvaluatorStore {
|
|
|
37
53
|
create(input: EvaluatorWriteInput): Promise<EvaluatorView>;
|
|
38
54
|
list(): Promise<EvaluatorView[]>;
|
|
39
55
|
get(id: string): Promise<EvaluatorView>;
|
|
56
|
+
private loadCalibrations;
|
|
40
57
|
getByName(name: string): Promise<EvaluatorView | undefined>;
|
|
41
58
|
update(id: string, input: Partial<EvaluatorWriteInput>): Promise<EvaluatorView>;
|
|
42
59
|
remove(id: string): Promise<void>;
|