@terreno/ai 58.0.0-beta.3 → 58.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -26,12 +26,15 @@ const createLocalPlugin = (): ObservabilityPlugin => {
|
|
|
26
26
|
};
|
|
27
27
|
};
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
// Only "local" is a typed primary; these cases check runtime validation of other ids.
|
|
30
|
+
const REMOTE = "remote" as never;
|
|
31
|
+
|
|
32
|
+
const createRemotePlugin = (): ObservabilityPlugin => {
|
|
30
33
|
return {
|
|
31
34
|
capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
|
|
32
35
|
datasetStore: {},
|
|
33
36
|
experimentRunner: {},
|
|
34
|
-
id: "
|
|
37
|
+
id: "remote",
|
|
35
38
|
promptRegistry: {get: async () => undefined},
|
|
36
39
|
scoreSink: {export: async () => {}},
|
|
37
40
|
traceSink: {export: async () => {}},
|
|
@@ -55,16 +58,16 @@ describe("ObservabilityApp config", () => {
|
|
|
55
58
|
it("rejects experiments.primary that does not equal datasets.primary", () => {
|
|
56
59
|
expect(() => {
|
|
57
60
|
new ObservabilityApp({
|
|
58
|
-
control: {datasets: "local", experiments:
|
|
59
|
-
plugins: [createLocalPlugin(),
|
|
61
|
+
control: {datasets: "local", experiments: REMOTE},
|
|
62
|
+
plugins: [createLocalPlugin(), createRemotePlugin()],
|
|
60
63
|
});
|
|
61
64
|
}).toThrow(/experiments\.primary must equal datasets\.primary/);
|
|
62
65
|
});
|
|
63
66
|
|
|
64
|
-
it("rejects reviewQueue.primary
|
|
67
|
+
it("rejects a reviewQueue.primary other than local", () => {
|
|
65
68
|
expect(() => {
|
|
66
69
|
new ObservabilityApp({
|
|
67
|
-
control: {reviewQueue:
|
|
70
|
+
control: {reviewQueue: REMOTE},
|
|
68
71
|
plugins: [createLocalPlugin()],
|
|
69
72
|
});
|
|
70
73
|
}).toThrow(/reviewQueue\.primary must be local/);
|
|
@@ -81,12 +84,12 @@ describe("ObservabilityApp config", () => {
|
|
|
81
84
|
expect(() => {
|
|
82
85
|
new ObservabilityApp({
|
|
83
86
|
control: {
|
|
84
|
-
datasets:
|
|
85
|
-
experiments:
|
|
86
|
-
prompts:
|
|
87
|
+
datasets: REMOTE,
|
|
88
|
+
experiments: REMOTE,
|
|
89
|
+
prompts: REMOTE,
|
|
87
90
|
},
|
|
88
91
|
plugins: [createLocalPlugin()],
|
|
89
92
|
});
|
|
90
|
-
}).toThrow(/prompts primary "
|
|
93
|
+
}).toThrow(/prompts primary "remote" has no plugin/);
|
|
91
94
|
});
|
|
92
95
|
});
|
|
@@ -6,6 +6,7 @@ import {LocalDatasetStore} from "./local/datasetStore";
|
|
|
6
6
|
import {LocalEvaluatorStore} from "./local/evaluatorStore";
|
|
7
7
|
import {LocalExperimentRunner} from "./local/experimentRunner";
|
|
8
8
|
import {LocalPromptStore} from "./local/promptStore";
|
|
9
|
+
import {LocalReviewBatchStore} from "./local/reviewBatchStore";
|
|
9
10
|
import {LocalReviewStore} from "./local/reviewStore";
|
|
10
11
|
import {LocalTraceSink} from "./local/traceStore";
|
|
11
12
|
import {
|
|
@@ -18,6 +19,7 @@ import {addObservabilityEvaluatorRoutes} from "./routes/evaluators";
|
|
|
18
19
|
import {addObservabilityExperimentRoutes} from "./routes/experiments";
|
|
19
20
|
import {addObservabilityPromptRoutes} from "./routes/prompts";
|
|
20
21
|
import {addObservabilityReviewRoutes} from "./routes/review";
|
|
22
|
+
import {addObservabilityReviewBatchRoutes} from "./routes/reviewBatches";
|
|
21
23
|
import {addObservabilityStatusRoutes} from "./routes/status";
|
|
22
24
|
import {addObservabilityTestMultiStageRoutes} from "./routes/testMultiStage";
|
|
23
25
|
import {addObservabilityTraceRoutes} from "./routes/traces";
|
|
@@ -149,9 +151,16 @@ export class ObservabilityApp implements TerrenoPlugin {
|
|
|
149
151
|
...routeAccess,
|
|
150
152
|
store: evaluatorStore,
|
|
151
153
|
});
|
|
154
|
+
const reviewStore = new LocalReviewStore();
|
|
155
|
+
addObservabilityReviewBatchRoutes(app, {
|
|
156
|
+
...routeAccess,
|
|
157
|
+
aiService: this.aiService,
|
|
158
|
+
requestAiServiceFactory: this.requestAiServiceFactory,
|
|
159
|
+
store: new LocalReviewBatchStore(promptStore ?? new LocalPromptStore(), reviewStore),
|
|
160
|
+
});
|
|
152
161
|
addObservabilityReviewRoutes(app, {
|
|
153
162
|
...routeAccess,
|
|
154
|
-
store:
|
|
163
|
+
store: reviewStore,
|
|
155
164
|
});
|
|
156
165
|
if (localPlugin.datasetStore instanceof LocalDatasetStore) {
|
|
157
166
|
addObservabilityDatasetRoutes(app, {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {questionSetFingerprint} from "./questionFingerprint";
|
|
4
|
+
|
|
5
|
+
const QUESTIONS = [
|
|
6
|
+
{dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
|
|
7
|
+
{dataType: "numeric" as const, key: "score", range: "1-5", required: false},
|
|
8
|
+
];
|
|
9
|
+
|
|
10
|
+
describe("questionSetFingerprint", () => {
|
|
11
|
+
it("is a stable sha256 that ignores dimension order and the required flag", () => {
|
|
12
|
+
const fingerprint = questionSetFingerprint(QUESTIONS);
|
|
13
|
+
expect(fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
|
14
|
+
expect(questionSetFingerprint([...QUESTIONS].reverse())).toBe(fingerprint);
|
|
15
|
+
expect(
|
|
16
|
+
questionSetFingerprint(QUESTIONS.map((row) => ({...row, required: !row.required})))
|
|
17
|
+
).toBe(fingerprint);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("changes when a key, data type, question, or range changes", () => {
|
|
21
|
+
const fingerprint = questionSetFingerprint(QUESTIONS);
|
|
22
|
+
const [first, second] = QUESTIONS;
|
|
23
|
+
const variants = [
|
|
24
|
+
[{...first, key: "grounded"}, second],
|
|
25
|
+
[{...first, dataType: "categorical" as const}, second],
|
|
26
|
+
[{...first, question: "Is every claim backed by the input?"}, second],
|
|
27
|
+
[first, {...second, range: "1-10"}],
|
|
28
|
+
];
|
|
29
|
+
for (const variant of variants) {
|
|
30
|
+
expect(questionSetFingerprint(variant)).not.toBe(fingerprint);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import {createHash} from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import type {EvaluatorDimension} from "../types/observability";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Stable fingerprint of a question set's questions. Human answers stay reusable while the
|
|
7
|
+
* fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
|
|
8
|
+
* stale. Dimension order and the `required` flag do not change the fingerprint.
|
|
9
|
+
*/
|
|
10
|
+
export const questionSetFingerprint = (
|
|
11
|
+
dimensions: Pick<EvaluatorDimension, "dataType" | "key" | "question" | "range">[]
|
|
12
|
+
): string => {
|
|
13
|
+
const canonical = [...dimensions]
|
|
14
|
+
.map((dimension) => {
|
|
15
|
+
return {
|
|
16
|
+
dataType: dimension.dataType,
|
|
17
|
+
key: dimension.key,
|
|
18
|
+
question: dimension.question?.trim() ?? null,
|
|
19
|
+
range: dimension.range?.trim() ?? null,
|
|
20
|
+
};
|
|
21
|
+
})
|
|
22
|
+
.sort((left, right) => left.key.localeCompare(right.key));
|
|
23
|
+
return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
24
|
+
};
|
|
@@ -240,6 +240,43 @@ describe("observability experiment routes", () => {
|
|
|
240
240
|
expect(emptyPromptName.body.total).toBeGreaterThanOrEqual(2);
|
|
241
241
|
});
|
|
242
242
|
|
|
243
|
+
it("filters experiments by kind and validates judge experiments", async () => {
|
|
244
|
+
const admin = await authAsUser(app, "admin");
|
|
245
|
+
const fixtures = await seedExperimentFixtures(admin);
|
|
246
|
+
const run = await admin.post("/ai/observability/experiments").send({
|
|
247
|
+
datasetId: fixtures.datasetId,
|
|
248
|
+
evaluatorIds: [fixtures.evaluatorId],
|
|
249
|
+
name: "prompt-run",
|
|
250
|
+
promptName: fixtures.promptName,
|
|
251
|
+
versions: [1, 2],
|
|
252
|
+
});
|
|
253
|
+
expect(run.status).toBe(201);
|
|
254
|
+
expect(run.body.data.kind).toBe("prompt");
|
|
255
|
+
expect(run.body.data.datasetName).toBe("route-exp-dataset");
|
|
256
|
+
|
|
257
|
+
const prompts = await admin.get("/ai/observability/experiments?kind=prompt");
|
|
258
|
+
expect(prompts.body.total).toBe(1);
|
|
259
|
+
const judges = await admin.get("/ai/observability/experiments?kind=judge");
|
|
260
|
+
expect(judges.body.total).toBe(0);
|
|
261
|
+
const badKind = await admin.get("/ai/observability/experiments?kind=other");
|
|
262
|
+
expect(badKind.status).toBe(400);
|
|
263
|
+
|
|
264
|
+
const notHuman = await admin.post("/ai/observability/experiments").send({
|
|
265
|
+
datasetId: fixtures.datasetId,
|
|
266
|
+
judgeEvaluatorId: fixtures.evaluatorId,
|
|
267
|
+
kind: "judge",
|
|
268
|
+
name: "judge-run",
|
|
269
|
+
questionSetId: fixtures.evaluatorId,
|
|
270
|
+
versions: [1, 2],
|
|
271
|
+
});
|
|
272
|
+
expect(notHuman.status).toBe(400);
|
|
273
|
+
expect(notHuman.body.title).toBe("questionSetId must be a human question set");
|
|
274
|
+
const unknownKind = await admin
|
|
275
|
+
.post("/ai/observability/experiments")
|
|
276
|
+
.send({datasetId: fixtures.datasetId, kind: "other", name: "x", versions: [1, 2]});
|
|
277
|
+
expect(unknownKind.status).toBe(400);
|
|
278
|
+
});
|
|
279
|
+
|
|
243
280
|
it("lists experiments and returns estimates", async () => {
|
|
244
281
|
const admin = await authAsUser(app, "admin");
|
|
245
282
|
const fixtures = await seedExperimentFixtures(admin);
|
|
@@ -27,6 +27,16 @@ const parseOptionalPromptName = (value: unknown): string | undefined => {
|
|
|
27
27
|
return trimmed;
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
+
const parseKind = (value: unknown): "judge" | "prompt" | undefined => {
|
|
31
|
+
if (value === undefined || value === "") {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
if (value !== "judge" && value !== "prompt") {
|
|
35
|
+
throw new APIError({status: 400, title: "kind must be prompt or judge"});
|
|
36
|
+
}
|
|
37
|
+
return value;
|
|
38
|
+
};
|
|
39
|
+
|
|
30
40
|
const parsePositiveInt = (value: unknown, fallback: number): number => {
|
|
31
41
|
if (value === undefined) {
|
|
32
42
|
return fallback;
|
|
@@ -73,6 +83,7 @@ export const addObservabilityExperimentRoutes = (
|
|
|
73
83
|
.withTags(["observability"])
|
|
74
84
|
.withSummary("List experiments")
|
|
75
85
|
.withQueryParameter("promptName", {type: "string"}, {required: false})
|
|
86
|
+
.withQueryParameter("kind", {enum: ["judge", "prompt"], type: "string"}, {required: false})
|
|
76
87
|
.withQueryParameter("page", {type: "number"}, {required: false})
|
|
77
88
|
.withQueryParameter("limit", {type: "number"}, {required: false})
|
|
78
89
|
.withResponse(200, {})
|
|
@@ -80,6 +91,7 @@ export const addObservabilityExperimentRoutes = (
|
|
|
80
91
|
),
|
|
81
92
|
asyncHandler(async (req, res) => {
|
|
82
93
|
const listed = await options.runner.list({
|
|
94
|
+
kind: parseKind(req.query.kind),
|
|
83
95
|
limit: parsePositiveInt(req.query.limit, EXPERIMENT_LIST_DEFAULT_LIMIT),
|
|
84
96
|
page: parsePositiveInt(req.query.page, 1),
|
|
85
97
|
promptName: parseOptionalPromptName(req.query.promptName),
|
|
@@ -102,7 +114,7 @@ export const addObservabilityExperimentRoutes = (
|
|
|
102
114
|
{action: "create", resource: "aiExperiment"},
|
|
103
115
|
builder()
|
|
104
116
|
.withTags(["observability"])
|
|
105
|
-
.withSummary("Create
|
|
117
|
+
.withSummary("Create a prompt comparison or judge calibration experiment")
|
|
106
118
|
.withResponse(201, {})
|
|
107
119
|
.build()
|
|
108
120
|
),
|
|
@@ -62,7 +62,8 @@ export const addObservabilityReviewRoutes = (
|
|
|
62
62
|
),
|
|
63
63
|
asyncHandler(async (req, res) => {
|
|
64
64
|
const status = req.query.status as ReviewStatus | undefined;
|
|
65
|
-
|
|
65
|
+
const batchId = typeof req.query.batchId === "string" ? req.query.batchId : undefined;
|
|
66
|
+
return res.json({...(await options.store.list(status, {batchId})), more: false});
|
|
66
67
|
})
|
|
67
68
|
);
|
|
68
69
|
|
|
@@ -108,6 +109,7 @@ export const addObservabilityReviewRoutes = (
|
|
|
108
109
|
id: req.params.id,
|
|
109
110
|
scores: body.scores ?? {},
|
|
110
111
|
sinks: getObservabilityApp()?.scoreSinks ?? [],
|
|
112
|
+
userId: req.user?._id ? String(req.user._id) : undefined,
|
|
111
113
|
});
|
|
112
114
|
} else if (body.action === "skip") {
|
|
113
115
|
await options.store.skip(req.params.id);
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import {TerrenoApp} from "@terreno/api";
|
|
3
|
+
import type express from "express";
|
|
4
|
+
import {DateTime} from "luxon";
|
|
5
|
+
|
|
6
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
7
|
+
import {LocalEvaluatorStore} from "../local/evaluatorStore";
|
|
8
|
+
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
|
+
import {registerObsDataset} from "../local/models/obsDataset";
|
|
10
|
+
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
11
|
+
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
12
|
+
import {registerObsJudgeRun} from "../local/models/obsJudgeRun";
|
|
13
|
+
import {registerObsReviewBatch} from "../local/models/obsReviewBatch";
|
|
14
|
+
import {registerObsReviewItem} from "../local/models/obsReviewItem";
|
|
15
|
+
import {registerObsTrace} from "../local/models/obsTrace";
|
|
16
|
+
import {LocalPromptStore} from "../local/promptStore";
|
|
17
|
+
import {LocalTraceStore} from "../local/traceStore";
|
|
18
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
19
|
+
import type {ObservabilityGenerateClient} from "../types";
|
|
20
|
+
|
|
21
|
+
const judgeClient: ObservabilityGenerateClient = {
|
|
22
|
+
generateJsonObject: async <OBJECT>() => ({confidence: 0.9, faithful: true}) as OBJECT,
|
|
23
|
+
generateText: async () => "",
|
|
24
|
+
modelId: "route-judge",
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const buildApp = (aiService?: ObservabilityGenerateClient): express.Application => {
|
|
28
|
+
return new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
29
|
+
.register(new ObservabilityApp({aiService, plugins: [createLocalObservabilityPlugin()]}))
|
|
30
|
+
.build();
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
describe("observability review batch routes", () => {
|
|
34
|
+
let questionSetId: string;
|
|
35
|
+
let judgeId: string;
|
|
36
|
+
|
|
37
|
+
beforeAll(async () => {
|
|
38
|
+
await ensureTestUsers();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
afterEach(() => {
|
|
42
|
+
resetObservabilityApp();
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
beforeEach(async () => {
|
|
46
|
+
await registerObsReviewItem().deleteMany({});
|
|
47
|
+
await registerObsReviewBatch().deleteMany({});
|
|
48
|
+
await registerObsDataset().deleteMany({});
|
|
49
|
+
await registerObsDatasetItem().deleteMany({});
|
|
50
|
+
await registerObsJudgeRun().deleteMany({});
|
|
51
|
+
await registerObsEvaluator().deleteMany({});
|
|
52
|
+
await registerObsTrace().deleteMany({});
|
|
53
|
+
// Register the models and default local stores before seeding fixtures.
|
|
54
|
+
buildApp();
|
|
55
|
+
resetObservabilityApp();
|
|
56
|
+
const promptStore = new LocalPromptStore();
|
|
57
|
+
if (!(await promptStore.getVersionByLabel("route-judge-prompt"))) {
|
|
58
|
+
await promptStore.create({
|
|
59
|
+
folder: "evaluators",
|
|
60
|
+
name: "route-judge-prompt",
|
|
61
|
+
outputSchema: {
|
|
62
|
+
properties: {confidence: {type: "number"}, faithful: {type: "boolean"}},
|
|
63
|
+
required: ["faithful"],
|
|
64
|
+
type: "object",
|
|
65
|
+
},
|
|
66
|
+
type: "chat",
|
|
67
|
+
});
|
|
68
|
+
await promptStore.moveLabel("route-judge-prompt", {label: "production", version: 1});
|
|
69
|
+
}
|
|
70
|
+
const evaluators = new LocalEvaluatorStore(promptStore);
|
|
71
|
+
questionSetId = (
|
|
72
|
+
await evaluators.create({
|
|
73
|
+
dimensions: [
|
|
74
|
+
{dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
|
|
75
|
+
],
|
|
76
|
+
name: "route-questions",
|
|
77
|
+
target: "full trace",
|
|
78
|
+
type: "human",
|
|
79
|
+
})
|
|
80
|
+
).id;
|
|
81
|
+
judgeId = (
|
|
82
|
+
await evaluators.create({
|
|
83
|
+
dimensions: [{dataType: "boolean", key: "faithful", required: true}],
|
|
84
|
+
judgePromptName: "route-judge-prompt",
|
|
85
|
+
name: "route-judge",
|
|
86
|
+
target: "full trace",
|
|
87
|
+
type: "llm-judge",
|
|
88
|
+
})
|
|
89
|
+
).id;
|
|
90
|
+
for (let index = 0; index < 3; index += 1) {
|
|
91
|
+
const startedAt = DateTime.utc().minus({hours: index}).toISO() ?? "";
|
|
92
|
+
await new LocalTraceStore().exportTrace({
|
|
93
|
+
id: `t${index}`,
|
|
94
|
+
input: `in ${index}`,
|
|
95
|
+
name: "summarize",
|
|
96
|
+
output: `out ${index}`,
|
|
97
|
+
prompts: [{name: "summarize", version: 1}],
|
|
98
|
+
sensitive: false,
|
|
99
|
+
spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: "ok"}],
|
|
100
|
+
startedAt,
|
|
101
|
+
status: "ok",
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("creates a batch from recent traces, answers it, and compares a judge", async () => {
|
|
107
|
+
const app = buildApp(judgeClient);
|
|
108
|
+
const agent = await authAsUser(app, "admin");
|
|
109
|
+
const created = await agent
|
|
110
|
+
.post("/ai/observability/review/batches")
|
|
111
|
+
.send({evaluatorId: questionSetId, limit: 2, promptName: "summarize"});
|
|
112
|
+
expect(created.status).toBe(201);
|
|
113
|
+
expect(created.body.data.counts.total).toBe(2);
|
|
114
|
+
const batchId = created.body.data.id as string;
|
|
115
|
+
|
|
116
|
+
const listed = await agent.get("/ai/observability/review/batches");
|
|
117
|
+
expect(listed.status).toBe(200);
|
|
118
|
+
expect(listed.body.data.map((row: {id: string}) => row.id)).toEqual([batchId]);
|
|
119
|
+
|
|
120
|
+
const pending = await agent.get(`/ai/observability/review?status=pending&batchId=${batchId}`);
|
|
121
|
+
expect(pending.body.data).toHaveLength(2);
|
|
122
|
+
for (const item of pending.body.data as Array<{id: string}>) {
|
|
123
|
+
const submitted = await agent
|
|
124
|
+
.post(`/ai/observability/review/${item.id}`)
|
|
125
|
+
.send({action: "submit", scores: {faithful: true}});
|
|
126
|
+
expect(submitted.status).toBe(200);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const judged = await agent
|
|
130
|
+
.post(`/ai/observability/review/batches/${batchId}/judgeRuns`)
|
|
131
|
+
.send({evaluatorId: judgeId});
|
|
132
|
+
expect(judged.status).toBe(201);
|
|
133
|
+
expect(judged.body.data.judges[0].agreement.rate).toBe(1);
|
|
134
|
+
expect(judged.body.data.judges[0].modelId).toBe("route-judge");
|
|
135
|
+
|
|
136
|
+
const detail = await agent.get(`/ai/observability/review/batches/${batchId}`);
|
|
137
|
+
expect(detail.status).toBe(200);
|
|
138
|
+
expect(detail.body.data.counts.done).toBe(2);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("requires an evaluator and an AI client for judge runs", async () => {
|
|
142
|
+
const app = buildApp();
|
|
143
|
+
const agent = await authAsUser(app, "admin");
|
|
144
|
+
const missing = await agent.post("/ai/observability/review/batches").send({});
|
|
145
|
+
expect(missing.status).toBe(400);
|
|
146
|
+
const created = await agent
|
|
147
|
+
.post("/ai/observability/review/batches")
|
|
148
|
+
.send({evaluatorId: questionSetId});
|
|
149
|
+
expect(created.status).toBe(201);
|
|
150
|
+
const judged = await agent
|
|
151
|
+
.post(`/ai/observability/review/batches/${created.body.data.id}/judgeRuns`)
|
|
152
|
+
.send({evaluatorId: judgeId});
|
|
153
|
+
expect(judged.status).toBe(503);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("builds a batch from a dataset's unanswered items and records answers on them", async () => {
|
|
157
|
+
const app = buildApp(judgeClient);
|
|
158
|
+
const agent = await authAsUser(app, "admin");
|
|
159
|
+
const dataset = await agent.post("/ai/observability/datasets").send({name: "route-set"});
|
|
160
|
+
const datasetId = dataset.body.data.id as string;
|
|
161
|
+
await agent
|
|
162
|
+
.post(`/ai/observability/datasets/${datasetId}/items`)
|
|
163
|
+
.send({expectedOutput: "summary", input: {text: "todo"}});
|
|
164
|
+
|
|
165
|
+
const mixed = await agent
|
|
166
|
+
.post("/ai/observability/review/batches")
|
|
167
|
+
.send({datasetId, evaluatorId: questionSetId, traceIds: []});
|
|
168
|
+
expect(mixed.status).toBe(400);
|
|
169
|
+
|
|
170
|
+
const created = await agent
|
|
171
|
+
.post("/ai/observability/review/batches")
|
|
172
|
+
.send({datasetId, evaluatorId: questionSetId});
|
|
173
|
+
expect(created.status).toBe(201);
|
|
174
|
+
expect(created.body.data.datasetId).toBe(datasetId);
|
|
175
|
+
const [item] = created.body.data.items as Array<{datasetItemId: string; reviewItemId: string}>;
|
|
176
|
+
const submitted = await agent
|
|
177
|
+
.post(`/ai/observability/review/${item?.reviewItemId}`)
|
|
178
|
+
.send({action: "submit", scores: {faithful: true}});
|
|
179
|
+
expect(submitted.status).toBe(200);
|
|
180
|
+
expect(submitted.body.data.datasetItemId).toBe(item?.datasetItemId);
|
|
181
|
+
|
|
182
|
+
const again = await agent
|
|
183
|
+
.post("/ai/observability/review/batches")
|
|
184
|
+
.send({datasetId, evaluatorId: questionSetId});
|
|
185
|
+
expect(again.status).toBe(400);
|
|
186
|
+
expect(again.body.title).toBe("Every item already has current answers");
|
|
187
|
+
|
|
188
|
+
const detail = await agent.get(`/ai/observability/datasets/${datasetId}`);
|
|
189
|
+
expect(detail.body.data.judgments).toEqual([
|
|
190
|
+
{
|
|
191
|
+
answered: 1,
|
|
192
|
+
current: 1,
|
|
193
|
+
evaluatorId: questionSetId,
|
|
194
|
+
evaluatorName: "route-questions",
|
|
195
|
+
stale: 0,
|
|
196
|
+
total: 1,
|
|
197
|
+
},
|
|
198
|
+
]);
|
|
199
|
+
const items = await agent.get(`/ai/observability/datasets/${datasetId}/items`);
|
|
200
|
+
expect(items.body.data[0].humanAnswers[questionSetId].scores).toEqual({faithful: true});
|
|
201
|
+
expect(items.body.data[0].humanAnswers[questionSetId].userId).toBeDefined();
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it("forbids non-admin users", async () => {
|
|
205
|
+
const agent = await authAsUser(buildApp(judgeClient), "notAdmin");
|
|
206
|
+
const listed = await agent.get("/ai/observability/review/batches");
|
|
207
|
+
expect(listed.status).toBe(403);
|
|
208
|
+
});
|
|
209
|
+
});
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
|
|
2
|
+
import type express from "express";
|
|
3
|
+
|
|
4
|
+
import type {LocalReviewBatchStore} from "../local/reviewBatchStore";
|
|
5
|
+
import type {ObservabilityGenerateClient, ObservabilityRequestAiServiceFactory} from "../types";
|
|
6
|
+
import {
|
|
7
|
+
type ObservabilityRouteAccessOptions,
|
|
8
|
+
observabilityRouteMiddleware,
|
|
9
|
+
} from "./observabilityRouteAccess";
|
|
10
|
+
import {resolveRequestAiService} from "./requestAiService";
|
|
11
|
+
|
|
12
|
+
const BASE_PATH = "/ai/observability";
|
|
13
|
+
|
|
14
|
+
export interface ObservabilityReviewBatchRouteOptions extends ObservabilityRouteAccessOptions {
|
|
15
|
+
aiService?: ObservabilityGenerateClient;
|
|
16
|
+
requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
|
|
17
|
+
store: LocalReviewBatchStore;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Register before the review item routes so `/review/batches` is not read as an item id. */
|
|
21
|
+
export const addObservabilityReviewBatchRoutes = (
|
|
22
|
+
router: express.Application,
|
|
23
|
+
options: ObservabilityReviewBatchRouteOptions
|
|
24
|
+
): void => {
|
|
25
|
+
const openApiOptions = options.openApi ? {openApi: options.openApi} : {};
|
|
26
|
+
const builder = (): ReturnType<typeof createOpenApiBuilder> => {
|
|
27
|
+
return createOpenApiBuilder(openApiOptions as Parameters<typeof createOpenApiBuilder>[0]);
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
router.get(
|
|
31
|
+
`${BASE_PATH}/review/batches`,
|
|
32
|
+
observabilityRouteMiddleware(
|
|
33
|
+
options.accessControl,
|
|
34
|
+
{action: "list", resource: "aiReview"},
|
|
35
|
+
builder()
|
|
36
|
+
.withTags(["observability"])
|
|
37
|
+
.withSummary("List human review batches with judge agreement")
|
|
38
|
+
.withResponse(200, {data: {type: "array"}})
|
|
39
|
+
.build()
|
|
40
|
+
),
|
|
41
|
+
asyncHandler(async (_req, res) => {
|
|
42
|
+
return res.json({data: await options.store.list(), more: false});
|
|
43
|
+
})
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
router.post(
|
|
47
|
+
`${BASE_PATH}/review/batches`,
|
|
48
|
+
observabilityRouteMiddleware(
|
|
49
|
+
options.accessControl,
|
|
50
|
+
{action: "assign", resource: "aiReview"},
|
|
51
|
+
builder()
|
|
52
|
+
.withTags(["observability"])
|
|
53
|
+
.withSummary(
|
|
54
|
+
"Create a review batch from traces, the most recent traces, or a dataset's unanswered items"
|
|
55
|
+
)
|
|
56
|
+
.withResponse(201, {data: {type: "object"}})
|
|
57
|
+
.build()
|
|
58
|
+
),
|
|
59
|
+
asyncHandler(async (req, res) => {
|
|
60
|
+
const body = req.body as {
|
|
61
|
+
datasetId?: string;
|
|
62
|
+
description?: string;
|
|
63
|
+
evaluatorId?: string;
|
|
64
|
+
limit?: number;
|
|
65
|
+
name?: string;
|
|
66
|
+
promptName?: string;
|
|
67
|
+
traceIds?: string[];
|
|
68
|
+
};
|
|
69
|
+
if (!body.evaluatorId) {
|
|
70
|
+
throw new APIError({status: 400, title: "evaluatorId is required"});
|
|
71
|
+
}
|
|
72
|
+
if (body.traceIds !== undefined && !Array.isArray(body.traceIds)) {
|
|
73
|
+
throw new APIError({status: 400, title: "traceIds must be an array"});
|
|
74
|
+
}
|
|
75
|
+
if (body.datasetId !== undefined && typeof body.datasetId !== "string") {
|
|
76
|
+
throw new APIError({status: 400, title: "datasetId must be a string"});
|
|
77
|
+
}
|
|
78
|
+
const data = await options.store.create({
|
|
79
|
+
datasetId: body.datasetId || undefined,
|
|
80
|
+
description: body.description,
|
|
81
|
+
evaluatorId: body.evaluatorId,
|
|
82
|
+
limit: body.limit,
|
|
83
|
+
name: body.name,
|
|
84
|
+
promptName: body.promptName || undefined,
|
|
85
|
+
traceIds: body.traceIds,
|
|
86
|
+
});
|
|
87
|
+
return res.status(201).json({data});
|
|
88
|
+
})
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
router.get(
|
|
92
|
+
`${BASE_PATH}/review/batches/:id`,
|
|
93
|
+
observabilityRouteMiddleware(
|
|
94
|
+
options.accessControl,
|
|
95
|
+
{action: "read", resource: "aiReview"},
|
|
96
|
+
builder()
|
|
97
|
+
.withTags(["observability"])
|
|
98
|
+
.withSummary("Get a review batch with human answers and judge agreement")
|
|
99
|
+
.withPathParameter("id", {type: "string"})
|
|
100
|
+
.withResponse(200, {data: {type: "object"}})
|
|
101
|
+
.build()
|
|
102
|
+
),
|
|
103
|
+
asyncHandler(async (req, res) => {
|
|
104
|
+
return res.json({data: await options.store.get(req.params.id)});
|
|
105
|
+
})
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
router.post(
|
|
109
|
+
`${BASE_PATH}/review/batches/:id/judgeRuns`,
|
|
110
|
+
observabilityRouteMiddleware(
|
|
111
|
+
options.accessControl,
|
|
112
|
+
{action: "assign", resource: "aiReview"},
|
|
113
|
+
builder()
|
|
114
|
+
.withTags(["observability"])
|
|
115
|
+
.withSummary("Run an LLM judge over a review batch")
|
|
116
|
+
.withPathParameter("id", {type: "string"})
|
|
117
|
+
.withResponse(201, {data: {type: "object"}})
|
|
118
|
+
.build()
|
|
119
|
+
),
|
|
120
|
+
asyncHandler(async (req, res) => {
|
|
121
|
+
const body = req.body as {evaluatorId?: string};
|
|
122
|
+
if (!body.evaluatorId) {
|
|
123
|
+
throw new APIError({status: 400, title: "evaluatorId is required"});
|
|
124
|
+
}
|
|
125
|
+
const ai = resolveRequestAiService({
|
|
126
|
+
aiService: options.aiService,
|
|
127
|
+
req,
|
|
128
|
+
requestAiServiceFactory: options.requestAiServiceFactory,
|
|
129
|
+
});
|
|
130
|
+
const data = await options.store.runJudge({
|
|
131
|
+
ai,
|
|
132
|
+
batchId: req.params.id,
|
|
133
|
+
evaluatorId: body.evaluatorId,
|
|
134
|
+
});
|
|
135
|
+
return res.status(201).json({data});
|
|
136
|
+
})
|
|
137
|
+
);
|
|
138
|
+
};
|
|
@@ -9,12 +9,12 @@ import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
|
9
9
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
10
10
|
import type {ObservabilityPlugin} from "../types";
|
|
11
11
|
|
|
12
|
-
const
|
|
12
|
+
const createRemotePlugin = (): ObservabilityPlugin => {
|
|
13
13
|
return {
|
|
14
14
|
capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
|
|
15
15
|
datasetStore: {},
|
|
16
16
|
experimentRunner: {},
|
|
17
|
-
id: "
|
|
17
|
+
id: "remote",
|
|
18
18
|
promptRegistry: {get: async () => undefined},
|
|
19
19
|
scoreSink: {export: async () => {}},
|
|
20
20
|
traceSink: {export: async () => {}},
|
|
@@ -36,7 +36,7 @@ describe("observability status routes", () => {
|
|
|
36
36
|
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
37
37
|
.register(
|
|
38
38
|
new ObservabilityApp({
|
|
39
|
-
plugins: [createLocalObservabilityPlugin(),
|
|
39
|
+
plugins: [createLocalObservabilityPlugin(), createRemotePlugin()],
|
|
40
40
|
})
|
|
41
41
|
)
|
|
42
42
|
.build();
|
|
@@ -55,7 +55,7 @@ describe("observability status routes", () => {
|
|
|
55
55
|
});
|
|
56
56
|
expect(response.body.data.plugins.map((plugin: {id: string}) => plugin.id)).toEqual([
|
|
57
57
|
"local",
|
|
58
|
-
"
|
|
58
|
+
"remote",
|
|
59
59
|
]);
|
|
60
60
|
expect(response.body.data.plugins[0].capabilities).toContain("reviewQueue");
|
|
61
61
|
});
|