@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -26,12 +26,15 @@ const createLocalPlugin = (): ObservabilityPlugin => {
|
|
|
26
26
|
};
|
|
27
27
|
};
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
// Only "local" is a typed primary; these cases check runtime validation of other ids.
|
|
30
|
+
const REMOTE = "remote" as never;
|
|
31
|
+
|
|
32
|
+
const createRemotePlugin = (): ObservabilityPlugin => {
|
|
30
33
|
return {
|
|
31
34
|
capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
|
|
32
35
|
datasetStore: {},
|
|
33
36
|
experimentRunner: {},
|
|
34
|
-
id: "
|
|
37
|
+
id: "remote",
|
|
35
38
|
promptRegistry: {get: async () => undefined},
|
|
36
39
|
scoreSink: {export: async () => {}},
|
|
37
40
|
traceSink: {export: async () => {}},
|
|
@@ -55,16 +58,16 @@ describe("ObservabilityApp config", () => {
|
|
|
55
58
|
it("rejects experiments.primary that does not equal datasets.primary", () => {
|
|
56
59
|
expect(() => {
|
|
57
60
|
new ObservabilityApp({
|
|
58
|
-
control: {datasets: "local", experiments:
|
|
59
|
-
plugins: [createLocalPlugin(),
|
|
61
|
+
control: {datasets: "local", experiments: REMOTE},
|
|
62
|
+
plugins: [createLocalPlugin(), createRemotePlugin()],
|
|
60
63
|
});
|
|
61
64
|
}).toThrow(/experiments\.primary must equal datasets\.primary/);
|
|
62
65
|
});
|
|
63
66
|
|
|
64
|
-
it("rejects reviewQueue.primary
|
|
67
|
+
it("rejects a reviewQueue.primary other than local", () => {
|
|
65
68
|
expect(() => {
|
|
66
69
|
new ObservabilityApp({
|
|
67
|
-
control: {reviewQueue:
|
|
70
|
+
control: {reviewQueue: REMOTE},
|
|
68
71
|
plugins: [createLocalPlugin()],
|
|
69
72
|
});
|
|
70
73
|
}).toThrow(/reviewQueue\.primary must be local/);
|
|
@@ -81,12 +84,12 @@ describe("ObservabilityApp config", () => {
|
|
|
81
84
|
expect(() => {
|
|
82
85
|
new ObservabilityApp({
|
|
83
86
|
control: {
|
|
84
|
-
datasets:
|
|
85
|
-
experiments:
|
|
86
|
-
prompts:
|
|
87
|
+
datasets: REMOTE,
|
|
88
|
+
experiments: REMOTE,
|
|
89
|
+
prompts: REMOTE,
|
|
87
90
|
},
|
|
88
91
|
plugins: [createLocalPlugin()],
|
|
89
92
|
});
|
|
90
|
-
}).toThrow(/prompts primary "
|
|
93
|
+
}).toThrow(/prompts primary "remote" has no plugin/);
|
|
91
94
|
});
|
|
92
95
|
});
|
|
@@ -6,6 +6,7 @@ import {LocalDatasetStore} from "./local/datasetStore";
|
|
|
6
6
|
import {LocalEvaluatorStore} from "./local/evaluatorStore";
|
|
7
7
|
import {LocalExperimentRunner} from "./local/experimentRunner";
|
|
8
8
|
import {LocalPromptStore} from "./local/promptStore";
|
|
9
|
+
import {LocalReviewBatchStore} from "./local/reviewBatchStore";
|
|
9
10
|
import {LocalReviewStore} from "./local/reviewStore";
|
|
10
11
|
import {LocalTraceSink} from "./local/traceStore";
|
|
11
12
|
import {
|
|
@@ -18,6 +19,7 @@ import {addObservabilityEvaluatorRoutes} from "./routes/evaluators";
|
|
|
18
19
|
import {addObservabilityExperimentRoutes} from "./routes/experiments";
|
|
19
20
|
import {addObservabilityPromptRoutes} from "./routes/prompts";
|
|
20
21
|
import {addObservabilityReviewRoutes} from "./routes/review";
|
|
22
|
+
import {addObservabilityReviewBatchRoutes} from "./routes/reviewBatches";
|
|
21
23
|
import {addObservabilityStatusRoutes} from "./routes/status";
|
|
22
24
|
import {addObservabilityTestMultiStageRoutes} from "./routes/testMultiStage";
|
|
23
25
|
import {addObservabilityTraceRoutes} from "./routes/traces";
|
|
@@ -149,9 +151,16 @@ export class ObservabilityApp implements TerrenoPlugin {
|
|
|
149
151
|
...routeAccess,
|
|
150
152
|
store: evaluatorStore,
|
|
151
153
|
});
|
|
154
|
+
const reviewStore = new LocalReviewStore();
|
|
155
|
+
addObservabilityReviewBatchRoutes(app, {
|
|
156
|
+
...routeAccess,
|
|
157
|
+
aiService: this.aiService,
|
|
158
|
+
requestAiServiceFactory: this.requestAiServiceFactory,
|
|
159
|
+
store: new LocalReviewBatchStore(promptStore ?? new LocalPromptStore(), reviewStore),
|
|
160
|
+
});
|
|
152
161
|
addObservabilityReviewRoutes(app, {
|
|
153
162
|
...routeAccess,
|
|
154
|
-
store:
|
|
163
|
+
store: reviewStore,
|
|
155
164
|
});
|
|
156
165
|
if (localPlugin.datasetStore instanceof LocalDatasetStore) {
|
|
157
166
|
addObservabilityDatasetRoutes(app, {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {questionSetFingerprint} from "./questionFingerprint";
|
|
4
|
+
|
|
5
|
+
const QUESTIONS = [
|
|
6
|
+
{dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
|
|
7
|
+
{dataType: "numeric" as const, key: "score", range: "1-5", required: false},
|
|
8
|
+
];
|
|
9
|
+
|
|
10
|
+
describe("questionSetFingerprint", () => {
|
|
11
|
+
it("is a stable sha256 that ignores dimension order and the required flag", () => {
|
|
12
|
+
const fingerprint = questionSetFingerprint(QUESTIONS);
|
|
13
|
+
expect(fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
|
14
|
+
expect(questionSetFingerprint([...QUESTIONS].reverse())).toBe(fingerprint);
|
|
15
|
+
expect(
|
|
16
|
+
questionSetFingerprint(QUESTIONS.map((row) => ({...row, required: !row.required})))
|
|
17
|
+
).toBe(fingerprint);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("changes when a key, data type, question, or range changes", () => {
|
|
21
|
+
const fingerprint = questionSetFingerprint(QUESTIONS);
|
|
22
|
+
const [first, second] = QUESTIONS;
|
|
23
|
+
const variants = [
|
|
24
|
+
[{...first, key: "grounded"}, second],
|
|
25
|
+
[{...first, dataType: "categorical" as const}, second],
|
|
26
|
+
[{...first, question: "Is every claim backed by the input?"}, second],
|
|
27
|
+
[first, {...second, range: "1-10"}],
|
|
28
|
+
];
|
|
29
|
+
for (const variant of variants) {
|
|
30
|
+
expect(questionSetFingerprint(variant)).not.toBe(fingerprint);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import {createHash} from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import type {EvaluatorDimension} from "../types/observability";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Stable fingerprint of a question set's questions. Human answers stay reusable while the
|
|
7
|
+
* fingerprint matches; editing a question's key, type, wording, or range makes earlier answers
|
|
8
|
+
* stale. Dimension order and the `required` flag do not change the fingerprint.
|
|
9
|
+
*/
|
|
10
|
+
export const questionSetFingerprint = (
|
|
11
|
+
dimensions: Pick<EvaluatorDimension, "dataType" | "key" | "question" | "range">[]
|
|
12
|
+
): string => {
|
|
13
|
+
const canonical = [...dimensions]
|
|
14
|
+
.map((dimension) => {
|
|
15
|
+
return {
|
|
16
|
+
dataType: dimension.dataType,
|
|
17
|
+
key: dimension.key,
|
|
18
|
+
question: dimension.question?.trim() ?? null,
|
|
19
|
+
range: dimension.range?.trim() ?? null,
|
|
20
|
+
};
|
|
21
|
+
})
|
|
22
|
+
.sort((left, right) => left.key.localeCompare(right.key));
|
|
23
|
+
return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
24
|
+
};
|
|
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
|
-
import {authAsUser, ensureTestUsers,
|
|
5
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
6
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
7
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
8
8
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
@@ -32,7 +32,7 @@ describe("observability dataset routes", () => {
|
|
|
32
32
|
await registerObsPromptLabel().deleteMany({});
|
|
33
33
|
promptStore = new LocalPromptStore();
|
|
34
34
|
const plugin = createLocalObservabilityPlugin();
|
|
35
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
35
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
36
36
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
37
37
|
.build();
|
|
38
38
|
});
|
|
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
|
-
import {authAsUser, ensureTestUsers,
|
|
5
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
6
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
7
|
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
8
8
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
@@ -40,7 +40,7 @@ describe("observability evaluator routes", () => {
|
|
|
40
40
|
type: "text",
|
|
41
41
|
});
|
|
42
42
|
await promptStore.moveLabel("eval-judge-correctness", {label: "production", version: 1});
|
|
43
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
43
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
44
44
|
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
45
45
|
.build();
|
|
46
46
|
});
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {AIService} from "../../service/aiService";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
9
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
10
10
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
@@ -70,7 +70,7 @@ describe("observability experiment routes", () => {
|
|
|
70
70
|
},
|
|
71
71
|
},
|
|
72
72
|
});
|
|
73
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
73
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
74
74
|
.register(
|
|
75
75
|
new ObservabilityApp({
|
|
76
76
|
aiService,
|
|
@@ -240,6 +240,43 @@ describe("observability experiment routes", () => {
|
|
|
240
240
|
expect(emptyPromptName.body.total).toBeGreaterThanOrEqual(2);
|
|
241
241
|
});
|
|
242
242
|
|
|
243
|
+
it("filters experiments by kind and validates judge experiments", async () => {
|
|
244
|
+
const admin = await authAsUser(app, "admin");
|
|
245
|
+
const fixtures = await seedExperimentFixtures(admin);
|
|
246
|
+
const run = await admin.post("/ai/observability/experiments").send({
|
|
247
|
+
datasetId: fixtures.datasetId,
|
|
248
|
+
evaluatorIds: [fixtures.evaluatorId],
|
|
249
|
+
name: "prompt-run",
|
|
250
|
+
promptName: fixtures.promptName,
|
|
251
|
+
versions: [1, 2],
|
|
252
|
+
});
|
|
253
|
+
expect(run.status).toBe(201);
|
|
254
|
+
expect(run.body.data.kind).toBe("prompt");
|
|
255
|
+
expect(run.body.data.datasetName).toBe("route-exp-dataset");
|
|
256
|
+
|
|
257
|
+
const prompts = await admin.get("/ai/observability/experiments?kind=prompt");
|
|
258
|
+
expect(prompts.body.total).toBe(1);
|
|
259
|
+
const judges = await admin.get("/ai/observability/experiments?kind=judge");
|
|
260
|
+
expect(judges.body.total).toBe(0);
|
|
261
|
+
const badKind = await admin.get("/ai/observability/experiments?kind=other");
|
|
262
|
+
expect(badKind.status).toBe(400);
|
|
263
|
+
|
|
264
|
+
const notHuman = await admin.post("/ai/observability/experiments").send({
|
|
265
|
+
datasetId: fixtures.datasetId,
|
|
266
|
+
judgeEvaluatorId: fixtures.evaluatorId,
|
|
267
|
+
kind: "judge",
|
|
268
|
+
name: "judge-run",
|
|
269
|
+
questionSetId: fixtures.evaluatorId,
|
|
270
|
+
versions: [1, 2],
|
|
271
|
+
});
|
|
272
|
+
expect(notHuman.status).toBe(400);
|
|
273
|
+
expect(notHuman.body.title).toBe("questionSetId must be a human question set");
|
|
274
|
+
const unknownKind = await admin
|
|
275
|
+
.post("/ai/observability/experiments")
|
|
276
|
+
.send({datasetId: fixtures.datasetId, kind: "other", name: "x", versions: [1, 2]});
|
|
277
|
+
expect(unknownKind.status).toBe(400);
|
|
278
|
+
});
|
|
279
|
+
|
|
243
280
|
it("lists experiments and returns estimates", async () => {
|
|
244
281
|
const admin = await authAsUser(app, "admin");
|
|
245
282
|
const fixtures = await seedExperimentFixtures(admin);
|
|
@@ -27,6 +27,16 @@ const parseOptionalPromptName = (value: unknown): string | undefined => {
|
|
|
27
27
|
return trimmed;
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
+
const parseKind = (value: unknown): "judge" | "prompt" | undefined => {
|
|
31
|
+
if (value === undefined || value === "") {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
if (value !== "judge" && value !== "prompt") {
|
|
35
|
+
throw new APIError({status: 400, title: "kind must be prompt or judge"});
|
|
36
|
+
}
|
|
37
|
+
return value;
|
|
38
|
+
};
|
|
39
|
+
|
|
30
40
|
const parsePositiveInt = (value: unknown, fallback: number): number => {
|
|
31
41
|
if (value === undefined) {
|
|
32
42
|
return fallback;
|
|
@@ -73,6 +83,7 @@ export const addObservabilityExperimentRoutes = (
|
|
|
73
83
|
.withTags(["observability"])
|
|
74
84
|
.withSummary("List experiments")
|
|
75
85
|
.withQueryParameter("promptName", {type: "string"}, {required: false})
|
|
86
|
+
.withQueryParameter("kind", {enum: ["judge", "prompt"], type: "string"}, {required: false})
|
|
76
87
|
.withQueryParameter("page", {type: "number"}, {required: false})
|
|
77
88
|
.withQueryParameter("limit", {type: "number"}, {required: false})
|
|
78
89
|
.withResponse(200, {})
|
|
@@ -80,6 +91,7 @@ export const addObservabilityExperimentRoutes = (
|
|
|
80
91
|
),
|
|
81
92
|
asyncHandler(async (req, res) => {
|
|
82
93
|
const listed = await options.runner.list({
|
|
94
|
+
kind: parseKind(req.query.kind),
|
|
83
95
|
limit: parsePositiveInt(req.query.limit, EXPERIMENT_LIST_DEFAULT_LIMIT),
|
|
84
96
|
page: parsePositiveInt(req.query.page, 1),
|
|
85
97
|
promptName: parseOptionalPromptName(req.query.promptName),
|
|
@@ -102,7 +114,7 @@ export const addObservabilityExperimentRoutes = (
|
|
|
102
114
|
{action: "create", resource: "aiExperiment"},
|
|
103
115
|
builder()
|
|
104
116
|
.withTags(["observability"])
|
|
105
|
-
.withSummary("Create
|
|
117
|
+
.withSummary("Create a prompt comparison or judge calibration experiment")
|
|
106
118
|
.withResponse(201, {})
|
|
107
119
|
.build()
|
|
108
120
|
),
|
|
@@ -8,7 +8,7 @@ import mongoose from "mongoose";
|
|
|
8
8
|
|
|
9
9
|
import {AIRequest} from "../../models/aiRequest";
|
|
10
10
|
import {AIService} from "../../service/aiService";
|
|
11
|
-
import {authAsUser, ensureTestUsers,
|
|
11
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
12
12
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
13
13
|
import {registerObsExperiment} from "../local/models/obsExperiment";
|
|
14
14
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
@@ -61,7 +61,7 @@ describe("observability prompt routes", () => {
|
|
|
61
61
|
const model = createMockModel();
|
|
62
62
|
doGenerate = model.doGenerate;
|
|
63
63
|
aiService = new AIService({model: model as unknown as LanguageModel});
|
|
64
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
64
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
65
65
|
.register(
|
|
66
66
|
new ObservabilityApp({
|
|
67
67
|
aiService,
|
|
@@ -134,7 +134,7 @@ describe("observability prompt routes", () => {
|
|
|
134
134
|
const requestAiServiceFactory = mock(({apiKey}: {apiKey?: string}) => {
|
|
135
135
|
return apiKey === "request-key" ? aiService : undefined;
|
|
136
136
|
});
|
|
137
|
-
const requestServiceApp = new TerrenoApp({skipListen: true, userModel:
|
|
137
|
+
const requestServiceApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
138
138
|
.register(
|
|
139
139
|
new ObservabilityApp({
|
|
140
140
|
plugins: [createLocalObservabilityPlugin()],
|
|
@@ -4,7 +4,7 @@ import {assert} from "chai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
import {DateTime} from "luxon";
|
|
6
6
|
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {LocalEvaluatorStore} from "../local/evaluatorStore";
|
|
9
9
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
10
10
|
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
@@ -28,7 +28,7 @@ describe("observability review routes", () => {
|
|
|
28
28
|
await registerObsReviewItem().deleteMany({});
|
|
29
29
|
await registerObsEvaluator().deleteMany({});
|
|
30
30
|
await registerObsTrace().deleteMany({});
|
|
31
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
31
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
32
32
|
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
33
33
|
.build();
|
|
34
34
|
});
|
|
@@ -62,7 +62,8 @@ export const addObservabilityReviewRoutes = (
|
|
|
62
62
|
),
|
|
63
63
|
asyncHandler(async (req, res) => {
|
|
64
64
|
const status = req.query.status as ReviewStatus | undefined;
|
|
65
|
-
|
|
65
|
+
const batchId = typeof req.query.batchId === "string" ? req.query.batchId : undefined;
|
|
66
|
+
return res.json({...(await options.store.list(status, {batchId})), more: false});
|
|
66
67
|
})
|
|
67
68
|
);
|
|
68
69
|
|
|
@@ -108,6 +109,7 @@ export const addObservabilityReviewRoutes = (
|
|
|
108
109
|
id: req.params.id,
|
|
109
110
|
scores: body.scores ?? {},
|
|
110
111
|
sinks: getObservabilityApp()?.scoreSinks ?? [],
|
|
112
|
+
userId: req.user?._id ? String(req.user._id) : undefined,
|
|
111
113
|
});
|
|
112
114
|
} else if (body.action === "skip") {
|
|
113
115
|
await options.store.skip(req.params.id);
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import {TerrenoApp} from "@terreno/api";
|
|
3
|
+
import type express from "express";
|
|
4
|
+
import {DateTime} from "luxon";
|
|
5
|
+
|
|
6
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
7
|
+
import {LocalEvaluatorStore} from "../local/evaluatorStore";
|
|
8
|
+
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
|
+
import {registerObsDataset} from "../local/models/obsDataset";
|
|
10
|
+
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
11
|
+
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
12
|
+
import {registerObsJudgeRun} from "../local/models/obsJudgeRun";
|
|
13
|
+
import {registerObsReviewBatch} from "../local/models/obsReviewBatch";
|
|
14
|
+
import {registerObsReviewItem} from "../local/models/obsReviewItem";
|
|
15
|
+
import {registerObsTrace} from "../local/models/obsTrace";
|
|
16
|
+
import {LocalPromptStore} from "../local/promptStore";
|
|
17
|
+
import {LocalTraceStore} from "../local/traceStore";
|
|
18
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
19
|
+
import type {ObservabilityGenerateClient} from "../types";
|
|
20
|
+
|
|
21
|
+
const judgeClient: ObservabilityGenerateClient = {
|
|
22
|
+
generateJsonObject: async <OBJECT>() => ({confidence: 0.9, faithful: true}) as OBJECT,
|
|
23
|
+
generateText: async () => "",
|
|
24
|
+
modelId: "route-judge",
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const buildApp = (aiService?: ObservabilityGenerateClient): express.Application => {
|
|
28
|
+
return new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
29
|
+
.register(new ObservabilityApp({aiService, plugins: [createLocalObservabilityPlugin()]}))
|
|
30
|
+
.build();
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
describe("observability review batch routes", () => {
|
|
34
|
+
let questionSetId: string;
|
|
35
|
+
let judgeId: string;
|
|
36
|
+
|
|
37
|
+
beforeAll(async () => {
|
|
38
|
+
await ensureTestUsers();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
afterEach(() => {
|
|
42
|
+
resetObservabilityApp();
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
beforeEach(async () => {
|
|
46
|
+
await registerObsReviewItem().deleteMany({});
|
|
47
|
+
await registerObsReviewBatch().deleteMany({});
|
|
48
|
+
await registerObsDataset().deleteMany({});
|
|
49
|
+
await registerObsDatasetItem().deleteMany({});
|
|
50
|
+
await registerObsJudgeRun().deleteMany({});
|
|
51
|
+
await registerObsEvaluator().deleteMany({});
|
|
52
|
+
await registerObsTrace().deleteMany({});
|
|
53
|
+
// Register the models and default local stores before seeding fixtures.
|
|
54
|
+
buildApp();
|
|
55
|
+
resetObservabilityApp();
|
|
56
|
+
const promptStore = new LocalPromptStore();
|
|
57
|
+
if (!(await promptStore.getVersionByLabel("route-judge-prompt"))) {
|
|
58
|
+
await promptStore.create({
|
|
59
|
+
folder: "evaluators",
|
|
60
|
+
name: "route-judge-prompt",
|
|
61
|
+
outputSchema: {
|
|
62
|
+
properties: {confidence: {type: "number"}, faithful: {type: "boolean"}},
|
|
63
|
+
required: ["faithful"],
|
|
64
|
+
type: "object",
|
|
65
|
+
},
|
|
66
|
+
type: "chat",
|
|
67
|
+
});
|
|
68
|
+
await promptStore.moveLabel("route-judge-prompt", {label: "production", version: 1});
|
|
69
|
+
}
|
|
70
|
+
const evaluators = new LocalEvaluatorStore(promptStore);
|
|
71
|
+
questionSetId = (
|
|
72
|
+
await evaluators.create({
|
|
73
|
+
dimensions: [
|
|
74
|
+
{dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
|
|
75
|
+
],
|
|
76
|
+
name: "route-questions",
|
|
77
|
+
target: "full trace",
|
|
78
|
+
type: "human",
|
|
79
|
+
})
|
|
80
|
+
).id;
|
|
81
|
+
judgeId = (
|
|
82
|
+
await evaluators.create({
|
|
83
|
+
dimensions: [{dataType: "boolean", key: "faithful", required: true}],
|
|
84
|
+
judgePromptName: "route-judge-prompt",
|
|
85
|
+
name: "route-judge",
|
|
86
|
+
target: "full trace",
|
|
87
|
+
type: "llm-judge",
|
|
88
|
+
})
|
|
89
|
+
).id;
|
|
90
|
+
for (let index = 0; index < 3; index += 1) {
|
|
91
|
+
const startedAt = DateTime.utc().minus({hours: index}).toISO() ?? "";
|
|
92
|
+
await new LocalTraceStore().exportTrace({
|
|
93
|
+
id: `t${index}`,
|
|
94
|
+
input: `in ${index}`,
|
|
95
|
+
name: "summarize",
|
|
96
|
+
output: `out ${index}`,
|
|
97
|
+
prompts: [{name: "summarize", version: 1}],
|
|
98
|
+
sensitive: false,
|
|
99
|
+
spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: "ok"}],
|
|
100
|
+
startedAt,
|
|
101
|
+
status: "ok",
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("creates a batch from recent traces, answers it, and compares a judge", async () => {
|
|
107
|
+
const app = buildApp(judgeClient);
|
|
108
|
+
const agent = await authAsUser(app, "admin");
|
|
109
|
+
const created = await agent
|
|
110
|
+
.post("/ai/observability/review/batches")
|
|
111
|
+
.send({evaluatorId: questionSetId, limit: 2, promptName: "summarize"});
|
|
112
|
+
expect(created.status).toBe(201);
|
|
113
|
+
expect(created.body.data.counts.total).toBe(2);
|
|
114
|
+
const batchId = created.body.data.id as string;
|
|
115
|
+
|
|
116
|
+
const listed = await agent.get("/ai/observability/review/batches");
|
|
117
|
+
expect(listed.status).toBe(200);
|
|
118
|
+
expect(listed.body.data.map((row: {id: string}) => row.id)).toEqual([batchId]);
|
|
119
|
+
|
|
120
|
+
const pending = await agent.get(`/ai/observability/review?status=pending&batchId=${batchId}`);
|
|
121
|
+
expect(pending.body.data).toHaveLength(2);
|
|
122
|
+
for (const item of pending.body.data as Array<{id: string}>) {
|
|
123
|
+
const submitted = await agent
|
|
124
|
+
.post(`/ai/observability/review/${item.id}`)
|
|
125
|
+
.send({action: "submit", scores: {faithful: true}});
|
|
126
|
+
expect(submitted.status).toBe(200);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const judged = await agent
|
|
130
|
+
.post(`/ai/observability/review/batches/${batchId}/judgeRuns`)
|
|
131
|
+
.send({evaluatorId: judgeId});
|
|
132
|
+
expect(judged.status).toBe(201);
|
|
133
|
+
expect(judged.body.data.judges[0].agreement.rate).toBe(1);
|
|
134
|
+
expect(judged.body.data.judges[0].modelId).toBe("route-judge");
|
|
135
|
+
|
|
136
|
+
const detail = await agent.get(`/ai/observability/review/batches/${batchId}`);
|
|
137
|
+
expect(detail.status).toBe(200);
|
|
138
|
+
expect(detail.body.data.counts.done).toBe(2);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("requires an evaluator and an AI client for judge runs", async () => {
|
|
142
|
+
const app = buildApp();
|
|
143
|
+
const agent = await authAsUser(app, "admin");
|
|
144
|
+
const missing = await agent.post("/ai/observability/review/batches").send({});
|
|
145
|
+
expect(missing.status).toBe(400);
|
|
146
|
+
const created = await agent
|
|
147
|
+
.post("/ai/observability/review/batches")
|
|
148
|
+
.send({evaluatorId: questionSetId});
|
|
149
|
+
expect(created.status).toBe(201);
|
|
150
|
+
const judged = await agent
|
|
151
|
+
.post(`/ai/observability/review/batches/${created.body.data.id}/judgeRuns`)
|
|
152
|
+
.send({evaluatorId: judgeId});
|
|
153
|
+
expect(judged.status).toBe(503);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("builds a batch from a dataset's unanswered items and records answers on them", async () => {
|
|
157
|
+
const app = buildApp(judgeClient);
|
|
158
|
+
const agent = await authAsUser(app, "admin");
|
|
159
|
+
const dataset = await agent.post("/ai/observability/datasets").send({name: "route-set"});
|
|
160
|
+
const datasetId = dataset.body.data.id as string;
|
|
161
|
+
await agent
|
|
162
|
+
.post(`/ai/observability/datasets/${datasetId}/items`)
|
|
163
|
+
.send({expectedOutput: "summary", input: {text: "todo"}});
|
|
164
|
+
|
|
165
|
+
const mixed = await agent
|
|
166
|
+
.post("/ai/observability/review/batches")
|
|
167
|
+
.send({datasetId, evaluatorId: questionSetId, traceIds: []});
|
|
168
|
+
expect(mixed.status).toBe(400);
|
|
169
|
+
|
|
170
|
+
const created = await agent
|
|
171
|
+
.post("/ai/observability/review/batches")
|
|
172
|
+
.send({datasetId, evaluatorId: questionSetId});
|
|
173
|
+
expect(created.status).toBe(201);
|
|
174
|
+
expect(created.body.data.datasetId).toBe(datasetId);
|
|
175
|
+
const [item] = created.body.data.items as Array<{datasetItemId: string; reviewItemId: string}>;
|
|
176
|
+
const submitted = await agent
|
|
177
|
+
.post(`/ai/observability/review/${item?.reviewItemId}`)
|
|
178
|
+
.send({action: "submit", scores: {faithful: true}});
|
|
179
|
+
expect(submitted.status).toBe(200);
|
|
180
|
+
expect(submitted.body.data.datasetItemId).toBe(item?.datasetItemId);
|
|
181
|
+
|
|
182
|
+
const again = await agent
|
|
183
|
+
.post("/ai/observability/review/batches")
|
|
184
|
+
.send({datasetId, evaluatorId: questionSetId});
|
|
185
|
+
expect(again.status).toBe(400);
|
|
186
|
+
expect(again.body.title).toBe("Every item already has current answers");
|
|
187
|
+
|
|
188
|
+
const detail = await agent.get(`/ai/observability/datasets/${datasetId}`);
|
|
189
|
+
expect(detail.body.data.judgments).toEqual([
|
|
190
|
+
{
|
|
191
|
+
answered: 1,
|
|
192
|
+
current: 1,
|
|
193
|
+
evaluatorId: questionSetId,
|
|
194
|
+
evaluatorName: "route-questions",
|
|
195
|
+
stale: 0,
|
|
196
|
+
total: 1,
|
|
197
|
+
},
|
|
198
|
+
]);
|
|
199
|
+
const items = await agent.get(`/ai/observability/datasets/${datasetId}/items`);
|
|
200
|
+
expect(items.body.data[0].humanAnswers[questionSetId].scores).toEqual({faithful: true});
|
|
201
|
+
expect(items.body.data[0].humanAnswers[questionSetId].userId).toBeDefined();
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it("forbids non-admin users", async () => {
|
|
205
|
+
const agent = await authAsUser(buildApp(judgeClient), "notAdmin");
|
|
206
|
+
const listed = await agent.get("/ai/observability/review/batches");
|
|
207
|
+
expect(listed.status).toBe(403);
|
|
208
|
+
});
|
|
209
|
+
});
|