@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -4,11 +4,14 @@ import {DateTime, Settings} from "luxon";
|
|
|
4
4
|
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
5
5
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
6
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
|
-
import type {HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
7
|
+
import type {HarnessTaskDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
8
8
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
9
9
|
import {type AnyHarnessTaskDefinition, defineTask, Harness, InProcessRunner} from "./harness";
|
|
10
10
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
11
11
|
|
|
12
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
13
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
14
|
+
|
|
12
15
|
const TaskModel = registerHarnessTask();
|
|
13
16
|
let SpanModel: ObsSpanModel;
|
|
14
17
|
let TraceModel: ObsTraceModel;
|
|
@@ -62,7 +65,7 @@ const flakyTask = ({
|
|
|
62
65
|
failures: number;
|
|
63
66
|
name: string;
|
|
64
67
|
retry: {backoffMs?: number; maxAttempts?: number; maxBackoffMs?: number};
|
|
65
|
-
}): {attemptsSeen: number[]; definition:
|
|
68
|
+
}): {attemptsSeen: number[]; definition: TestTaskDefinition} => {
|
|
66
69
|
const attemptsSeen: number[] = [];
|
|
67
70
|
const definition = defineTask<unknown, unknown, {ok: boolean}>({
|
|
68
71
|
initial: () => ({phase: "call"}),
|
|
@@ -81,7 +84,7 @@ const flakyTask = ({
|
|
|
81
84
|
retry,
|
|
82
85
|
version: 1,
|
|
83
86
|
});
|
|
84
|
-
return {attemptsSeen, definition: definition as unknown as
|
|
87
|
+
return {attemptsSeen, definition: definition as unknown as TestTaskDefinition};
|
|
85
88
|
};
|
|
86
89
|
|
|
87
90
|
describe("Harness phase retries", () => {
|
|
@@ -10,7 +10,7 @@ import {createLocalObservabilityPlugin} from "../../observability/local/localPlu
|
|
|
10
10
|
import {registerObsSpan} from "../../observability/local/models/obsSpan";
|
|
11
11
|
import {registerObsTrace} from "../../observability/local/models/obsTrace";
|
|
12
12
|
import {gatedStreamingModel} from "../../tests/harnessStreaming";
|
|
13
|
-
import {ensureTestUsers, UserModel} from "../../tests/helpers";
|
|
13
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
|
|
14
14
|
import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
|
|
15
15
|
import {defineAgent, defineTool, Harness, HarnessApp, InProcessRunner} from "../harness";
|
|
16
16
|
import {registerHarnessConversation} from "../models/harnessConversation";
|
|
@@ -88,7 +88,7 @@ const setup = async () => {
|
|
|
88
88
|
});
|
|
89
89
|
openHarnesses.push(harness);
|
|
90
90
|
await harness.start();
|
|
91
|
-
const app: express.Application = new TerrenoApp({skipListen: true, userModel:
|
|
91
|
+
const app: express.Application = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
92
92
|
.register(new HarnessApp({harness}))
|
|
93
93
|
.build();
|
|
94
94
|
const login = async ({email, password}: {email: string; password: string}) => {
|
|
@@ -8,7 +8,7 @@ import type TestAgent from "supertest/lib/agent";
|
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../../observability/local/localPlugin";
|
|
9
9
|
import {registerObsSpan} from "../../observability/local/models/obsSpan";
|
|
10
10
|
import {registerObsTrace} from "../../observability/local/models/obsTrace";
|
|
11
|
-
import {ensureTestUsers, UserModel} from "../../tests/helpers";
|
|
11
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
|
|
12
12
|
import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
|
|
13
13
|
import {defineTask, Harness, HarnessApp, InProcessRunner} from "../harness";
|
|
14
14
|
import {registerHarnessEvent, registerHarnessEventStream} from "../models/harnessEvent";
|
|
@@ -55,7 +55,7 @@ const setup = async ({start = false}: {start?: boolean} = {}) => {
|
|
|
55
55
|
if (start) {
|
|
56
56
|
await harness.start();
|
|
57
57
|
}
|
|
58
|
-
const app: express.Application = new TerrenoApp({skipListen: true, userModel:
|
|
58
|
+
const app: express.Application = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
59
59
|
.register(new HarnessApp({harness}))
|
|
60
60
|
.build();
|
|
61
61
|
const login = async ({email, password}: {email: string; password: string}) => {
|
|
@@ -676,7 +676,7 @@ describe("JobsRunner", () => {
|
|
|
676
676
|
});
|
|
677
677
|
|
|
678
678
|
it("is not reachable from the root @terreno/ai or @terreno/ai/harness imports", () => {
|
|
679
|
-
const srcRoot = resolve(
|
|
679
|
+
const srcRoot = resolve(__dirname, "../..");
|
|
680
680
|
const seen = new Set<string>();
|
|
681
681
|
const importsJobs: string[] = [];
|
|
682
682
|
const visit = (file: string): void => {
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from "bun:test";
|
|
11
11
|
import {z} from "@terreno/api";
|
|
12
12
|
import type {LanguageModel} from "ai";
|
|
13
|
+
import type mongoose from "mongoose";
|
|
13
14
|
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
14
15
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
15
16
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
@@ -246,9 +247,6 @@ const parentTask = ({
|
|
|
246
247
|
|
|
247
248
|
describe("rt.runAgent (subagents)", () => {
|
|
248
249
|
beforeAll(async () => {
|
|
249
|
-
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
250
|
-
// cold first import can exceed it, so load it here first.
|
|
251
|
-
await import("../langfuseClient");
|
|
252
250
|
createLocalObservabilityPlugin();
|
|
253
251
|
SpanModel = registerObsSpan();
|
|
254
252
|
TraceModel = registerObsTrace();
|
|
@@ -377,6 +375,16 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
377
375
|
).rejects.toThrow(
|
|
378
376
|
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" label must be a string')
|
|
379
377
|
);
|
|
378
|
+
await expect(
|
|
379
|
+
harness.createTask(parent, {}, {prompts: [{includes: {}, name: "plan", version: 1} as never]})
|
|
380
|
+
).rejects.toThrow(
|
|
381
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" includes must be an array')
|
|
382
|
+
);
|
|
383
|
+
await expect(
|
|
384
|
+
harness.createTask(parent, {}, {prompts: [{includes: {}, version: 1} as never]})
|
|
385
|
+
).rejects.toThrow(
|
|
386
|
+
harnessErrorMatching("invalidRequest", "Prompt ref includes must be an array")
|
|
387
|
+
);
|
|
380
388
|
await expect(harness.createTask(parent, {}, {trace: {tags: ["ok", " "]}})).rejects.toThrow(
|
|
381
389
|
harnessErrorMatching("invalidRequest", "trace.tags must be an array of non-empty strings")
|
|
382
390
|
);
|
|
@@ -396,6 +404,23 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
396
404
|
expect(plain(trace.prompts)).toEqual([plan]);
|
|
397
405
|
});
|
|
398
406
|
|
|
407
|
+
it("records a registry prompt's shared includes after it", async () => {
|
|
408
|
+
const agent = summarizerWith();
|
|
409
|
+
const parent = parentTask({agent, inputs: []});
|
|
410
|
+
const {model} = scriptedModel([]);
|
|
411
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
412
|
+
|
|
413
|
+
const include = {includedBy: "plan", label: "production", name: "base-rules", version: 4};
|
|
414
|
+
const created = await harness.createTask(
|
|
415
|
+
parent,
|
|
416
|
+
{},
|
|
417
|
+
{prompts: [{includes: [include], name: "plan", version: 2}]}
|
|
418
|
+
);
|
|
419
|
+
|
|
420
|
+
const trace = await TraceModel.findExactlyOne({_id: created.traceId});
|
|
421
|
+
expect(plain(trace.prompts)).toEqual([{name: "plan", version: 2}, include]);
|
|
422
|
+
});
|
|
423
|
+
|
|
399
424
|
it("fails the caller at once for an instructions prompt without a body or a bad prompt ref", async () => {
|
|
400
425
|
const agent = summarizerWith();
|
|
401
426
|
const noBody = parentTask({
|
|
@@ -653,7 +678,7 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
653
678
|
expect(await TaskModel.countDocuments({name: AGENT_TURN_TASK_NAME})).toBe(2);
|
|
654
679
|
// The first child is found again, not re-run, when the phase resumes for the second.
|
|
655
680
|
expect(calls).toHaveLength(2);
|
|
656
|
-
const firstMessages = async (conversationId:
|
|
681
|
+
const firstMessages = async (conversationId: mongoose.Types.ObjectId | string) =>
|
|
657
682
|
(await MessageModel.find({conversationId}).sort({seq: 1})).map(
|
|
658
683
|
({parts}) => (parts[0] as {text: string}).text
|
|
659
684
|
);
|
|
@@ -6,7 +6,7 @@ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin
|
|
|
6
6
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
7
7
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
8
8
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
9
|
-
import type {HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
9
|
+
import type {HarnessTaskDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
10
10
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
11
11
|
import {commitWaiting, type HarnessModels} from "./commit";
|
|
12
12
|
import {errorMessage} from "./errors";
|
|
@@ -16,6 +16,9 @@ import {registerHarnessInboxEvent} from "./models/harnessInboxEvent";
|
|
|
16
16
|
import {registerHarnessOwner} from "./models/harnessOwner";
|
|
17
17
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
18
18
|
|
|
19
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
20
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
21
|
+
|
|
19
22
|
const TaskModel = registerHarnessTask();
|
|
20
23
|
const OwnerModel = registerHarnessOwner();
|
|
21
24
|
const InboxModel = registerHarnessInboxEvent();
|
|
@@ -58,8 +61,7 @@ const findTask = (id: unknown): Promise<HarnessTaskDocument> => TaskModel.findEx
|
|
|
58
61
|
const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
|
|
59
62
|
(await findTask(id)).status === status;
|
|
60
63
|
|
|
61
|
-
const asAny = (definition: unknown):
|
|
62
|
-
definition as AnyHarnessTaskDefinition;
|
|
64
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
63
65
|
|
|
64
66
|
/** One phase that waits on `event` (with `timeout`) and completes with what it got. */
|
|
65
67
|
const waitsOnce = ({
|
|
@@ -72,7 +74,7 @@ const waitsOnce = ({
|
|
|
72
74
|
name: string;
|
|
73
75
|
runs: {count: number};
|
|
74
76
|
timeout?: {minutes: number};
|
|
75
|
-
}):
|
|
77
|
+
}): TestTaskDefinition =>
|
|
76
78
|
asAny(
|
|
77
79
|
defineTask<unknown, unknown, unknown>({
|
|
78
80
|
initial: () => ({phase: "await"}),
|
package/src/index.test.ts
CHANGED
|
@@ -18,21 +18,10 @@ describe("@terreno/ai public exports", () => {
|
|
|
18
18
|
"addMcpRoutes",
|
|
19
19
|
"addProjectRoutes",
|
|
20
20
|
"assertVertexModelsEnabled",
|
|
21
|
-
"compilePrompt",
|
|
22
21
|
"createAskTools",
|
|
23
|
-
"createPrompt",
|
|
24
|
-
"createTelemetryConfig",
|
|
25
22
|
"createVertexProvider",
|
|
26
23
|
"findAgentBlock",
|
|
27
|
-
"getCached",
|
|
28
24
|
"getMCPTools",
|
|
29
|
-
"getLangfuseClient",
|
|
30
|
-
"getPrompt",
|
|
31
|
-
"initLangfuseClient",
|
|
32
|
-
"initTracing",
|
|
33
|
-
"invalidateCache",
|
|
34
|
-
"invalidatePromptCache",
|
|
35
|
-
"isLangfuseInitialized",
|
|
36
25
|
"isVertexModelAllowed",
|
|
37
26
|
"jsonSchema",
|
|
38
27
|
"listEnabledVertexModels",
|
|
@@ -41,17 +30,13 @@ describe("@terreno/ai public exports", () => {
|
|
|
41
30
|
"normalizeLlmJsonTextForStructuredOutput",
|
|
42
31
|
"normalizeVertexModelId",
|
|
43
32
|
"parseAiJson",
|
|
44
|
-
"preparePromptForAI",
|
|
45
33
|
"getObservabilityApp",
|
|
46
34
|
"resetObservabilityApp",
|
|
47
35
|
"resolveObservabilityControl",
|
|
48
36
|
"runBufferedChatTurn",
|
|
49
37
|
"validateObservabilityConfig",
|
|
50
|
-
"setCached",
|
|
51
|
-
"shutdownLangfuseClient",
|
|
52
|
-
"shutdownTracing",
|
|
53
38
|
"verifyVertexModelsEnabled",
|
|
54
|
-
]
|
|
39
|
+
];
|
|
55
40
|
|
|
56
41
|
const expectedClasses = [
|
|
57
42
|
"AIRequest",
|
|
@@ -60,14 +45,12 @@ describe("@terreno/ai public exports", () => {
|
|
|
60
45
|
"FileAttachment",
|
|
61
46
|
"FileStorageService",
|
|
62
47
|
"GptHistory",
|
|
63
|
-
"LangfuseApp",
|
|
64
|
-
"LangfuseCache",
|
|
65
48
|
"MCPService",
|
|
66
49
|
"MemoryScoreSink",
|
|
67
50
|
"MemoryTraceSink",
|
|
68
51
|
"ObservabilityApp",
|
|
69
52
|
"Project",
|
|
70
|
-
]
|
|
53
|
+
];
|
|
71
54
|
|
|
72
55
|
const expectedConstants = [
|
|
73
56
|
"COMPACT_SURFACE_SYSTEM_PROMPT",
|
|
@@ -85,7 +68,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
85
68
|
"toggleChecklistHostAction",
|
|
86
69
|
"TRANSLATION_PROMPT",
|
|
87
70
|
"TemperaturePresets",
|
|
88
|
-
]
|
|
71
|
+
];
|
|
89
72
|
|
|
90
73
|
it.each(expectedFunctions)("exports %s as a function", (name) => {
|
|
91
74
|
expect(typeof exported[name]).toBe("function");
|
package/src/index.ts
CHANGED
|
@@ -12,44 +12,40 @@ export type {AIAdminAppOptions} from "./aiAdminApp";
|
|
|
12
12
|
export {AIAdminApp} from "./aiAdminApp";
|
|
13
13
|
export type {AiAppOptions} from "./aiApp";
|
|
14
14
|
export {AiApp} from "./aiApp";
|
|
15
|
-
export {LangfuseApp} from "./langfuseApp";
|
|
16
|
-
export {getCached, invalidateCache, LangfuseCache, setCached} from "./langfuseCache";
|
|
17
|
-
export {
|
|
18
|
-
getLangfuseClient,
|
|
19
|
-
initLangfuseClient,
|
|
20
|
-
isLangfuseInitialized,
|
|
21
|
-
shutdownLangfuseClient,
|
|
22
|
-
} from "./langfuseClient";
|
|
23
|
-
export {compilePrompt, createPrompt, getPrompt, invalidatePromptCache} from "./langfusePrompts";
|
|
24
|
-
export {initTracing, shutdownTracing} from "./langfuseTracing";
|
|
25
|
-
export type {
|
|
26
|
-
ChatMessage,
|
|
27
|
-
GetPromptOptions,
|
|
28
|
-
LangfuseAppOptions,
|
|
29
|
-
LangfuseCachedPrompt,
|
|
30
|
-
PaginatedResult,
|
|
31
|
-
PreparePromptResult,
|
|
32
|
-
PromptListItem,
|
|
33
|
-
ScoreSubmission,
|
|
34
|
-
ScoringFunction,
|
|
35
|
-
TelemetrySettings,
|
|
36
|
-
TraceListItem,
|
|
37
|
-
} from "./langfuseTypes";
|
|
38
|
-
export {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
39
15
|
export {AIDataset} from "./models/aiDataset";
|
|
40
16
|
export {AIRequest} from "./models/aiRequest";
|
|
41
17
|
export {FileAttachment} from "./models/fileAttachment";
|
|
42
18
|
export {GptHistory} from "./models/gptHistory";
|
|
43
19
|
export {getProjectModel, Project} from "./models/project";
|
|
44
20
|
export {AI_OBSERVABILITY_GROUP, observabilityAdminScreens} from "./observability/adminScreens";
|
|
21
|
+
export {
|
|
22
|
+
answersAgree,
|
|
23
|
+
computeJudgeAgreement,
|
|
24
|
+
type JudgeAgreement,
|
|
25
|
+
} from "./observability/judgeAgreement";
|
|
45
26
|
export {LocalDatasetStore} from "./observability/local/datasetStore";
|
|
46
|
-
export {LocalEvaluatorStore} from "./observability/local/evaluatorStore";
|
|
47
|
-
export {
|
|
27
|
+
export {type EvaluatorCalibration, LocalEvaluatorStore} from "./observability/local/evaluatorStore";
|
|
28
|
+
export {
|
|
29
|
+
DEFAULT_TARGET_AGREEMENT,
|
|
30
|
+
type JudgeExperimentCreateInput,
|
|
31
|
+
LocalExperimentRunner,
|
|
32
|
+
} from "./observability/local/experimentRunner";
|
|
33
|
+
export {
|
|
34
|
+
type DatasetJudgmentSummary,
|
|
35
|
+
loadCurrentHumanAnswers,
|
|
36
|
+
} from "./observability/local/humanAnswers";
|
|
48
37
|
export {
|
|
49
38
|
createLocalObservabilityBundle,
|
|
50
39
|
createLocalObservabilityPlugin,
|
|
51
40
|
} from "./observability/local/localPlugin";
|
|
52
41
|
export {LocalPromptStore} from "./observability/local/promptStore";
|
|
42
|
+
export {
|
|
43
|
+
DEFAULT_REVIEW_BATCH_SIZE,
|
|
44
|
+
LocalReviewBatchStore,
|
|
45
|
+
MAX_REVIEW_BATCH_SIZE,
|
|
46
|
+
type ReviewBatchDetail,
|
|
47
|
+
type ReviewBatchListItem,
|
|
48
|
+
} from "./observability/local/reviewBatchStore";
|
|
53
49
|
export {LocalReviewStore} from "./observability/local/reviewStore";
|
|
54
50
|
export {
|
|
55
51
|
LocalScoreSink,
|
|
@@ -63,6 +59,7 @@ export {
|
|
|
63
59
|
ObservabilityApp,
|
|
64
60
|
resetObservabilityApp,
|
|
65
61
|
} from "./observability/observabilityApp";
|
|
62
|
+
export {questionSetFingerprint} from "./observability/questionFingerprint";
|
|
66
63
|
export type {
|
|
67
64
|
ObservabilityPluginStatus,
|
|
68
65
|
ObservabilityStatus,
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {GptHistory} from "../models/gptHistory";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../tests/helpers";
|
|
8
8
|
|
|
9
9
|
type StreamPart = {type: string; [key: string]: unknown};
|
|
10
10
|
|
|
@@ -13,32 +13,29 @@ type StreamPart = {type: string; [key: string]: unknown};
|
|
|
13
13
|
let streamParts: StreamPart[] = [];
|
|
14
14
|
let streamFiles: Array<{base64: string; mediaType: string}> = [];
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
})
|
|
38
|
-
})
|
|
39
|
-
|
|
40
|
-
const {addGptRoutes} = await import("../routes/gpt");
|
|
41
|
-
const {AIService} = await import("../service/aiService");
|
|
16
|
+
let addGptRoutes: typeof import("../routes/gpt.js").addGptRoutes;
|
|
17
|
+
let AIService: typeof import("../service/aiService.js").AIService;
|
|
18
|
+
|
|
19
|
+
beforeAll(async () => {
|
|
20
|
+
// Mock the AI SDK so we control exactly which fullStream parts the GPT prompt
|
|
21
|
+
// handler iterates over. All other exports are preserved so AIService and the
|
|
22
|
+
// route helpers keep working.
|
|
23
|
+
const realAi = await import("ai");
|
|
24
|
+
mock.module("ai", () => ({
|
|
25
|
+
...realAi,
|
|
26
|
+
streamText: () => ({
|
|
27
|
+
files: Promise.resolve(streamFiles),
|
|
28
|
+
fullStream: (async function* fullStream(): AsyncGenerator<StreamPart> {
|
|
29
|
+
for (const part of streamParts) {
|
|
30
|
+
yield part;
|
|
31
|
+
}
|
|
32
|
+
})(),
|
|
33
|
+
}),
|
|
34
|
+
}));
|
|
35
|
+
|
|
36
|
+
({addGptRoutes} = await import("../routes/gpt.js"));
|
|
37
|
+
({AIService} = await import("../service/aiService.js"));
|
|
38
|
+
});
|
|
42
39
|
|
|
43
40
|
const createMockModel = () => ({
|
|
44
41
|
doGenerate: mock(async () => ({
|
|
@@ -87,7 +84,7 @@ describe("GPT prompt streaming edge cases", () => {
|
|
|
87
84
|
addGptRoutes(router, {aiService, openApiOptions: options});
|
|
88
85
|
},
|
|
89
86
|
skipListen: true,
|
|
90
|
-
userModel:
|
|
87
|
+
userModel: terrenoUserModel,
|
|
91
88
|
}).build();
|
|
92
89
|
});
|
|
93
90
|
|
|
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
|
|
|
3
3
|
|
|
4
4
|
describe("@terreno/ai root entry", () => {
|
|
5
5
|
it("does not register the Project model until it is used", async () => {
|
|
6
|
-
const ai = await import("../index");
|
|
6
|
+
const ai = await import("../index.js");
|
|
7
7
|
expect(mongoose.models.Project).toBeUndefined();
|
|
8
8
|
|
|
9
9
|
expect(ai.Project.modelName).toBe("Project");
|
|
@@ -11,7 +11,7 @@ describe("@terreno/ai root entry", () => {
|
|
|
11
11
|
});
|
|
12
12
|
|
|
13
13
|
it("forwards statics, construction and instanceof through the deprecated Project export", async () => {
|
|
14
|
-
const {Project} = await import("../index");
|
|
14
|
+
const {Project} = await import("../index.js");
|
|
15
15
|
const userId = new mongoose.Types.ObjectId();
|
|
16
16
|
|
|
17
17
|
const project = new Project({name: "Roadmap", userId});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
2
|
import mongoose from "mongoose";
|
|
3
3
|
|
|
4
|
+
import type {GptHistoryDocument, GptHistoryPendingAsk, GptHistoryPrompt} from "../types/index";
|
|
4
5
|
import {GptHistory} from "./gptHistory";
|
|
5
6
|
|
|
6
7
|
describe("GptHistory Model", () => {
|
|
@@ -217,9 +218,9 @@ describe("GptHistory Model", () => {
|
|
|
217
218
|
{id: "team", label: "Team"},
|
|
218
219
|
],
|
|
219
220
|
prompt: "Which plan should I set up?",
|
|
220
|
-
select: "one",
|
|
221
|
+
select: "one" as const,
|
|
221
222
|
};
|
|
222
|
-
const pendingAsk = {
|
|
223
|
+
const pendingAsk: GptHistoryPendingAsk = {
|
|
223
224
|
created: new Date("2026-09-27T12:00:00.000Z"),
|
|
224
225
|
input: planAsk,
|
|
225
226
|
kind: "choice",
|
|
@@ -256,7 +257,10 @@ describe("GptHistory Model", () => {
|
|
|
256
257
|
},
|
|
257
258
|
toolCallId: "call_plan",
|
|
258
259
|
};
|
|
259
|
-
|
|
260
|
+
/** Create from deliberately invalid input, so the schema rather than the compiler rejects it. */
|
|
261
|
+
const createUnchecked = (doc: Record<string, unknown>) =>
|
|
262
|
+
GptHistory.create(doc as Partial<GptHistoryDocument>);
|
|
263
|
+
const askRow: GptHistoryPrompt = {
|
|
260
264
|
args: planAsk,
|
|
261
265
|
ask: {kind: "choice", status: "pending"},
|
|
262
266
|
text: "Tool call: ask_choice",
|
|
@@ -309,7 +313,7 @@ describe("GptHistory Model", () => {
|
|
|
309
313
|
const userId = new mongoose.Types.ObjectId();
|
|
310
314
|
const history = await GptHistory.create({prompts: [], userId});
|
|
311
315
|
|
|
312
|
-
history.pendingAsk = pendingAsk
|
|
316
|
+
history.pendingAsk = pendingAsk;
|
|
313
317
|
await history.save();
|
|
314
318
|
|
|
315
319
|
const stored = await GptHistory.collection.findOne({_id: history._id});
|
|
@@ -334,7 +338,7 @@ describe("GptHistory Model", () => {
|
|
|
334
338
|
},
|
|
335
339
|
])("rejects an unknown key in $field", async ({expected, history}) => {
|
|
336
340
|
await expect(
|
|
337
|
-
|
|
341
|
+
createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
|
|
338
342
|
).rejects.toThrow(expected);
|
|
339
343
|
});
|
|
340
344
|
|
|
@@ -361,7 +365,7 @@ describe("GptHistory Model", () => {
|
|
|
361
365
|
},
|
|
362
366
|
])("rejects $label", async ({expected, history}) => {
|
|
363
367
|
await expect(
|
|
364
|
-
|
|
368
|
+
createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
|
|
365
369
|
).rejects.toThrow(expected);
|
|
366
370
|
});
|
|
367
371
|
});
|
|
@@ -24,6 +24,7 @@ describe("observabilityAdminScreens", () => {
|
|
|
24
24
|
"ai-datasets",
|
|
25
25
|
"ai-experiments",
|
|
26
26
|
"ai-review",
|
|
27
|
+
"ai-review-batches",
|
|
27
28
|
]);
|
|
28
29
|
});
|
|
29
30
|
|
|
@@ -66,6 +67,12 @@ describe("observabilityAdminScreens", () => {
|
|
|
66
67
|
group: AI_OBSERVABILITY_GROUP,
|
|
67
68
|
name: "ai-review",
|
|
68
69
|
},
|
|
70
|
+
{
|
|
71
|
+
adminAccess: {action: "list", resource: "aiReview"},
|
|
72
|
+
displayName: "Review batches",
|
|
73
|
+
group: AI_OBSERVABILITY_GROUP,
|
|
74
|
+
name: "ai-review-batches",
|
|
75
|
+
},
|
|
69
76
|
]);
|
|
70
77
|
});
|
|
71
78
|
});
|
|
@@ -35,7 +35,12 @@ export const observabilityAdminScreens = ({localOn}: {localOn: boolean}): AdminC
|
|
|
35
35
|
];
|
|
36
36
|
if (localOn) {
|
|
37
37
|
screens.push(
|
|
38
|
-
observabilityScreen({displayName: "Review queue", name: "ai-review", resource: "aiReview"})
|
|
38
|
+
observabilityScreen({displayName: "Review queue", name: "ai-review", resource: "aiReview"}),
|
|
39
|
+
observabilityScreen({
|
|
40
|
+
displayName: "Review batches",
|
|
41
|
+
name: "ai-review-batches",
|
|
42
|
+
resource: "aiReview",
|
|
43
|
+
})
|
|
39
44
|
);
|
|
40
45
|
}
|
|
41
46
|
return screens;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
2
2
|
import {assert} from "chai";
|
|
3
|
+
import {judgeAnswering} from "../tests/fakeJudge";
|
|
3
4
|
import {
|
|
4
5
|
judgeSchemaMissingDimensions,
|
|
5
6
|
runEvaluator,
|
|
@@ -95,9 +96,7 @@ describe("evaluate", () => {
|
|
|
95
96
|
output: "4",
|
|
96
97
|
},
|
|
97
98
|
{
|
|
98
|
-
generateJsonObject:
|
|
99
|
-
return {confidence: 0.95, correct: true};
|
|
100
|
-
},
|
|
99
|
+
generateJsonObject: judgeAnswering(() => ({confidence: 0.95, correct: true})),
|
|
101
100
|
},
|
|
102
101
|
{
|
|
103
102
|
properties: {
|
|
@@ -152,7 +151,7 @@ describe("evaluate", () => {
|
|
|
152
151
|
const base = {
|
|
153
152
|
evaluator: {
|
|
154
153
|
assertion: {constraint: "exists", path: "answer"},
|
|
155
|
-
dimensions: [{dataType: "boolean", key: "matches", required: true}],
|
|
154
|
+
dimensions: [{dataType: "boolean" as const, key: "matches", required: true}],
|
|
156
155
|
name: "path-assert",
|
|
157
156
|
type: "json-assert" as const,
|
|
158
157
|
},
|
|
@@ -389,7 +388,7 @@ describe("evaluate", () => {
|
|
|
389
388
|
input: {},
|
|
390
389
|
output: "4",
|
|
391
390
|
},
|
|
392
|
-
{generateJsonObject:
|
|
391
|
+
{generateJsonObject: judgeAnswering(() => ({correct: true}))},
|
|
393
392
|
{properties: {correct: {type: "boolean"}}, type: "object"}
|
|
394
393
|
);
|
|
395
394
|
assert.equal(missingJudgePrompt.error, "llm-judge evaluator is missing judgePromptName");
|
|
@@ -408,7 +407,7 @@ describe("evaluate", () => {
|
|
|
408
407
|
},
|
|
409
408
|
{
|
|
410
409
|
ai: {
|
|
411
|
-
generateJsonObject:
|
|
410
|
+
generateJsonObject: judgeAnswering(() => ({confidence: "not-a-number", correct: 1})),
|
|
412
411
|
},
|
|
413
412
|
judgeOutputSchema: {
|
|
414
413
|
properties: {
|
|
@@ -461,9 +460,12 @@ describe("evaluate", () => {
|
|
|
461
460
|
output: "4",
|
|
462
461
|
},
|
|
463
462
|
{
|
|
464
|
-
generateJsonObject:
|
|
465
|
-
|
|
466
|
-
|
|
463
|
+
generateJsonObject: judgeAnswering(() => ({
|
|
464
|
+
confidence: "0.8",
|
|
465
|
+
correct: "true",
|
|
466
|
+
helpfulness: "not-a-number",
|
|
467
|
+
tone: 1,
|
|
468
|
+
})),
|
|
467
469
|
},
|
|
468
470
|
{
|
|
469
471
|
properties: {
|
|
@@ -489,7 +491,7 @@ describe("LocalEvaluatorStore phase 2", () => {
|
|
|
489
491
|
beforeEach(async () => {
|
|
490
492
|
new ObservabilityApp({plugins: [createLocalObservabilityBundle().plugin]});
|
|
491
493
|
promptStore = new LocalPromptStore();
|
|
492
|
-
store = new (await import("./local/evaluatorStore")).LocalEvaluatorStore(promptStore);
|
|
494
|
+
store = new (await import("./local/evaluatorStore.js")).LocalEvaluatorStore(promptStore);
|
|
493
495
|
await registerObsEvaluator().deleteMany({});
|
|
494
496
|
await clearObsPrompts();
|
|
495
497
|
await promptStore.create({
|
|
@@ -170,10 +170,16 @@ export const runJsonAssertEvaluator = (
|
|
|
170
170
|
};
|
|
171
171
|
};
|
|
172
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Run an llm-judge evaluator. By default the AI client resolves the judge prompt's production
|
|
175
|
+
* version by name. Pass `judgeSystemPrompt` (a specific version's resolved body) to run that
|
|
176
|
+
* version instead, as judge calibration experiments do.
|
|
177
|
+
*/
|
|
173
178
|
export const runLlmJudgeEvaluator = async (
|
|
174
179
|
context: EvaluatorExecutionContext,
|
|
175
180
|
ai: EvaluatorAiClient,
|
|
176
|
-
judgeOutputSchema: Record<string, unknown
|
|
181
|
+
judgeOutputSchema: Record<string, unknown>,
|
|
182
|
+
options: {judgeSystemPrompt?: string} = {}
|
|
177
183
|
): Promise<EvaluatorExecutionResult> => {
|
|
178
184
|
const judgePromptName = context.evaluator.judgePromptName;
|
|
179
185
|
if (!judgePromptName) {
|
|
@@ -181,9 +187,13 @@ export const runLlmJudgeEvaluator = async (
|
|
|
181
187
|
}
|
|
182
188
|
|
|
183
189
|
try {
|
|
190
|
+
const promptSource =
|
|
191
|
+
options.judgeSystemPrompt === undefined
|
|
192
|
+
? {promptName: judgePromptName}
|
|
193
|
+
: {systemPrompt: options.judgeSystemPrompt};
|
|
184
194
|
const payload = (await ai.generateJsonObject({
|
|
185
195
|
prompt: buildJudgePrompt(context),
|
|
186
|
-
|
|
196
|
+
...promptSource,
|
|
187
197
|
schema: jsonSchema(judgeOutputSchema) as FlexibleSchema<Record<string, unknown>>,
|
|
188
198
|
schemaName: `${context.evaluator.name}-judge`,
|
|
189
199
|
skipTrace: true,
|