@terreno/ai 57.20.0 → 58.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +20 -18
- package/CHANGELOG.md +93 -0
- package/biome.jsonc +3 -2
- package/dist/aiApp.d.ts +5 -2
- package/dist/aiApp.js +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/migrations/gptHistoryStringIds.d.ts +41 -0
- package/dist/migrations/gptHistoryStringIds.js +94 -0
- package/dist/models/aiDataset.js +2 -2
- package/dist/models/gptHistory.js +11 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/routes/files.d.ts +7 -1
- package/dist/routes/files.js +62 -13
- package/dist/routes/gptActions.js +0 -7
- package/dist/routes/gptHistories.js +8 -2
- package/dist/service/aiDatasets.js +2 -1
- package/dist/service/chatTurn.js +4 -0
- package/dist/service/vertex.d.ts +5 -2
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +19 -16
- package/src/aiApp.test.ts +43 -18
- package/src/aiApp.ts +6 -3
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/index.ts +7 -1
- package/src/isolated/gptStream.isolated.ts +35 -30
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/migrations/gptHistoryStringIds.test.ts +231 -0
- package/src/migrations/gptHistoryStringIds.ts +150 -0
- package/src/models/aiDataset.test.ts +1 -1
- package/src/models/aiDataset.ts +2 -2
- package/src/models/gptHistory.test.ts +22 -11
- package/src/models/gptHistory.ts +19 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +126 -3
- package/src/routes/files.ts +90 -15
- package/src/routes/gpt.test.ts +23 -16
- package/src/routes/gptActions.test.ts +15 -15
- package/src/routes/gptActions.ts +1 -4
- package/src/routes/gptApprovals.test.ts +1 -1
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +2 -2
- package/src/routes/gptHistories.test.ts +17 -11
- package/src/routes/gptHistories.ts +59 -55
- package/src/routes/gptHistoriesSync.test.ts +141 -0
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +64 -47
- package/src/service/aiDatasets.ts +2 -1
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +4 -0
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/service/vertex.test.ts +1 -1
- package/src/service/vertex.ts +7 -4
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/src/types/index.ts +5 -2
- package/tsconfig.typecheck.json +18 -0
|
@@ -676,7 +676,7 @@ describe("JobsRunner", () => {
|
|
|
676
676
|
});
|
|
677
677
|
|
|
678
678
|
it("is not reachable from the root @terreno/ai or @terreno/ai/harness imports", () => {
|
|
679
|
-
const srcRoot = resolve(
|
|
679
|
+
const srcRoot = resolve(__dirname, "../..");
|
|
680
680
|
const seen = new Set<string>();
|
|
681
681
|
const importsJobs: string[] = [];
|
|
682
682
|
const visit = (file: string): void => {
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from "bun:test";
|
|
11
11
|
import {z} from "@terreno/api";
|
|
12
12
|
import type {LanguageModel} from "ai";
|
|
13
|
+
import type mongoose from "mongoose";
|
|
13
14
|
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
14
15
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
15
16
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
@@ -248,7 +249,7 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
248
249
|
beforeAll(async () => {
|
|
249
250
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
250
251
|
// cold first import can exceed it, so load it here first.
|
|
251
|
-
await import("../langfuseClient");
|
|
252
|
+
await import("../langfuseClient.js");
|
|
252
253
|
createLocalObservabilityPlugin();
|
|
253
254
|
SpanModel = registerObsSpan();
|
|
254
255
|
TraceModel = registerObsTrace();
|
|
@@ -653,7 +654,7 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
653
654
|
expect(await TaskModel.countDocuments({name: AGENT_TURN_TASK_NAME})).toBe(2);
|
|
654
655
|
// The first child is found again, not re-run, when the phase resumes for the second.
|
|
655
656
|
expect(calls).toHaveLength(2);
|
|
656
|
-
const firstMessages = async (conversationId:
|
|
657
|
+
const firstMessages = async (conversationId: mongoose.Types.ObjectId | string) =>
|
|
657
658
|
(await MessageModel.find({conversationId}).sort({seq: 1})).map(
|
|
658
659
|
({parts}) => (parts[0] as {text: string}).text
|
|
659
660
|
);
|
|
@@ -6,7 +6,7 @@ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin
|
|
|
6
6
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
7
7
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
8
8
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
9
|
-
import type {HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
9
|
+
import type {HarnessTaskDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
10
10
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
11
11
|
import {commitWaiting, type HarnessModels} from "./commit";
|
|
12
12
|
import {errorMessage} from "./errors";
|
|
@@ -16,6 +16,9 @@ import {registerHarnessInboxEvent} from "./models/harnessInboxEvent";
|
|
|
16
16
|
import {registerHarnessOwner} from "./models/harnessOwner";
|
|
17
17
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
18
18
|
|
|
19
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
20
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
21
|
+
|
|
19
22
|
const TaskModel = registerHarnessTask();
|
|
20
23
|
const OwnerModel = registerHarnessOwner();
|
|
21
24
|
const InboxModel = registerHarnessInboxEvent();
|
|
@@ -58,8 +61,7 @@ const findTask = (id: unknown): Promise<HarnessTaskDocument> => TaskModel.findEx
|
|
|
58
61
|
const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
|
|
59
62
|
(await findTask(id)).status === status;
|
|
60
63
|
|
|
61
|
-
const asAny = (definition: unknown):
|
|
62
|
-
definition as AnyHarnessTaskDefinition;
|
|
64
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
63
65
|
|
|
64
66
|
/** One phase that waits on `event` (with `timeout`) and completes with what it got. */
|
|
65
67
|
const waitsOnce = ({
|
|
@@ -72,7 +74,7 @@ const waitsOnce = ({
|
|
|
72
74
|
name: string;
|
|
73
75
|
runs: {count: number};
|
|
74
76
|
timeout?: {minutes: number};
|
|
75
|
-
}):
|
|
77
|
+
}): TestTaskDefinition =>
|
|
76
78
|
asAny(
|
|
77
79
|
defineTask<unknown, unknown, unknown>({
|
|
78
80
|
initial: () => ({phase: "await"}),
|
package/src/index.test.ts
CHANGED
|
@@ -51,7 +51,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
51
51
|
"shutdownLangfuseClient",
|
|
52
52
|
"shutdownTracing",
|
|
53
53
|
"verifyVertexModelsEnabled",
|
|
54
|
-
]
|
|
54
|
+
];
|
|
55
55
|
|
|
56
56
|
const expectedClasses = [
|
|
57
57
|
"AIRequest",
|
|
@@ -67,7 +67,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
67
67
|
"MemoryTraceSink",
|
|
68
68
|
"ObservabilityApp",
|
|
69
69
|
"Project",
|
|
70
|
-
]
|
|
70
|
+
];
|
|
71
71
|
|
|
72
72
|
const expectedConstants = [
|
|
73
73
|
"COMPACT_SURFACE_SYSTEM_PROMPT",
|
|
@@ -85,7 +85,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
85
85
|
"toggleChecklistHostAction",
|
|
86
86
|
"TRANSLATION_PROMPT",
|
|
87
87
|
"TemperaturePresets",
|
|
88
|
-
]
|
|
88
|
+
];
|
|
89
89
|
|
|
90
90
|
it.each(expectedFunctions)("exports %s as a function", (name) => {
|
|
91
91
|
expect(typeof exported[name]).toBe("function");
|
package/src/index.ts
CHANGED
|
@@ -36,6 +36,12 @@ export type {
|
|
|
36
36
|
TraceListItem,
|
|
37
37
|
} from "./langfuseTypes";
|
|
38
38
|
export {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
39
|
+
export {
|
|
40
|
+
GPT_HISTORY_STRING_IDS_MIGRATION,
|
|
41
|
+
type GptHistoryStringIdsResult,
|
|
42
|
+
gptHistoryStringIdsMigration,
|
|
43
|
+
migrateGptHistoryStringIds,
|
|
44
|
+
} from "./migrations/gptHistoryStringIds";
|
|
39
45
|
export {AIDataset} from "./models/aiDataset";
|
|
40
46
|
export {AIRequest} from "./models/aiRequest";
|
|
41
47
|
export {FileAttachment} from "./models/fileAttachment";
|
|
@@ -98,7 +104,7 @@ export {
|
|
|
98
104
|
validateObservabilityConfig,
|
|
99
105
|
} from "./observability/types";
|
|
100
106
|
export {addAiRequestsExplorerRoutes} from "./routes/aiRequestsExplorer";
|
|
101
|
-
export {addFileRoutes} from "./routes/files";
|
|
107
|
+
export {addFileRoutes, type FileStorageServiceSource} from "./routes/files";
|
|
102
108
|
export {addGptRoutes} from "./routes/gpt";
|
|
103
109
|
export {addGptHistoryRoutes} from "./routes/gptHistories";
|
|
104
110
|
export {addMcpRoutes} from "./routes/mcp";
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {GptHistory} from "../models/gptHistory";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../tests/helpers";
|
|
8
8
|
|
|
9
9
|
type StreamPart = {type: string; [key: string]: unknown};
|
|
10
10
|
|
|
@@ -13,32 +13,37 @@ type StreamPart = {type: string; [key: string]: unknown};
|
|
|
13
13
|
let streamParts: StreamPart[] = [];
|
|
14
14
|
let streamFiles: Array<{base64: string; mediaType: string}> = [];
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
16
|
+
let addGptRoutes: typeof import("../routes/gpt.js").addGptRoutes;
|
|
17
|
+
let AIService: typeof import("../service/aiService.js").AIService;
|
|
18
|
+
|
|
19
|
+
beforeAll(async () => {
|
|
20
|
+
// Mock langfuseVercelAi to avoid transitive langfuse SDK imports.
|
|
21
|
+
const realLangfuseVercelAi = await import("../langfuseVercelAi.js");
|
|
22
|
+
mock.module("../langfuseVercelAi", () => ({
|
|
23
|
+
...realLangfuseVercelAi,
|
|
24
|
+
createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
|
|
25
|
+
preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
// Mock the AI SDK so we control exactly which fullStream parts the GPT prompt
|
|
29
|
+
// handler iterates over. All other exports are preserved so AIService and the
|
|
30
|
+
// route helpers keep working.
|
|
31
|
+
const realAi = await import("ai");
|
|
32
|
+
mock.module("ai", () => ({
|
|
33
|
+
...realAi,
|
|
34
|
+
streamText: () => ({
|
|
35
|
+
files: Promise.resolve(streamFiles),
|
|
36
|
+
fullStream: (async function* fullStream(): AsyncGenerator<StreamPart> {
|
|
37
|
+
for (const part of streamParts) {
|
|
38
|
+
yield part;
|
|
39
|
+
}
|
|
40
|
+
})(),
|
|
41
|
+
}),
|
|
42
|
+
}));
|
|
43
|
+
|
|
44
|
+
({addGptRoutes} = await import("../routes/gpt.js"));
|
|
45
|
+
({AIService} = await import("../service/aiService.js"));
|
|
46
|
+
});
|
|
42
47
|
|
|
43
48
|
const createMockModel = () => ({
|
|
44
49
|
doGenerate: mock(async () => ({
|
|
@@ -78,7 +83,7 @@ describe("GPT prompt streaming edge cases", () => {
|
|
|
78
83
|
|
|
79
84
|
beforeAll(async () => {
|
|
80
85
|
await ensureTestUsers();
|
|
81
|
-
await GptHistory.deleteMany({});
|
|
86
|
+
await GptHistory.collection.deleteMany({});
|
|
82
87
|
const aiService = new AIService({
|
|
83
88
|
model: createMockModel() as unknown as LanguageModel,
|
|
84
89
|
});
|
|
@@ -87,12 +92,12 @@ describe("GPT prompt streaming edge cases", () => {
|
|
|
87
92
|
addGptRoutes(router, {aiService, openApiOptions: options});
|
|
88
93
|
},
|
|
89
94
|
skipListen: true,
|
|
90
|
-
userModel:
|
|
95
|
+
userModel: terrenoUserModel,
|
|
91
96
|
}).build();
|
|
92
97
|
});
|
|
93
98
|
|
|
94
99
|
afterAll(async () => {
|
|
95
|
-
await GptHistory.deleteMany({});
|
|
100
|
+
await GptHistory.collection.deleteMany({});
|
|
96
101
|
});
|
|
97
102
|
|
|
98
103
|
it("emits an inline image and flushes buffered text with no finish-step", async () => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
1
|
+
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
3
|
const shutdownMock = mock(async () => {});
|
|
4
4
|
class FakeLangfuseClient {
|
|
@@ -12,8 +12,14 @@ mock.module("@langfuse/client", () => ({
|
|
|
12
12
|
LangfuseClient: FakeLangfuseClient,
|
|
13
13
|
}));
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
let getLangfuseClient: typeof import("../langfuseClient.js").getLangfuseClient;
|
|
16
|
+
let initLangfuseClient: typeof import("../langfuseClient.js").initLangfuseClient;
|
|
17
|
+
let isLangfuseInitialized: typeof import("../langfuseClient.js").isLangfuseInitialized;
|
|
18
|
+
let shutdownLangfuseClient: typeof import("../langfuseClient.js").shutdownLangfuseClient;
|
|
19
|
+
beforeAll(async () => {
|
|
20
|
+
({getLangfuseClient, initLangfuseClient, isLangfuseInitialized, shutdownLangfuseClient} =
|
|
21
|
+
await import("../langfuseClient.js"));
|
|
22
|
+
});
|
|
17
23
|
|
|
18
24
|
describe("langfuseClient", () => {
|
|
19
25
|
beforeEach(async () => {
|
|
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
|
|
|
3
3
|
|
|
4
4
|
describe("@terreno/ai root entry", () => {
|
|
5
5
|
it("does not register the Project model until it is used", async () => {
|
|
6
|
-
const ai = await import("../index");
|
|
6
|
+
const ai = await import("../index.js");
|
|
7
7
|
expect(mongoose.models.Project).toBeUndefined();
|
|
8
8
|
|
|
9
9
|
expect(ai.Project.modelName).toBe("Project");
|
|
@@ -11,7 +11,7 @@ describe("@terreno/ai root entry", () => {
|
|
|
11
11
|
});
|
|
12
12
|
|
|
13
13
|
it("forwards statics, construction and instanceof through the deprecated Project export", async () => {
|
|
14
|
-
const {Project} = await import("../index");
|
|
14
|
+
const {Project} = await import("../index.js");
|
|
15
15
|
const userId = new mongoose.Types.ObjectId();
|
|
16
16
|
|
|
17
17
|
const project = new Project({name: "Roadmap", userId});
|
package/src/langfuseApp.test.ts
CHANGED
|
@@ -1,34 +1,18 @@
|
|
|
1
1
|
import {beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
-
import {
|
|
2
|
+
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
|
-
import mongoose from "mongoose";
|
|
5
|
-
import passportLocalMongoose from "passport-local-mongoose";
|
|
6
4
|
import supertest from "supertest";
|
|
7
5
|
|
|
8
6
|
import {LangfuseApp} from "./langfuseApp";
|
|
7
|
+
import {terrenoUserModel, UserModel} from "./tests/helpers";
|
|
9
8
|
|
|
10
9
|
type PasswordedUser = {setPassword: (password: string) => Promise<void>};
|
|
11
10
|
|
|
12
|
-
const userSchema = new mongoose.Schema({
|
|
13
|
-
admin: {default: false, type: Boolean},
|
|
14
|
-
email: {index: true, type: String},
|
|
15
|
-
name: String,
|
|
16
|
-
});
|
|
17
|
-
userSchema.plugin(
|
|
18
|
-
passportLocalMongoose as unknown as (
|
|
19
|
-
schema: mongoose.Schema,
|
|
20
|
-
options: {usernameField: string}
|
|
21
|
-
) => void,
|
|
22
|
-
{usernameField: "email"}
|
|
23
|
-
);
|
|
24
|
-
userSchema.plugin(createdUpdatedPlugin);
|
|
25
|
-
const UserModel = mongoose.models.User || mongoose.model("User", userSchema);
|
|
26
|
-
|
|
27
11
|
const buildApp = (plugin: LangfuseApp) =>
|
|
28
12
|
new TerrenoApp({
|
|
29
13
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
30
14
|
skipListen: true,
|
|
31
|
-
userModel:
|
|
15
|
+
userModel: terrenoUserModel,
|
|
32
16
|
}).build();
|
|
33
17
|
|
|
34
18
|
describe("LangfuseApp", () => {
|
|
@@ -40,7 +24,7 @@ describe("LangfuseApp", () => {
|
|
|
40
24
|
});
|
|
41
25
|
|
|
42
26
|
beforeEach(async () => {
|
|
43
|
-
const {shutdownLangfuseClient} = await import("./langfuseClient");
|
|
27
|
+
const {shutdownLangfuseClient} = await import("./langfuseClient.js");
|
|
44
28
|
await shutdownLangfuseClient();
|
|
45
29
|
});
|
|
46
30
|
|
|
@@ -7,11 +7,12 @@ import {addEvaluationRoutes} from "./langfuseRoutesEvaluations";
|
|
|
7
7
|
import {addPlaygroundRoutes} from "./langfuseRoutesPlayground";
|
|
8
8
|
import {addPromptRoutes} from "./langfuseRoutesPrompts";
|
|
9
9
|
import {addTraceRoutes} from "./langfuseRoutesTraces";
|
|
10
|
+
import type {FakeLangfusePrompt} from "./tests/fakeLangfusePrompt";
|
|
10
11
|
import {
|
|
11
12
|
authAsUserWithCredentials,
|
|
12
13
|
ensureTestUsers,
|
|
13
14
|
type StandardAiTestUserRole,
|
|
14
|
-
|
|
15
|
+
terrenoUserModel,
|
|
15
16
|
} from "./tests/helpers";
|
|
16
17
|
|
|
17
18
|
const LANGFUSE_TEST_USERS = [
|
|
@@ -26,15 +27,17 @@ const authAsUser = async (appInstance: express.Application, type: StandardAiTest
|
|
|
26
27
|
|
|
27
28
|
const scoreCreate = mock(() => {});
|
|
28
29
|
const promptCreate = mock(async (_params: Record<string, unknown>) => ({}));
|
|
29
|
-
const promptGet = mock(
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
30
|
+
const promptGet = mock(
|
|
31
|
+
async (name: string): Promise<FakeLangfusePrompt> => ({
|
|
32
|
+
config: {temperature: 0.5},
|
|
33
|
+
labels: ["production"],
|
|
34
|
+
name,
|
|
35
|
+
prompt: "Hello {{name}}, welcome to {{place}}",
|
|
36
|
+
tags: ["tag"],
|
|
37
|
+
type: "text",
|
|
38
|
+
version: 1,
|
|
39
|
+
})
|
|
40
|
+
);
|
|
38
41
|
const promptsList = mock(async () => ({
|
|
39
42
|
data: [{name: "p1", versions: [1]}],
|
|
40
43
|
meta: {limit: 20, page: 1, total: 1, totalPages: 1},
|
|
@@ -77,13 +80,15 @@ describe("Langfuse routes", () => {
|
|
|
77
80
|
|
|
78
81
|
app = new TerrenoApp({
|
|
79
82
|
configureApp: (router) => {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
// The route helpers only register handlers, which a Router supports as well.
|
|
84
|
+
const routes = router as unknown as express.Application;
|
|
85
|
+
addPromptRoutes(routes, "/admin/langfuse");
|
|
86
|
+
addTraceRoutes(routes, "/admin/langfuse");
|
|
87
|
+
addPlaygroundRoutes(routes, "/admin/langfuse");
|
|
88
|
+
addEvaluationRoutes(routes, "/admin/langfuse", []);
|
|
84
89
|
},
|
|
85
90
|
skipListen: true,
|
|
86
|
-
userModel:
|
|
91
|
+
userModel: terrenoUserModel as unknown as TerrenoAppOptions["userModel"],
|
|
87
92
|
}).build();
|
|
88
93
|
});
|
|
89
94
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
1
|
+
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
3
|
interface FakeSdkOptions {
|
|
4
4
|
serviceName?: string;
|
|
@@ -28,7 +28,11 @@ class FakeLangfuseSpanProcessor {
|
|
|
28
28
|
mock.module("@opentelemetry/sdk-node", () => ({NodeSDK: FakeNodeSDK}));
|
|
29
29
|
mock.module("@langfuse/otel", () => ({LangfuseSpanProcessor: FakeLangfuseSpanProcessor}));
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
let initTracing: typeof import("./langfuseTracing.js").initTracing;
|
|
32
|
+
let shutdownTracing: typeof import("./langfuseTracing.js").shutdownTracing;
|
|
33
|
+
beforeAll(async () => {
|
|
34
|
+
({initTracing, shutdownTracing} = await import("./langfuseTracing.js"));
|
|
35
|
+
});
|
|
32
36
|
|
|
33
37
|
describe("langfuseTracing", () => {
|
|
34
38
|
beforeEach(async () => {
|
|
@@ -3,16 +3,19 @@ import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
|
3
3
|
import {LangfuseCache} from "./langfuseCache";
|
|
4
4
|
import {getLangfuseClient, initLangfuseClient} from "./langfuseClient";
|
|
5
5
|
import {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
6
|
+
import type {FakeLangfusePrompt} from "./tests/fakeLangfusePrompt";
|
|
6
7
|
|
|
7
|
-
const mockPromptGet = mock(
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
8
|
+
const mockPromptGet = mock(
|
|
9
|
+
async (_name: string, _options?: Record<string, unknown>): Promise<FakeLangfusePrompt> => ({
|
|
10
|
+
config: {temperature: 0.5},
|
|
11
|
+
labels: [],
|
|
12
|
+
name: "welcome",
|
|
13
|
+
prompt: "Hi {{name}}",
|
|
14
|
+
tags: [],
|
|
15
|
+
type: "text" as const,
|
|
16
|
+
version: 3,
|
|
17
|
+
})
|
|
18
|
+
);
|
|
16
19
|
|
|
17
20
|
describe("langfuseVercelAi", () => {
|
|
18
21
|
beforeEach(async () => {
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import type {MigrationContext} from "@terreno/api";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import {AIDataset} from "../models/aiDataset";
|
|
6
|
+
import {GptHistory} from "../models/gptHistory";
|
|
7
|
+
import {gptHistoryStringIdsMigration, migrateGptHistoryStringIds} from "./gptHistoryStringIds";
|
|
8
|
+
|
|
9
|
+
const userId = new mongoose.Types.ObjectId();
|
|
10
|
+
|
|
11
|
+
const historiesCollection = (): mongoose.mongo.Collection =>
|
|
12
|
+
mongoose.connection.collection(GptHistory.collection.collectionName);
|
|
13
|
+
|
|
14
|
+
const datasetsCollection = (): mongoose.mongo.Collection =>
|
|
15
|
+
mongoose.connection.collection(AIDataset.collection.collectionName);
|
|
16
|
+
|
|
17
|
+
const legacyHistory = (title: string): mongoose.mongo.Document => ({
|
|
18
|
+
_id: new mongoose.Types.ObjectId(),
|
|
19
|
+
created: new Date("2026-01-05T09:00:00.000Z"),
|
|
20
|
+
deleted: false,
|
|
21
|
+
prompts: [
|
|
22
|
+
{text: `Question about ${title}`, type: "user"},
|
|
23
|
+
{model: "gemini", text: `Answer about ${title}`, type: "assistant"},
|
|
24
|
+
],
|
|
25
|
+
title,
|
|
26
|
+
updated: new Date("2026-01-05T09:01:00.000Z"),
|
|
27
|
+
userId,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const legacyDataset = (historyId: unknown): mongoose.mongo.Document => ({
|
|
31
|
+
_id: new mongoose.Types.ObjectId(),
|
|
32
|
+
columns: [{name: "count", type: "number"}],
|
|
33
|
+
created: new Date("2026-01-05T09:02:00.000Z"),
|
|
34
|
+
deleted: false,
|
|
35
|
+
historyId,
|
|
36
|
+
rowCount: 1,
|
|
37
|
+
rows: [[1]],
|
|
38
|
+
updated: new Date("2026-01-05T09:02:00.000Z"),
|
|
39
|
+
userId,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
interface Seeded {
|
|
43
|
+
histories: mongoose.mongo.Document[];
|
|
44
|
+
stringHistoryId: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Five ObjectId histories (two batches at batchSize 2, plus one), a String one, and refs. */
|
|
48
|
+
const seed = async (): Promise<Seeded> => {
|
|
49
|
+
const histories = ["a", "b", "c", "d", "e"].map(legacyHistory);
|
|
50
|
+
const stringHistoryId = "already-a-string-id";
|
|
51
|
+
await historiesCollection().insertMany([
|
|
52
|
+
...histories,
|
|
53
|
+
{...legacyHistory("string"), _id: stringHistoryId},
|
|
54
|
+
]);
|
|
55
|
+
await datasetsCollection().insertMany([
|
|
56
|
+
legacyDataset(histories[0]._id),
|
|
57
|
+
legacyDataset(histories[0]._id),
|
|
58
|
+
legacyDataset(histories[3]._id),
|
|
59
|
+
// A ref whose history was deleted long ago still becomes a String.
|
|
60
|
+
legacyDataset(new mongoose.Types.ObjectId()),
|
|
61
|
+
legacyDataset(stringHistoryId),
|
|
62
|
+
]);
|
|
63
|
+
return {histories, stringHistoryId};
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const idTypes = async (collection: mongoose.mongo.Collection, field: string): Promise<string[]> => {
|
|
67
|
+
const rows = await collection
|
|
68
|
+
.aggregate<{type: string}>([{$project: {_id: 0, type: {$type: `$${field}`}}}])
|
|
69
|
+
.toArray();
|
|
70
|
+
return [...new Set(rows.map((row) => row.type))].sort();
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const migrationContext = (dryRun: boolean, logs: string[] = []): MigrationContext => ({
|
|
74
|
+
addLog: async (_level, message): Promise<void> => {
|
|
75
|
+
logs.push(message);
|
|
76
|
+
},
|
|
77
|
+
dryRun,
|
|
78
|
+
logger: {debug: () => {}, error: () => {}, info: () => {}, warn: () => {}},
|
|
79
|
+
mongoose,
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe("gpt-history-string-ids migration", () => {
|
|
83
|
+
beforeEach(async () => {
|
|
84
|
+
await historiesCollection().deleteMany({});
|
|
85
|
+
await datasetsCollection().deleteMany({});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
afterEach(async () => {
|
|
89
|
+
await historiesCollection().deleteMany({});
|
|
90
|
+
await datasetsCollection().deleteMany({});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("moves ObjectId histories to their hex string ids, rewrites refs, and is a no-op the second time", async () => {
|
|
94
|
+
const {histories, stringHistoryId} = await seed();
|
|
95
|
+
const historyCount = await historiesCollection().countDocuments();
|
|
96
|
+
const datasetCount = await datasetsCollection().countDocuments();
|
|
97
|
+
|
|
98
|
+
const first = await migrateGptHistoryStringIds({
|
|
99
|
+
batchSize: 2,
|
|
100
|
+
connection: mongoose.connection,
|
|
101
|
+
dryRun: false,
|
|
102
|
+
});
|
|
103
|
+
const second = await migrateGptHistoryStringIds({
|
|
104
|
+
batchSize: 2,
|
|
105
|
+
connection: mongoose.connection,
|
|
106
|
+
dryRun: false,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(first).toEqual({datasetsUpdated: 4, historiesMigrated: 5});
|
|
110
|
+
expect(second).toEqual({datasetsUpdated: 0, historiesMigrated: 0});
|
|
111
|
+
expect(await historiesCollection().countDocuments()).toBe(historyCount);
|
|
112
|
+
expect(await datasetsCollection().countDocuments()).toBe(datasetCount);
|
|
113
|
+
expect(await idTypes(historiesCollection(), "_id")).toEqual(["string"]);
|
|
114
|
+
expect(await idTypes(datasetsCollection(), "historyId")).toEqual(["string"]);
|
|
115
|
+
|
|
116
|
+
const ids = (await historiesCollection().find({}).toArray()).map((doc) => doc._id).sort();
|
|
117
|
+
expect(ids).toEqual([...histories.map((doc) => doc._id.toHexString()), stringHistoryId].sort());
|
|
118
|
+
const firstHex = histories[0]._id.toHexString();
|
|
119
|
+
expect(await datasetsCollection().countDocuments({historyId: firstHex})).toBe(2);
|
|
120
|
+
expect(
|
|
121
|
+
await datasetsCollection().countDocuments({historyId: histories[3]._id.toHexString()})
|
|
122
|
+
).toBe(1);
|
|
123
|
+
expect(await datasetsCollection().countDocuments({historyId: stringHistoryId})).toBe(1);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("keeps every stored field, so the moved history loads through the model", async () => {
|
|
127
|
+
const {histories} = await seed();
|
|
128
|
+
|
|
129
|
+
await migrateGptHistoryStringIds({connection: mongoose.connection, dryRun: false});
|
|
130
|
+
|
|
131
|
+
const hexId = histories[1]._id.toHexString();
|
|
132
|
+
const history = await GptHistory.findById(hexId);
|
|
133
|
+
expect(history?._id).toBe(hexId);
|
|
134
|
+
expect(history?.title).toBe("b");
|
|
135
|
+
expect(history?.userId.toString()).toBe(userId.toString());
|
|
136
|
+
expect(history?.prompts.map((prompt) => prompt.text)).toEqual([
|
|
137
|
+
"Question about b",
|
|
138
|
+
"Answer about b",
|
|
139
|
+
]);
|
|
140
|
+
const datasets = await AIDataset.find({historyId: histories[0]._id.toHexString()});
|
|
141
|
+
expect(datasets).toHaveLength(2);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it("finishes a run that stopped after copying a history but before deleting the original", async () => {
|
|
145
|
+
const {histories} = await seed();
|
|
146
|
+
const interrupted = histories[2];
|
|
147
|
+
await historiesCollection().insertOne({...interrupted, _id: interrupted._id.toHexString()});
|
|
148
|
+
|
|
149
|
+
const result = await migrateGptHistoryStringIds({
|
|
150
|
+
connection: mongoose.connection,
|
|
151
|
+
dryRun: false,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
expect(result.historiesMigrated).toBe(5);
|
|
155
|
+
expect(await historiesCollection().countDocuments()).toBe(6);
|
|
156
|
+
expect(await idTypes(historiesCollection(), "_id")).toEqual(["string"]);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("rewrites refs left as ObjectIds when a run stopped after moving their history", async () => {
|
|
160
|
+
const orphanedHistory = new mongoose.Types.ObjectId();
|
|
161
|
+
const movedHistory: mongoose.mongo.Document = {
|
|
162
|
+
...legacyHistory("moved"),
|
|
163
|
+
_id: orphanedHistory.toHexString(),
|
|
164
|
+
};
|
|
165
|
+
await historiesCollection().insertOne(movedHistory);
|
|
166
|
+
await datasetsCollection().insertOne(legacyDataset(orphanedHistory));
|
|
167
|
+
|
|
168
|
+
const result = await migrateGptHistoryStringIds({
|
|
169
|
+
connection: mongoose.connection,
|
|
170
|
+
dryRun: false,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
expect(result).toEqual({datasetsUpdated: 1, historiesMigrated: 0});
|
|
174
|
+
expect(
|
|
175
|
+
await datasetsCollection().countDocuments({historyId: orphanedHistory.toHexString()})
|
|
176
|
+
).toBe(1);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it("only counts in a dry run", async () => {
|
|
180
|
+
await seed();
|
|
181
|
+
|
|
182
|
+
const result = await migrateGptHistoryStringIds({
|
|
183
|
+
connection: mongoose.connection,
|
|
184
|
+
dryRun: true,
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
expect(result).toEqual({datasetsUpdated: 4, historiesMigrated: 5});
|
|
188
|
+
expect(await idTypes(historiesCollection(), "_id")).toEqual(["objectId", "string"]);
|
|
189
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(5);
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
it("runs as a migration up through the migration context", async () => {
|
|
193
|
+
await seed();
|
|
194
|
+
|
|
195
|
+
const logs: string[] = [];
|
|
196
|
+
await gptHistoryStringIdsMigration.up(migrationContext(true, logs));
|
|
197
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(5);
|
|
198
|
+
|
|
199
|
+
await gptHistoryStringIdsMigration.up(migrationContext(false, logs));
|
|
200
|
+
await gptHistoryStringIdsMigration.up(migrationContext(false, logs));
|
|
201
|
+
expect(gptHistoryStringIdsMigration.slug).toBe("gpt-history-string-ids");
|
|
202
|
+
expect(logs).toEqual([
|
|
203
|
+
"[gpt-history-string-ids] would move 5 histories and 4 dataset refs",
|
|
204
|
+
"[gpt-history-string-ids] moved 5 histories and 4 dataset refs",
|
|
205
|
+
"[gpt-history-string-ids] moved 0 histories and 0 dataset refs",
|
|
206
|
+
]);
|
|
207
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(0);
|
|
208
|
+
expect(await datasetsCollection().countDocuments({historyId: {$type: "objectId"}})).toBe(0);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it("stops between batches when the run is cancelled", async () => {
|
|
212
|
+
await seed();
|
|
213
|
+
let checks = 0;
|
|
214
|
+
const checkCancellation = async (): Promise<void> => {
|
|
215
|
+
checks += 1;
|
|
216
|
+
if (checks > 1) {
|
|
217
|
+
throw new Error("cancelled");
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
await expect(
|
|
222
|
+
migrateGptHistoryStringIds({
|
|
223
|
+
batchSize: 2,
|
|
224
|
+
checkCancellation,
|
|
225
|
+
connection: mongoose.connection,
|
|
226
|
+
dryRun: false,
|
|
227
|
+
})
|
|
228
|
+
).rejects.toThrow("cancelled");
|
|
229
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(3);
|
|
230
|
+
});
|
|
231
|
+
});
|