@terreno/ai 57.20.0 → 57.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/biome.jsonc +1 -0
- package/dist/harness/agentLoop.js +1 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/types/harness.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -12
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/isolated/gptStream.isolated.ts +33 -28
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +58 -42
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/tsconfig.typecheck.json +18 -0
|
@@ -21,10 +21,11 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
|
21
21
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
22
22
|
import {withGenerateStreaming} from "../tests/generateStream";
|
|
23
23
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
24
|
-
import {authAsUser, ensureTestUsers, UserModel} from "../tests/helpers";
|
|
24
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
25
25
|
import type {
|
|
26
26
|
HarnessApprovalDocument,
|
|
27
27
|
HarnessApprovalResult,
|
|
28
|
+
HarnessTaskDefinition,
|
|
28
29
|
HarnessTaskDocument,
|
|
29
30
|
HarnessTestHooks,
|
|
30
31
|
} from "../types/harness";
|
|
@@ -32,7 +33,6 @@ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
|
32
33
|
import type {HarnessModels} from "./commit";
|
|
33
34
|
import {
|
|
34
35
|
AGENT_TOOL_TASK_NAME,
|
|
35
|
-
type AnyHarnessTaskDefinition,
|
|
36
36
|
approvalGate,
|
|
37
37
|
approvalTaskInput,
|
|
38
38
|
defineAgent,
|
|
@@ -55,6 +55,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
|
|
|
55
55
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
56
56
|
import {HarnessWaitRaceError, sendEventRecords} from "./waits";
|
|
57
57
|
|
|
58
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
59
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
60
|
+
|
|
58
61
|
// Whole tasks and turns on a real replica set; the first test also pays for index creation.
|
|
59
62
|
setDefaultTimeout(30_000);
|
|
60
63
|
|
|
@@ -107,8 +110,7 @@ const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
|
|
|
107
110
|
const approvalOf = (taskId: unknown): Promise<HarnessApprovalDocument> =>
|
|
108
111
|
ApprovalModel.findExactlyOne({taskId});
|
|
109
112
|
|
|
110
|
-
const asAny = (definition: unknown):
|
|
111
|
-
definition as AnyHarnessTaskDefinition;
|
|
113
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
112
114
|
|
|
113
115
|
const openHarness = async (
|
|
114
116
|
registry: ReadonlyArray<HarnessRegistryEntry>,
|
|
@@ -138,7 +140,7 @@ const openHarness = async (
|
|
|
138
140
|
};
|
|
139
141
|
|
|
140
142
|
const buildApp = (harness: Harness): express.Application =>
|
|
141
|
-
new TerrenoApp({skipListen: true, userModel:
|
|
143
|
+
new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
142
144
|
.register(new HarnessApp({harness}))
|
|
143
145
|
.build();
|
|
144
146
|
|
|
@@ -176,7 +178,7 @@ const asksApproval = ({
|
|
|
176
178
|
retry?: {maxAttempts: number};
|
|
177
179
|
runs?: {count: number};
|
|
178
180
|
timeout?: {hours: number};
|
|
179
|
-
}):
|
|
181
|
+
}): TestTaskDefinition =>
|
|
180
182
|
asAny(
|
|
181
183
|
defineTask<{patientId: string}, unknown, HarnessApprovalResult>({
|
|
182
184
|
approvals,
|
|
@@ -563,7 +565,7 @@ describe("Harness approvals", () => {
|
|
|
563
565
|
|
|
564
566
|
describe("races and forged events", () => {
|
|
565
567
|
/** Past the approval's timeout with the runner stopped, so the next owner resolves it. */
|
|
566
|
-
const parkThenPassTimeout = async (definition:
|
|
568
|
+
const parkThenPassTimeout = async (definition: TestTaskDefinition) => {
|
|
567
569
|
const start = DateTime.now();
|
|
568
570
|
Settings.now = () => start.toMillis();
|
|
569
571
|
const first = await openHarness([definition]);
|
|
@@ -762,7 +764,7 @@ describe("Harness approvals", () => {
|
|
|
762
764
|
});
|
|
763
765
|
|
|
764
766
|
it("fails the task on rt.approval misuse and rejects invalid approval policies", async () => {
|
|
765
|
-
const misuse = (options: unknown, key = "k"):
|
|
767
|
+
const misuse = (options: unknown, key = "k"): TestTaskDefinition =>
|
|
766
768
|
asAny(
|
|
767
769
|
defineTask<unknown, unknown, unknown>({
|
|
768
770
|
initial: () => ({phase: "ask"}),
|
|
@@ -778,7 +780,7 @@ describe("Harness approvals", () => {
|
|
|
778
780
|
version: 1,
|
|
779
781
|
})
|
|
780
782
|
);
|
|
781
|
-
const cases: Array<[
|
|
783
|
+
const cases: Array<[TestTaskDefinition, string]> = [
|
|
782
784
|
[misuse({title: "t"}, " "), "rt.approval requires a key"],
|
|
783
785
|
[misuse({}), 'rt.approval("k") requires a title'],
|
|
784
786
|
[misuse({summary: 3, title: "t"}), 'rt.approval("k") summary must be a string'],
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
type SseClient,
|
|
19
19
|
type SseFrame,
|
|
20
20
|
} from "../tests/harnessStreaming";
|
|
21
|
-
import {ensureTestUsers, UserModel} from "../tests/helpers";
|
|
21
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
22
22
|
import type {HarnessStreamingOptions} from "../types/harness";
|
|
23
23
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
24
24
|
import type {HarnessModels} from "./commit";
|
|
@@ -102,7 +102,9 @@ const serve = async (
|
|
|
102
102
|
harness: Harness
|
|
103
103
|
): Promise<{app: express.Application; plugin: HarnessApp; url: string}> => {
|
|
104
104
|
const plugin = new HarnessApp({harness, heartbeatInterval: {milliseconds: 50}});
|
|
105
|
-
const app = new TerrenoApp({skipListen: true, userModel:
|
|
105
|
+
const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
106
|
+
.register(plugin)
|
|
107
|
+
.build();
|
|
106
108
|
const {stop, url} = await listen(app);
|
|
107
109
|
cleanups.push(stop);
|
|
108
110
|
return {app, plugin, url};
|
|
@@ -111,7 +113,8 @@ const serve = async (
|
|
|
111
113
|
/** Open change-stream cursors on the server, idle ones included. */
|
|
112
114
|
const openChangeStreamOps = async (): Promise<Array<Record<string, unknown>>> => {
|
|
113
115
|
const dbName = mongoose.connection.db?.databaseName;
|
|
114
|
-
return mongoose.connection
|
|
116
|
+
return mongoose.connection
|
|
117
|
+
.getClient()
|
|
115
118
|
.db("admin")
|
|
116
119
|
.aggregate([
|
|
117
120
|
{$currentOp: {allUsers: true, idleCursors: true}},
|
|
@@ -168,10 +171,11 @@ const sse = async (
|
|
|
168
171
|
/** Status and body of a request that must fail before streaming starts. */
|
|
169
172
|
const refused = async (url: string, headers: Record<string, string> = {}) => {
|
|
170
173
|
const response = await fetch(url, {headers});
|
|
171
|
-
return {body: await response.json(), status: response.status};
|
|
174
|
+
return {body: (await response.json()) as {title?: string}, status: response.status};
|
|
172
175
|
};
|
|
173
176
|
|
|
174
|
-
const storedEvents = (streamId:
|
|
177
|
+
const storedEvents = (streamId: mongoose.Types.ObjectId | string) =>
|
|
178
|
+
EventModel.find({streamId}).sort({seq: 1}).lean();
|
|
175
179
|
|
|
176
180
|
const deltaText = (frames: ReadonlyArray<SseFrame>): string =>
|
|
177
181
|
frames
|
|
@@ -372,7 +376,7 @@ describe("Harness event stream", () => {
|
|
|
372
376
|
});
|
|
373
377
|
|
|
374
378
|
it("rejects a heartbeat interval that is not positive", () => {
|
|
375
|
-
const app = new TerrenoApp({skipListen: true, userModel:
|
|
379
|
+
const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel});
|
|
376
380
|
expect(() =>
|
|
377
381
|
app
|
|
378
382
|
.register(new HarnessApp({harness: {} as Harness, heartbeatInterval: {seconds: 0}}))
|
|
@@ -200,7 +200,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
200
200
|
beforeAll(async () => {
|
|
201
201
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
202
202
|
// cold first import can exceed it, so load it here first.
|
|
203
|
-
await import("../langfuseClient");
|
|
203
|
+
await import("../langfuseClient.js");
|
|
204
204
|
createLocalObservabilityPlugin();
|
|
205
205
|
SpanModel = registerObsSpan();
|
|
206
206
|
TraceModel = registerObsTrace();
|
|
@@ -233,7 +233,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
233
233
|
expect(() => section("s", "x" as never)).toThrow(
|
|
234
234
|
harnessErrorMatching("definitionInvalid", "section(s): build must be a function")
|
|
235
235
|
);
|
|
236
|
-
expect(() => hook("onEverything" as never, () => undefined)).toThrow(
|
|
236
|
+
expect(() => hook("onEverything" as never, (() => undefined) as never)).toThrow(
|
|
237
237
|
harnessErrorMatching(
|
|
238
238
|
"definitionInvalid",
|
|
239
239
|
'hook: kind must be one of afterTool, beforeModelRequest, beforeTool, not "onEverything"'
|
|
@@ -774,7 +774,8 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
774
774
|
...tool,
|
|
775
775
|
execute: async (args, api) => {
|
|
776
776
|
ran.push(`${tag}:before`);
|
|
777
|
-
|
|
777
|
+
// `args` was parsed by the wrapped tool's own `parameters`.
|
|
778
|
+
return {[tag]: await tool.execute(args as never, api)};
|
|
778
779
|
},
|
|
779
780
|
});
|
|
780
781
|
// The wrapping extension comes before the overriding one, yet wraps its tool.
|
|
@@ -1053,7 +1054,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1053
1054
|
version: 1,
|
|
1054
1055
|
});
|
|
1055
1056
|
const harness = await openHarness({registry: [task]});
|
|
1056
|
-
const ids = [];
|
|
1057
|
+
const ids: mongoose.Types.ObjectId[] = [];
|
|
1057
1058
|
for (let n = 0; n < count; n++) {
|
|
1058
1059
|
ids.push((await harness.createTask(task, {n}))._id);
|
|
1059
1060
|
}
|
|
@@ -1106,7 +1107,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1106
1107
|
it("lets exactly one of two independent runs win a key (first write wins)", async () => {
|
|
1107
1108
|
const {release, rows} = await hangTasks(2);
|
|
1108
1109
|
try {
|
|
1109
|
-
const [first, second] = rows
|
|
1110
|
+
const [first, second] = rows;
|
|
1110
1111
|
// Two runs fencing different task rows race on one shared key, so neither queues
|
|
1111
1112
|
// behind the other's lease write and the unique index decides.
|
|
1112
1113
|
const scope = first._id;
|
|
@@ -1128,7 +1129,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1128
1129
|
it("refuses a write from a run whose lease token is stale", async () => {
|
|
1129
1130
|
const {release, rows} = await hangTasks(1);
|
|
1130
1131
|
try {
|
|
1131
|
-
const [row] = rows
|
|
1132
|
+
const [row] = rows;
|
|
1132
1133
|
const stale = memoFor(row, row._id, "stale-token")("choice", "a");
|
|
1133
1134
|
await expect(stale).rejects.toBeInstanceOf(HarnessCommitConflictError);
|
|
1134
1135
|
await expect(stale).rejects.toThrow(
|
|
@@ -5,7 +5,12 @@ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin
|
|
|
5
5
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
6
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
7
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
8
|
-
import type {
|
|
8
|
+
import type {
|
|
9
|
+
HarnessChildOutcome,
|
|
10
|
+
HarnessTaskDefinition,
|
|
11
|
+
HarnessTaskDocument,
|
|
12
|
+
HarnessTestHooks,
|
|
13
|
+
} from "../types/harness";
|
|
9
14
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
10
15
|
import type {HarnessModels} from "./commit";
|
|
11
16
|
import {
|
|
@@ -20,6 +25,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
|
|
|
20
25
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
21
26
|
import {type HarnessEngine, sweepWaitingTasks} from "./ownership";
|
|
22
27
|
|
|
28
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
29
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
30
|
+
|
|
23
31
|
// Multi-task trees under a loaded full run need more than Bun's 5s default.
|
|
24
32
|
setDefaultTimeout(30_000);
|
|
25
33
|
|
|
@@ -61,10 +69,9 @@ const findTask = (id: mongoose.Types.ObjectId | string): Promise<HarnessTaskDocu
|
|
|
61
69
|
const childrenOf = (id: mongoose.Types.ObjectId): Promise<HarnessTaskDocument[]> =>
|
|
62
70
|
TaskModel.find({"ownership.id": id, "ownership.kind": "task"}).sort({created: 1});
|
|
63
71
|
|
|
64
|
-
const asAny = (definition: unknown):
|
|
65
|
-
definition as AnyHarnessTaskDefinition;
|
|
72
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
66
73
|
|
|
67
|
-
const completesWith = (name: string, result: unknown):
|
|
74
|
+
const completesWith = (name: string, result: unknown): TestTaskDefinition =>
|
|
68
75
|
asAny(
|
|
69
76
|
defineTask<unknown, unknown, unknown>({
|
|
70
77
|
initial: () => ({phase: "work"}),
|
|
@@ -76,7 +83,7 @@ const completesWith = (name: string, result: unknown): AnyHarnessTaskDefinition
|
|
|
76
83
|
})
|
|
77
84
|
);
|
|
78
85
|
|
|
79
|
-
const failsWith = (name: string, error: string):
|
|
86
|
+
const failsWith = (name: string, error: string): TestTaskDefinition =>
|
|
80
87
|
asAny(
|
|
81
88
|
defineTask<unknown, unknown, unknown>({
|
|
82
89
|
initial: () => ({phase: "work"}),
|
|
@@ -103,11 +110,11 @@ const fanOut = ({
|
|
|
103
110
|
policy,
|
|
104
111
|
runs,
|
|
105
112
|
}: {
|
|
106
|
-
children:
|
|
113
|
+
children: TestTaskDefinition[];
|
|
107
114
|
name: string;
|
|
108
115
|
policy?: "all" | "failFast";
|
|
109
116
|
runs: {count: number};
|
|
110
|
-
}):
|
|
117
|
+
}): TestTaskDefinition =>
|
|
111
118
|
asAny(
|
|
112
119
|
defineTask<unknown, unknown, HarnessChildOutcome[]>({
|
|
113
120
|
initial: () => ({phase: "fanOut"}),
|
|
@@ -790,7 +797,9 @@ describe("Harness ownership tree", () => {
|
|
|
790
797
|
expect(second.status).toBe("aborted");
|
|
791
798
|
expect(await SpanModel.countDocuments({name: "abort", traceId: created.traceId})).toBe(1);
|
|
792
799
|
// The first request recorded wins; the other waited for its handler to finish.
|
|
793
|
-
expect(["first", "second"]).toContain(
|
|
800
|
+
expect<Array<string | undefined>>(["first", "second"]).toContain(
|
|
801
|
+
(await findTask(created._id)).abortRequested?.reason
|
|
802
|
+
);
|
|
794
803
|
});
|
|
795
804
|
|
|
796
805
|
it("keeps aborting the tree when a child's handler throws", async () => {
|
|
@@ -940,7 +949,7 @@ describe("Harness ownership tree", () => {
|
|
|
940
949
|
|
|
941
950
|
describe("waiting sweep", () => {
|
|
942
951
|
const buildEngine = (
|
|
943
|
-
definitions: ReadonlyArray<
|
|
952
|
+
definitions: ReadonlyArray<TestTaskDefinition> = []
|
|
944
953
|
): {engine: HarnessEngine; wakes: {count: number}} => {
|
|
945
954
|
const wakes = {count: 0};
|
|
946
955
|
const models = {
|
|
@@ -953,8 +962,10 @@ describe("Harness ownership tree", () => {
|
|
|
953
962
|
} as unknown as HarnessModels;
|
|
954
963
|
return {
|
|
955
964
|
engine: {
|
|
965
|
+
agents: new Map(),
|
|
956
966
|
controllers: new Map(),
|
|
957
967
|
definitions: new Map(definitions.map((definition) => [definition.key, definition])),
|
|
968
|
+
extensions: new Map(),
|
|
958
969
|
models,
|
|
959
970
|
wake: () => {
|
|
960
971
|
wakes.count += 1;
|
|
@@ -4,11 +4,14 @@ import {DateTime, Settings} from "luxon";
|
|
|
4
4
|
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
5
5
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
6
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
|
-
import type {HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
7
|
+
import type {HarnessTaskDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
8
8
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
9
9
|
import {type AnyHarnessTaskDefinition, defineTask, Harness, InProcessRunner} from "./harness";
|
|
10
10
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
11
11
|
|
|
12
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
13
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
14
|
+
|
|
12
15
|
const TaskModel = registerHarnessTask();
|
|
13
16
|
let SpanModel: ObsSpanModel;
|
|
14
17
|
let TraceModel: ObsTraceModel;
|
|
@@ -62,7 +65,7 @@ const flakyTask = ({
|
|
|
62
65
|
failures: number;
|
|
63
66
|
name: string;
|
|
64
67
|
retry: {backoffMs?: number; maxAttempts?: number; maxBackoffMs?: number};
|
|
65
|
-
}): {attemptsSeen: number[]; definition:
|
|
68
|
+
}): {attemptsSeen: number[]; definition: TestTaskDefinition} => {
|
|
66
69
|
const attemptsSeen: number[] = [];
|
|
67
70
|
const definition = defineTask<unknown, unknown, {ok: boolean}>({
|
|
68
71
|
initial: () => ({phase: "call"}),
|
|
@@ -81,7 +84,7 @@ const flakyTask = ({
|
|
|
81
84
|
retry,
|
|
82
85
|
version: 1,
|
|
83
86
|
});
|
|
84
|
-
return {attemptsSeen, definition: definition as unknown as
|
|
87
|
+
return {attemptsSeen, definition: definition as unknown as TestTaskDefinition};
|
|
85
88
|
};
|
|
86
89
|
|
|
87
90
|
describe("Harness phase retries", () => {
|
|
@@ -10,7 +10,7 @@ import {createLocalObservabilityPlugin} from "../../observability/local/localPlu
|
|
|
10
10
|
import {registerObsSpan} from "../../observability/local/models/obsSpan";
|
|
11
11
|
import {registerObsTrace} from "../../observability/local/models/obsTrace";
|
|
12
12
|
import {gatedStreamingModel} from "../../tests/harnessStreaming";
|
|
13
|
-
import {ensureTestUsers, UserModel} from "../../tests/helpers";
|
|
13
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
|
|
14
14
|
import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
|
|
15
15
|
import {defineAgent, defineTool, Harness, HarnessApp, InProcessRunner} from "../harness";
|
|
16
16
|
import {registerHarnessConversation} from "../models/harnessConversation";
|
|
@@ -88,7 +88,7 @@ const setup = async () => {
|
|
|
88
88
|
});
|
|
89
89
|
openHarnesses.push(harness);
|
|
90
90
|
await harness.start();
|
|
91
|
-
const app: express.Application = new TerrenoApp({skipListen: true, userModel:
|
|
91
|
+
const app: express.Application = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
92
92
|
.register(new HarnessApp({harness}))
|
|
93
93
|
.build();
|
|
94
94
|
const login = async ({email, password}: {email: string; password: string}) => {
|
|
@@ -8,7 +8,7 @@ import type TestAgent from "supertest/lib/agent";
|
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../../observability/local/localPlugin";
|
|
9
9
|
import {registerObsSpan} from "../../observability/local/models/obsSpan";
|
|
10
10
|
import {registerObsTrace} from "../../observability/local/models/obsTrace";
|
|
11
|
-
import {ensureTestUsers, UserModel} from "../../tests/helpers";
|
|
11
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
|
|
12
12
|
import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
|
|
13
13
|
import {defineTask, Harness, HarnessApp, InProcessRunner} from "../harness";
|
|
14
14
|
import {registerHarnessEvent, registerHarnessEventStream} from "../models/harnessEvent";
|
|
@@ -55,7 +55,7 @@ const setup = async ({start = false}: {start?: boolean} = {}) => {
|
|
|
55
55
|
if (start) {
|
|
56
56
|
await harness.start();
|
|
57
57
|
}
|
|
58
|
-
const app: express.Application = new TerrenoApp({skipListen: true, userModel:
|
|
58
|
+
const app: express.Application = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
59
59
|
.register(new HarnessApp({harness}))
|
|
60
60
|
.build();
|
|
61
61
|
const login = async ({email, password}: {email: string; password: string}) => {
|
|
@@ -676,7 +676,7 @@ describe("JobsRunner", () => {
|
|
|
676
676
|
});
|
|
677
677
|
|
|
678
678
|
it("is not reachable from the root @terreno/ai or @terreno/ai/harness imports", () => {
|
|
679
|
-
const srcRoot = resolve(
|
|
679
|
+
const srcRoot = resolve(__dirname, "../..");
|
|
680
680
|
const seen = new Set<string>();
|
|
681
681
|
const importsJobs: string[] = [];
|
|
682
682
|
const visit = (file: string): void => {
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from "bun:test";
|
|
11
11
|
import {z} from "@terreno/api";
|
|
12
12
|
import type {LanguageModel} from "ai";
|
|
13
|
+
import type mongoose from "mongoose";
|
|
13
14
|
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
14
15
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
15
16
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
@@ -248,7 +249,7 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
248
249
|
beforeAll(async () => {
|
|
249
250
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
250
251
|
// cold first import can exceed it, so load it here first.
|
|
251
|
-
await import("../langfuseClient");
|
|
252
|
+
await import("../langfuseClient.js");
|
|
252
253
|
createLocalObservabilityPlugin();
|
|
253
254
|
SpanModel = registerObsSpan();
|
|
254
255
|
TraceModel = registerObsTrace();
|
|
@@ -653,7 +654,7 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
653
654
|
expect(await TaskModel.countDocuments({name: AGENT_TURN_TASK_NAME})).toBe(2);
|
|
654
655
|
// The first child is found again, not re-run, when the phase resumes for the second.
|
|
655
656
|
expect(calls).toHaveLength(2);
|
|
656
|
-
const firstMessages = async (conversationId:
|
|
657
|
+
const firstMessages = async (conversationId: mongoose.Types.ObjectId | string) =>
|
|
657
658
|
(await MessageModel.find({conversationId}).sort({seq: 1})).map(
|
|
658
659
|
({parts}) => (parts[0] as {text: string}).text
|
|
659
660
|
);
|
|
@@ -6,7 +6,7 @@ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin
|
|
|
6
6
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
7
7
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
8
8
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
9
|
-
import type {HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
9
|
+
import type {HarnessTaskDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
10
10
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
11
11
|
import {commitWaiting, type HarnessModels} from "./commit";
|
|
12
12
|
import {errorMessage} from "./errors";
|
|
@@ -16,6 +16,9 @@ import {registerHarnessInboxEvent} from "./models/harnessInboxEvent";
|
|
|
16
16
|
import {registerHarnessOwner} from "./models/harnessOwner";
|
|
17
17
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
18
18
|
|
|
19
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
20
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
21
|
+
|
|
19
22
|
const TaskModel = registerHarnessTask();
|
|
20
23
|
const OwnerModel = registerHarnessOwner();
|
|
21
24
|
const InboxModel = registerHarnessInboxEvent();
|
|
@@ -58,8 +61,7 @@ const findTask = (id: unknown): Promise<HarnessTaskDocument> => TaskModel.findEx
|
|
|
58
61
|
const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
|
|
59
62
|
(await findTask(id)).status === status;
|
|
60
63
|
|
|
61
|
-
const asAny = (definition: unknown):
|
|
62
|
-
definition as AnyHarnessTaskDefinition;
|
|
64
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
63
65
|
|
|
64
66
|
/** One phase that waits on `event` (with `timeout`) and completes with what it got. */
|
|
65
67
|
const waitsOnce = ({
|
|
@@ -72,7 +74,7 @@ const waitsOnce = ({
|
|
|
72
74
|
name: string;
|
|
73
75
|
runs: {count: number};
|
|
74
76
|
timeout?: {minutes: number};
|
|
75
|
-
}):
|
|
77
|
+
}): TestTaskDefinition =>
|
|
76
78
|
asAny(
|
|
77
79
|
defineTask<unknown, unknown, unknown>({
|
|
78
80
|
initial: () => ({phase: "await"}),
|
package/src/index.test.ts
CHANGED
|
@@ -51,7 +51,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
51
51
|
"shutdownLangfuseClient",
|
|
52
52
|
"shutdownTracing",
|
|
53
53
|
"verifyVertexModelsEnabled",
|
|
54
|
-
]
|
|
54
|
+
];
|
|
55
55
|
|
|
56
56
|
const expectedClasses = [
|
|
57
57
|
"AIRequest",
|
|
@@ -67,7 +67,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
67
67
|
"MemoryTraceSink",
|
|
68
68
|
"ObservabilityApp",
|
|
69
69
|
"Project",
|
|
70
|
-
]
|
|
70
|
+
];
|
|
71
71
|
|
|
72
72
|
const expectedConstants = [
|
|
73
73
|
"COMPACT_SURFACE_SYSTEM_PROMPT",
|
|
@@ -85,7 +85,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
85
85
|
"toggleChecklistHostAction",
|
|
86
86
|
"TRANSLATION_PROMPT",
|
|
87
87
|
"TemperaturePresets",
|
|
88
|
-
]
|
|
88
|
+
];
|
|
89
89
|
|
|
90
90
|
it.each(expectedFunctions)("exports %s as a function", (name) => {
|
|
91
91
|
expect(typeof exported[name]).toBe("function");
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {GptHistory} from "../models/gptHistory";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../tests/helpers";
|
|
8
8
|
|
|
9
9
|
type StreamPart = {type: string; [key: string]: unknown};
|
|
10
10
|
|
|
@@ -13,32 +13,37 @@ type StreamPart = {type: string; [key: string]: unknown};
|
|
|
13
13
|
let streamParts: StreamPart[] = [];
|
|
14
14
|
let streamFiles: Array<{base64: string; mediaType: string}> = [];
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
16
|
+
let addGptRoutes: typeof import("../routes/gpt.js").addGptRoutes;
|
|
17
|
+
let AIService: typeof import("../service/aiService.js").AIService;
|
|
18
|
+
|
|
19
|
+
beforeAll(async () => {
|
|
20
|
+
// Mock langfuseVercelAi to avoid transitive langfuse SDK imports.
|
|
21
|
+
const realLangfuseVercelAi = await import("../langfuseVercelAi.js");
|
|
22
|
+
mock.module("../langfuseVercelAi", () => ({
|
|
23
|
+
...realLangfuseVercelAi,
|
|
24
|
+
createTelemetryConfig: () => ({functionId: "test", isEnabled: false}),
|
|
25
|
+
preparePromptForAI: async () => ({config: {}, prompt: "test", telemetry: {isEnabled: false}}),
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
// Mock the AI SDK so we control exactly which fullStream parts the GPT prompt
|
|
29
|
+
// handler iterates over. All other exports are preserved so AIService and the
|
|
30
|
+
// route helpers keep working.
|
|
31
|
+
const realAi = await import("ai");
|
|
32
|
+
mock.module("ai", () => ({
|
|
33
|
+
...realAi,
|
|
34
|
+
streamText: () => ({
|
|
35
|
+
files: Promise.resolve(streamFiles),
|
|
36
|
+
fullStream: (async function* fullStream(): AsyncGenerator<StreamPart> {
|
|
37
|
+
for (const part of streamParts) {
|
|
38
|
+
yield part;
|
|
39
|
+
}
|
|
40
|
+
})(),
|
|
41
|
+
}),
|
|
42
|
+
}));
|
|
43
|
+
|
|
44
|
+
({addGptRoutes} = await import("../routes/gpt.js"));
|
|
45
|
+
({AIService} = await import("../service/aiService.js"));
|
|
46
|
+
});
|
|
42
47
|
|
|
43
48
|
const createMockModel = () => ({
|
|
44
49
|
doGenerate: mock(async () => ({
|
|
@@ -87,7 +92,7 @@ describe("GPT prompt streaming edge cases", () => {
|
|
|
87
92
|
addGptRoutes(router, {aiService, openApiOptions: options});
|
|
88
93
|
},
|
|
89
94
|
skipListen: true,
|
|
90
|
-
userModel:
|
|
95
|
+
userModel: terrenoUserModel,
|
|
91
96
|
}).build();
|
|
92
97
|
});
|
|
93
98
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
1
|
+
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
3
|
const shutdownMock = mock(async () => {});
|
|
4
4
|
class FakeLangfuseClient {
|
|
@@ -12,8 +12,14 @@ mock.module("@langfuse/client", () => ({
|
|
|
12
12
|
LangfuseClient: FakeLangfuseClient,
|
|
13
13
|
}));
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
let getLangfuseClient: typeof import("../langfuseClient.js").getLangfuseClient;
|
|
16
|
+
let initLangfuseClient: typeof import("../langfuseClient.js").initLangfuseClient;
|
|
17
|
+
let isLangfuseInitialized: typeof import("../langfuseClient.js").isLangfuseInitialized;
|
|
18
|
+
let shutdownLangfuseClient: typeof import("../langfuseClient.js").shutdownLangfuseClient;
|
|
19
|
+
beforeAll(async () => {
|
|
20
|
+
({getLangfuseClient, initLangfuseClient, isLangfuseInitialized, shutdownLangfuseClient} =
|
|
21
|
+
await import("../langfuseClient.js"));
|
|
22
|
+
});
|
|
17
23
|
|
|
18
24
|
describe("langfuseClient", () => {
|
|
19
25
|
beforeEach(async () => {
|
|
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
|
|
|
3
3
|
|
|
4
4
|
describe("@terreno/ai root entry", () => {
|
|
5
5
|
it("does not register the Project model until it is used", async () => {
|
|
6
|
-
const ai = await import("../index");
|
|
6
|
+
const ai = await import("../index.js");
|
|
7
7
|
expect(mongoose.models.Project).toBeUndefined();
|
|
8
8
|
|
|
9
9
|
expect(ai.Project.modelName).toBe("Project");
|
|
@@ -11,7 +11,7 @@ describe("@terreno/ai root entry", () => {
|
|
|
11
11
|
});
|
|
12
12
|
|
|
13
13
|
it("forwards statics, construction and instanceof through the deprecated Project export", async () => {
|
|
14
|
-
const {Project} = await import("../index");
|
|
14
|
+
const {Project} = await import("../index.js");
|
|
15
15
|
const userId = new mongoose.Types.ObjectId();
|
|
16
16
|
|
|
17
17
|
const project = new Project({name: "Roadmap", userId});
|
package/src/langfuseApp.test.ts
CHANGED
|
@@ -1,34 +1,18 @@
|
|
|
1
1
|
import {beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
-
import {
|
|
2
|
+
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
|
-
import mongoose from "mongoose";
|
|
5
|
-
import passportLocalMongoose from "passport-local-mongoose";
|
|
6
4
|
import supertest from "supertest";
|
|
7
5
|
|
|
8
6
|
import {LangfuseApp} from "./langfuseApp";
|
|
7
|
+
import {terrenoUserModel, UserModel} from "./tests/helpers";
|
|
9
8
|
|
|
10
9
|
type PasswordedUser = {setPassword: (password: string) => Promise<void>};
|
|
11
10
|
|
|
12
|
-
const userSchema = new mongoose.Schema({
|
|
13
|
-
admin: {default: false, type: Boolean},
|
|
14
|
-
email: {index: true, type: String},
|
|
15
|
-
name: String,
|
|
16
|
-
});
|
|
17
|
-
userSchema.plugin(
|
|
18
|
-
passportLocalMongoose as unknown as (
|
|
19
|
-
schema: mongoose.Schema,
|
|
20
|
-
options: {usernameField: string}
|
|
21
|
-
) => void,
|
|
22
|
-
{usernameField: "email"}
|
|
23
|
-
);
|
|
24
|
-
userSchema.plugin(createdUpdatedPlugin);
|
|
25
|
-
const UserModel = mongoose.models.User || mongoose.model("User", userSchema);
|
|
26
|
-
|
|
27
11
|
const buildApp = (plugin: LangfuseApp) =>
|
|
28
12
|
new TerrenoApp({
|
|
29
13
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
30
14
|
skipListen: true,
|
|
31
|
-
userModel:
|
|
15
|
+
userModel: terrenoUserModel,
|
|
32
16
|
}).build();
|
|
33
17
|
|
|
34
18
|
describe("LangfuseApp", () => {
|
|
@@ -40,7 +24,7 @@ describe("LangfuseApp", () => {
|
|
|
40
24
|
});
|
|
41
25
|
|
|
42
26
|
beforeEach(async () => {
|
|
43
|
-
const {shutdownLangfuseClient} = await import("./langfuseClient");
|
|
27
|
+
const {shutdownLangfuseClient} = await import("./langfuseClient.js");
|
|
44
28
|
await shutdownLangfuseClient();
|
|
45
29
|
});
|
|
46
30
|
|