@terreno/ai 57.20.0 → 58.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +20 -18
- package/CHANGELOG.md +93 -0
- package/biome.jsonc +3 -2
- package/dist/aiApp.d.ts +5 -2
- package/dist/aiApp.js +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/migrations/gptHistoryStringIds.d.ts +41 -0
- package/dist/migrations/gptHistoryStringIds.js +94 -0
- package/dist/models/aiDataset.js +2 -2
- package/dist/models/gptHistory.js +11 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/routes/files.d.ts +7 -1
- package/dist/routes/files.js +62 -13
- package/dist/routes/gptActions.js +0 -7
- package/dist/routes/gptHistories.js +8 -2
- package/dist/service/aiDatasets.js +2 -1
- package/dist/service/chatTurn.js +4 -0
- package/dist/service/vertex.d.ts +5 -2
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +19 -16
- package/src/aiApp.test.ts +43 -18
- package/src/aiApp.ts +6 -3
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/index.ts +7 -1
- package/src/isolated/gptStream.isolated.ts +35 -30
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/migrations/gptHistoryStringIds.test.ts +231 -0
- package/src/migrations/gptHistoryStringIds.ts +150 -0
- package/src/models/aiDataset.test.ts +1 -1
- package/src/models/aiDataset.ts +2 -2
- package/src/models/gptHistory.test.ts +22 -11
- package/src/models/gptHistory.ts +19 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +126 -3
- package/src/routes/files.ts +90 -15
- package/src/routes/gpt.test.ts +23 -16
- package/src/routes/gptActions.test.ts +15 -15
- package/src/routes/gptActions.ts +1 -4
- package/src/routes/gptApprovals.test.ts +1 -1
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +2 -2
- package/src/routes/gptHistories.test.ts +17 -11
- package/src/routes/gptHistories.ts +59 -55
- package/src/routes/gptHistoriesSync.test.ts +141 -0
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +64 -47
- package/src/service/aiDatasets.ts +2 -1
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +4 -0
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/service/vertex.test.ts +1 -1
- package/src/service/vertex.ts +7 -4
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/src/types/index.ts +5 -2
- package/tsconfig.typecheck.json +18 -0
package/src/aiApp.test.ts
CHANGED
|
@@ -10,7 +10,12 @@ import {GptHistory} from "./models/gptHistory";
|
|
|
10
10
|
import type {FileStorageService} from "./service/fileStorage";
|
|
11
11
|
import type {MCPService} from "./service/mcpService";
|
|
12
12
|
import {createScriptedModel, modelCall, textStep, toolCallStep} from "./tests/chatHarness";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
authAsUserWithCredentials,
|
|
15
|
+
ensureTestUsers,
|
|
16
|
+
terrenoUserModel,
|
|
17
|
+
UserModel,
|
|
18
|
+
} from "./tests/helpers";
|
|
14
19
|
|
|
15
20
|
const AI_APP_TEST_USER = {
|
|
16
21
|
admin: false,
|
|
@@ -50,7 +55,7 @@ describe("AiApp", () => {
|
|
|
50
55
|
|
|
51
56
|
afterEach(async () => {
|
|
52
57
|
await AIRequest.deleteMany({});
|
|
53
|
-
await GptHistory.deleteMany({});
|
|
58
|
+
await GptHistory.collection.deleteMany({});
|
|
54
59
|
});
|
|
55
60
|
|
|
56
61
|
const authAsUser = async (app: express.Application) => {
|
|
@@ -61,13 +66,13 @@ describe("AiApp", () => {
|
|
|
61
66
|
};
|
|
62
67
|
|
|
63
68
|
it("registers gpt and project routes by default", async () => {
|
|
64
|
-
const {AIService} = await import("./service/aiService");
|
|
69
|
+
const {AIService} = await import("./service/aiService.js");
|
|
65
70
|
const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
66
71
|
const plugin = new AiApp({aiService});
|
|
67
72
|
const app = new TerrenoApp({
|
|
68
73
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
69
74
|
skipListen: true,
|
|
70
|
-
userModel:
|
|
75
|
+
userModel: terrenoUserModel,
|
|
71
76
|
}).build();
|
|
72
77
|
|
|
73
78
|
const agent = await authAsUser(app);
|
|
@@ -80,13 +85,13 @@ describe("AiApp", () => {
|
|
|
80
85
|
});
|
|
81
86
|
|
|
82
87
|
it("skips the project routes with projects: false", async () => {
|
|
83
|
-
const {AIService} = await import("./service/aiService");
|
|
88
|
+
const {AIService} = await import("./service/aiService.js");
|
|
84
89
|
const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
85
90
|
const plugin = new AiApp({aiService, projects: false});
|
|
86
91
|
const app = new TerrenoApp({
|
|
87
92
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
88
93
|
skipListen: true,
|
|
89
|
-
userModel:
|
|
94
|
+
userModel: terrenoUserModel,
|
|
90
95
|
}).build();
|
|
91
96
|
|
|
92
97
|
const agent = await authAsUser(app);
|
|
@@ -95,14 +100,14 @@ describe("AiApp", () => {
|
|
|
95
100
|
});
|
|
96
101
|
|
|
97
102
|
it("passes asks to the gpt routes, so the model is offered the ask tools", async () => {
|
|
98
|
-
const {AIService} = await import("./service/aiService");
|
|
103
|
+
const {AIService} = await import("./service/aiService.js");
|
|
99
104
|
const model = createMockModel();
|
|
100
105
|
const aiService = new AIService({model: model as unknown as LanguageModel});
|
|
101
106
|
const plugin = new AiApp({aiService, asks: true});
|
|
102
107
|
const app = new TerrenoApp({
|
|
103
108
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
104
109
|
skipListen: true,
|
|
105
|
-
userModel:
|
|
110
|
+
userModel: terrenoUserModel,
|
|
106
111
|
}).build();
|
|
107
112
|
const agent = await authAsUser(app);
|
|
108
113
|
|
|
@@ -120,7 +125,7 @@ describe("AiApp", () => {
|
|
|
120
125
|
});
|
|
121
126
|
|
|
122
127
|
it("adds the documented headless turn actions to the history routes", async () => {
|
|
123
|
-
const {AIService} = await import("./service/aiService");
|
|
128
|
+
const {AIService} = await import("./service/aiService.js");
|
|
124
129
|
const model = createMockModel();
|
|
125
130
|
const aiService = new AIService({model: model as unknown as LanguageModel});
|
|
126
131
|
const app = new TerrenoApp({
|
|
@@ -129,7 +134,7 @@ describe("AiApp", () => {
|
|
|
129
134
|
router as unknown as express.Application
|
|
130
135
|
),
|
|
131
136
|
skipListen: true,
|
|
132
|
-
userModel:
|
|
137
|
+
userModel: terrenoUserModel,
|
|
133
138
|
}).build();
|
|
134
139
|
const agent = await authAsUser(app);
|
|
135
140
|
|
|
@@ -159,7 +164,7 @@ describe("AiApp", () => {
|
|
|
159
164
|
});
|
|
160
165
|
|
|
161
166
|
it("adds the headless actions only with asks on, so hosts without asks get no new endpoints", async () => {
|
|
162
|
-
const {AIService} = await import("./service/aiService");
|
|
167
|
+
const {AIService} = await import("./service/aiService.js");
|
|
163
168
|
const historyRoutes = async (asks?: boolean) => {
|
|
164
169
|
const model = createMockModel();
|
|
165
170
|
const aiService = new AIService({model: model as unknown as LanguageModel});
|
|
@@ -169,7 +174,7 @@ describe("AiApp", () => {
|
|
|
169
174
|
router as unknown as express.Application
|
|
170
175
|
),
|
|
171
176
|
skipListen: true,
|
|
172
|
-
userModel:
|
|
177
|
+
userModel: terrenoUserModel,
|
|
173
178
|
}).build();
|
|
174
179
|
const agent = await authAsUser(app);
|
|
175
180
|
const openApi = await agent.get("/openapi.json");
|
|
@@ -226,7 +231,7 @@ describe("AiApp", () => {
|
|
|
226
231
|
const app = new TerrenoApp({
|
|
227
232
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
228
233
|
skipListen: true,
|
|
229
|
-
userModel:
|
|
234
|
+
userModel: terrenoUserModel,
|
|
230
235
|
}).build();
|
|
231
236
|
// Unauthenticated access to upload route: ensure it exists (returns 401, not 404).
|
|
232
237
|
const upload = await supertest(app).post("/files/upload");
|
|
@@ -234,8 +239,8 @@ describe("AiApp", () => {
|
|
|
234
239
|
});
|
|
235
240
|
|
|
236
241
|
it("caps uploads at asks.maxFileSizeBytes and loads a files answer's uploads from storage", async () => {
|
|
237
|
-
const {AIService} = await import("./service/aiService");
|
|
238
|
-
const {FileAttachment} = await import("./models/fileAttachment");
|
|
242
|
+
const {AIService} = await import("./service/aiService.js");
|
|
243
|
+
const {FileAttachment} = await import("./models/fileAttachment.js");
|
|
239
244
|
const text = Buffer.from("Coffee, 4.50\n");
|
|
240
245
|
const upload = mock(async () => {
|
|
241
246
|
const [user] = await UserModel.find({email: AI_APP_TEST_USER.email});
|
|
@@ -278,7 +283,7 @@ describe("AiApp", () => {
|
|
|
278
283
|
const app = new TerrenoApp({
|
|
279
284
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
280
285
|
skipListen: true,
|
|
281
|
-
userModel:
|
|
286
|
+
userModel: terrenoUserModel,
|
|
282
287
|
}).build();
|
|
283
288
|
const agent = await authAsUser(app);
|
|
284
289
|
|
|
@@ -329,7 +334,7 @@ describe("AiApp", () => {
|
|
|
329
334
|
const app = new TerrenoApp({
|
|
330
335
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
331
336
|
skipListen: true,
|
|
332
|
-
userModel:
|
|
337
|
+
userModel: terrenoUserModel,
|
|
333
338
|
}).build();
|
|
334
339
|
const upload = await supertest(app).post("/files/upload");
|
|
335
340
|
expect(upload.status).toBe(404);
|
|
@@ -345,10 +350,30 @@ describe("AiApp", () => {
|
|
|
345
350
|
const app = new TerrenoApp({
|
|
346
351
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
347
352
|
skipListen: true,
|
|
348
|
-
userModel:
|
|
353
|
+
userModel: terrenoUserModel,
|
|
349
354
|
}).build();
|
|
350
355
|
const agent = await authAsUser(app);
|
|
351
356
|
const servers = await agent.get("/mcp/servers");
|
|
352
357
|
expect(servers.status).not.toBe(404);
|
|
353
358
|
});
|
|
359
|
+
it("documents its routes with the OpenAPI instance TerrenoApp passes to register", async () => {
|
|
360
|
+
const {AIService} = await import("./service/aiService.js");
|
|
361
|
+
const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
362
|
+
const fileStorageService = {
|
|
363
|
+
delete: mock(async () => {}),
|
|
364
|
+
getSignedUrl: mock(async () => "https://example.com/signed"),
|
|
365
|
+
upload: mock(async () => {
|
|
366
|
+
throw new Error("not called");
|
|
367
|
+
}),
|
|
368
|
+
} as unknown as FileStorageService;
|
|
369
|
+
const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
370
|
+
.register(new AiApp({aiService, fileStorageService, gcsBucket: "bucket"}))
|
|
371
|
+
.build();
|
|
372
|
+
|
|
373
|
+
const res = await supertest(app).get("/openapi.json").expect(200);
|
|
374
|
+
expect(res.body.paths["/files/upload"]?.post?.requestBody?.content).toHaveProperty(
|
|
375
|
+
"multipart/form-data"
|
|
376
|
+
);
|
|
377
|
+
expect(res.body.paths["/gpt/prompt"]?.post).toBeDefined();
|
|
378
|
+
});
|
|
354
379
|
});
|
package/src/aiApp.ts
CHANGED
|
@@ -40,7 +40,10 @@ export interface AiAppOptions {
|
|
|
40
40
|
maxSteps?: number;
|
|
41
41
|
/** MCP service for connecting to external tool servers. */
|
|
42
42
|
mcpService?: MCPService;
|
|
43
|
-
/**
|
|
43
|
+
/**
|
|
44
|
+
* OpenAPI options passed through to route builders for spec generation. Defaults to the
|
|
45
|
+
* OpenAPI instance TerrenoApp passes to `register`.
|
|
46
|
+
*/
|
|
44
47
|
openApiOptions?: Record<string, unknown>;
|
|
45
48
|
/**
|
|
46
49
|
* Mount the `/gpt/projects` routes, which register the `Project` model. Defaults to `true`;
|
|
@@ -88,7 +91,7 @@ export class AiApp implements TerrenoPlugin {
|
|
|
88
91
|
this.options = options;
|
|
89
92
|
}
|
|
90
93
|
|
|
91
|
-
register(app: express.Application): void {
|
|
94
|
+
register(app: express.Application, openApi?: unknown): void {
|
|
92
95
|
const router = app;
|
|
93
96
|
const {
|
|
94
97
|
aiService,
|
|
@@ -101,13 +104,13 @@ export class AiApp implements TerrenoPlugin {
|
|
|
101
104
|
gcsBucket,
|
|
102
105
|
maxSteps,
|
|
103
106
|
mcpService,
|
|
104
|
-
openApiOptions,
|
|
105
107
|
projects,
|
|
106
108
|
titleModelId,
|
|
107
109
|
toolChoice,
|
|
108
110
|
tools,
|
|
109
111
|
} = this.options;
|
|
110
112
|
|
|
113
|
+
const openApiOptions = this.options.openApiOptions ?? (openApi ? {openApi} : undefined);
|
|
111
114
|
const hasFileRoutes = Boolean(fileStorageService && gcsBucket);
|
|
112
115
|
const chat = {
|
|
113
116
|
aiService,
|
|
@@ -180,7 +180,7 @@ describe("Agent turns", () => {
|
|
|
180
180
|
beforeAll(async () => {
|
|
181
181
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
182
182
|
// cold first import can exceed it, so load it here first.
|
|
183
|
-
await import("../langfuseClient");
|
|
183
|
+
await import("../langfuseClient.js");
|
|
184
184
|
createLocalObservabilityPlugin();
|
|
185
185
|
SpanModel = registerObsSpan();
|
|
186
186
|
TraceModel = registerObsTrace();
|
package/src/harness/agentLoop.ts
CHANGED
|
@@ -653,7 +653,8 @@ export const createAgentTasks = (context: AgentLoopContext): AgentTasks => {
|
|
|
653
653
|
}
|
|
654
654
|
let returned: unknown;
|
|
655
655
|
try {
|
|
656
|
-
|
|
656
|
+
// `args.data` was parsed by this tool's own `parameters`, which match its `execute` args.
|
|
657
|
+
returned = await tool.execute(args.data as never, {
|
|
657
658
|
conversationId,
|
|
658
659
|
env: rt.env,
|
|
659
660
|
output: (text: string) => {
|
|
@@ -44,38 +44,41 @@ type Reply = {
|
|
|
44
44
|
};
|
|
45
45
|
|
|
46
46
|
/** LanguageModelV2 mock answering each request with the next reply; `hang` blocks it first. */
|
|
47
|
-
const scriptedModel = (replies: Reply[]) =>
|
|
48
|
-
withGenerateStreaming
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
47
|
+
const scriptedModel = (replies: Reply[]) => {
|
|
48
|
+
// Declared before the call so `withGenerateStreaming` infers the mock's own type.
|
|
49
|
+
const doGenerate = mock(async () => {
|
|
50
|
+
const reply = replies.shift();
|
|
51
|
+
if (!reply) {
|
|
52
|
+
throw new Error("script exhausted");
|
|
53
|
+
}
|
|
54
|
+
reply.started?.release();
|
|
55
|
+
if (reply.hang) {
|
|
56
|
+
await reply.hang.promise;
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
content: reply.toolCall
|
|
60
|
+
? [
|
|
61
|
+
{
|
|
62
|
+
input: "{}",
|
|
63
|
+
toolCallId: reply.toolCall.id,
|
|
64
|
+
toolName: reply.toolCall.name,
|
|
65
|
+
type: "tool-call" as const,
|
|
66
|
+
},
|
|
67
|
+
]
|
|
68
|
+
: [{text: reply.text ?? "", type: "text" as const}],
|
|
69
|
+
finishReason: reply.toolCall ? ("tool-calls" as const) : ("stop" as const),
|
|
70
|
+
usage: {inputTokens: 1, outputTokens: 1, totalTokens: 2},
|
|
71
|
+
warnings: [],
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
return withGenerateStreaming({
|
|
75
|
+
doGenerate,
|
|
74
76
|
modelId: "mock-model",
|
|
75
77
|
provider: "mock",
|
|
76
78
|
specificationVersion: "v2" as const,
|
|
77
79
|
supportedUrls: {},
|
|
78
80
|
});
|
|
81
|
+
};
|
|
79
82
|
|
|
80
83
|
interface ProcessHandle {
|
|
81
84
|
die: () => void;
|
|
@@ -169,7 +172,7 @@ describe("Agent turns across a crash", () => {
|
|
|
169
172
|
beforeAll(async () => {
|
|
170
173
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
171
174
|
// cold first import can exceed it, so load it here first.
|
|
172
|
-
await import("../langfuseClient");
|
|
175
|
+
await import("../langfuseClient.js");
|
|
173
176
|
createLocalObservabilityPlugin();
|
|
174
177
|
SpanModel = registerObsSpan();
|
|
175
178
|
TraceModel = registerObsTrace();
|
|
@@ -21,10 +21,11 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
|
21
21
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
22
22
|
import {withGenerateStreaming} from "../tests/generateStream";
|
|
23
23
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
24
|
-
import {authAsUser, ensureTestUsers, UserModel} from "../tests/helpers";
|
|
24
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
25
25
|
import type {
|
|
26
26
|
HarnessApprovalDocument,
|
|
27
27
|
HarnessApprovalResult,
|
|
28
|
+
HarnessTaskDefinition,
|
|
28
29
|
HarnessTaskDocument,
|
|
29
30
|
HarnessTestHooks,
|
|
30
31
|
} from "../types/harness";
|
|
@@ -32,7 +33,6 @@ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
|
32
33
|
import type {HarnessModels} from "./commit";
|
|
33
34
|
import {
|
|
34
35
|
AGENT_TOOL_TASK_NAME,
|
|
35
|
-
type AnyHarnessTaskDefinition,
|
|
36
36
|
approvalGate,
|
|
37
37
|
approvalTaskInput,
|
|
38
38
|
defineAgent,
|
|
@@ -55,6 +55,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
|
|
|
55
55
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
56
56
|
import {HarnessWaitRaceError, sendEventRecords} from "./waits";
|
|
57
57
|
|
|
58
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
59
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
60
|
+
|
|
58
61
|
// Whole tasks and turns on a real replica set; the first test also pays for index creation.
|
|
59
62
|
setDefaultTimeout(30_000);
|
|
60
63
|
|
|
@@ -107,8 +110,7 @@ const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
|
|
|
107
110
|
const approvalOf = (taskId: unknown): Promise<HarnessApprovalDocument> =>
|
|
108
111
|
ApprovalModel.findExactlyOne({taskId});
|
|
109
112
|
|
|
110
|
-
const asAny = (definition: unknown):
|
|
111
|
-
definition as AnyHarnessTaskDefinition;
|
|
113
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
112
114
|
|
|
113
115
|
const openHarness = async (
|
|
114
116
|
registry: ReadonlyArray<HarnessRegistryEntry>,
|
|
@@ -138,7 +140,7 @@ const openHarness = async (
|
|
|
138
140
|
};
|
|
139
141
|
|
|
140
142
|
const buildApp = (harness: Harness): express.Application =>
|
|
141
|
-
new TerrenoApp({skipListen: true, userModel:
|
|
143
|
+
new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
142
144
|
.register(new HarnessApp({harness}))
|
|
143
145
|
.build();
|
|
144
146
|
|
|
@@ -176,7 +178,7 @@ const asksApproval = ({
|
|
|
176
178
|
retry?: {maxAttempts: number};
|
|
177
179
|
runs?: {count: number};
|
|
178
180
|
timeout?: {hours: number};
|
|
179
|
-
}):
|
|
181
|
+
}): TestTaskDefinition =>
|
|
180
182
|
asAny(
|
|
181
183
|
defineTask<{patientId: string}, unknown, HarnessApprovalResult>({
|
|
182
184
|
approvals,
|
|
@@ -563,7 +565,7 @@ describe("Harness approvals", () => {
|
|
|
563
565
|
|
|
564
566
|
describe("races and forged events", () => {
|
|
565
567
|
/** Past the approval's timeout with the runner stopped, so the next owner resolves it. */
|
|
566
|
-
const parkThenPassTimeout = async (definition:
|
|
568
|
+
const parkThenPassTimeout = async (definition: TestTaskDefinition) => {
|
|
567
569
|
const start = DateTime.now();
|
|
568
570
|
Settings.now = () => start.toMillis();
|
|
569
571
|
const first = await openHarness([definition]);
|
|
@@ -762,7 +764,7 @@ describe("Harness approvals", () => {
|
|
|
762
764
|
});
|
|
763
765
|
|
|
764
766
|
it("fails the task on rt.approval misuse and rejects invalid approval policies", async () => {
|
|
765
|
-
const misuse = (options: unknown, key = "k"):
|
|
767
|
+
const misuse = (options: unknown, key = "k"): TestTaskDefinition =>
|
|
766
768
|
asAny(
|
|
767
769
|
defineTask<unknown, unknown, unknown>({
|
|
768
770
|
initial: () => ({phase: "ask"}),
|
|
@@ -778,7 +780,7 @@ describe("Harness approvals", () => {
|
|
|
778
780
|
version: 1,
|
|
779
781
|
})
|
|
780
782
|
);
|
|
781
|
-
const cases: Array<[
|
|
783
|
+
const cases: Array<[TestTaskDefinition, string]> = [
|
|
782
784
|
[misuse({title: "t"}, " "), "rt.approval requires a key"],
|
|
783
785
|
[misuse({}), 'rt.approval("k") requires a title'],
|
|
784
786
|
[misuse({summary: 3, title: "t"}), 'rt.approval("k") summary must be a string'],
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
type SseClient,
|
|
19
19
|
type SseFrame,
|
|
20
20
|
} from "../tests/harnessStreaming";
|
|
21
|
-
import {ensureTestUsers, UserModel} from "../tests/helpers";
|
|
21
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
22
22
|
import type {HarnessStreamingOptions} from "../types/harness";
|
|
23
23
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
24
24
|
import type {HarnessModels} from "./commit";
|
|
@@ -102,7 +102,9 @@ const serve = async (
|
|
|
102
102
|
harness: Harness
|
|
103
103
|
): Promise<{app: express.Application; plugin: HarnessApp; url: string}> => {
|
|
104
104
|
const plugin = new HarnessApp({harness, heartbeatInterval: {milliseconds: 50}});
|
|
105
|
-
const app = new TerrenoApp({skipListen: true, userModel:
|
|
105
|
+
const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
106
|
+
.register(plugin)
|
|
107
|
+
.build();
|
|
106
108
|
const {stop, url} = await listen(app);
|
|
107
109
|
cleanups.push(stop);
|
|
108
110
|
return {app, plugin, url};
|
|
@@ -111,7 +113,8 @@ const serve = async (
|
|
|
111
113
|
/** Open change-stream cursors on the server, idle ones included. */
|
|
112
114
|
const openChangeStreamOps = async (): Promise<Array<Record<string, unknown>>> => {
|
|
113
115
|
const dbName = mongoose.connection.db?.databaseName;
|
|
114
|
-
return mongoose.connection
|
|
116
|
+
return mongoose.connection
|
|
117
|
+
.getClient()
|
|
115
118
|
.db("admin")
|
|
116
119
|
.aggregate([
|
|
117
120
|
{$currentOp: {allUsers: true, idleCursors: true}},
|
|
@@ -168,10 +171,11 @@ const sse = async (
|
|
|
168
171
|
/** Status and body of a request that must fail before streaming starts. */
|
|
169
172
|
const refused = async (url: string, headers: Record<string, string> = {}) => {
|
|
170
173
|
const response = await fetch(url, {headers});
|
|
171
|
-
return {body: await response.json(), status: response.status};
|
|
174
|
+
return {body: (await response.json()) as {title?: string}, status: response.status};
|
|
172
175
|
};
|
|
173
176
|
|
|
174
|
-
const storedEvents = (streamId:
|
|
177
|
+
const storedEvents = (streamId: mongoose.Types.ObjectId | string) =>
|
|
178
|
+
EventModel.find({streamId}).sort({seq: 1}).lean();
|
|
175
179
|
|
|
176
180
|
const deltaText = (frames: ReadonlyArray<SseFrame>): string =>
|
|
177
181
|
frames
|
|
@@ -372,7 +376,7 @@ describe("Harness event stream", () => {
|
|
|
372
376
|
});
|
|
373
377
|
|
|
374
378
|
it("rejects a heartbeat interval that is not positive", () => {
|
|
375
|
-
const app = new TerrenoApp({skipListen: true, userModel:
|
|
379
|
+
const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel});
|
|
376
380
|
expect(() =>
|
|
377
381
|
app
|
|
378
382
|
.register(new HarnessApp({harness: {} as Harness, heartbeatInterval: {seconds: 0}}))
|
|
@@ -200,7 +200,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
200
200
|
beforeAll(async () => {
|
|
201
201
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
202
202
|
// cold first import can exceed it, so load it here first.
|
|
203
|
-
await import("../langfuseClient");
|
|
203
|
+
await import("../langfuseClient.js");
|
|
204
204
|
createLocalObservabilityPlugin();
|
|
205
205
|
SpanModel = registerObsSpan();
|
|
206
206
|
TraceModel = registerObsTrace();
|
|
@@ -233,7 +233,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
233
233
|
expect(() => section("s", "x" as never)).toThrow(
|
|
234
234
|
harnessErrorMatching("definitionInvalid", "section(s): build must be a function")
|
|
235
235
|
);
|
|
236
|
-
expect(() => hook("onEverything" as never, () => undefined)).toThrow(
|
|
236
|
+
expect(() => hook("onEverything" as never, (() => undefined) as never)).toThrow(
|
|
237
237
|
harnessErrorMatching(
|
|
238
238
|
"definitionInvalid",
|
|
239
239
|
'hook: kind must be one of afterTool, beforeModelRequest, beforeTool, not "onEverything"'
|
|
@@ -774,7 +774,8 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
774
774
|
...tool,
|
|
775
775
|
execute: async (args, api) => {
|
|
776
776
|
ran.push(`${tag}:before`);
|
|
777
|
-
|
|
777
|
+
// `args` was parsed by the wrapped tool's own `parameters`.
|
|
778
|
+
return {[tag]: await tool.execute(args as never, api)};
|
|
778
779
|
},
|
|
779
780
|
});
|
|
780
781
|
// The wrapping extension comes before the overriding one, yet wraps its tool.
|
|
@@ -1053,7 +1054,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1053
1054
|
version: 1,
|
|
1054
1055
|
});
|
|
1055
1056
|
const harness = await openHarness({registry: [task]});
|
|
1056
|
-
const ids = [];
|
|
1057
|
+
const ids: mongoose.Types.ObjectId[] = [];
|
|
1057
1058
|
for (let n = 0; n < count; n++) {
|
|
1058
1059
|
ids.push((await harness.createTask(task, {n}))._id);
|
|
1059
1060
|
}
|
|
@@ -1106,7 +1107,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1106
1107
|
it("lets exactly one of two independent runs win a key (first write wins)", async () => {
|
|
1107
1108
|
const {release, rows} = await hangTasks(2);
|
|
1108
1109
|
try {
|
|
1109
|
-
const [first, second] = rows
|
|
1110
|
+
const [first, second] = rows;
|
|
1110
1111
|
// Two runs fencing different task rows race on one shared key, so neither queues
|
|
1111
1112
|
// behind the other's lease write and the unique index decides.
|
|
1112
1113
|
const scope = first._id;
|
|
@@ -1128,7 +1129,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1128
1129
|
it("refuses a write from a run whose lease token is stale", async () => {
|
|
1129
1130
|
const {release, rows} = await hangTasks(1);
|
|
1130
1131
|
try {
|
|
1131
|
-
const [row] = rows
|
|
1132
|
+
const [row] = rows;
|
|
1132
1133
|
const stale = memoFor(row, row._id, "stale-token")("choice", "a");
|
|
1133
1134
|
await expect(stale).rejects.toBeInstanceOf(HarnessCommitConflictError);
|
|
1134
1135
|
await expect(stale).rejects.toThrow(
|
|
@@ -5,7 +5,12 @@ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin
|
|
|
5
5
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
6
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
7
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
8
|
-
import type {
|
|
8
|
+
import type {
|
|
9
|
+
HarnessChildOutcome,
|
|
10
|
+
HarnessTaskDefinition,
|
|
11
|
+
HarnessTaskDocument,
|
|
12
|
+
HarnessTestHooks,
|
|
13
|
+
} from "../types/harness";
|
|
9
14
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
10
15
|
import type {HarnessModels} from "./commit";
|
|
11
16
|
import {
|
|
@@ -20,6 +25,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
|
|
|
20
25
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
21
26
|
import {type HarnessEngine, sweepWaitingTasks} from "./ownership";
|
|
22
27
|
|
|
28
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
29
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
30
|
+
|
|
23
31
|
// Multi-task trees under a loaded full run need more than Bun's 5s default.
|
|
24
32
|
setDefaultTimeout(30_000);
|
|
25
33
|
|
|
@@ -61,10 +69,9 @@ const findTask = (id: mongoose.Types.ObjectId | string): Promise<HarnessTaskDocu
|
|
|
61
69
|
const childrenOf = (id: mongoose.Types.ObjectId): Promise<HarnessTaskDocument[]> =>
|
|
62
70
|
TaskModel.find({"ownership.id": id, "ownership.kind": "task"}).sort({created: 1});
|
|
63
71
|
|
|
64
|
-
const asAny = (definition: unknown):
|
|
65
|
-
definition as AnyHarnessTaskDefinition;
|
|
72
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
66
73
|
|
|
67
|
-
const completesWith = (name: string, result: unknown):
|
|
74
|
+
const completesWith = (name: string, result: unknown): TestTaskDefinition =>
|
|
68
75
|
asAny(
|
|
69
76
|
defineTask<unknown, unknown, unknown>({
|
|
70
77
|
initial: () => ({phase: "work"}),
|
|
@@ -76,7 +83,7 @@ const completesWith = (name: string, result: unknown): AnyHarnessTaskDefinition
|
|
|
76
83
|
})
|
|
77
84
|
);
|
|
78
85
|
|
|
79
|
-
const failsWith = (name: string, error: string):
|
|
86
|
+
const failsWith = (name: string, error: string): TestTaskDefinition =>
|
|
80
87
|
asAny(
|
|
81
88
|
defineTask<unknown, unknown, unknown>({
|
|
82
89
|
initial: () => ({phase: "work"}),
|
|
@@ -103,11 +110,11 @@ const fanOut = ({
|
|
|
103
110
|
policy,
|
|
104
111
|
runs,
|
|
105
112
|
}: {
|
|
106
|
-
children:
|
|
113
|
+
children: TestTaskDefinition[];
|
|
107
114
|
name: string;
|
|
108
115
|
policy?: "all" | "failFast";
|
|
109
116
|
runs: {count: number};
|
|
110
|
-
}):
|
|
117
|
+
}): TestTaskDefinition =>
|
|
111
118
|
asAny(
|
|
112
119
|
defineTask<unknown, unknown, HarnessChildOutcome[]>({
|
|
113
120
|
initial: () => ({phase: "fanOut"}),
|
|
@@ -790,7 +797,9 @@ describe("Harness ownership tree", () => {
|
|
|
790
797
|
expect(second.status).toBe("aborted");
|
|
791
798
|
expect(await SpanModel.countDocuments({name: "abort", traceId: created.traceId})).toBe(1);
|
|
792
799
|
// The first request recorded wins; the other waited for its handler to finish.
|
|
793
|
-
expect(["first", "second"]).toContain(
|
|
800
|
+
expect<Array<string | undefined>>(["first", "second"]).toContain(
|
|
801
|
+
(await findTask(created._id)).abortRequested?.reason
|
|
802
|
+
);
|
|
794
803
|
});
|
|
795
804
|
|
|
796
805
|
it("keeps aborting the tree when a child's handler throws", async () => {
|
|
@@ -940,7 +949,7 @@ describe("Harness ownership tree", () => {
|
|
|
940
949
|
|
|
941
950
|
describe("waiting sweep", () => {
|
|
942
951
|
const buildEngine = (
|
|
943
|
-
definitions: ReadonlyArray<
|
|
952
|
+
definitions: ReadonlyArray<TestTaskDefinition> = []
|
|
944
953
|
): {engine: HarnessEngine; wakes: {count: number}} => {
|
|
945
954
|
const wakes = {count: 0};
|
|
946
955
|
const models = {
|
|
@@ -953,8 +962,10 @@ describe("Harness ownership tree", () => {
|
|
|
953
962
|
} as unknown as HarnessModels;
|
|
954
963
|
return {
|
|
955
964
|
engine: {
|
|
965
|
+
agents: new Map(),
|
|
956
966
|
controllers: new Map(),
|
|
957
967
|
definitions: new Map(definitions.map((definition) => [definition.key, definition])),
|
|
968
|
+
extensions: new Map(),
|
|
958
969
|
models,
|
|
959
970
|
wake: () => {
|
|
960
971
|
wakes.count += 1;
|
|
@@ -4,11 +4,14 @@ import {DateTime, Settings} from "luxon";
|
|
|
4
4
|
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
5
5
|
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
6
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
|
-
import type {HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
7
|
+
import type {HarnessTaskDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
8
8
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
9
9
|
import {type AnyHarnessTaskDefinition, defineTask, Harness, InProcessRunner} from "./harness";
|
|
10
10
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
11
11
|
|
|
12
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
13
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
14
|
+
|
|
12
15
|
const TaskModel = registerHarnessTask();
|
|
13
16
|
let SpanModel: ObsSpanModel;
|
|
14
17
|
let TraceModel: ObsTraceModel;
|
|
@@ -62,7 +65,7 @@ const flakyTask = ({
|
|
|
62
65
|
failures: number;
|
|
63
66
|
name: string;
|
|
64
67
|
retry: {backoffMs?: number; maxAttempts?: number; maxBackoffMs?: number};
|
|
65
|
-
}): {attemptsSeen: number[]; definition:
|
|
68
|
+
}): {attemptsSeen: number[]; definition: TestTaskDefinition} => {
|
|
66
69
|
const attemptsSeen: number[] = [];
|
|
67
70
|
const definition = defineTask<unknown, unknown, {ok: boolean}>({
|
|
68
71
|
initial: () => ({phase: "call"}),
|
|
@@ -81,7 +84,7 @@ const flakyTask = ({
|
|
|
81
84
|
retry,
|
|
82
85
|
version: 1,
|
|
83
86
|
});
|
|
84
|
-
return {attemptsSeen, definition: definition as unknown as
|
|
87
|
+
return {attemptsSeen, definition: definition as unknown as TestTaskDefinition};
|
|
85
88
|
};
|
|
86
89
|
|
|
87
90
|
describe("Harness phase retries", () => {
|
|
@@ -10,7 +10,7 @@ import {createLocalObservabilityPlugin} from "../../observability/local/localPlu
|
|
|
10
10
|
import {registerObsSpan} from "../../observability/local/models/obsSpan";
|
|
11
11
|
import {registerObsTrace} from "../../observability/local/models/obsTrace";
|
|
12
12
|
import {gatedStreamingModel} from "../../tests/harnessStreaming";
|
|
13
|
-
import {ensureTestUsers, UserModel} from "../../tests/helpers";
|
|
13
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
|
|
14
14
|
import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
|
|
15
15
|
import {defineAgent, defineTool, Harness, HarnessApp, InProcessRunner} from "../harness";
|
|
16
16
|
import {registerHarnessConversation} from "../models/harnessConversation";
|
|
@@ -88,7 +88,7 @@ const setup = async () => {
|
|
|
88
88
|
});
|
|
89
89
|
openHarnesses.push(harness);
|
|
90
90
|
await harness.start();
|
|
91
|
-
const app: express.Application = new TerrenoApp({skipListen: true, userModel:
|
|
91
|
+
const app: express.Application = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
92
92
|
.register(new HarnessApp({harness}))
|
|
93
93
|
.build();
|
|
94
94
|
const login = async ({email, password}: {email: string; password: string}) => {
|
|
@@ -8,7 +8,7 @@ import type TestAgent from "supertest/lib/agent";
|
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../../observability/local/localPlugin";
|
|
9
9
|
import {registerObsSpan} from "../../observability/local/models/obsSpan";
|
|
10
10
|
import {registerObsTrace} from "../../observability/local/models/obsTrace";
|
|
11
|
-
import {ensureTestUsers, UserModel} from "../../tests/helpers";
|
|
11
|
+
import {ensureTestUsers, terrenoUserModel, UserModel} from "../../tests/helpers";
|
|
12
12
|
import type {ObsSpanModel, ObsTraceModel} from "../../types/observability";
|
|
13
13
|
import {defineTask, Harness, HarnessApp, InProcessRunner} from "../harness";
|
|
14
14
|
import {registerHarnessEvent, registerHarnessEventStream} from "../models/harnessEvent";
|
|
@@ -55,7 +55,7 @@ const setup = async ({start = false}: {start?: boolean} = {}) => {
|
|
|
55
55
|
if (start) {
|
|
56
56
|
await harness.start();
|
|
57
57
|
}
|
|
58
|
-
const app: express.Application = new TerrenoApp({skipListen: true, userModel:
|
|
58
|
+
const app: express.Application = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
59
59
|
.register(new HarnessApp({harness}))
|
|
60
60
|
.build();
|
|
61
61
|
const login = async ({email, password}: {email: string; password: string}) => {
|