@terreno/ai 57.19.2 → 57.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/CLAUDE.local.md +1 -1
- package/.cursor/rules/00-root.mdc +1 -1
- package/.github/copilot-instructions.md +1 -1
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +443 -0
- package/biome.jsonc +1 -0
- package/dist/harness/agentLoop.js +1 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/types/harness.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -12
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/isolated/gptStream.isolated.ts +33 -28
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +58 -42
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/tsconfig.typecheck.json +18 -0
package/dist/types/harness.d.ts
CHANGED
|
@@ -536,9 +536,12 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown> extends
|
|
|
536
536
|
}
|
|
537
537
|
/**
|
|
538
538
|
* Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
|
|
539
|
-
* `never` accepts every tool in heterogeneous lists without an explicit `any`.
|
|
539
|
+
* `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
|
|
540
|
+
* output is covariant, so `parameters` widens to `unknown` instead.
|
|
540
541
|
*/
|
|
541
|
-
export
|
|
542
|
+
export interface AnyHarnessToolDefinition extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
|
|
543
|
+
parameters: z.ZodType<unknown>;
|
|
544
|
+
}
|
|
542
545
|
export interface HarnessAgentDefinitionInput {
|
|
543
546
|
/** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
|
|
544
547
|
fallbackModels?: HarnessModelRef[];
|
package/e2e/smoke.ts
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Real-process smoke for @terreno/ai. Run `bun e2e/smoke.ts` after `bun run compile`.
|
|
3
|
+
*
|
|
4
|
+
* The smoke re-runs this file with `serve` as a child process: a minimal TerrenoApp from the
|
|
5
|
+
* compiled `@terreno/api` with the compiled `AiApp` registered, on a free port, against
|
|
6
|
+
* `smokeMongoUri()`. The child's `AIService` wraps a deterministic fake provider
|
|
7
|
+
* (`MockLanguageModelV3` from `ai/test`) and runs with every provider key unset. The parent then
|
|
8
|
+
* checks over real HTTP that a signed-in user's chat request streams back the fake provider's
|
|
9
|
+
* text. Exits non-zero on the first failed check.
|
|
10
|
+
*/
|
|
11
|
+
import {
|
|
12
|
+
freePort,
|
|
13
|
+
type SpawnedProcess,
|
|
14
|
+
smokeMongoUri,
|
|
15
|
+
spawnUntilReady,
|
|
16
|
+
stopSmokeMongo,
|
|
17
|
+
} from "@terreno/test";
|
|
18
|
+
import type express from "express";
|
|
19
|
+
import {DateTime} from "luxon";
|
|
20
|
+
import type {Schema} from "mongoose";
|
|
21
|
+
|
|
22
|
+
const SERVE_ARG = "serve";
|
|
23
|
+
const FAKE_REPLY = "Hello from the ai smoke's fake provider.";
|
|
24
|
+
const FAKE_TITLE = "Smoke chat";
|
|
25
|
+
|
|
26
|
+
/** Provider credentials the smoke unsets in the child, so no real provider can answer. */
|
|
27
|
+
const PROVIDER_ENV_KEYS = [
|
|
28
|
+
"ANTHROPIC_API_KEY",
|
|
29
|
+
"GEMINI_API_KEY",
|
|
30
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
31
|
+
"GOOGLE_GENERATIVE_AI_API_KEY",
|
|
32
|
+
"GOOGLE_VERTEX_PROJECT",
|
|
33
|
+
"LANGFUSE_PUBLIC_KEY",
|
|
34
|
+
"LANGFUSE_SECRET_KEY",
|
|
35
|
+
"OPENAI_API_KEY",
|
|
36
|
+
];
|
|
37
|
+
|
|
38
|
+
interface ChatEvent {
|
|
39
|
+
done?: boolean;
|
|
40
|
+
error?: string;
|
|
41
|
+
historyId?: string;
|
|
42
|
+
text?: string;
|
|
43
|
+
title?: string;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const ZERO_USAGE = {
|
|
47
|
+
inputTokens: {cacheRead: 0, cacheWrite: 0, noCache: 0, total: 0},
|
|
48
|
+
outputTokens: {reasoning: 0, text: 0, total: 0},
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Child: the compiled API with auth, the compiled AiApp on a fake model, and a health plugin.
|
|
53
|
+
* The packages are imported here, not at the top, so the parent process never loads them.
|
|
54
|
+
*/
|
|
55
|
+
const serve = async (): Promise<void> => {
|
|
56
|
+
const {createdUpdatedPlugin, logger, TerrenoApp} = await import("@terreno/api");
|
|
57
|
+
const {AIService, AiApp} = await import("@terreno/ai");
|
|
58
|
+
const {MockLanguageModelV3} = await import("ai/test");
|
|
59
|
+
const {default: mongoose} = await import("mongoose");
|
|
60
|
+
const {default: passportLocalMongoose} = await import("passport-local-mongoose");
|
|
61
|
+
|
|
62
|
+
const userSchema = new mongoose.Schema({
|
|
63
|
+
admin: {default: false, description: "Whether the user is an admin", type: Boolean},
|
|
64
|
+
email: {description: "Sign-in email", type: String},
|
|
65
|
+
name: {description: "Display name", type: String},
|
|
66
|
+
});
|
|
67
|
+
userSchema.plugin(passportLocalMongoose as unknown as (schema: Schema, options: object) => void, {
|
|
68
|
+
usernameCaseInsensitive: true,
|
|
69
|
+
usernameField: "email",
|
|
70
|
+
});
|
|
71
|
+
userSchema.plugin(createdUpdatedPlugin);
|
|
72
|
+
const User = mongoose.model("User", userSchema);
|
|
73
|
+
|
|
74
|
+
// Streams the chat reply and answers the title request, the two calls one chat turn makes.
|
|
75
|
+
const model = new MockLanguageModelV3({
|
|
76
|
+
doGenerate: async () => ({
|
|
77
|
+
content: [{text: FAKE_TITLE, type: "text"}],
|
|
78
|
+
finishReason: {raw: "stop", unified: "stop"},
|
|
79
|
+
usage: ZERO_USAGE,
|
|
80
|
+
warnings: [],
|
|
81
|
+
}),
|
|
82
|
+
doStream: async () => ({
|
|
83
|
+
stream: new ReadableStream({
|
|
84
|
+
start(controller): void {
|
|
85
|
+
controller.enqueue({type: "stream-start", warnings: []});
|
|
86
|
+
controller.enqueue({id: "text-1", type: "text-start"});
|
|
87
|
+
controller.enqueue({delta: FAKE_REPLY, id: "text-1", type: "text-delta"});
|
|
88
|
+
controller.enqueue({id: "text-1", type: "text-end"});
|
|
89
|
+
controller.enqueue({
|
|
90
|
+
finishReason: {raw: "stop", unified: "stop"},
|
|
91
|
+
type: "finish",
|
|
92
|
+
usage: ZERO_USAGE,
|
|
93
|
+
});
|
|
94
|
+
controller.close();
|
|
95
|
+
},
|
|
96
|
+
}),
|
|
97
|
+
}),
|
|
98
|
+
modelId: "ai-smoke-fake-model",
|
|
99
|
+
provider: "ai-smoke-fake-provider",
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const healthPlugin = {
|
|
103
|
+
register: (app: express.Application): void => {
|
|
104
|
+
app.get("/health", (_req, res) => {
|
|
105
|
+
res.json({healthy: mongoose.connection.readyState === 1});
|
|
106
|
+
});
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
await mongoose.connect(process.env.MONGO_URI as string);
|
|
111
|
+
const app = new TerrenoApp({
|
|
112
|
+
loggingOptions: {logRequests: false},
|
|
113
|
+
userModel: User as unknown as ConstructorParameters<typeof TerrenoApp>[0]["userModel"],
|
|
114
|
+
})
|
|
115
|
+
.register(new AiApp({aiService: new AIService({model})}))
|
|
116
|
+
.register(healthPlugin);
|
|
117
|
+
app.start();
|
|
118
|
+
await app.whenReady();
|
|
119
|
+
logger.info(`[aiSmoke] serving on port ${process.env.PORT}`);
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
const expectStatus = async (res: Response, status: number, step: string): Promise<string> => {
|
|
123
|
+
const body = await res.text();
|
|
124
|
+
if (res.status !== status) {
|
|
125
|
+
throw new Error(`${step}: expected ${status}, got ${res.status}: ${body}`);
|
|
126
|
+
}
|
|
127
|
+
return body;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/** Splits a finished `text/event-stream` body into its JSON `data:` events. */
|
|
131
|
+
const parseEvents = (body: string): ChatEvent[] =>
|
|
132
|
+
body
|
|
133
|
+
.split("\n\n")
|
|
134
|
+
.map((chunk) => chunk.trim())
|
|
135
|
+
.filter((chunk) => chunk.startsWith("data: "))
|
|
136
|
+
.map((chunk) => JSON.parse(chunk.slice("data: ".length)) as ChatEvent);
|
|
137
|
+
|
|
138
|
+
/** Parent: boots the child and sends one chat request over HTTP. */
|
|
139
|
+
const run = async (): Promise<void> => {
|
|
140
|
+
const startedAt = DateTime.now();
|
|
141
|
+
const port = await freePort();
|
|
142
|
+
const baseUrl = `http://127.0.0.1:${port}`;
|
|
143
|
+
let server: SpawnedProcess | undefined;
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
server = await spawnUntilReady({
|
|
147
|
+
cmd: [process.execPath, __filename, SERVE_ARG],
|
|
148
|
+
env: {
|
|
149
|
+
...Object.fromEntries(PROVIDER_ENV_KEYS.map((key) => [key, undefined])),
|
|
150
|
+
MONGO_URI: await smokeMongoUri({databaseName: "aiSmoke"}),
|
|
151
|
+
PORT: String(port),
|
|
152
|
+
REFRESH_TOKEN_SECRET: "ai-smoke-refresh-secret",
|
|
153
|
+
SESSION_SECRET: "ai-smoke-session-secret",
|
|
154
|
+
TOKEN_ISSUER: "ai-smoke",
|
|
155
|
+
TOKEN_SECRET: "ai-smoke-token-secret",
|
|
156
|
+
},
|
|
157
|
+
timeoutMs: 30000,
|
|
158
|
+
url: `${baseUrl}/health`,
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
const signup = JSON.parse(
|
|
162
|
+
await expectStatus(
|
|
163
|
+
await fetch(`${baseUrl}/auth/signup`, {
|
|
164
|
+
body: JSON.stringify({
|
|
165
|
+
email: `ai-smoke-${startedAt.toMillis()}@example.com`,
|
|
166
|
+
password: "ai-smoke-password-1",
|
|
167
|
+
}),
|
|
168
|
+
headers: {"Content-Type": "application/json"},
|
|
169
|
+
method: "POST",
|
|
170
|
+
}),
|
|
171
|
+
200,
|
|
172
|
+
"POST /auth/signup"
|
|
173
|
+
)
|
|
174
|
+
) as {data: {token: string}};
|
|
175
|
+
console.info("✅ POST /auth/signup signs in a user");
|
|
176
|
+
|
|
177
|
+
const chat = await fetch(`${baseUrl}/gpt/prompt`, {
|
|
178
|
+
body: JSON.stringify({prompt: "Say hello."}),
|
|
179
|
+
headers: {
|
|
180
|
+
Authorization: `Bearer ${signup.data.token}`,
|
|
181
|
+
"Content-Type": "application/json",
|
|
182
|
+
},
|
|
183
|
+
method: "POST",
|
|
184
|
+
});
|
|
185
|
+
const events = parseEvents(await expectStatus(chat, 200, "POST /gpt/prompt"));
|
|
186
|
+
const failed = events.find((event) => event.error);
|
|
187
|
+
if (failed) {
|
|
188
|
+
throw new Error(`POST /gpt/prompt: the stream reported an error: ${failed.error}`);
|
|
189
|
+
}
|
|
190
|
+
const reply = events.map((event) => event.text ?? "").join("");
|
|
191
|
+
if (reply !== FAKE_REPLY) {
|
|
192
|
+
throw new Error(
|
|
193
|
+
`POST /gpt/prompt: expected the fake provider's text, got ${JSON.stringify(events)}`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
const done = events.find((event) => event.done);
|
|
197
|
+
if (!done?.historyId || done.title !== FAKE_TITLE) {
|
|
198
|
+
throw new Error(
|
|
199
|
+
`POST /gpt/prompt: expected a done event with a history and the fake title, got ${JSON.stringify(done)}`
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
console.info("✅ POST /gpt/prompt streams the fake provider's reply");
|
|
203
|
+
} catch (error: unknown) {
|
|
204
|
+
if (server) {
|
|
205
|
+
console.error(`--- server output ---\n${server.output()}`);
|
|
206
|
+
}
|
|
207
|
+
throw error;
|
|
208
|
+
} finally {
|
|
209
|
+
await server?.stop();
|
|
210
|
+
await stopSmokeMongo();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
console.info(`ai smoke passed in ${DateTime.now().diff(startedAt).toMillis()} ms`);
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
const main = process.argv[2] === SERVE_ARG ? serve : run;
|
|
217
|
+
main().then(
|
|
218
|
+
() => {
|
|
219
|
+
if (main === run) {
|
|
220
|
+
process.exit(0);
|
|
221
|
+
}
|
|
222
|
+
},
|
|
223
|
+
(error: unknown) => {
|
|
224
|
+
console.error(`❌ ${error instanceof Error ? error.message : String(error)}`);
|
|
225
|
+
process.exit(1);
|
|
226
|
+
}
|
|
227
|
+
);
|
package/package.json
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"@langfuse/otel": "^5.0.1",
|
|
11
11
|
"@opentelemetry/api": "^1.9.0",
|
|
12
12
|
"@opentelemetry/sdk-node": "0.217.0",
|
|
13
|
-
"@terreno/blocks": "57.
|
|
13
|
+
"@terreno/blocks": "57.21.0",
|
|
14
14
|
"ai": "^6.0.97",
|
|
15
15
|
"ajv": "8.20.0",
|
|
16
16
|
"csv-parse": "^6.1.0",
|
|
@@ -22,10 +22,10 @@
|
|
|
22
22
|
"description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
|
|
23
23
|
"devDependencies": {
|
|
24
24
|
"@biomejs/biome": "^2.3.6",
|
|
25
|
-
"@terreno/admin-backend": "57.
|
|
26
|
-
"@terreno/api": "57.
|
|
27
|
-
"@terreno/jobs": "57.
|
|
28
|
-
"@terreno/test": "57.
|
|
25
|
+
"@terreno/admin-backend": "57.21.0",
|
|
26
|
+
"@terreno/api": "57.21.0",
|
|
27
|
+
"@terreno/jobs": "57.21.0",
|
|
28
|
+
"@terreno/test": "57.21.0",
|
|
29
29
|
"@types/bun": "^1.4.0",
|
|
30
30
|
"@types/express": "^5.0.6",
|
|
31
31
|
"@types/luxon": "^3.7.1",
|
|
@@ -136,8 +136,8 @@
|
|
|
136
136
|
"name": "@terreno/ai",
|
|
137
137
|
"peerDependencies": {
|
|
138
138
|
"@ai-sdk/google-vertex": "^4.0.142",
|
|
139
|
-
"@terreno/api": "57.
|
|
140
|
-
"@terreno/jobs": "57.
|
|
139
|
+
"@terreno/api": "57.21.0",
|
|
140
|
+
"@terreno/jobs": "57.21.0",
|
|
141
141
|
"mongoose": "^8.0.0 || ^9.0.0"
|
|
142
142
|
},
|
|
143
143
|
"peerDependenciesMeta": {
|
|
@@ -159,14 +159,16 @@
|
|
|
159
159
|
"compile": "bun tsc",
|
|
160
160
|
"compile:watch": "bun tsc -w",
|
|
161
161
|
"dev": "bun tsc -w",
|
|
162
|
-
"lint": "biome check ./src",
|
|
163
|
-
"lint:fix": "biome check --write ./src",
|
|
164
|
-
"lint:unsafefix": "biome check --fix --unsafe ./src",
|
|
162
|
+
"lint": "biome check ./src ./e2e",
|
|
163
|
+
"lint:fix": "biome check --write ./src ./e2e",
|
|
164
|
+
"lint:unsafefix": "biome check --fix --unsafe ./src ./e2e",
|
|
165
|
+
"smoke": "bun e2e/smoke.ts",
|
|
165
166
|
"test": "bun test --preload ./src/tests/bunSetup.ts && for f in ./src/isolated/*.isolated.ts; do bun test --preload ./src/tests/bunSetup.ts \"$f\" || exit 1; done",
|
|
166
167
|
"test:agent": "AGENT=1 bun test --only-failures --preload ./src/tests/bunSetup.ts && for f in ./src/isolated/*.isolated.ts; do AGENT=1 bun test --only-failures --preload ./src/tests/bunSetup.ts \"$f\" || exit 1; done",
|
|
167
168
|
"test:ci": "bun test --preload ./src/tests/bunSetup.ts && for f in ./src/isolated/*.isolated.ts; do bun test --preload ./src/tests/bunSetup.ts \"$f\" || exit 1; done",
|
|
168
|
-
"test:coverage": "bun run ../scripts/check-coverage.ts"
|
|
169
|
+
"test:coverage": "bun run ../scripts/check-coverage.ts",
|
|
170
|
+
"typecheck": "tsc -p tsconfig.typecheck.json"
|
|
169
171
|
},
|
|
170
172
|
"types": "dist/index.d.ts",
|
|
171
|
-
"version": "57.
|
|
173
|
+
"version": "57.21.0"
|
|
172
174
|
}
|
package/src/aiApp.test.ts
CHANGED
|
@@ -10,7 +10,12 @@ import {GptHistory} from "./models/gptHistory";
|
|
|
10
10
|
import type {FileStorageService} from "./service/fileStorage";
|
|
11
11
|
import type {MCPService} from "./service/mcpService";
|
|
12
12
|
import {createScriptedModel, modelCall, textStep, toolCallStep} from "./tests/chatHarness";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
authAsUserWithCredentials,
|
|
15
|
+
ensureTestUsers,
|
|
16
|
+
terrenoUserModel,
|
|
17
|
+
UserModel,
|
|
18
|
+
} from "./tests/helpers";
|
|
14
19
|
|
|
15
20
|
const AI_APP_TEST_USER = {
|
|
16
21
|
admin: false,
|
|
@@ -61,13 +66,13 @@ describe("AiApp", () => {
|
|
|
61
66
|
};
|
|
62
67
|
|
|
63
68
|
it("registers gpt and project routes by default", async () => {
|
|
64
|
-
const {AIService} = await import("./service/aiService");
|
|
69
|
+
const {AIService} = await import("./service/aiService.js");
|
|
65
70
|
const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
66
71
|
const plugin = new AiApp({aiService});
|
|
67
72
|
const app = new TerrenoApp({
|
|
68
73
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
69
74
|
skipListen: true,
|
|
70
|
-
userModel:
|
|
75
|
+
userModel: terrenoUserModel,
|
|
71
76
|
}).build();
|
|
72
77
|
|
|
73
78
|
const agent = await authAsUser(app);
|
|
@@ -80,13 +85,13 @@ describe("AiApp", () => {
|
|
|
80
85
|
});
|
|
81
86
|
|
|
82
87
|
it("skips the project routes with projects: false", async () => {
|
|
83
|
-
const {AIService} = await import("./service/aiService");
|
|
88
|
+
const {AIService} = await import("./service/aiService.js");
|
|
84
89
|
const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
85
90
|
const plugin = new AiApp({aiService, projects: false});
|
|
86
91
|
const app = new TerrenoApp({
|
|
87
92
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
88
93
|
skipListen: true,
|
|
89
|
-
userModel:
|
|
94
|
+
userModel: terrenoUserModel,
|
|
90
95
|
}).build();
|
|
91
96
|
|
|
92
97
|
const agent = await authAsUser(app);
|
|
@@ -95,14 +100,14 @@ describe("AiApp", () => {
|
|
|
95
100
|
});
|
|
96
101
|
|
|
97
102
|
it("passes asks to the gpt routes, so the model is offered the ask tools", async () => {
|
|
98
|
-
const {AIService} = await import("./service/aiService");
|
|
103
|
+
const {AIService} = await import("./service/aiService.js");
|
|
99
104
|
const model = createMockModel();
|
|
100
105
|
const aiService = new AIService({model: model as unknown as LanguageModel});
|
|
101
106
|
const plugin = new AiApp({aiService, asks: true});
|
|
102
107
|
const app = new TerrenoApp({
|
|
103
108
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
104
109
|
skipListen: true,
|
|
105
|
-
userModel:
|
|
110
|
+
userModel: terrenoUserModel,
|
|
106
111
|
}).build();
|
|
107
112
|
const agent = await authAsUser(app);
|
|
108
113
|
|
|
@@ -120,7 +125,7 @@ describe("AiApp", () => {
|
|
|
120
125
|
});
|
|
121
126
|
|
|
122
127
|
it("adds the documented headless turn actions to the history routes", async () => {
|
|
123
|
-
const {AIService} = await import("./service/aiService");
|
|
128
|
+
const {AIService} = await import("./service/aiService.js");
|
|
124
129
|
const model = createMockModel();
|
|
125
130
|
const aiService = new AIService({model: model as unknown as LanguageModel});
|
|
126
131
|
const app = new TerrenoApp({
|
|
@@ -129,7 +134,7 @@ describe("AiApp", () => {
|
|
|
129
134
|
router as unknown as express.Application
|
|
130
135
|
),
|
|
131
136
|
skipListen: true,
|
|
132
|
-
userModel:
|
|
137
|
+
userModel: terrenoUserModel,
|
|
133
138
|
}).build();
|
|
134
139
|
const agent = await authAsUser(app);
|
|
135
140
|
|
|
@@ -159,7 +164,7 @@ describe("AiApp", () => {
|
|
|
159
164
|
});
|
|
160
165
|
|
|
161
166
|
it("adds the headless actions only with asks on, so hosts without asks get no new endpoints", async () => {
|
|
162
|
-
const {AIService} = await import("./service/aiService");
|
|
167
|
+
const {AIService} = await import("./service/aiService.js");
|
|
163
168
|
const historyRoutes = async (asks?: boolean) => {
|
|
164
169
|
const model = createMockModel();
|
|
165
170
|
const aiService = new AIService({model: model as unknown as LanguageModel});
|
|
@@ -169,7 +174,7 @@ describe("AiApp", () => {
|
|
|
169
174
|
router as unknown as express.Application
|
|
170
175
|
),
|
|
171
176
|
skipListen: true,
|
|
172
|
-
userModel:
|
|
177
|
+
userModel: terrenoUserModel,
|
|
173
178
|
}).build();
|
|
174
179
|
const agent = await authAsUser(app);
|
|
175
180
|
const openApi = await agent.get("/openapi.json");
|
|
@@ -226,7 +231,7 @@ describe("AiApp", () => {
|
|
|
226
231
|
const app = new TerrenoApp({
|
|
227
232
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
228
233
|
skipListen: true,
|
|
229
|
-
userModel:
|
|
234
|
+
userModel: terrenoUserModel,
|
|
230
235
|
}).build();
|
|
231
236
|
// Unauthenticated access to upload route: ensure it exists (returns 401, not 404).
|
|
232
237
|
const upload = await supertest(app).post("/files/upload");
|
|
@@ -234,8 +239,8 @@ describe("AiApp", () => {
|
|
|
234
239
|
});
|
|
235
240
|
|
|
236
241
|
it("caps uploads at asks.maxFileSizeBytes and loads a files answer's uploads from storage", async () => {
|
|
237
|
-
const {AIService} = await import("./service/aiService");
|
|
238
|
-
const {FileAttachment} = await import("./models/fileAttachment");
|
|
242
|
+
const {AIService} = await import("./service/aiService.js");
|
|
243
|
+
const {FileAttachment} = await import("./models/fileAttachment.js");
|
|
239
244
|
const text = Buffer.from("Coffee, 4.50\n");
|
|
240
245
|
const upload = mock(async () => {
|
|
241
246
|
const [user] = await UserModel.find({email: AI_APP_TEST_USER.email});
|
|
@@ -278,7 +283,7 @@ describe("AiApp", () => {
|
|
|
278
283
|
const app = new TerrenoApp({
|
|
279
284
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
280
285
|
skipListen: true,
|
|
281
|
-
userModel:
|
|
286
|
+
userModel: terrenoUserModel,
|
|
282
287
|
}).build();
|
|
283
288
|
const agent = await authAsUser(app);
|
|
284
289
|
|
|
@@ -329,7 +334,7 @@ describe("AiApp", () => {
|
|
|
329
334
|
const app = new TerrenoApp({
|
|
330
335
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
331
336
|
skipListen: true,
|
|
332
|
-
userModel:
|
|
337
|
+
userModel: terrenoUserModel,
|
|
333
338
|
}).build();
|
|
334
339
|
const upload = await supertest(app).post("/files/upload");
|
|
335
340
|
expect(upload.status).toBe(404);
|
|
@@ -345,7 +350,7 @@ describe("AiApp", () => {
|
|
|
345
350
|
const app = new TerrenoApp({
|
|
346
351
|
configureApp: (router) => plugin.register(router as unknown as express.Application),
|
|
347
352
|
skipListen: true,
|
|
348
|
-
userModel:
|
|
353
|
+
userModel: terrenoUserModel,
|
|
349
354
|
}).build();
|
|
350
355
|
const agent = await authAsUser(app);
|
|
351
356
|
const servers = await agent.get("/mcp/servers");
|
|
@@ -180,7 +180,7 @@ describe("Agent turns", () => {
|
|
|
180
180
|
beforeAll(async () => {
|
|
181
181
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
182
182
|
// cold first import can exceed it, so load it here first.
|
|
183
|
-
await import("../langfuseClient");
|
|
183
|
+
await import("../langfuseClient.js");
|
|
184
184
|
createLocalObservabilityPlugin();
|
|
185
185
|
SpanModel = registerObsSpan();
|
|
186
186
|
TraceModel = registerObsTrace();
|
package/src/harness/agentLoop.ts
CHANGED
|
@@ -653,7 +653,8 @@ export const createAgentTasks = (context: AgentLoopContext): AgentTasks => {
|
|
|
653
653
|
}
|
|
654
654
|
let returned: unknown;
|
|
655
655
|
try {
|
|
656
|
-
|
|
656
|
+
// `args.data` was parsed by this tool's own `parameters`, which match its `execute` args.
|
|
657
|
+
returned = await tool.execute(args.data as never, {
|
|
657
658
|
conversationId,
|
|
658
659
|
env: rt.env,
|
|
659
660
|
output: (text: string) => {
|
|
@@ -44,38 +44,41 @@ type Reply = {
|
|
|
44
44
|
};
|
|
45
45
|
|
|
46
46
|
/** LanguageModelV2 mock answering each request with the next reply; `hang` blocks it first. */
|
|
47
|
-
const scriptedModel = (replies: Reply[]) =>
|
|
48
|
-
withGenerateStreaming
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
47
|
+
const scriptedModel = (replies: Reply[]) => {
|
|
48
|
+
// Declared before the call so `withGenerateStreaming` infers the mock's own type.
|
|
49
|
+
const doGenerate = mock(async () => {
|
|
50
|
+
const reply = replies.shift();
|
|
51
|
+
if (!reply) {
|
|
52
|
+
throw new Error("script exhausted");
|
|
53
|
+
}
|
|
54
|
+
reply.started?.release();
|
|
55
|
+
if (reply.hang) {
|
|
56
|
+
await reply.hang.promise;
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
content: reply.toolCall
|
|
60
|
+
? [
|
|
61
|
+
{
|
|
62
|
+
input: "{}",
|
|
63
|
+
toolCallId: reply.toolCall.id,
|
|
64
|
+
toolName: reply.toolCall.name,
|
|
65
|
+
type: "tool-call" as const,
|
|
66
|
+
},
|
|
67
|
+
]
|
|
68
|
+
: [{text: reply.text ?? "", type: "text" as const}],
|
|
69
|
+
finishReason: reply.toolCall ? ("tool-calls" as const) : ("stop" as const),
|
|
70
|
+
usage: {inputTokens: 1, outputTokens: 1, totalTokens: 2},
|
|
71
|
+
warnings: [],
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
return withGenerateStreaming({
|
|
75
|
+
doGenerate,
|
|
74
76
|
modelId: "mock-model",
|
|
75
77
|
provider: "mock",
|
|
76
78
|
specificationVersion: "v2" as const,
|
|
77
79
|
supportedUrls: {},
|
|
78
80
|
});
|
|
81
|
+
};
|
|
79
82
|
|
|
80
83
|
interface ProcessHandle {
|
|
81
84
|
die: () => void;
|
|
@@ -169,7 +172,7 @@ describe("Agent turns across a crash", () => {
|
|
|
169
172
|
beforeAll(async () => {
|
|
170
173
|
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
171
174
|
// cold first import can exceed it, so load it here first.
|
|
172
|
-
await import("../langfuseClient");
|
|
175
|
+
await import("../langfuseClient.js");
|
|
173
176
|
createLocalObservabilityPlugin();
|
|
174
177
|
SpanModel = registerObsSpan();
|
|
175
178
|
TraceModel = registerObsTrace();
|
|
@@ -21,10 +21,11 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
|
21
21
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
22
22
|
import {withGenerateStreaming} from "../tests/generateStream";
|
|
23
23
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
24
|
-
import {authAsUser, ensureTestUsers, UserModel} from "../tests/helpers";
|
|
24
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
25
25
|
import type {
|
|
26
26
|
HarnessApprovalDocument,
|
|
27
27
|
HarnessApprovalResult,
|
|
28
|
+
HarnessTaskDefinition,
|
|
28
29
|
HarnessTaskDocument,
|
|
29
30
|
HarnessTestHooks,
|
|
30
31
|
} from "../types/harness";
|
|
@@ -32,7 +33,6 @@ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
|
32
33
|
import type {HarnessModels} from "./commit";
|
|
33
34
|
import {
|
|
34
35
|
AGENT_TOOL_TASK_NAME,
|
|
35
|
-
type AnyHarnessTaskDefinition,
|
|
36
36
|
approvalGate,
|
|
37
37
|
approvalTaskInput,
|
|
38
38
|
defineAgent,
|
|
@@ -55,6 +55,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
|
|
|
55
55
|
import {registerHarnessTask} from "./models/harnessTask";
|
|
56
56
|
import {HarnessWaitRaceError, sendEventRecords} from "./waits";
|
|
57
57
|
|
|
58
|
+
/** A task definition whose input is not checked, so tests can create it with any input. */
|
|
59
|
+
type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
|
|
60
|
+
|
|
58
61
|
// Whole tasks and turns on a real replica set; the first test also pays for index creation.
|
|
59
62
|
setDefaultTimeout(30_000);
|
|
60
63
|
|
|
@@ -107,8 +110,7 @@ const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
|
|
|
107
110
|
const approvalOf = (taskId: unknown): Promise<HarnessApprovalDocument> =>
|
|
108
111
|
ApprovalModel.findExactlyOne({taskId});
|
|
109
112
|
|
|
110
|
-
const asAny = (definition: unknown):
|
|
111
|
-
definition as AnyHarnessTaskDefinition;
|
|
113
|
+
const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
|
|
112
114
|
|
|
113
115
|
const openHarness = async (
|
|
114
116
|
registry: ReadonlyArray<HarnessRegistryEntry>,
|
|
@@ -138,7 +140,7 @@ const openHarness = async (
|
|
|
138
140
|
};
|
|
139
141
|
|
|
140
142
|
const buildApp = (harness: Harness): express.Application =>
|
|
141
|
-
new TerrenoApp({skipListen: true, userModel:
|
|
143
|
+
new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
142
144
|
.register(new HarnessApp({harness}))
|
|
143
145
|
.build();
|
|
144
146
|
|
|
@@ -176,7 +178,7 @@ const asksApproval = ({
|
|
|
176
178
|
retry?: {maxAttempts: number};
|
|
177
179
|
runs?: {count: number};
|
|
178
180
|
timeout?: {hours: number};
|
|
179
|
-
}):
|
|
181
|
+
}): TestTaskDefinition =>
|
|
180
182
|
asAny(
|
|
181
183
|
defineTask<{patientId: string}, unknown, HarnessApprovalResult>({
|
|
182
184
|
approvals,
|
|
@@ -563,7 +565,7 @@ describe("Harness approvals", () => {
|
|
|
563
565
|
|
|
564
566
|
describe("races and forged events", () => {
|
|
565
567
|
/** Past the approval's timeout with the runner stopped, so the next owner resolves it. */
|
|
566
|
-
const parkThenPassTimeout = async (definition:
|
|
568
|
+
const parkThenPassTimeout = async (definition: TestTaskDefinition) => {
|
|
567
569
|
const start = DateTime.now();
|
|
568
570
|
Settings.now = () => start.toMillis();
|
|
569
571
|
const first = await openHarness([definition]);
|
|
@@ -762,7 +764,7 @@ describe("Harness approvals", () => {
|
|
|
762
764
|
});
|
|
763
765
|
|
|
764
766
|
it("fails the task on rt.approval misuse and rejects invalid approval policies", async () => {
|
|
765
|
-
const misuse = (options: unknown, key = "k"):
|
|
767
|
+
const misuse = (options: unknown, key = "k"): TestTaskDefinition =>
|
|
766
768
|
asAny(
|
|
767
769
|
defineTask<unknown, unknown, unknown>({
|
|
768
770
|
initial: () => ({phase: "ask"}),
|
|
@@ -778,7 +780,7 @@ describe("Harness approvals", () => {
|
|
|
778
780
|
version: 1,
|
|
779
781
|
})
|
|
780
782
|
);
|
|
781
|
-
const cases: Array<[
|
|
783
|
+
const cases: Array<[TestTaskDefinition, string]> = [
|
|
782
784
|
[misuse({title: "t"}, " "), "rt.approval requires a key"],
|
|
783
785
|
[misuse({}), 'rt.approval("k") requires a title'],
|
|
784
786
|
[misuse({summary: 3, title: "t"}), 'rt.approval("k") summary must be a string'],
|