@terreno/ai 57.19.2 → 57.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/CLAUDE.local.md +1 -1
- package/.cursor/rules/00-root.mdc +1 -1
- package/.github/copilot-instructions.md +1 -1
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +443 -0
- package/biome.jsonc +1 -0
- package/dist/harness/agentLoop.js +1 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/types/harness.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -12
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/isolated/gptStream.isolated.ts +33 -28
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +58 -42
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/tsconfig.typecheck.json +18 -0
|
@@ -7,12 +7,16 @@ import {
|
|
|
7
7
|
registerMCPTool,
|
|
8
8
|
type User,
|
|
9
9
|
} from "@terreno/api";
|
|
10
|
+
import type {ToolExecutionOptions} from "ai";
|
|
10
11
|
import {assert} from "chai";
|
|
11
12
|
import mongoose, {Schema} from "mongoose";
|
|
12
13
|
import {z} from "zod";
|
|
13
14
|
|
|
14
15
|
import {getMCPTools} from "./getMCPTools";
|
|
15
16
|
|
|
17
|
+
/** `execute` options as the AI SDK passes them; getMCPTools ignores them. */
|
|
18
|
+
const callOptions: ToolExecutionOptions = {messages: [], toolCallId: "call-1"};
|
|
19
|
+
|
|
16
20
|
interface NoteFields {
|
|
17
21
|
_id: mongoose.Types.ObjectId;
|
|
18
22
|
title: string;
|
|
@@ -67,7 +71,7 @@ describe("getMCPTools", () => {
|
|
|
67
71
|
|
|
68
72
|
expect(Object.keys(tools).sort()).toEqual(["notes_create", "notes_list"]);
|
|
69
73
|
|
|
70
|
-
const created = await tools.notes_create.execute?.({title: "From AI tools"});
|
|
74
|
+
const created = await tools.notes_create.execute?.({title: "From AI tools"}, callOptions);
|
|
71
75
|
expect(created).toEqual(
|
|
72
76
|
expect.objectContaining({
|
|
73
77
|
data: expect.objectContaining({title: "From AI tools"}),
|
|
@@ -88,7 +92,7 @@ describe("getMCPTools", () => {
|
|
|
88
92
|
const tools = getMCPTools();
|
|
89
93
|
assert.deepEqual(Object.keys(tools), ["echo_json"]);
|
|
90
94
|
assert.equal(tools.echo_json.description, "Echoes structured json back to the caller");
|
|
91
|
-
assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}), {echoed: "hi"});
|
|
95
|
+
assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}, callOptions), {echoed: "hi"});
|
|
92
96
|
});
|
|
93
97
|
|
|
94
98
|
it("returns joined plain text when the tool response is not json", async () => {
|
|
@@ -105,7 +109,7 @@ describe("getMCPTools", () => {
|
|
|
105
109
|
});
|
|
106
110
|
|
|
107
111
|
const tools = getMCPTools();
|
|
108
|
-
assert.equal(await tools.prose.execute?.({}), "first line\nsecond line");
|
|
112
|
+
assert.equal(await tools.prose.execute?.({}, callOptions), "first line\nsecond line");
|
|
109
113
|
});
|
|
110
114
|
|
|
111
115
|
it("passes the authenticated user through to custom tool handlers", async () => {
|
|
@@ -121,8 +125,8 @@ describe("getMCPTools", () => {
|
|
|
121
125
|
});
|
|
122
126
|
|
|
123
127
|
const user = asUser();
|
|
124
|
-
await getMCPTools(user).whoami.execute?.({});
|
|
125
|
-
await getMCPTools().whoami.execute?.({});
|
|
128
|
+
await getMCPTools(user).whoami.execute?.({}, callOptions);
|
|
129
|
+
await getMCPTools().whoami.execute?.({}, callOptions);
|
|
126
130
|
assert.deepEqual(seenUserIds, [user.id, undefined]);
|
|
127
131
|
});
|
|
128
132
|
});
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
1
|
+
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
3
|
const clientImpl = {
|
|
4
4
|
close: mock(async () => {}),
|
|
5
5
|
tools: mock(async () => ({search: {description: "Search"}})),
|
|
6
6
|
};
|
|
7
|
-
const createMCPClientMock = mock(async () => clientImpl);
|
|
7
|
+
const createMCPClientMock = mock(async (_opts: Record<string, unknown>) => clientImpl);
|
|
8
8
|
mock.module("@ai-sdk/mcp", () => ({
|
|
9
9
|
createMCPClient: (opts: Record<string, unknown>) => createMCPClientMock(opts),
|
|
10
10
|
}));
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
let MCPService: typeof import("./mcpService.js").MCPService;
|
|
13
|
+
beforeAll(async () => {
|
|
14
|
+
({MCPService} = await import("./mcpService.js"));
|
|
15
|
+
});
|
|
13
16
|
|
|
14
17
|
const makeConfig = (overrides?: Partial<Record<string, unknown>>) => ({
|
|
15
18
|
name: "test-server",
|
package/src/tests/bunSetup.ts
CHANGED
|
@@ -7,7 +7,7 @@ registerBackendPreload({
|
|
|
7
7
|
connectMongoInBeforeAll: true,
|
|
8
8
|
mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
|
|
9
9
|
onBeforeEach: async () => {
|
|
10
|
-
const {shutdownLangfuseClient} = await import("../langfuseClient");
|
|
10
|
+
const {shutdownLangfuseClient} = await import("../langfuseClient.js");
|
|
11
11
|
await shutdownLangfuseClient();
|
|
12
12
|
},
|
|
13
13
|
testEnv: {
|
package/src/tests/chatHarness.ts
CHANGED
|
@@ -11,7 +11,7 @@ import {addGptRoutes} from "../routes/gpt";
|
|
|
11
11
|
import {addGptHistoryRoutes} from "../routes/gptHistories";
|
|
12
12
|
import {AIService} from "../service/aiService";
|
|
13
13
|
import type {GptHistoryDocument, GptRouteOptions} from "../types";
|
|
14
|
-
import {type authAsUser,
|
|
14
|
+
import {type authAsUser, historyRouterOptions, terrenoUserModel} from "./helpers";
|
|
15
15
|
|
|
16
16
|
type ModelStreamPart = {type: string; [key: string]: unknown};
|
|
17
17
|
|
|
@@ -232,7 +232,7 @@ export const createScriptedModel = ({
|
|
|
232
232
|
const remaining = [...steps];
|
|
233
233
|
let callCount = 0;
|
|
234
234
|
return {
|
|
235
|
-
doGenerate: mock(async () => ({
|
|
235
|
+
doGenerate: mock(async (_options: ModelCallOptions) => ({
|
|
236
236
|
content: [{text: "Workspace setup", type: "text" as const}],
|
|
237
237
|
finishReason: "stop" as const,
|
|
238
238
|
usage: USAGE,
|
|
@@ -348,11 +348,11 @@ export const buildApp = ({
|
|
|
348
348
|
openApiOptions: options,
|
|
349
349
|
...routeOptions,
|
|
350
350
|
};
|
|
351
|
-
addGptHistoryRoutes(router, {...options, chat});
|
|
351
|
+
addGptHistoryRoutes(router, {...historyRouterOptions(options), chat});
|
|
352
352
|
addGptRoutes(router, chat);
|
|
353
353
|
},
|
|
354
354
|
skipListen: true,
|
|
355
|
-
userModel:
|
|
355
|
+
userModel: terrenoUserModel,
|
|
356
356
|
}).build();
|
|
357
357
|
|
|
358
358
|
export interface ServedApp {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A stub for `AIService.generateJsonObject` that answers every call with `answer()`, whatever
|
|
3
|
+
* schema the caller asks for. The real method is generic over the parsed object's type.
|
|
4
|
+
*/
|
|
5
|
+
export const judgeAnswering =
|
|
6
|
+
(answer: () => Record<string, unknown>) =>
|
|
7
|
+
async <OBJECT>(): Promise<OBJECT> =>
|
|
8
|
+
answer() as OBJECT;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** The fields of a Langfuse prompt that tests stub `client.prompt.get` with. */
|
|
2
|
+
export interface FakeLangfusePrompt {
|
|
3
|
+
config: Record<string, unknown>;
|
|
4
|
+
labels: string[];
|
|
5
|
+
name: string;
|
|
6
|
+
prompt: string | Array<{content: string; role: string}>;
|
|
7
|
+
tags: string[];
|
|
8
|
+
type: "chat" | "text";
|
|
9
|
+
version: number;
|
|
10
|
+
}
|
package/src/tests/helpers.ts
CHANGED
|
@@ -1,10 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type UserModel as ApiUserModel,
|
|
3
|
+
createdUpdatedPlugin,
|
|
4
|
+
type ModelRouterOptions,
|
|
5
|
+
} from "@terreno/api";
|
|
2
6
|
import {authAsUser as authAsUserWithCredentials} from "@terreno/test";
|
|
3
7
|
import type express from "express";
|
|
4
|
-
import mongoose, {type Model} from "mongoose";
|
|
8
|
+
import mongoose, {type HydratedDocument, type Model} from "mongoose";
|
|
5
9
|
import passportLocalMongoose from "passport-local-mongoose";
|
|
6
10
|
import type TestAgent from "supertest/lib/agent";
|
|
7
11
|
|
|
12
|
+
import type {GptHistoryDocument} from "../types/index";
|
|
13
|
+
|
|
8
14
|
type PasswordedUser = {setPassword: (password: string) => Promise<void>};
|
|
9
15
|
|
|
10
16
|
export interface AiTestUser {
|
|
@@ -43,6 +49,22 @@ const createTestUserModel = (): Model<AiTestUserDoc> => {
|
|
|
43
49
|
|
|
44
50
|
export const UserModel = createTestUserModel();
|
|
45
51
|
|
|
52
|
+
/**
|
|
53
|
+
* `UserModel` as `TerrenoApp({userModel})` expects it. The passport-local-mongoose plugin adds
|
|
54
|
+
* the auth statics at runtime, and `@terreno/api`'s `User._id` uses mongoose's schema `ObjectId`
|
|
55
|
+
* type, which a hydrated document's `Types.ObjectId` does not satisfy.
|
|
56
|
+
*/
|
|
57
|
+
export const terrenoUserModel = UserModel as unknown as ApiUserModel;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* `TerrenoApp`'s `configureApp` options, typed as the GptHistory model router options tests
|
|
61
|
+
* spread them into. `configureApp` types them for no particular model.
|
|
62
|
+
*/
|
|
63
|
+
export const historyRouterOptions = (
|
|
64
|
+
options: Partial<ModelRouterOptions<unknown>> | undefined
|
|
65
|
+
): Partial<ModelRouterOptions<GptHistoryDocument>> =>
|
|
66
|
+
options as Partial<ModelRouterOptions<GptHistoryDocument>>;
|
|
67
|
+
|
|
46
68
|
const STANDARD_AI_TEST_USERS = {
|
|
47
69
|
admin: {admin: true, email: "admin@example.com", name: "Admin", password: "securePassword"},
|
|
48
70
|
notAdmin: {admin: false, email: "notAdmin@example.com", name: "User", password: "password"},
|
|
@@ -52,10 +74,10 @@ export type StandardAiTestUserRole = keyof typeof STANDARD_AI_TEST_USERS;
|
|
|
52
74
|
|
|
53
75
|
export const ensureTestUsers = async (
|
|
54
76
|
users: AiTestUser[] = [STANDARD_AI_TEST_USERS.admin, STANDARD_AI_TEST_USERS.notAdmin]
|
|
55
|
-
): Promise<AiTestUserDoc[]> => {
|
|
77
|
+
): Promise<HydratedDocument<AiTestUserDoc>[]> => {
|
|
56
78
|
await UserModel.deleteMany({email: {$in: users.map((user) => user.email)}});
|
|
57
79
|
|
|
58
|
-
const createdUsers: AiTestUserDoc[] = [];
|
|
80
|
+
const createdUsers: HydratedDocument<AiTestUserDoc>[] = [];
|
|
59
81
|
for (const user of users) {
|
|
60
82
|
const doc = await UserModel.create({
|
|
61
83
|
admin: user.admin ?? false,
|
package/src/types/harness.ts
CHANGED
|
@@ -625,9 +625,13 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown>
|
|
|
625
625
|
|
|
626
626
|
/**
|
|
627
627
|
* Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
|
|
628
|
-
* `never` accepts every tool in heterogeneous lists without an explicit `any`.
|
|
628
|
+
* `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
|
|
629
|
+
* output is covariant, so `parameters` widens to `unknown` instead.
|
|
629
630
|
*/
|
|
630
|
-
export
|
|
631
|
+
export interface AnyHarnessToolDefinition
|
|
632
|
+
extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
|
|
633
|
+
parameters: z.ZodType<unknown>;
|
|
634
|
+
}
|
|
631
635
|
|
|
632
636
|
export interface HarnessAgentDefinitionInput {
|
|
633
637
|
/** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "./tsconfig.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"noEmit": true,
|
|
5
|
+
"composite": false,
|
|
6
|
+
"incremental": false,
|
|
7
|
+
"declaration": false,
|
|
8
|
+
"declarationMap": false,
|
|
9
|
+
"types": [
|
|
10
|
+
"node",
|
|
11
|
+
"bun"
|
|
12
|
+
]
|
|
13
|
+
},
|
|
14
|
+
"exclude": [
|
|
15
|
+
"node_modules",
|
|
16
|
+
"dist"
|
|
17
|
+
]
|
|
18
|
+
}
|