@terreno/ai 57.20.0 → 57.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/biome.jsonc +1 -0
  3. package/dist/harness/agentLoop.js +1 -0
  4. package/dist/observability/local/experimentRunner.d.ts +2 -5
  5. package/dist/types/harness.d.ts +5 -2
  6. package/e2e/smoke.ts +227 -0
  7. package/package.json +14 -12
  8. package/src/aiApp.test.ts +22 -17
  9. package/src/harness/agentLoop.test.ts +1 -1
  10. package/src/harness/agentLoop.ts +2 -1
  11. package/src/harness/agentRecovery.test.ts +31 -28
  12. package/src/harness/approvals.test.ts +11 -9
  13. package/src/harness/events.test.ts +10 -6
  14. package/src/harness/extensions.test.ts +7 -6
  15. package/src/harness/ownership.test.ts +20 -9
  16. package/src/harness/retries.test.ts +6 -3
  17. package/src/harness/routes/conversations.test.ts +2 -2
  18. package/src/harness/routes/tasks.test.ts +2 -2
  19. package/src/harness/runners/jobsRunner.test.ts +1 -1
  20. package/src/harness/subagent.test.ts +3 -2
  21. package/src/harness/waits.test.ts +6 -4
  22. package/src/index.test.ts +3 -3
  23. package/src/isolated/gptStream.isolated.ts +33 -28
  24. package/src/isolated/langfuseClient.isolated.ts +9 -3
  25. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  26. package/src/langfuseApp.test.ts +4 -20
  27. package/src/langfuseRoutes.test.ts +20 -15
  28. package/src/langfuseTracing.test.ts +6 -2
  29. package/src/langfuseVercelAi.test.ts +12 -9
  30. package/src/models/gptHistory.test.ts +10 -6
  31. package/src/observability/evaluate.test.ts +12 -10
  32. package/src/observability/local/experimentRunner.test.ts +9 -12
  33. package/src/observability/local/experimentRunner.ts +2 -2
  34. package/src/observability/local/promptStore.test.ts +4 -3
  35. package/src/observability/routes/datasets.test.ts +2 -2
  36. package/src/observability/routes/evaluators.test.ts +2 -2
  37. package/src/observability/routes/experiments.test.ts +2 -2
  38. package/src/observability/routes/prompts.test.ts +3 -3
  39. package/src/observability/routes/review.test.ts +2 -2
  40. package/src/observability/routes/status.test.ts +4 -4
  41. package/src/observability/routes/testMultiStage.test.ts +6 -6
  42. package/src/observability/routes/traces.test.ts +2 -2
  43. package/src/routes/files.test.ts +5 -3
  44. package/src/routes/gpt.test.ts +21 -14
  45. package/src/routes/gptActions.test.ts +7 -8
  46. package/src/routes/gptDatasets.test.ts +1 -1
  47. package/src/routes/gptFiles.test.ts +1 -1
  48. package/src/routes/gptHistories.test.ts +15 -9
  49. package/src/routes/mcp.test.ts +3 -3
  50. package/src/routes/projects.test.ts +5 -3
  51. package/src/routes/routes.test.ts +58 -42
  52. package/src/service/aiService.test.ts +1 -1
  53. package/src/service/asks.test.ts +14 -6
  54. package/src/service/fileStorage.test.ts +5 -2
  55. package/src/service/getMCPTools.test.ts +9 -5
  56. package/src/service/mcpService.test.ts +6 -3
  57. package/src/tests/bunSetup.ts +1 -1
  58. package/src/tests/chatHarness.ts +4 -4
  59. package/src/tests/fakeJudge.ts +8 -0
  60. package/src/tests/fakeLangfusePrompt.ts +10 -0
  61. package/src/tests/helpers.ts +26 -4
  62. package/src/types/harness.ts +6 -2
  63. package/tsconfig.typecheck.json +18 -0
@@ -13,7 +13,7 @@ import {AIService, TemperaturePresets} from "./aiService";
13
13
  // Create a mock LanguageModelV2
14
14
  const createMockModel = (responseText = "Mock response") => {
15
15
  return {
16
- doGenerate: mock(async () => ({
16
+ doGenerate: mock(async (_options: unknown) => ({
17
17
  content: [{text: responseText, type: "text" as const}],
18
18
  finishReason: "stop" as const,
19
19
  usage: {inputTokens: 5, outputTokens: 10},
@@ -1,6 +1,14 @@
1
1
  import {describe, expect, it} from "bun:test";
2
2
  import {ASK_LIMITS, type AskKind, type AskResponse, askPromptSection} from "@terreno/blocks";
3
- import {asSchema, jsonSchema, type ModelMessage, type Tool, tool} from "ai";
3
+ import {
4
+ asSchema,
5
+ type JSONValue,
6
+ jsonSchema,
7
+ type ModelMessage,
8
+ type Tool,
9
+ type ToolResultPart,
10
+ tool,
11
+ } from "ai";
4
12
 
5
13
  import {
6
14
  approvalAskInput,
@@ -24,7 +32,7 @@ const PLAN_ASK_INPUT = {
24
32
  {id: "team", label: "Team"},
25
33
  ],
26
34
  prompt: "Which plan should I set up?",
27
- select: "one",
35
+ select: "one" as const,
28
36
  };
29
37
 
30
38
  const TEAM_ANSWER = {action: "accept" as const, content: {selected: ["team"]}};
@@ -57,8 +65,8 @@ const lookupResult = {
57
65
  type: "tool-result" as const,
58
66
  };
59
67
 
60
- const answerResult = (toolCallId: string, value: unknown) => ({
61
- output: {type: "json", value},
68
+ const answerResult = (toolCallId: string, value: unknown): ToolResultPart => ({
69
+ output: {type: "json", value: value as JSONValue},
62
70
  toolCallId,
63
71
  toolName: "ask_choice",
64
72
  type: "tool-result",
@@ -76,14 +84,14 @@ describe("asks", () => {
76
84
  expected: ["confirm"],
77
85
  label: '{kinds: ["confirm"]}',
78
86
  },
79
- {asks: {kinds: []}, expected: [], label: "{kinds: []}"},
87
+ {asks: {kinds: [] as AskKind[]}, expected: [], label: "{kinds: []}"},
80
88
  {
81
89
  asks: {kinds: ["choice", "choice"] as AskKind[]},
82
90
  expected: ["choice"],
83
91
  label: '{kinds: ["choice", "choice"]}',
84
92
  },
85
93
  ])("resolves $label", ({asks, expected}) => {
86
- expect(resolveAskKinds(asks)).toEqual(expected);
94
+ expect(resolveAskKinds(asks)).toEqual([...expected]);
87
95
  });
88
96
 
89
97
  it("throws on an unknown kind and names the known kinds", () => {
@@ -1,4 +1,4 @@
1
- import {afterEach, beforeEach, describe, expect, it} from "bun:test";
1
+ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
2
  import {registerGcsMock} from "@terreno/test";
3
3
  import mongoose from "mongoose";
4
4
 
@@ -7,7 +7,10 @@ import {FileAttachment} from "../models/fileAttachment";
7
7
  // In-memory @google-cloud/storage so the service doesn't try to authenticate
8
8
  const gcs = registerGcsMock();
9
9
 
10
- const {FileStorageService} = await import("./fileStorage");
10
+ let FileStorageService: typeof import("./fileStorage.js").FileStorageService;
11
+ beforeAll(async () => {
12
+ ({FileStorageService} = await import("./fileStorage.js"));
13
+ });
11
14
 
12
15
  describe("FileStorageService", () => {
13
16
  beforeEach(async () => {
@@ -7,12 +7,16 @@ import {
7
7
  registerMCPTool,
8
8
  type User,
9
9
  } from "@terreno/api";
10
+ import type {ToolExecutionOptions} from "ai";
10
11
  import {assert} from "chai";
11
12
  import mongoose, {Schema} from "mongoose";
12
13
  import {z} from "zod";
13
14
 
14
15
  import {getMCPTools} from "./getMCPTools";
15
16
 
17
+ /** `execute` options as the AI SDK passes them; getMCPTools ignores them. */
18
+ const callOptions: ToolExecutionOptions = {messages: [], toolCallId: "call-1"};
19
+
16
20
  interface NoteFields {
17
21
  _id: mongoose.Types.ObjectId;
18
22
  title: string;
@@ -67,7 +71,7 @@ describe("getMCPTools", () => {
67
71
 
68
72
  expect(Object.keys(tools).sort()).toEqual(["notes_create", "notes_list"]);
69
73
 
70
- const created = await tools.notes_create.execute?.({title: "From AI tools"});
74
+ const created = await tools.notes_create.execute?.({title: "From AI tools"}, callOptions);
71
75
  expect(created).toEqual(
72
76
  expect.objectContaining({
73
77
  data: expect.objectContaining({title: "From AI tools"}),
@@ -88,7 +92,7 @@ describe("getMCPTools", () => {
88
92
  const tools = getMCPTools();
89
93
  assert.deepEqual(Object.keys(tools), ["echo_json"]);
90
94
  assert.equal(tools.echo_json.description, "Echoes structured json back to the caller");
91
- assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}), {echoed: "hi"});
95
+ assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}, callOptions), {echoed: "hi"});
92
96
  });
93
97
 
94
98
  it("returns joined plain text when the tool response is not json", async () => {
@@ -105,7 +109,7 @@ describe("getMCPTools", () => {
105
109
  });
106
110
 
107
111
  const tools = getMCPTools();
108
- assert.equal(await tools.prose.execute?.({}), "first line\nsecond line");
112
+ assert.equal(await tools.prose.execute?.({}, callOptions), "first line\nsecond line");
109
113
  });
110
114
 
111
115
  it("passes the authenticated user through to custom tool handlers", async () => {
@@ -121,8 +125,8 @@ describe("getMCPTools", () => {
121
125
  });
122
126
 
123
127
  const user = asUser();
124
- await getMCPTools(user).whoami.execute?.({});
125
- await getMCPTools().whoami.execute?.({});
128
+ await getMCPTools(user).whoami.execute?.({}, callOptions);
129
+ await getMCPTools().whoami.execute?.({}, callOptions);
126
130
  assert.deepEqual(seenUserIds, [user.id, undefined]);
127
131
  });
128
132
  });
@@ -1,15 +1,18 @@
1
- import {beforeEach, describe, expect, it, mock} from "bun:test";
1
+ import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
2
2
 
3
3
  const clientImpl = {
4
4
  close: mock(async () => {}),
5
5
  tools: mock(async () => ({search: {description: "Search"}})),
6
6
  };
7
- const createMCPClientMock = mock(async () => clientImpl);
7
+ const createMCPClientMock = mock(async (_opts: Record<string, unknown>) => clientImpl);
8
8
  mock.module("@ai-sdk/mcp", () => ({
9
9
  createMCPClient: (opts: Record<string, unknown>) => createMCPClientMock(opts),
10
10
  }));
11
11
 
12
- const {MCPService} = await import("./mcpService");
12
+ let MCPService: typeof import("./mcpService.js").MCPService;
13
+ beforeAll(async () => {
14
+ ({MCPService} = await import("./mcpService.js"));
15
+ });
13
16
 
14
17
  const makeConfig = (overrides?: Partial<Record<string, unknown>>) => ({
15
18
  name: "test-server",
@@ -7,7 +7,7 @@ registerBackendPreload({
7
7
  connectMongoInBeforeAll: true,
8
8
  mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
9
9
  onBeforeEach: async () => {
10
- const {shutdownLangfuseClient} = await import("../langfuseClient");
10
+ const {shutdownLangfuseClient} = await import("../langfuseClient.js");
11
11
  await shutdownLangfuseClient();
12
12
  },
13
13
  testEnv: {
@@ -11,7 +11,7 @@ import {addGptRoutes} from "../routes/gpt";
11
11
  import {addGptHistoryRoutes} from "../routes/gptHistories";
12
12
  import {AIService} from "../service/aiService";
13
13
  import type {GptHistoryDocument, GptRouteOptions} from "../types";
14
- import {type authAsUser, UserModel} from "./helpers";
14
+ import {type authAsUser, historyRouterOptions, terrenoUserModel} from "./helpers";
15
15
 
16
16
  type ModelStreamPart = {type: string; [key: string]: unknown};
17
17
 
@@ -232,7 +232,7 @@ export const createScriptedModel = ({
232
232
  const remaining = [...steps];
233
233
  let callCount = 0;
234
234
  return {
235
- doGenerate: mock(async () => ({
235
+ doGenerate: mock(async (_options: ModelCallOptions) => ({
236
236
  content: [{text: "Workspace setup", type: "text" as const}],
237
237
  finishReason: "stop" as const,
238
238
  usage: USAGE,
@@ -348,11 +348,11 @@ export const buildApp = ({
348
348
  openApiOptions: options,
349
349
  ...routeOptions,
350
350
  };
351
- addGptHistoryRoutes(router, {...options, chat});
351
+ addGptHistoryRoutes(router, {...historyRouterOptions(options), chat});
352
352
  addGptRoutes(router, chat);
353
353
  },
354
354
  skipListen: true,
355
- userModel: UserModel,
355
+ userModel: terrenoUserModel,
356
356
  }).build();
357
357
 
358
358
  export interface ServedApp {
@@ -0,0 +1,8 @@
1
+ /**
2
+ * A stub for `AIService.generateJsonObject` that answers every call with `answer()`, whatever
3
+ * schema the caller asks for. The real method is generic over the parsed object's type.
4
+ */
5
+ export const judgeAnswering =
6
+ (answer: () => Record<string, unknown>) =>
7
+ async <OBJECT>(): Promise<OBJECT> =>
8
+ answer() as OBJECT;
@@ -0,0 +1,10 @@
1
+ /** The fields of a Langfuse prompt that tests stub `client.prompt.get` with. */
2
+ export interface FakeLangfusePrompt {
3
+ config: Record<string, unknown>;
4
+ labels: string[];
5
+ name: string;
6
+ prompt: string | Array<{content: string; role: string}>;
7
+ tags: string[];
8
+ type: "chat" | "text";
9
+ version: number;
10
+ }
@@ -1,10 +1,16 @@
1
- import {createdUpdatedPlugin} from "@terreno/api";
1
+ import {
2
+ type UserModel as ApiUserModel,
3
+ createdUpdatedPlugin,
4
+ type ModelRouterOptions,
5
+ } from "@terreno/api";
2
6
  import {authAsUser as authAsUserWithCredentials} from "@terreno/test";
3
7
  import type express from "express";
4
- import mongoose, {type Model} from "mongoose";
8
+ import mongoose, {type HydratedDocument, type Model} from "mongoose";
5
9
  import passportLocalMongoose from "passport-local-mongoose";
6
10
  import type TestAgent from "supertest/lib/agent";
7
11
 
12
+ import type {GptHistoryDocument} from "../types/index";
13
+
8
14
  type PasswordedUser = {setPassword: (password: string) => Promise<void>};
9
15
 
10
16
  export interface AiTestUser {
@@ -43,6 +49,22 @@ const createTestUserModel = (): Model<AiTestUserDoc> => {
43
49
 
44
50
  export const UserModel = createTestUserModel();
45
51
 
52
+ /**
53
+ * `UserModel` as `TerrenoApp({userModel})` expects it. The passport-local-mongoose plugin adds
54
+ * the auth statics at runtime, and `@terreno/api`'s `User._id` uses mongoose's schema `ObjectId`
55
+ * type, which a hydrated document's `Types.ObjectId` does not satisfy.
56
+ */
57
+ export const terrenoUserModel = UserModel as unknown as ApiUserModel;
58
+
59
+ /**
60
+ * `TerrenoApp`'s `configureApp` options, typed as the GptHistory model router options tests
61
+ * spread them into. `configureApp` types them for no particular model.
62
+ */
63
+ export const historyRouterOptions = (
64
+ options: Partial<ModelRouterOptions<unknown>> | undefined
65
+ ): Partial<ModelRouterOptions<GptHistoryDocument>> =>
66
+ options as Partial<ModelRouterOptions<GptHistoryDocument>>;
67
+
46
68
  const STANDARD_AI_TEST_USERS = {
47
69
  admin: {admin: true, email: "admin@example.com", name: "Admin", password: "securePassword"},
48
70
  notAdmin: {admin: false, email: "notAdmin@example.com", name: "User", password: "password"},
@@ -52,10 +74,10 @@ export type StandardAiTestUserRole = keyof typeof STANDARD_AI_TEST_USERS;
52
74
 
53
75
  export const ensureTestUsers = async (
54
76
  users: AiTestUser[] = [STANDARD_AI_TEST_USERS.admin, STANDARD_AI_TEST_USERS.notAdmin]
55
- ): Promise<AiTestUserDoc[]> => {
77
+ ): Promise<HydratedDocument<AiTestUserDoc>[]> => {
56
78
  await UserModel.deleteMany({email: {$in: users.map((user) => user.email)}});
57
79
 
58
- const createdUsers: AiTestUserDoc[] = [];
80
+ const createdUsers: HydratedDocument<AiTestUserDoc>[] = [];
59
81
  for (const user of users) {
60
82
  const doc = await UserModel.create({
61
83
  admin: user.admin ?? false,
@@ -625,9 +625,13 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown>
625
625
 
626
626
  /**
627
627
  * Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
628
- * `never` accepts every tool in heterogeneous lists without an explicit `any`.
628
+ * `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
629
+ * output is covariant, so `parameters` widens to `unknown` instead.
629
630
  */
630
- export type AnyHarnessToolDefinition = HarnessToolDefinition<never, unknown>;
631
+ export interface AnyHarnessToolDefinition
632
+ extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
633
+ parameters: z.ZodType<unknown>;
634
+ }
631
635
 
632
636
  export interface HarnessAgentDefinitionInput {
633
637
  /** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
@@ -0,0 +1,18 @@
1
+ {
2
+ "extends": "./tsconfig.json",
3
+ "compilerOptions": {
4
+ "noEmit": true,
5
+ "composite": false,
6
+ "incremental": false,
7
+ "declaration": false,
8
+ "declarationMap": false,
9
+ "types": [
10
+ "node",
11
+ "bun"
12
+ ]
13
+ },
14
+ "exclude": [
15
+ "node_modules",
16
+ "dist"
17
+ ]
18
+ }