@terreno/ai 57.20.0 → 58.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/AGENTS.md +20 -18
  2. package/CHANGELOG.md +93 -0
  3. package/biome.jsonc +3 -2
  4. package/dist/aiApp.d.ts +5 -2
  5. package/dist/aiApp.js +3 -2
  6. package/dist/harness/agentLoop.js +1 -0
  7. package/dist/index.d.ts +2 -1
  8. package/dist/index.js +6 -2
  9. package/dist/migrations/gptHistoryStringIds.d.ts +41 -0
  10. package/dist/migrations/gptHistoryStringIds.js +94 -0
  11. package/dist/models/aiDataset.js +2 -2
  12. package/dist/models/gptHistory.js +11 -0
  13. package/dist/observability/local/experimentRunner.d.ts +2 -5
  14. package/dist/routes/files.d.ts +7 -1
  15. package/dist/routes/files.js +62 -13
  16. package/dist/routes/gptActions.js +0 -7
  17. package/dist/routes/gptHistories.js +8 -2
  18. package/dist/service/aiDatasets.js +2 -1
  19. package/dist/service/chatTurn.js +4 -0
  20. package/dist/service/vertex.d.ts +5 -2
  21. package/dist/types/harness.d.ts +5 -2
  22. package/dist/types/index.d.ts +5 -2
  23. package/e2e/smoke.ts +227 -0
  24. package/package.json +19 -16
  25. package/src/aiApp.test.ts +43 -18
  26. package/src/aiApp.ts +6 -3
  27. package/src/harness/agentLoop.test.ts +1 -1
  28. package/src/harness/agentLoop.ts +2 -1
  29. package/src/harness/agentRecovery.test.ts +31 -28
  30. package/src/harness/approvals.test.ts +11 -9
  31. package/src/harness/events.test.ts +10 -6
  32. package/src/harness/extensions.test.ts +7 -6
  33. package/src/harness/ownership.test.ts +20 -9
  34. package/src/harness/retries.test.ts +6 -3
  35. package/src/harness/routes/conversations.test.ts +2 -2
  36. package/src/harness/routes/tasks.test.ts +2 -2
  37. package/src/harness/runners/jobsRunner.test.ts +1 -1
  38. package/src/harness/subagent.test.ts +3 -2
  39. package/src/harness/waits.test.ts +6 -4
  40. package/src/index.test.ts +3 -3
  41. package/src/index.ts +7 -1
  42. package/src/isolated/gptStream.isolated.ts +35 -30
  43. package/src/isolated/langfuseClient.isolated.ts +9 -3
  44. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  45. package/src/langfuseApp.test.ts +4 -20
  46. package/src/langfuseRoutes.test.ts +20 -15
  47. package/src/langfuseTracing.test.ts +6 -2
  48. package/src/langfuseVercelAi.test.ts +12 -9
  49. package/src/migrations/gptHistoryStringIds.test.ts +231 -0
  50. package/src/migrations/gptHistoryStringIds.ts +150 -0
  51. package/src/models/aiDataset.test.ts +1 -1
  52. package/src/models/aiDataset.ts +2 -2
  53. package/src/models/gptHistory.test.ts +22 -11
  54. package/src/models/gptHistory.ts +19 -1
  55. package/src/observability/evaluate.test.ts +12 -10
  56. package/src/observability/local/experimentRunner.test.ts +9 -12
  57. package/src/observability/local/experimentRunner.ts +2 -2
  58. package/src/observability/local/promptStore.test.ts +4 -3
  59. package/src/observability/routes/datasets.test.ts +2 -2
  60. package/src/observability/routes/evaluators.test.ts +2 -2
  61. package/src/observability/routes/experiments.test.ts +2 -2
  62. package/src/observability/routes/prompts.test.ts +3 -3
  63. package/src/observability/routes/review.test.ts +2 -2
  64. package/src/observability/routes/status.test.ts +4 -4
  65. package/src/observability/routes/testMultiStage.test.ts +6 -6
  66. package/src/observability/routes/traces.test.ts +2 -2
  67. package/src/routes/files.test.ts +126 -3
  68. package/src/routes/files.ts +90 -15
  69. package/src/routes/gpt.test.ts +23 -16
  70. package/src/routes/gptActions.test.ts +15 -15
  71. package/src/routes/gptActions.ts +1 -4
  72. package/src/routes/gptApprovals.test.ts +1 -1
  73. package/src/routes/gptDatasets.test.ts +1 -1
  74. package/src/routes/gptFiles.test.ts +2 -2
  75. package/src/routes/gptHistories.test.ts +17 -11
  76. package/src/routes/gptHistories.ts +59 -55
  77. package/src/routes/gptHistoriesSync.test.ts +141 -0
  78. package/src/routes/mcp.test.ts +3 -3
  79. package/src/routes/projects.test.ts +5 -3
  80. package/src/routes/routes.test.ts +64 -47
  81. package/src/service/aiDatasets.ts +2 -1
  82. package/src/service/aiService.test.ts +1 -1
  83. package/src/service/asks.test.ts +14 -6
  84. package/src/service/chatTurn.ts +4 -0
  85. package/src/service/fileStorage.test.ts +5 -2
  86. package/src/service/getMCPTools.test.ts +9 -5
  87. package/src/service/mcpService.test.ts +6 -3
  88. package/src/service/vertex.test.ts +1 -1
  89. package/src/service/vertex.ts +7 -4
  90. package/src/tests/bunSetup.ts +1 -1
  91. package/src/tests/chatHarness.ts +4 -4
  92. package/src/tests/fakeJudge.ts +8 -0
  93. package/src/tests/fakeLangfusePrompt.ts +10 -0
  94. package/src/tests/helpers.ts +26 -4
  95. package/src/types/harness.ts +6 -2
  96. package/src/types/index.ts +5 -2
  97. package/tsconfig.typecheck.json +18 -0
@@ -0,0 +1,150 @@
1
+ /**
2
+ * Data migration for Terreno 58: GptHistory became a synced collection with String `_id`s.
3
+ * Histories saved before 58 have ObjectId `_id`s, which sync clients and the String-typed
4
+ * schema cannot address. This migration re-inserts each one with `_id` set to the same id's
5
+ * hex string, so URLs and client caches that hold the hex id keep working, rewrites
6
+ * `AIDataset.historyId` refs to that hex string, and then deletes the ObjectId originals.
7
+ *
8
+ * It writes through the native driver: `syncPlugin` forbids the hard deletes and
9
+ * multi-document writes a move needs, and the copies keep every stored field as-is.
10
+ * Running it again is a no-op.
11
+ */
12
+ import type {MigrationContext, MigrationLogger} from "@terreno/api";
13
+ import type mongoose from "mongoose";
14
+
15
+ import {AIDataset} from "../models/aiDataset";
16
+ import {GptHistory} from "../models/gptHistory";
17
+
18
+ /** Slug for the timestamped file an app adds to its `migrations/` directory. */
19
+ export const GPT_HISTORY_STRING_IDS_MIGRATION = "gpt-history-string-ids";
20
+
21
+ const DEFAULT_BATCH_SIZE = 500;
22
+
23
+ const DUPLICATE_KEY_CODE = 11000;
24
+
25
+ /** Only ObjectId `_id`s still need a move; String ids are already migrated or new. */
26
+ const OBJECT_ID_FILTER = {_id: {$type: "objectId" as const}};
27
+
28
+ /** Server-side ObjectId → hex string, so refs move without reading each dataset. */
29
+ const HISTORY_ID_TO_STRING = [{$set: {historyId: {$toString: "$historyId"}}}];
30
+
31
+ export interface GptHistoryStringIdsResult {
32
+ /** AIDataset rows whose ObjectId `historyId` was rewritten (or would be, in a dry run). */
33
+ datasetsUpdated: number;
34
+ /** Histories moved to a String `_id` (or that would move, in a dry run). */
35
+ historiesMigrated: number;
36
+ }
37
+
38
+ interface WriteErrorLike {
39
+ code?: number;
40
+ writeErrors?: Array<{code?: number}>;
41
+ }
42
+
43
+ /** An insert that failed only because the hex-id copy exists from an interrupted earlier run. */
44
+ const isOnlyDuplicateKeyError = (error: unknown): boolean => {
45
+ const writeError = error as WriteErrorLike;
46
+ if (writeError.writeErrors?.length) {
47
+ return writeError.writeErrors.every((item) => item.code === DUPLICATE_KEY_CODE);
48
+ }
49
+ return writeError.code === DUPLICATE_KEY_CODE;
50
+ };
51
+
52
+ const insertCopies = async ({
53
+ collection,
54
+ copies,
55
+ }: {
56
+ collection: mongoose.mongo.Collection;
57
+ copies: mongoose.mongo.Document[];
58
+ }): Promise<void> => {
59
+ try {
60
+ await collection.insertMany(copies, {ordered: false});
61
+ } catch (error) {
62
+ if (!isOnlyDuplicateKeyError(error)) {
63
+ throw error;
64
+ }
65
+ }
66
+ };
67
+
68
+ /**
69
+ * Moves every ObjectId-id GptHistory to its hex-string id in batches and rewrites the
70
+ * AIDataset refs. In a dry run it only counts what it would change.
71
+ */
72
+ export const migrateGptHistoryStringIds = async ({
73
+ batchSize = DEFAULT_BATCH_SIZE,
74
+ checkCancellation,
75
+ connection,
76
+ dryRun,
77
+ logger,
78
+ }: {
79
+ batchSize?: number;
80
+ checkCancellation?: () => Promise<void>;
81
+ connection: mongoose.Connection;
82
+ dryRun: boolean;
83
+ logger?: MigrationLogger;
84
+ }): Promise<GptHistoryStringIdsResult> => {
85
+ const histories = connection.collection(GptHistory.collection.collectionName);
86
+ const datasets = connection.collection(AIDataset.collection.collectionName);
87
+ const datasetFilter = {historyId: {$type: "objectId" as const}};
88
+
89
+ if (dryRun) {
90
+ const result = {
91
+ datasetsUpdated: await datasets.countDocuments(datasetFilter),
92
+ historiesMigrated: await histories.countDocuments(OBJECT_ID_FILTER),
93
+ };
94
+ logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] dry run`, result);
95
+ return result;
96
+ }
97
+
98
+ let historiesMigrated = 0;
99
+ let datasetsUpdated = 0;
100
+ while (true) {
101
+ await checkCancellation?.();
102
+ const batch = await histories.find(OBJECT_ID_FILTER).limit(batchSize).toArray();
103
+ if (batch.length === 0) {
104
+ break;
105
+ }
106
+ const originalIds = batch.map((doc) => doc._id);
107
+ // Copy first and delete second, so an interrupted run never loses a history: the next
108
+ // run finds the original again and the copy's duplicate key is ignored.
109
+ await insertCopies({
110
+ collection: histories,
111
+ copies: batch.map((doc) => ({...doc, _id: String(doc._id)})),
112
+ });
113
+ const moved = await datasets.updateMany({historyId: {$in: originalIds}}, HISTORY_ID_TO_STRING);
114
+ datasetsUpdated += moved.modifiedCount;
115
+ await histories.deleteMany({_id: {$in: originalIds}});
116
+ historiesMigrated += batch.length;
117
+ logger?.info(
118
+ `[${GPT_HISTORY_STRING_IDS_MIGRATION}] moved ${historiesMigrated} histories to String ids`
119
+ );
120
+ }
121
+
122
+ // Refs whose history was moved by an interrupted run, or never existed, still hold ObjectIds.
123
+ const remaining = await datasets.updateMany(datasetFilter, HISTORY_ID_TO_STRING);
124
+ datasetsUpdated += remaining.modifiedCount;
125
+ const result = {datasetsUpdated, historiesMigrated};
126
+ logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] done`, result);
127
+ return result;
128
+ };
129
+
130
+ /**
131
+ * Ready-made `up` for an app's `migrations/<YYYYMMDDHHmmss>-gpt-history-string-ids.ts` file.
132
+ * Run it before deploying Terreno 58 frontends, which address histories by String id. It has
133
+ * no `down`: new histories get UUIDs, which have no ObjectId to go back to.
134
+ */
135
+ export const gptHistoryStringIdsMigration = {
136
+ slug: GPT_HISTORY_STRING_IDS_MIGRATION,
137
+ up: async (ctx: MigrationContext): Promise<void> => {
138
+ const result = await migrateGptHistoryStringIds({
139
+ checkCancellation: ctx.checkCancellation,
140
+ connection: ctx.mongoose.connection,
141
+ dryRun: ctx.dryRun,
142
+ logger: ctx.logger,
143
+ });
144
+ await ctx.addLog?.(
145
+ "info",
146
+ `[${GPT_HISTORY_STRING_IDS_MIGRATION}] ${ctx.dryRun ? "would move" : "moved"} ` +
147
+ `${result.historiesMigrated} histories and ${result.datasetsUpdated} dataset refs`
148
+ );
149
+ },
150
+ };
@@ -12,7 +12,7 @@ describe("AIDataset model", () => {
12
12
  const userId = new mongoose.Types.ObjectId();
13
13
  const dataset = await AIDataset.create({
14
14
  columns: [{name: "count", type: "number"}],
15
- historyId: new mongoose.Types.ObjectId(),
15
+ historyId: new mongoose.Types.ObjectId().toString(),
16
16
  rowCount: 1,
17
17
  rows: [[3]],
18
18
  userId,
@@ -32,11 +32,11 @@ const aiDatasetSchema: AIDatasetSchema = new mongoose.Schema(
32
32
  type: Date,
33
33
  },
34
34
  historyId: {
35
- description: "The conversation whose tool stored these rows",
35
+ description: "The conversation whose tool stored these rows (a GptHistory String id)",
36
36
  index: true,
37
37
  ref: "GptHistory",
38
38
  required: true,
39
- type: mongoose.Schema.Types.ObjectId,
39
+ type: String,
40
40
  },
41
41
  rowCount: {
42
42
  description: "How many rows were stored",
@@ -1,15 +1,22 @@
1
1
  import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
2
  import mongoose from "mongoose";
3
3
 
4
+ import type {GptHistoryDocument, GptHistoryPendingAsk, GptHistoryPrompt} from "../types/index";
4
5
  import {GptHistory} from "./gptHistory";
5
6
 
7
+ // GptHistory ids are strings, but the driver types the raw collection with ObjectId ids.
8
+ const stringIdCollection = (): mongoose.mongo.Collection<{_id: string} & mongoose.mongo.Document> =>
9
+ GptHistory.collection as unknown as mongoose.mongo.Collection<
10
+ {_id: string} & mongoose.mongo.Document
11
+ >;
12
+
6
13
  describe("GptHistory Model", () => {
7
14
  beforeEach(async () => {
8
- await GptHistory.deleteMany({});
15
+ await GptHistory.collection.deleteMany({});
9
16
  });
10
17
 
11
18
  afterEach(async () => {
12
- await GptHistory.deleteMany({});
19
+ await GptHistory.collection.deleteMany({});
13
20
  });
14
21
 
15
22
  describe("schema", () => {
@@ -217,9 +224,9 @@ describe("GptHistory Model", () => {
217
224
  {id: "team", label: "Team"},
218
225
  ],
219
226
  prompt: "Which plan should I set up?",
220
- select: "one",
227
+ select: "one" as const,
221
228
  };
222
- const pendingAsk = {
229
+ const pendingAsk: GptHistoryPendingAsk = {
223
230
  created: new Date("2026-09-27T12:00:00.000Z"),
224
231
  input: planAsk,
225
232
  kind: "choice",
@@ -256,7 +263,10 @@ describe("GptHistory Model", () => {
256
263
  },
257
264
  toolCallId: "call_plan",
258
265
  };
259
- const askRow = {
266
+ /** Create from deliberately invalid input, so the schema rather than the compiler rejects it. */
267
+ const createUnchecked = (doc: Record<string, unknown>) =>
268
+ GptHistory.create(doc as Partial<GptHistoryDocument>);
269
+ const askRow: GptHistoryPrompt = {
260
270
  args: planAsk,
261
271
  ask: {kind: "choice", status: "pending"},
262
272
  text: "Tool call: ask_choice",
@@ -267,7 +277,8 @@ describe("GptHistory Model", () => {
267
277
 
268
278
  it("loads a history saved before asks existed", async () => {
269
279
  const userId = new mongoose.Types.ObjectId();
270
- const {insertedId} = await GptHistory.collection.insertOne({
280
+ const {insertedId} = await stringIdCollection().insertOne({
281
+ _id: new mongoose.Types.ObjectId().toHexString(),
271
282
  created: new Date("2026-01-05T09:00:00.000Z"),
272
283
  deleted: false,
273
284
  prompts: [
@@ -299,7 +310,7 @@ describe("GptHistory Model", () => {
299
310
  userId,
300
311
  });
301
312
 
302
- const stored = await GptHistory.collection.findOne({_id: history._id});
313
+ const stored = await stringIdCollection().findOne({_id: history._id});
303
314
 
304
315
  expect(stored?.pendingAsk).toEqual(pendingAsk);
305
316
  expect(stored?.prompts[1].ask).toEqual({kind: "choice", status: "pending"});
@@ -309,10 +320,10 @@ describe("GptHistory Model", () => {
309
320
  const userId = new mongoose.Types.ObjectId();
310
321
  const history = await GptHistory.create({prompts: [], userId});
311
322
 
312
- history.pendingAsk = pendingAsk as unknown as NonNullable<typeof history.pendingAsk>;
323
+ history.pendingAsk = pendingAsk;
313
324
  await history.save();
314
325
 
315
- const stored = await GptHistory.collection.findOne({_id: history._id});
326
+ const stored = await stringIdCollection().findOne({_id: history._id});
316
327
  expect(stored?.pendingAsk.responseMessages[0].content[0]).toEqual({
317
328
  input: {},
318
329
  toolCallId: "call_lookup",
@@ -334,7 +345,7 @@ describe("GptHistory Model", () => {
334
345
  },
335
346
  ])("rejects an unknown key in $field", async ({expected, history}) => {
336
347
  await expect(
337
- GptHistory.create({...history, userId: new mongoose.Types.ObjectId()})
348
+ createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
338
349
  ).rejects.toThrow(expected);
339
350
  });
340
351
 
@@ -361,7 +372,7 @@ describe("GptHistory Model", () => {
361
372
  },
362
373
  ])("rejects $label", async ({expected, history}) => {
363
374
  await expect(
364
- GptHistory.create({...history, userId: new mongoose.Types.ObjectId()})
375
+ createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
365
376
  ).rejects.toThrow(expected);
366
377
  });
367
378
  });
@@ -1,4 +1,12 @@
1
- import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
1
+ import {randomUUID} from "node:crypto";
2
+
3
+ import {
4
+ createdUpdatedPlugin,
5
+ findExactlyOne,
6
+ findOneOrNone,
7
+ isDeletedPlugin,
8
+ syncPlugin,
9
+ } from "@terreno/api";
2
10
  import {ASK_KINDS} from "@terreno/blocks";
3
11
  import mongoose from "mongoose";
4
12
 
@@ -153,6 +161,14 @@ const gptHistoryPromptSchema = new mongoose.Schema(
153
161
 
154
162
  const gptHistorySchema = new mongoose.Schema<GptHistoryDocument, GptHistoryModel>(
155
163
  {
164
+ _id: {
165
+ // Synced through @terreno/syncdb: offline clients mint entity ids locally and the sync
166
+ // mutation channel writes them through as _id. Histories created before Terreno 58 keep
167
+ // their ObjectId hex string after the gpt-history-string-ids migration.
168
+ default: (): string => randomUUID(),
169
+ description: "The conversation id (a String so offline sync clients can mint ids)",
170
+ type: String,
171
+ },
156
172
  pendingAsk: {
157
173
  description:
158
174
  "The ask this conversation is waiting on; cleared when the user answers or the ask is cancelled",
@@ -185,6 +201,8 @@ gptHistorySchema.plugin(createdUpdatedPlugin);
185
201
  gptHistorySchema.plugin(isDeletedPlugin);
186
202
  gptHistorySchema.plugin(findOneOrNone);
187
203
  gptHistorySchema.plugin(findExactlyOne);
204
+ // Stamps a per-stream _syncSeq on every write; required by the /gpt/histories sync config.
205
+ gptHistorySchema.plugin(syncPlugin);
188
206
 
189
207
  // Virtual ownerId alias so Permissions.IsOwner works with userId field
190
208
  gptHistorySchema.virtual("ownerId").get(function () {
@@ -1,5 +1,6 @@
1
1
  import {afterEach, beforeEach, describe, it} from "bun:test";
2
2
  import {assert} from "chai";
3
+ import {judgeAnswering} from "../tests/fakeJudge";
3
4
  import {
4
5
  judgeSchemaMissingDimensions,
5
6
  runEvaluator,
@@ -95,9 +96,7 @@ describe("evaluate", () => {
95
96
  output: "4",
96
97
  },
97
98
  {
98
- generateJsonObject: async () => {
99
- return {confidence: 0.95, correct: true};
100
- },
99
+ generateJsonObject: judgeAnswering(() => ({confidence: 0.95, correct: true})),
101
100
  },
102
101
  {
103
102
  properties: {
@@ -152,7 +151,7 @@ describe("evaluate", () => {
152
151
  const base = {
153
152
  evaluator: {
154
153
  assertion: {constraint: "exists", path: "answer"},
155
- dimensions: [{dataType: "boolean", key: "matches", required: true}],
154
+ dimensions: [{dataType: "boolean" as const, key: "matches", required: true}],
156
155
  name: "path-assert",
157
156
  type: "json-assert" as const,
158
157
  },
@@ -389,7 +388,7 @@ describe("evaluate", () => {
389
388
  input: {},
390
389
  output: "4",
391
390
  },
392
- {generateJsonObject: async () => ({correct: true})},
391
+ {generateJsonObject: judgeAnswering(() => ({correct: true}))},
393
392
  {properties: {correct: {type: "boolean"}}, type: "object"}
394
393
  );
395
394
  assert.equal(missingJudgePrompt.error, "llm-judge evaluator is missing judgePromptName");
@@ -408,7 +407,7 @@ describe("evaluate", () => {
408
407
  },
409
408
  {
410
409
  ai: {
411
- generateJsonObject: async () => ({confidence: "not-a-number", correct: 1}),
410
+ generateJsonObject: judgeAnswering(() => ({confidence: "not-a-number", correct: 1})),
412
411
  },
413
412
  judgeOutputSchema: {
414
413
  properties: {
@@ -461,9 +460,12 @@ describe("evaluate", () => {
461
460
  output: "4",
462
461
  },
463
462
  {
464
- generateJsonObject: async () => {
465
- return {confidence: "0.8", correct: "true", helpfulness: "not-a-number", tone: 1};
466
- },
463
+ generateJsonObject: judgeAnswering(() => ({
464
+ confidence: "0.8",
465
+ correct: "true",
466
+ helpfulness: "not-a-number",
467
+ tone: 1,
468
+ })),
467
469
  },
468
470
  {
469
471
  properties: {
@@ -489,7 +491,7 @@ describe("LocalEvaluatorStore phase 2", () => {
489
491
  beforeEach(async () => {
490
492
  new ObservabilityApp({plugins: [createLocalObservabilityBundle().plugin]});
491
493
  promptStore = new LocalPromptStore();
492
- store = new (await import("./local/evaluatorStore")).LocalEvaluatorStore(promptStore);
494
+ store = new (await import("./local/evaluatorStore.js")).LocalEvaluatorStore(promptStore);
493
495
  await registerObsEvaluator().deleteMany({});
494
496
  await clearObsPrompts();
495
497
  await promptStore.create({
@@ -1,8 +1,8 @@
1
1
  import {afterEach, beforeEach, describe, it} from "bun:test";
2
- import {BackgroundTask} from "@terreno/api";
2
+ import {BackgroundTask, type BackgroundTaskDocument, type FindExactlyOnePlugin} from "@terreno/api";
3
3
  import {assert} from "chai";
4
4
  import mongoose from "mongoose";
5
-
5
+ import {judgeAnswering} from "../../tests/fakeJudge";
6
6
  import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
7
7
  import {LocalDatasetStore} from "./datasetStore";
8
8
  import {LocalEvaluatorStore} from "./evaluatorStore";
@@ -37,9 +37,7 @@ const createTestRunner = (params: {
37
37
  });
38
38
  runner.configureAi({
39
39
  aiService: {
40
- generateJsonObject: async () => {
41
- return {correct: true};
42
- },
40
+ generateJsonObject: judgeAnswering(() => ({correct: true})),
43
41
  generateText: params.generateText,
44
42
  modelId: "mock-model",
45
43
  },
@@ -340,9 +338,7 @@ describe("LocalExperimentRunner", () => {
340
338
  });
341
339
  defaultRunner.configureAi({
342
340
  aiService: {
343
- generateJsonObject: async () => {
344
- return {correct: true};
345
- },
341
+ generateJsonObject: judgeAnswering(() => ({correct: true})),
346
342
  generateText: async () => {
347
343
  return JSON.stringify({answer: "ok"});
348
344
  },
@@ -399,9 +395,7 @@ describe("LocalExperimentRunner", () => {
399
395
  });
400
396
  failingRunner.configureAi({
401
397
  aiService: {
402
- generateJsonObject: async () => {
403
- return {correct: true};
404
- },
398
+ generateJsonObject: judgeAnswering(() => ({correct: true})),
405
399
  generateText: async () => {
406
400
  throw new Error("generation failed");
407
401
  },
@@ -427,7 +421,10 @@ describe("LocalExperimentRunner", () => {
427
421
  });
428
422
  detail = await failingRunner.get(experiment.id);
429
423
  }
430
- const task = await BackgroundTask.findExactlyOne({_id: detail.backgroundTaskId});
424
+ // @terreno/api applies findExactlyOne to BackgroundTask, but its model type does not declare it.
425
+ const tasks = BackgroundTask as typeof BackgroundTask &
426
+ FindExactlyOnePlugin<BackgroundTaskDocument>;
427
+ const task = await tasks.findExactlyOne({_id: detail.backgroundTaskId});
431
428
  assert.equal(task.status, "failed");
432
429
  assert.include(task.error ?? "", "generation failed");
433
430
  });
@@ -22,7 +22,7 @@ import type {LocalDatasetStore} from "./datasetStore";
22
22
  import type {LocalEvaluatorStore} from "./evaluatorStore";
23
23
  import {registerObsExperiment} from "./models/obsExperiment";
24
24
  import {registerObsExperimentItem} from "./models/obsExperimentItem";
25
- import type {LocalPromptStore} from "./promptStore";
25
+ import type {LocalPromptStore, MoveLabelResult} from "./promptStore";
26
26
 
27
27
  export interface ExperimentCreateInput {
28
28
  datasetId: string;
@@ -354,7 +354,7 @@ export class LocalExperimentRunner {
354
354
  };
355
355
  }
356
356
 
357
- async promote(id: string, version: number): Promise<{label: string; outgoingVersion?: number}> {
357
+ async promote(id: string, version: number): Promise<MoveLabelResult> {
358
358
  const experiment = await this.get(id);
359
359
  if (!experiment.results) {
360
360
  throw new APIError({status: 409, title: "Experiment has not finished running"});
@@ -10,7 +10,7 @@ import {registerObsPrompt} from "./models/obsPrompt";
10
10
  import {registerObsPromptLabel} from "./models/obsPromptLabel";
11
11
  import {registerObsPromptVersion} from "./models/obsPromptVersion";
12
12
  import {registerObsTrace} from "./models/obsTrace";
13
- import {LocalPromptStore} from "./promptStore";
13
+ import {type CreatePromptInput, LocalPromptStore} from "./promptStore";
14
14
 
15
15
  describe("LocalPromptStore", () => {
16
16
  let store: LocalPromptStore;
@@ -253,7 +253,7 @@ describe("LocalPromptStore", () => {
253
253
 
254
254
  it("validates create/moveLabel inputs and resolves versions by label or number", async () => {
255
255
  try {
256
- await store.create({folder: "", name: "bad"});
256
+ await store.create({folder: "", name: "bad"} as CreatePromptInput);
257
257
  expect.unreachable();
258
258
  } catch (error) {
259
259
  expect(String(error)).toMatch(/folder and name/);
@@ -334,9 +334,10 @@ describe("LocalPromptStore", () => {
334
334
  const latest = await store.runPlayground({
335
335
  generator: {
336
336
  generate: async () => {
337
- return {output: "latest"};
337
+ return {latencyMs: 0, output: "latest"};
338
338
  },
339
339
  },
340
+ modelId: "mock-model",
340
341
  name: "playground",
341
342
  variables: {name: "Grace"},
342
343
  });
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
2
  import {TerrenoApp} from "@terreno/api";
3
3
  import type express from "express";
4
4
 
5
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
5
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
6
6
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
7
7
  import {registerObsDataset} from "../local/models/obsDataset";
8
8
  import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
@@ -32,7 +32,7 @@ describe("observability dataset routes", () => {
32
32
  await registerObsPromptLabel().deleteMany({});
33
33
  promptStore = new LocalPromptStore();
34
34
  const plugin = createLocalObservabilityPlugin();
35
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
35
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
36
36
  .register(new ObservabilityApp({plugins: [plugin]}))
37
37
  .build();
38
38
  });
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
2
2
  import {TerrenoApp} from "@terreno/api";
3
3
  import type express from "express";
4
4
 
5
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
5
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
6
6
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
7
7
  import {registerObsEvaluator} from "../local/models/obsEvaluator";
8
8
  import {registerObsPrompt} from "../local/models/obsPrompt";
@@ -40,7 +40,7 @@ describe("observability evaluator routes", () => {
40
40
  type: "text",
41
41
  });
42
42
  await promptStore.moveLabel("eval-judge-correctness", {label: "production", version: 1});
43
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
43
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
44
44
  .register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
45
45
  .build();
46
46
  });
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
4
4
  import type express from "express";
5
5
 
6
6
  import {AIService} from "../../service/aiService";
7
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
7
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
8
8
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
9
  import {registerObsDataset} from "../local/models/obsDataset";
10
10
  import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
@@ -70,7 +70,7 @@ describe("observability experiment routes", () => {
70
70
  },
71
71
  },
72
72
  });
73
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
73
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
74
74
  .register(
75
75
  new ObservabilityApp({
76
76
  aiService,
@@ -8,7 +8,7 @@ import mongoose from "mongoose";
8
8
 
9
9
  import {AIRequest} from "../../models/aiRequest";
10
10
  import {AIService} from "../../service/aiService";
11
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
11
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
12
12
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
13
13
  import {registerObsExperiment} from "../local/models/obsExperiment";
14
14
  import {registerObsPrompt} from "../local/models/obsPrompt";
@@ -61,7 +61,7 @@ describe("observability prompt routes", () => {
61
61
  const model = createMockModel();
62
62
  doGenerate = model.doGenerate;
63
63
  aiService = new AIService({model: model as unknown as LanguageModel});
64
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
64
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
65
65
  .register(
66
66
  new ObservabilityApp({
67
67
  aiService,
@@ -134,7 +134,7 @@ describe("observability prompt routes", () => {
134
134
  const requestAiServiceFactory = mock(({apiKey}: {apiKey?: string}) => {
135
135
  return apiKey === "request-key" ? aiService : undefined;
136
136
  });
137
- const requestServiceApp = new TerrenoApp({skipListen: true, userModel: UserModel})
137
+ const requestServiceApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
138
138
  .register(
139
139
  new ObservabilityApp({
140
140
  plugins: [createLocalObservabilityPlugin()],
@@ -4,7 +4,7 @@ import {assert} from "chai";
4
4
  import type express from "express";
5
5
  import {DateTime} from "luxon";
6
6
 
7
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
7
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
8
8
  import {LocalEvaluatorStore} from "../local/evaluatorStore";
9
9
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
10
10
  import {registerObsEvaluator} from "../local/models/obsEvaluator";
@@ -28,7 +28,7 @@ describe("observability review routes", () => {
28
28
  await registerObsReviewItem().deleteMany({});
29
29
  await registerObsEvaluator().deleteMany({});
30
30
  await registerObsTrace().deleteMany({});
31
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
31
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
32
32
  .register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
33
33
  .build();
34
34
  });
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
4
4
  import type express from "express";
5
5
 
6
6
  import {AIService} from "../../service/aiService";
7
- import {authAsUser, ensureTestUsers, UserModel} from "../../tests/helpers";
7
+ import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
8
8
  import {createLocalObservabilityPlugin} from "../local/localPlugin";
9
9
  import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
10
10
  import type {ObservabilityPlugin} from "../types";
@@ -33,7 +33,7 @@ describe("observability status routes", () => {
33
33
  });
34
34
 
35
35
  beforeEach(() => {
36
- app = new TerrenoApp({skipListen: true, userModel: UserModel})
36
+ app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
37
37
  .register(
38
38
  new ObservabilityApp({
39
39
  plugins: [createLocalObservabilityPlugin(), createLangfusePlugin()],
@@ -89,7 +89,7 @@ describe("observability status routes", () => {
89
89
  specificationVersion: "v2" as const,
90
90
  supportedUrls: {},
91
91
  };
92
- const serverApp = new TerrenoApp({skipListen: true, userModel: UserModel})
92
+ const serverApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
93
93
  .register(
94
94
  new ObservabilityApp({
95
95
  aiService: new AIService({model: model as unknown as LanguageModel}),
@@ -101,7 +101,7 @@ describe("observability status routes", () => {
101
101
  const serverStatus = await serverAgent.get("/ai/observability/status");
102
102
  expect(serverStatus.body.data.playgroundAi).toEqual({source: "server"});
103
103
 
104
- const requestKeyApp = new TerrenoApp({skipListen: true, userModel: UserModel})
104
+ const requestKeyApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
105
105
  .register(
106
106
  new ObservabilityApp({
107
107
  plugins: [createLocalObservabilityPlugin()],