@terreno/ai 57.21.0 → 58.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/AGENTS.md +20 -18
  2. package/CHANGELOG.md +43 -0
  3. package/biome.jsonc +2 -2
  4. package/dist/aiApp.d.ts +5 -2
  5. package/dist/aiApp.js +3 -2
  6. package/dist/index.d.ts +2 -1
  7. package/dist/index.js +6 -2
  8. package/dist/migrations/gptHistoryStringIds.d.ts +41 -0
  9. package/dist/migrations/gptHistoryStringIds.js +94 -0
  10. package/dist/models/aiDataset.js +2 -2
  11. package/dist/models/gptHistory.js +11 -0
  12. package/dist/routes/files.d.ts +7 -1
  13. package/dist/routes/files.js +62 -13
  14. package/dist/routes/gptActions.js +0 -7
  15. package/dist/routes/gptHistories.js +8 -2
  16. package/dist/service/aiDatasets.js +2 -1
  17. package/dist/service/chatTurn.js +4 -0
  18. package/dist/service/vertex.d.ts +5 -2
  19. package/dist/types/index.d.ts +5 -2
  20. package/package.json +13 -12
  21. package/src/aiApp.test.ts +21 -1
  22. package/src/aiApp.ts +6 -3
  23. package/src/index.ts +7 -1
  24. package/src/isolated/gptStream.isolated.ts +2 -2
  25. package/src/migrations/gptHistoryStringIds.test.ts +231 -0
  26. package/src/migrations/gptHistoryStringIds.ts +150 -0
  27. package/src/models/aiDataset.test.ts +1 -1
  28. package/src/models/aiDataset.ts +2 -2
  29. package/src/models/gptHistory.test.ts +12 -5
  30. package/src/models/gptHistory.ts +19 -1
  31. package/src/routes/files.test.ts +121 -0
  32. package/src/routes/files.ts +90 -15
  33. package/src/routes/gpt.test.ts +2 -2
  34. package/src/routes/gptActions.test.ts +8 -7
  35. package/src/routes/gptActions.ts +1 -4
  36. package/src/routes/gptApprovals.test.ts +1 -1
  37. package/src/routes/gptFiles.test.ts +1 -1
  38. package/src/routes/gptHistories.test.ts +2 -2
  39. package/src/routes/gptHistories.ts +59 -55
  40. package/src/routes/gptHistoriesSync.test.ts +141 -0
  41. package/src/routes/routes.test.ts +6 -5
  42. package/src/service/aiDatasets.ts +2 -1
  43. package/src/service/chatTurn.ts +4 -0
  44. package/src/service/vertex.test.ts +1 -1
  45. package/src/service/vertex.ts +7 -4
  46. package/src/types/index.ts +5 -2
@@ -91,7 +91,8 @@ const registerAiDataset = async ({ columns, historyId, rows, userId, }) => {
91
91
  assertRows(columns, rows);
92
92
  const created = await aiDataset_1.AIDataset.create({
93
93
  columns,
94
- historyId,
94
+ // GptHistory ids are Strings; an ObjectId from an older caller is stored as its hex string.
95
+ historyId: String(historyId),
95
96
  rowCount: rows.length,
96
97
  rows,
97
98
  userId,
@@ -155,6 +155,10 @@ const loadHistory = async ({ historyId, projectId, userId, }) => {
155
155
  if (!historyId) {
156
156
  return new gptHistory_1.GptHistory({ prompts: [], userId, ...(projectId ? { projectId } : {}) });
157
157
  }
158
+ // History ids are Strings (synced collection); an object here would be read as a query operator.
159
+ if (typeof historyId !== "string") {
160
+ throw new api_1.APIError({ status: 400, title: "historyId must be a string" });
161
+ }
158
162
  const history = await gptHistory_1.GptHistory.findById(historyId);
159
163
  if (!history) {
160
164
  throw new api_1.APIError({ status: 404, title: "History not found" });
@@ -1,3 +1,4 @@
1
+ import type { ImageModelV4, LanguageModelV4 } from "@ai-sdk/provider";
1
2
  import type { ImageModel, LanguageModel } from "ai";
2
3
  /**
3
4
  * Helper for Google's Vertex AI provider, now part of the "Gemini Enterprise Agent Platform"
@@ -16,8 +17,10 @@ import type { ImageModel, LanguageModel } from "ai";
16
17
  export declare const DEFAULT_VERTEX_LOCATION = "global";
17
18
  /** Provider returned by `@ai-sdk/google-vertex` `createVertex`. */
18
19
  export interface VertexLanguageModelProvider {
19
- (modelId: string): LanguageModel;
20
- image: (modelId: string) => ImageModel;
20
+ /** `@ai-sdk/google-vertex` v4+ returns `LanguageModelV4`; older `ai` majors still type `LanguageModel` as v2/v3 only. */
21
+ (modelId: string): LanguageModel | LanguageModelV4;
22
+ /** `@ai-sdk/google-vertex` v4+ returns `ImageModelV4`; older `ai` majors still type `ImageModel` as v2/v3 only. */
23
+ image: (modelId: string) => ImageModel | ImageModelV4;
21
24
  }
22
25
  interface VertexModule {
23
26
  createVertex: (opts: {
@@ -113,7 +113,9 @@ interface GptHistoryPendingAskState {
113
113
  }
114
114
  /** The ask a history is waiting on. At most one per history. */
115
115
  export type GptHistoryPendingAsk = Ask & GptHistoryPendingAskState;
116
- export interface GptHistoryDocument extends mongoose.Document<mongoose.Types.ObjectId> {
116
+ /** Synced through @terreno/syncdb, so `_id` is a String (UUID for new rows, ObjectId hex for migrated rows). */
117
+ export interface GptHistoryDocument extends mongoose.Document<string> {
118
+ _id: string;
117
119
  created: Date;
118
120
  deleted: boolean;
119
121
  pendingAsk?: GptHistoryPendingAsk;
@@ -435,7 +437,8 @@ export interface AIDatasetDocument extends mongoose.Document<mongoose.Types.Obje
435
437
  created: Date;
436
438
  deleted: boolean;
437
439
  expiresAt?: Date;
438
- historyId: mongoose.Types.ObjectId;
440
+ /** The GptHistory String `_id`. */
441
+ historyId: string;
439
442
  rowCount: number;
440
443
  rows: AIDatasetCell[][];
441
444
  updated: Date;
package/package.json CHANGED
@@ -4,17 +4,17 @@
4
4
  "url": "https://github.com/TerrenoLabs/terreno/issues"
5
5
  },
6
6
  "dependencies": {
7
- "@ai-sdk/mcp": "^1.0.21",
7
+ "@ai-sdk/mcp": "^2.0.57",
8
8
  "@google-cloud/storage": "^7.15.0",
9
9
  "@langfuse/client": "^5.0.1",
10
10
  "@langfuse/otel": "^5.0.1",
11
11
  "@opentelemetry/api": "^1.9.0",
12
- "@opentelemetry/sdk-node": "0.217.0",
13
- "@terreno/blocks": "57.21.0",
12
+ "@opentelemetry/sdk-node": "0.222.0",
13
+ "@terreno/blocks": "58.0.0-beta.1",
14
14
  "ai": "^6.0.97",
15
15
  "ajv": "8.20.0",
16
16
  "csv-parse": "^6.1.0",
17
- "google-auth-library": "^10.5.0",
17
+ "google-auth-library": "^11.1.0",
18
18
  "luxon": "^3.7.2",
19
19
  "multer": "2.2.0",
20
20
  "sanitize-html": "^2.17.0"
@@ -22,10 +22,10 @@
22
22
  "description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
23
23
  "devDependencies": {
24
24
  "@biomejs/biome": "^2.3.6",
25
- "@terreno/admin-backend": "57.21.0",
26
- "@terreno/api": "57.21.0",
27
- "@terreno/jobs": "57.21.0",
28
- "@terreno/test": "57.21.0",
25
+ "@terreno/admin-backend": "58.0.0-beta.1",
26
+ "@terreno/api": "58.0.0-beta.1",
27
+ "@terreno/jobs": "58.0.0-beta.1",
28
+ "@terreno/test": "58.0.0-beta.1",
29
29
  "@types/bun": "^1.4.0",
30
30
  "@types/express": "^5.0.6",
31
31
  "@types/luxon": "^3.7.1",
@@ -135,9 +135,10 @@
135
135
  "main": "dist/index.js",
136
136
  "name": "@terreno/ai",
137
137
  "peerDependencies": {
138
- "@ai-sdk/google-vertex": "^4.0.142",
139
- "@terreno/api": "57.21.0",
140
- "@terreno/jobs": "57.21.0",
138
+ "@ai-sdk/google-vertex": "^5.0.92",
139
+ "@ai-sdk/provider": "^4.0.26",
140
+ "@terreno/api": "58.0.0-beta.1",
141
+ "@terreno/jobs": "58.0.0-beta.1",
141
142
  "mongoose": "^8.0.0 || ^9.0.0"
142
143
  },
143
144
  "peerDependenciesMeta": {
@@ -170,5 +171,5 @@
170
171
  "typecheck": "tsc -p tsconfig.typecheck.json"
171
172
  },
172
173
  "types": "dist/index.d.ts",
173
- "version": "57.21.0"
174
+ "version": "58.0.0-beta.1"
174
175
  }
package/src/aiApp.test.ts CHANGED
@@ -55,7 +55,7 @@ describe("AiApp", () => {
55
55
 
56
56
  afterEach(async () => {
57
57
  await AIRequest.deleteMany({});
58
- await GptHistory.deleteMany({});
58
+ await GptHistory.collection.deleteMany({});
59
59
  });
60
60
 
61
61
  const authAsUser = async (app: express.Application) => {
@@ -356,4 +356,24 @@ describe("AiApp", () => {
356
356
  const servers = await agent.get("/mcp/servers");
357
357
  expect(servers.status).not.toBe(404);
358
358
  });
359
+ it("documents its routes with the OpenAPI instance TerrenoApp passes to register", async () => {
360
+ const {AIService} = await import("./service/aiService.js");
361
+ const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
362
+ const fileStorageService = {
363
+ delete: mock(async () => {}),
364
+ getSignedUrl: mock(async () => "https://example.com/signed"),
365
+ upload: mock(async () => {
366
+ throw new Error("not called");
367
+ }),
368
+ } as unknown as FileStorageService;
369
+ const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
370
+ .register(new AiApp({aiService, fileStorageService, gcsBucket: "bucket"}))
371
+ .build();
372
+
373
+ const res = await supertest(app).get("/openapi.json").expect(200);
374
+ expect(res.body.paths["/files/upload"]?.post?.requestBody?.content).toHaveProperty(
375
+ "multipart/form-data"
376
+ );
377
+ expect(res.body.paths["/gpt/prompt"]?.post).toBeDefined();
378
+ });
359
379
  });
package/src/aiApp.ts CHANGED
@@ -40,7 +40,10 @@ export interface AiAppOptions {
40
40
  maxSteps?: number;
41
41
  /** MCP service for connecting to external tool servers. */
42
42
  mcpService?: MCPService;
43
- /** OpenAPI options passed through to route builders for spec generation. */
43
+ /**
44
+ * OpenAPI options passed through to route builders for spec generation. Defaults to the
45
+ * OpenAPI instance TerrenoApp passes to `register`.
46
+ */
44
47
  openApiOptions?: Record<string, unknown>;
45
48
  /**
46
49
  * Mount the `/gpt/projects` routes, which register the `Project` model. Defaults to `true`;
@@ -88,7 +91,7 @@ export class AiApp implements TerrenoPlugin {
88
91
  this.options = options;
89
92
  }
90
93
 
91
- register(app: express.Application): void {
94
+ register(app: express.Application, openApi?: unknown): void {
92
95
  const router = app;
93
96
  const {
94
97
  aiService,
@@ -101,13 +104,13 @@ export class AiApp implements TerrenoPlugin {
101
104
  gcsBucket,
102
105
  maxSteps,
103
106
  mcpService,
104
- openApiOptions,
105
107
  projects,
106
108
  titleModelId,
107
109
  toolChoice,
108
110
  tools,
109
111
  } = this.options;
110
112
 
113
+ const openApiOptions = this.options.openApiOptions ?? (openApi ? {openApi} : undefined);
111
114
  const hasFileRoutes = Boolean(fileStorageService && gcsBucket);
112
115
  const chat = {
113
116
  aiService,
package/src/index.ts CHANGED
@@ -36,6 +36,12 @@ export type {
36
36
  TraceListItem,
37
37
  } from "./langfuseTypes";
38
38
  export {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
39
+ export {
40
+ GPT_HISTORY_STRING_IDS_MIGRATION,
41
+ type GptHistoryStringIdsResult,
42
+ gptHistoryStringIdsMigration,
43
+ migrateGptHistoryStringIds,
44
+ } from "./migrations/gptHistoryStringIds";
39
45
  export {AIDataset} from "./models/aiDataset";
40
46
  export {AIRequest} from "./models/aiRequest";
41
47
  export {FileAttachment} from "./models/fileAttachment";
@@ -98,7 +104,7 @@ export {
98
104
  validateObservabilityConfig,
99
105
  } from "./observability/types";
100
106
  export {addAiRequestsExplorerRoutes} from "./routes/aiRequestsExplorer";
101
- export {addFileRoutes} from "./routes/files";
107
+ export {addFileRoutes, type FileStorageServiceSource} from "./routes/files";
102
108
  export {addGptRoutes} from "./routes/gpt";
103
109
  export {addGptHistoryRoutes} from "./routes/gptHistories";
104
110
  export {addMcpRoutes} from "./routes/mcp";
@@ -83,7 +83,7 @@ describe("GPT prompt streaming edge cases", () => {
83
83
 
84
84
  beforeAll(async () => {
85
85
  await ensureTestUsers();
86
- await GptHistory.deleteMany({});
86
+ await GptHistory.collection.deleteMany({});
87
87
  const aiService = new AIService({
88
88
  model: createMockModel() as unknown as LanguageModel,
89
89
  });
@@ -97,7 +97,7 @@ describe("GPT prompt streaming edge cases", () => {
97
97
  });
98
98
 
99
99
  afterAll(async () => {
100
- await GptHistory.deleteMany({});
100
+ await GptHistory.collection.deleteMany({});
101
101
  });
102
102
 
103
103
  it("emits an inline image and flushes buffered text with no finish-step", async () => {
@@ -0,0 +1,231 @@
1
+ import {afterEach, beforeEach, describe, expect, it} from "bun:test";
2
+ import type {MigrationContext} from "@terreno/api";
3
+ import mongoose from "mongoose";
4
+
5
+ import {AIDataset} from "../models/aiDataset";
6
+ import {GptHistory} from "../models/gptHistory";
7
+ import {gptHistoryStringIdsMigration, migrateGptHistoryStringIds} from "./gptHistoryStringIds";
8
+
9
+ const userId = new mongoose.Types.ObjectId();
10
+
11
+ const historiesCollection = (): mongoose.mongo.Collection =>
12
+ mongoose.connection.collection(GptHistory.collection.collectionName);
13
+
14
+ const datasetsCollection = (): mongoose.mongo.Collection =>
15
+ mongoose.connection.collection(AIDataset.collection.collectionName);
16
+
17
+ const legacyHistory = (title: string): mongoose.mongo.Document => ({
18
+ _id: new mongoose.Types.ObjectId(),
19
+ created: new Date("2026-01-05T09:00:00.000Z"),
20
+ deleted: false,
21
+ prompts: [
22
+ {text: `Question about ${title}`, type: "user"},
23
+ {model: "gemini", text: `Answer about ${title}`, type: "assistant"},
24
+ ],
25
+ title,
26
+ updated: new Date("2026-01-05T09:01:00.000Z"),
27
+ userId,
28
+ });
29
+
30
+ const legacyDataset = (historyId: unknown): mongoose.mongo.Document => ({
31
+ _id: new mongoose.Types.ObjectId(),
32
+ columns: [{name: "count", type: "number"}],
33
+ created: new Date("2026-01-05T09:02:00.000Z"),
34
+ deleted: false,
35
+ historyId,
36
+ rowCount: 1,
37
+ rows: [[1]],
38
+ updated: new Date("2026-01-05T09:02:00.000Z"),
39
+ userId,
40
+ });
41
+
42
+ interface Seeded {
43
+ histories: mongoose.mongo.Document[];
44
+ stringHistoryId: string;
45
+ }
46
+
47
+ /** Five ObjectId histories (two batches at batchSize 2, plus one), a String one, and refs. */
48
+ const seed = async (): Promise<Seeded> => {
49
+ const histories = ["a", "b", "c", "d", "e"].map(legacyHistory);
50
+ const stringHistoryId = "already-a-string-id";
51
+ await historiesCollection().insertMany([
52
+ ...histories,
53
+ {...legacyHistory("string"), _id: stringHistoryId},
54
+ ]);
55
+ await datasetsCollection().insertMany([
56
+ legacyDataset(histories[0]._id),
57
+ legacyDataset(histories[0]._id),
58
+ legacyDataset(histories[3]._id),
59
+ // A ref whose history was deleted long ago still becomes a String.
60
+ legacyDataset(new mongoose.Types.ObjectId()),
61
+ legacyDataset(stringHistoryId),
62
+ ]);
63
+ return {histories, stringHistoryId};
64
+ };
65
+
66
+ const idTypes = async (collection: mongoose.mongo.Collection, field: string): Promise<string[]> => {
67
+ const rows = await collection
68
+ .aggregate<{type: string}>([{$project: {_id: 0, type: {$type: `$${field}`}}}])
69
+ .toArray();
70
+ return [...new Set(rows.map((row) => row.type))].sort();
71
+ };
72
+
73
+ const migrationContext = (dryRun: boolean, logs: string[] = []): MigrationContext => ({
74
+ addLog: async (_level, message): Promise<void> => {
75
+ logs.push(message);
76
+ },
77
+ dryRun,
78
+ logger: {debug: () => {}, error: () => {}, info: () => {}, warn: () => {}},
79
+ mongoose,
80
+ });
81
+
82
+ describe("gpt-history-string-ids migration", () => {
83
+ beforeEach(async () => {
84
+ await historiesCollection().deleteMany({});
85
+ await datasetsCollection().deleteMany({});
86
+ });
87
+
88
+ afterEach(async () => {
89
+ await historiesCollection().deleteMany({});
90
+ await datasetsCollection().deleteMany({});
91
+ });
92
+
93
+ it("moves ObjectId histories to their hex string ids, rewrites refs, and is a no-op the second time", async () => {
94
+ const {histories, stringHistoryId} = await seed();
95
+ const historyCount = await historiesCollection().countDocuments();
96
+ const datasetCount = await datasetsCollection().countDocuments();
97
+
98
+ const first = await migrateGptHistoryStringIds({
99
+ batchSize: 2,
100
+ connection: mongoose.connection,
101
+ dryRun: false,
102
+ });
103
+ const second = await migrateGptHistoryStringIds({
104
+ batchSize: 2,
105
+ connection: mongoose.connection,
106
+ dryRun: false,
107
+ });
108
+
109
+ expect(first).toEqual({datasetsUpdated: 4, historiesMigrated: 5});
110
+ expect(second).toEqual({datasetsUpdated: 0, historiesMigrated: 0});
111
+ expect(await historiesCollection().countDocuments()).toBe(historyCount);
112
+ expect(await datasetsCollection().countDocuments()).toBe(datasetCount);
113
+ expect(await idTypes(historiesCollection(), "_id")).toEqual(["string"]);
114
+ expect(await idTypes(datasetsCollection(), "historyId")).toEqual(["string"]);
115
+
116
+ const ids = (await historiesCollection().find({}).toArray()).map((doc) => doc._id).sort();
117
+ expect(ids).toEqual([...histories.map((doc) => doc._id.toHexString()), stringHistoryId].sort());
118
+ const firstHex = histories[0]._id.toHexString();
119
+ expect(await datasetsCollection().countDocuments({historyId: firstHex})).toBe(2);
120
+ expect(
121
+ await datasetsCollection().countDocuments({historyId: histories[3]._id.toHexString()})
122
+ ).toBe(1);
123
+ expect(await datasetsCollection().countDocuments({historyId: stringHistoryId})).toBe(1);
124
+ });
125
+
126
+ it("keeps every stored field, so the moved history loads through the model", async () => {
127
+ const {histories} = await seed();
128
+
129
+ await migrateGptHistoryStringIds({connection: mongoose.connection, dryRun: false});
130
+
131
+ const hexId = histories[1]._id.toHexString();
132
+ const history = await GptHistory.findById(hexId);
133
+ expect(history?._id).toBe(hexId);
134
+ expect(history?.title).toBe("b");
135
+ expect(history?.userId.toString()).toBe(userId.toString());
136
+ expect(history?.prompts.map((prompt) => prompt.text)).toEqual([
137
+ "Question about b",
138
+ "Answer about b",
139
+ ]);
140
+ const datasets = await AIDataset.find({historyId: histories[0]._id.toHexString()});
141
+ expect(datasets).toHaveLength(2);
142
+ });
143
+
144
+ it("finishes a run that stopped after copying a history but before deleting the original", async () => {
145
+ const {histories} = await seed();
146
+ const interrupted = histories[2];
147
+ await historiesCollection().insertOne({...interrupted, _id: interrupted._id.toHexString()});
148
+
149
+ const result = await migrateGptHistoryStringIds({
150
+ connection: mongoose.connection,
151
+ dryRun: false,
152
+ });
153
+
154
+ expect(result.historiesMigrated).toBe(5);
155
+ expect(await historiesCollection().countDocuments()).toBe(6);
156
+ expect(await idTypes(historiesCollection(), "_id")).toEqual(["string"]);
157
+ });
158
+
159
+ it("rewrites refs left as ObjectIds when a run stopped after moving their history", async () => {
160
+ const orphanedHistory = new mongoose.Types.ObjectId();
161
+ const movedHistory: mongoose.mongo.Document = {
162
+ ...legacyHistory("moved"),
163
+ _id: orphanedHistory.toHexString(),
164
+ };
165
+ await historiesCollection().insertOne(movedHistory);
166
+ await datasetsCollection().insertOne(legacyDataset(orphanedHistory));
167
+
168
+ const result = await migrateGptHistoryStringIds({
169
+ connection: mongoose.connection,
170
+ dryRun: false,
171
+ });
172
+
173
+ expect(result).toEqual({datasetsUpdated: 1, historiesMigrated: 0});
174
+ expect(
175
+ await datasetsCollection().countDocuments({historyId: orphanedHistory.toHexString()})
176
+ ).toBe(1);
177
+ });
178
+
179
+ it("only counts in a dry run", async () => {
180
+ await seed();
181
+
182
+ const result = await migrateGptHistoryStringIds({
183
+ connection: mongoose.connection,
184
+ dryRun: true,
185
+ });
186
+
187
+ expect(result).toEqual({datasetsUpdated: 4, historiesMigrated: 5});
188
+ expect(await idTypes(historiesCollection(), "_id")).toEqual(["objectId", "string"]);
189
+ expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(5);
190
+ });
191
+
192
+ it("runs as a migration up through the migration context", async () => {
193
+ await seed();
194
+
195
+ const logs: string[] = [];
196
+ await gptHistoryStringIdsMigration.up(migrationContext(true, logs));
197
+ expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(5);
198
+
199
+ await gptHistoryStringIdsMigration.up(migrationContext(false, logs));
200
+ await gptHistoryStringIdsMigration.up(migrationContext(false, logs));
201
+ expect(gptHistoryStringIdsMigration.slug).toBe("gpt-history-string-ids");
202
+ expect(logs).toEqual([
203
+ "[gpt-history-string-ids] would move 5 histories and 4 dataset refs",
204
+ "[gpt-history-string-ids] moved 5 histories and 4 dataset refs",
205
+ "[gpt-history-string-ids] moved 0 histories and 0 dataset refs",
206
+ ]);
207
+ expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(0);
208
+ expect(await datasetsCollection().countDocuments({historyId: {$type: "objectId"}})).toBe(0);
209
+ });
210
+
211
+ it("stops between batches when the run is cancelled", async () => {
212
+ await seed();
213
+ let checks = 0;
214
+ const checkCancellation = async (): Promise<void> => {
215
+ checks += 1;
216
+ if (checks > 1) {
217
+ throw new Error("cancelled");
218
+ }
219
+ };
220
+
221
+ await expect(
222
+ migrateGptHistoryStringIds({
223
+ batchSize: 2,
224
+ checkCancellation,
225
+ connection: mongoose.connection,
226
+ dryRun: false,
227
+ })
228
+ ).rejects.toThrow("cancelled");
229
+ expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(3);
230
+ });
231
+ });
@@ -0,0 +1,150 @@
1
+ /**
2
+ * Data migration for Terreno 58: GptHistory became a synced collection with String `_id`s.
3
+ * Histories saved before 58 have ObjectId `_id`s, which sync clients and the String-typed
4
+ * schema cannot address. This migration re-inserts each one with `_id` set to the same id's
5
+ * hex string, so URLs and client caches that hold the hex id keep working, rewrites
6
+ * `AIDataset.historyId` refs to that hex string, and then deletes the ObjectId originals.
7
+ *
8
+ * It writes through the native driver: `syncPlugin` forbids the hard deletes and
9
+ * multi-document writes a move needs, and the copies keep every stored field as-is.
10
+ * Running it again is a no-op.
11
+ */
12
+ import type {MigrationContext, MigrationLogger} from "@terreno/api";
13
+ import type mongoose from "mongoose";
14
+
15
+ import {AIDataset} from "../models/aiDataset";
16
+ import {GptHistory} from "../models/gptHistory";
17
+
18
+ /** Slug for the timestamped file an app adds to its `migrations/` directory. */
19
+ export const GPT_HISTORY_STRING_IDS_MIGRATION = "gpt-history-string-ids";
20
+
21
+ const DEFAULT_BATCH_SIZE = 500;
22
+
23
+ const DUPLICATE_KEY_CODE = 11000;
24
+
25
+ /** Only ObjectId `_id`s still need a move; String ids are already migrated or new. */
26
+ const OBJECT_ID_FILTER = {_id: {$type: "objectId" as const}};
27
+
28
+ /** Server-side ObjectId → hex string, so refs move without reading each dataset. */
29
+ const HISTORY_ID_TO_STRING = [{$set: {historyId: {$toString: "$historyId"}}}];
30
+
31
+ export interface GptHistoryStringIdsResult {
32
+ /** AIDataset rows whose ObjectId `historyId` was rewritten (or would be, in a dry run). */
33
+ datasetsUpdated: number;
34
+ /** Histories moved to a String `_id` (or that would move, in a dry run). */
35
+ historiesMigrated: number;
36
+ }
37
+
38
+ interface WriteErrorLike {
39
+ code?: number;
40
+ writeErrors?: Array<{code?: number}>;
41
+ }
42
+
43
+ /** An insert that failed only because the hex-id copy exists from an interrupted earlier run. */
44
+ const isOnlyDuplicateKeyError = (error: unknown): boolean => {
45
+ const writeError = error as WriteErrorLike;
46
+ if (writeError.writeErrors?.length) {
47
+ return writeError.writeErrors.every((item) => item.code === DUPLICATE_KEY_CODE);
48
+ }
49
+ return writeError.code === DUPLICATE_KEY_CODE;
50
+ };
51
+
52
+ const insertCopies = async ({
53
+ collection,
54
+ copies,
55
+ }: {
56
+ collection: mongoose.mongo.Collection;
57
+ copies: mongoose.mongo.Document[];
58
+ }): Promise<void> => {
59
+ try {
60
+ await collection.insertMany(copies, {ordered: false});
61
+ } catch (error) {
62
+ if (!isOnlyDuplicateKeyError(error)) {
63
+ throw error;
64
+ }
65
+ }
66
+ };
67
+
68
+ /**
69
+ * Moves every ObjectId-id GptHistory to its hex-string id in batches and rewrites the
70
+ * AIDataset refs. In a dry run it only counts what it would change.
71
+ */
72
+ export const migrateGptHistoryStringIds = async ({
73
+ batchSize = DEFAULT_BATCH_SIZE,
74
+ checkCancellation,
75
+ connection,
76
+ dryRun,
77
+ logger,
78
+ }: {
79
+ batchSize?: number;
80
+ checkCancellation?: () => Promise<void>;
81
+ connection: mongoose.Connection;
82
+ dryRun: boolean;
83
+ logger?: MigrationLogger;
84
+ }): Promise<GptHistoryStringIdsResult> => {
85
+ const histories = connection.collection(GptHistory.collection.collectionName);
86
+ const datasets = connection.collection(AIDataset.collection.collectionName);
87
+ const datasetFilter = {historyId: {$type: "objectId" as const}};
88
+
89
+ if (dryRun) {
90
+ const result = {
91
+ datasetsUpdated: await datasets.countDocuments(datasetFilter),
92
+ historiesMigrated: await histories.countDocuments(OBJECT_ID_FILTER),
93
+ };
94
+ logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] dry run`, result);
95
+ return result;
96
+ }
97
+
98
+ let historiesMigrated = 0;
99
+ let datasetsUpdated = 0;
100
+ while (true) {
101
+ await checkCancellation?.();
102
+ const batch = await histories.find(OBJECT_ID_FILTER).limit(batchSize).toArray();
103
+ if (batch.length === 0) {
104
+ break;
105
+ }
106
+ const originalIds = batch.map((doc) => doc._id);
107
+ // Copy first and delete second, so an interrupted run never loses a history: the next
108
+ // run finds the original again and the copy's duplicate key is ignored.
109
+ await insertCopies({
110
+ collection: histories,
111
+ copies: batch.map((doc) => ({...doc, _id: String(doc._id)})),
112
+ });
113
+ const moved = await datasets.updateMany({historyId: {$in: originalIds}}, HISTORY_ID_TO_STRING);
114
+ datasetsUpdated += moved.modifiedCount;
115
+ await histories.deleteMany({_id: {$in: originalIds}});
116
+ historiesMigrated += batch.length;
117
+ logger?.info(
118
+ `[${GPT_HISTORY_STRING_IDS_MIGRATION}] moved ${historiesMigrated} histories to String ids`
119
+ );
120
+ }
121
+
122
+ // Refs whose history was moved by an interrupted run, or never existed, still hold ObjectIds.
123
+ const remaining = await datasets.updateMany(datasetFilter, HISTORY_ID_TO_STRING);
124
+ datasetsUpdated += remaining.modifiedCount;
125
+ const result = {datasetsUpdated, historiesMigrated};
126
+ logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] done`, result);
127
+ return result;
128
+ };
129
+
130
+ /**
131
+ * Ready-made `up` for an app's `migrations/<YYYYMMDDHHmmss>-gpt-history-string-ids.ts` file.
132
+ * Run it before deploying Terreno 58 frontends, which address histories by String id. It has
133
+ * no `down`: new histories get UUIDs, which have no ObjectId to go back to.
134
+ */
135
+ export const gptHistoryStringIdsMigration = {
136
+ slug: GPT_HISTORY_STRING_IDS_MIGRATION,
137
+ up: async (ctx: MigrationContext): Promise<void> => {
138
+ const result = await migrateGptHistoryStringIds({
139
+ checkCancellation: ctx.checkCancellation,
140
+ connection: ctx.mongoose.connection,
141
+ dryRun: ctx.dryRun,
142
+ logger: ctx.logger,
143
+ });
144
+ await ctx.addLog?.(
145
+ "info",
146
+ `[${GPT_HISTORY_STRING_IDS_MIGRATION}] ${ctx.dryRun ? "would move" : "moved"} ` +
147
+ `${result.historiesMigrated} histories and ${result.datasetsUpdated} dataset refs`
148
+ );
149
+ },
150
+ };
@@ -12,7 +12,7 @@ describe("AIDataset model", () => {
12
12
  const userId = new mongoose.Types.ObjectId();
13
13
  const dataset = await AIDataset.create({
14
14
  columns: [{name: "count", type: "number"}],
15
- historyId: new mongoose.Types.ObjectId(),
15
+ historyId: new mongoose.Types.ObjectId().toString(),
16
16
  rowCount: 1,
17
17
  rows: [[3]],
18
18
  userId,
@@ -32,11 +32,11 @@ const aiDatasetSchema: AIDatasetSchema = new mongoose.Schema(
32
32
  type: Date,
33
33
  },
34
34
  historyId: {
35
- description: "The conversation whose tool stored these rows",
35
+ description: "The conversation whose tool stored these rows (a GptHistory String id)",
36
36
  index: true,
37
37
  ref: "GptHistory",
38
38
  required: true,
39
- type: mongoose.Schema.Types.ObjectId,
39
+ type: String,
40
40
  },
41
41
  rowCount: {
42
42
  description: "How many rows were stored",