@terreno/ai 57.21.0 → 58.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +20 -18
- package/CHANGELOG.md +43 -0
- package/biome.jsonc +2 -2
- package/dist/aiApp.d.ts +5 -2
- package/dist/aiApp.js +3 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/migrations/gptHistoryStringIds.d.ts +41 -0
- package/dist/migrations/gptHistoryStringIds.js +94 -0
- package/dist/models/aiDataset.js +2 -2
- package/dist/models/gptHistory.js +11 -0
- package/dist/routes/files.d.ts +7 -1
- package/dist/routes/files.js +62 -13
- package/dist/routes/gptActions.js +0 -7
- package/dist/routes/gptHistories.js +8 -2
- package/dist/service/aiDatasets.js +2 -1
- package/dist/service/chatTurn.js +4 -0
- package/dist/service/vertex.d.ts +5 -2
- package/dist/types/index.d.ts +5 -2
- package/package.json +13 -12
- package/src/aiApp.test.ts +21 -1
- package/src/aiApp.ts +6 -3
- package/src/index.ts +7 -1
- package/src/isolated/gptStream.isolated.ts +2 -2
- package/src/migrations/gptHistoryStringIds.test.ts +231 -0
- package/src/migrations/gptHistoryStringIds.ts +150 -0
- package/src/models/aiDataset.test.ts +1 -1
- package/src/models/aiDataset.ts +2 -2
- package/src/models/gptHistory.test.ts +12 -5
- package/src/models/gptHistory.ts +19 -1
- package/src/routes/files.test.ts +121 -0
- package/src/routes/files.ts +90 -15
- package/src/routes/gpt.test.ts +2 -2
- package/src/routes/gptActions.test.ts +8 -7
- package/src/routes/gptActions.ts +1 -4
- package/src/routes/gptApprovals.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +2 -2
- package/src/routes/gptHistories.ts +59 -55
- package/src/routes/gptHistoriesSync.test.ts +141 -0
- package/src/routes/routes.test.ts +6 -5
- package/src/service/aiDatasets.ts +2 -1
- package/src/service/chatTurn.ts +4 -0
- package/src/service/vertex.test.ts +1 -1
- package/src/service/vertex.ts +7 -4
- package/src/types/index.ts +5 -2
|
@@ -91,7 +91,8 @@ const registerAiDataset = async ({ columns, historyId, rows, userId, }) => {
|
|
|
91
91
|
assertRows(columns, rows);
|
|
92
92
|
const created = await aiDataset_1.AIDataset.create({
|
|
93
93
|
columns,
|
|
94
|
-
|
|
94
|
+
// GptHistory ids are Strings; an ObjectId from an older caller is stored as its hex string.
|
|
95
|
+
historyId: String(historyId),
|
|
95
96
|
rowCount: rows.length,
|
|
96
97
|
rows,
|
|
97
98
|
userId,
|
package/dist/service/chatTurn.js
CHANGED
|
@@ -155,6 +155,10 @@ const loadHistory = async ({ historyId, projectId, userId, }) => {
|
|
|
155
155
|
if (!historyId) {
|
|
156
156
|
return new gptHistory_1.GptHistory({ prompts: [], userId, ...(projectId ? { projectId } : {}) });
|
|
157
157
|
}
|
|
158
|
+
// History ids are Strings (synced collection); an object here would be read as a query operator.
|
|
159
|
+
if (typeof historyId !== "string") {
|
|
160
|
+
throw new api_1.APIError({ status: 400, title: "historyId must be a string" });
|
|
161
|
+
}
|
|
158
162
|
const history = await gptHistory_1.GptHistory.findById(historyId);
|
|
159
163
|
if (!history) {
|
|
160
164
|
throw new api_1.APIError({ status: 404, title: "History not found" });
|
package/dist/service/vertex.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ImageModelV4, LanguageModelV4 } from "@ai-sdk/provider";
|
|
1
2
|
import type { ImageModel, LanguageModel } from "ai";
|
|
2
3
|
/**
|
|
3
4
|
* Helper for Google's Vertex AI provider, now part of the "Gemini Enterprise Agent Platform"
|
|
@@ -16,8 +17,10 @@ import type { ImageModel, LanguageModel } from "ai";
|
|
|
16
17
|
export declare const DEFAULT_VERTEX_LOCATION = "global";
|
|
17
18
|
/** Provider returned by `@ai-sdk/google-vertex` `createVertex`. */
|
|
18
19
|
export interface VertexLanguageModelProvider {
|
|
19
|
-
|
|
20
|
-
|
|
20
|
+
/** `@ai-sdk/google-vertex` v4+ returns `LanguageModelV4`; older `ai` majors still type `LanguageModel` as v2/v3 only. */
|
|
21
|
+
(modelId: string): LanguageModel | LanguageModelV4;
|
|
22
|
+
/** `@ai-sdk/google-vertex` v4+ returns `ImageModelV4`; older `ai` majors still type `ImageModel` as v2/v3 only. */
|
|
23
|
+
image: (modelId: string) => ImageModel | ImageModelV4;
|
|
21
24
|
}
|
|
22
25
|
interface VertexModule {
|
|
23
26
|
createVertex: (opts: {
|
package/dist/types/index.d.ts
CHANGED
|
@@ -113,7 +113,9 @@ interface GptHistoryPendingAskState {
|
|
|
113
113
|
}
|
|
114
114
|
/** The ask a history is waiting on. At most one per history. */
|
|
115
115
|
export type GptHistoryPendingAsk = Ask & GptHistoryPendingAskState;
|
|
116
|
-
|
|
116
|
+
/** Synced through @terreno/syncdb, so `_id` is a String (UUID for new rows, ObjectId hex for migrated rows). */
|
|
117
|
+
export interface GptHistoryDocument extends mongoose.Document<string> {
|
|
118
|
+
_id: string;
|
|
117
119
|
created: Date;
|
|
118
120
|
deleted: boolean;
|
|
119
121
|
pendingAsk?: GptHistoryPendingAsk;
|
|
@@ -435,7 +437,8 @@ export interface AIDatasetDocument extends mongoose.Document<mongoose.Types.Obje
|
|
|
435
437
|
created: Date;
|
|
436
438
|
deleted: boolean;
|
|
437
439
|
expiresAt?: Date;
|
|
438
|
-
|
|
440
|
+
/** The GptHistory String `_id`. */
|
|
441
|
+
historyId: string;
|
|
439
442
|
rowCount: number;
|
|
440
443
|
rows: AIDatasetCell[][];
|
|
441
444
|
updated: Date;
|
package/package.json
CHANGED
|
@@ -4,17 +4,17 @@
|
|
|
4
4
|
"url": "https://github.com/TerrenoLabs/terreno/issues"
|
|
5
5
|
},
|
|
6
6
|
"dependencies": {
|
|
7
|
-
"@ai-sdk/mcp": "^
|
|
7
|
+
"@ai-sdk/mcp": "^2.0.57",
|
|
8
8
|
"@google-cloud/storage": "^7.15.0",
|
|
9
9
|
"@langfuse/client": "^5.0.1",
|
|
10
10
|
"@langfuse/otel": "^5.0.1",
|
|
11
11
|
"@opentelemetry/api": "^1.9.0",
|
|
12
|
-
"@opentelemetry/sdk-node": "0.
|
|
13
|
-
"@terreno/blocks": "
|
|
12
|
+
"@opentelemetry/sdk-node": "0.222.0",
|
|
13
|
+
"@terreno/blocks": "58.0.0-beta.1",
|
|
14
14
|
"ai": "^6.0.97",
|
|
15
15
|
"ajv": "8.20.0",
|
|
16
16
|
"csv-parse": "^6.1.0",
|
|
17
|
-
"google-auth-library": "^
|
|
17
|
+
"google-auth-library": "^11.1.0",
|
|
18
18
|
"luxon": "^3.7.2",
|
|
19
19
|
"multer": "2.2.0",
|
|
20
20
|
"sanitize-html": "^2.17.0"
|
|
@@ -22,10 +22,10 @@
|
|
|
22
22
|
"description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
|
|
23
23
|
"devDependencies": {
|
|
24
24
|
"@biomejs/biome": "^2.3.6",
|
|
25
|
-
"@terreno/admin-backend": "
|
|
26
|
-
"@terreno/api": "
|
|
27
|
-
"@terreno/jobs": "
|
|
28
|
-
"@terreno/test": "
|
|
25
|
+
"@terreno/admin-backend": "58.0.0-beta.1",
|
|
26
|
+
"@terreno/api": "58.0.0-beta.1",
|
|
27
|
+
"@terreno/jobs": "58.0.0-beta.1",
|
|
28
|
+
"@terreno/test": "58.0.0-beta.1",
|
|
29
29
|
"@types/bun": "^1.4.0",
|
|
30
30
|
"@types/express": "^5.0.6",
|
|
31
31
|
"@types/luxon": "^3.7.1",
|
|
@@ -135,9 +135,10 @@
|
|
|
135
135
|
"main": "dist/index.js",
|
|
136
136
|
"name": "@terreno/ai",
|
|
137
137
|
"peerDependencies": {
|
|
138
|
-
"@ai-sdk/google-vertex": "^
|
|
139
|
-
"@
|
|
140
|
-
"@terreno/
|
|
138
|
+
"@ai-sdk/google-vertex": "^5.0.92",
|
|
139
|
+
"@ai-sdk/provider": "^4.0.26",
|
|
140
|
+
"@terreno/api": "58.0.0-beta.1",
|
|
141
|
+
"@terreno/jobs": "58.0.0-beta.1",
|
|
141
142
|
"mongoose": "^8.0.0 || ^9.0.0"
|
|
142
143
|
},
|
|
143
144
|
"peerDependenciesMeta": {
|
|
@@ -170,5 +171,5 @@
|
|
|
170
171
|
"typecheck": "tsc -p tsconfig.typecheck.json"
|
|
171
172
|
},
|
|
172
173
|
"types": "dist/index.d.ts",
|
|
173
|
-
"version": "
|
|
174
|
+
"version": "58.0.0-beta.1"
|
|
174
175
|
}
|
package/src/aiApp.test.ts
CHANGED
|
@@ -55,7 +55,7 @@ describe("AiApp", () => {
|
|
|
55
55
|
|
|
56
56
|
afterEach(async () => {
|
|
57
57
|
await AIRequest.deleteMany({});
|
|
58
|
-
await GptHistory.deleteMany({});
|
|
58
|
+
await GptHistory.collection.deleteMany({});
|
|
59
59
|
});
|
|
60
60
|
|
|
61
61
|
const authAsUser = async (app: express.Application) => {
|
|
@@ -356,4 +356,24 @@ describe("AiApp", () => {
|
|
|
356
356
|
const servers = await agent.get("/mcp/servers");
|
|
357
357
|
expect(servers.status).not.toBe(404);
|
|
358
358
|
});
|
|
359
|
+
it("documents its routes with the OpenAPI instance TerrenoApp passes to register", async () => {
|
|
360
|
+
const {AIService} = await import("./service/aiService.js");
|
|
361
|
+
const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
362
|
+
const fileStorageService = {
|
|
363
|
+
delete: mock(async () => {}),
|
|
364
|
+
getSignedUrl: mock(async () => "https://example.com/signed"),
|
|
365
|
+
upload: mock(async () => {
|
|
366
|
+
throw new Error("not called");
|
|
367
|
+
}),
|
|
368
|
+
} as unknown as FileStorageService;
|
|
369
|
+
const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
370
|
+
.register(new AiApp({aiService, fileStorageService, gcsBucket: "bucket"}))
|
|
371
|
+
.build();
|
|
372
|
+
|
|
373
|
+
const res = await supertest(app).get("/openapi.json").expect(200);
|
|
374
|
+
expect(res.body.paths["/files/upload"]?.post?.requestBody?.content).toHaveProperty(
|
|
375
|
+
"multipart/form-data"
|
|
376
|
+
);
|
|
377
|
+
expect(res.body.paths["/gpt/prompt"]?.post).toBeDefined();
|
|
378
|
+
});
|
|
359
379
|
});
|
package/src/aiApp.ts
CHANGED
|
@@ -40,7 +40,10 @@ export interface AiAppOptions {
|
|
|
40
40
|
maxSteps?: number;
|
|
41
41
|
/** MCP service for connecting to external tool servers. */
|
|
42
42
|
mcpService?: MCPService;
|
|
43
|
-
/**
|
|
43
|
+
/**
|
|
44
|
+
* OpenAPI options passed through to route builders for spec generation. Defaults to the
|
|
45
|
+
* OpenAPI instance TerrenoApp passes to `register`.
|
|
46
|
+
*/
|
|
44
47
|
openApiOptions?: Record<string, unknown>;
|
|
45
48
|
/**
|
|
46
49
|
* Mount the `/gpt/projects` routes, which register the `Project` model. Defaults to `true`;
|
|
@@ -88,7 +91,7 @@ export class AiApp implements TerrenoPlugin {
|
|
|
88
91
|
this.options = options;
|
|
89
92
|
}
|
|
90
93
|
|
|
91
|
-
register(app: express.Application): void {
|
|
94
|
+
register(app: express.Application, openApi?: unknown): void {
|
|
92
95
|
const router = app;
|
|
93
96
|
const {
|
|
94
97
|
aiService,
|
|
@@ -101,13 +104,13 @@ export class AiApp implements TerrenoPlugin {
|
|
|
101
104
|
gcsBucket,
|
|
102
105
|
maxSteps,
|
|
103
106
|
mcpService,
|
|
104
|
-
openApiOptions,
|
|
105
107
|
projects,
|
|
106
108
|
titleModelId,
|
|
107
109
|
toolChoice,
|
|
108
110
|
tools,
|
|
109
111
|
} = this.options;
|
|
110
112
|
|
|
113
|
+
const openApiOptions = this.options.openApiOptions ?? (openApi ? {openApi} : undefined);
|
|
111
114
|
const hasFileRoutes = Boolean(fileStorageService && gcsBucket);
|
|
112
115
|
const chat = {
|
|
113
116
|
aiService,
|
package/src/index.ts
CHANGED
|
@@ -36,6 +36,12 @@ export type {
|
|
|
36
36
|
TraceListItem,
|
|
37
37
|
} from "./langfuseTypes";
|
|
38
38
|
export {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
39
|
+
export {
|
|
40
|
+
GPT_HISTORY_STRING_IDS_MIGRATION,
|
|
41
|
+
type GptHistoryStringIdsResult,
|
|
42
|
+
gptHistoryStringIdsMigration,
|
|
43
|
+
migrateGptHistoryStringIds,
|
|
44
|
+
} from "./migrations/gptHistoryStringIds";
|
|
39
45
|
export {AIDataset} from "./models/aiDataset";
|
|
40
46
|
export {AIRequest} from "./models/aiRequest";
|
|
41
47
|
export {FileAttachment} from "./models/fileAttachment";
|
|
@@ -98,7 +104,7 @@ export {
|
|
|
98
104
|
validateObservabilityConfig,
|
|
99
105
|
} from "./observability/types";
|
|
100
106
|
export {addAiRequestsExplorerRoutes} from "./routes/aiRequestsExplorer";
|
|
101
|
-
export {addFileRoutes} from "./routes/files";
|
|
107
|
+
export {addFileRoutes, type FileStorageServiceSource} from "./routes/files";
|
|
102
108
|
export {addGptRoutes} from "./routes/gpt";
|
|
103
109
|
export {addGptHistoryRoutes} from "./routes/gptHistories";
|
|
104
110
|
export {addMcpRoutes} from "./routes/mcp";
|
|
@@ -83,7 +83,7 @@ describe("GPT prompt streaming edge cases", () => {
|
|
|
83
83
|
|
|
84
84
|
beforeAll(async () => {
|
|
85
85
|
await ensureTestUsers();
|
|
86
|
-
await GptHistory.deleteMany({});
|
|
86
|
+
await GptHistory.collection.deleteMany({});
|
|
87
87
|
const aiService = new AIService({
|
|
88
88
|
model: createMockModel() as unknown as LanguageModel,
|
|
89
89
|
});
|
|
@@ -97,7 +97,7 @@ describe("GPT prompt streaming edge cases", () => {
|
|
|
97
97
|
});
|
|
98
98
|
|
|
99
99
|
afterAll(async () => {
|
|
100
|
-
await GptHistory.deleteMany({});
|
|
100
|
+
await GptHistory.collection.deleteMany({});
|
|
101
101
|
});
|
|
102
102
|
|
|
103
103
|
it("emits an inline image and flushes buffered text with no finish-step", async () => {
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import type {MigrationContext} from "@terreno/api";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import {AIDataset} from "../models/aiDataset";
|
|
6
|
+
import {GptHistory} from "../models/gptHistory";
|
|
7
|
+
import {gptHistoryStringIdsMigration, migrateGptHistoryStringIds} from "./gptHistoryStringIds";
|
|
8
|
+
|
|
9
|
+
const userId = new mongoose.Types.ObjectId();
|
|
10
|
+
|
|
11
|
+
const historiesCollection = (): mongoose.mongo.Collection =>
|
|
12
|
+
mongoose.connection.collection(GptHistory.collection.collectionName);
|
|
13
|
+
|
|
14
|
+
const datasetsCollection = (): mongoose.mongo.Collection =>
|
|
15
|
+
mongoose.connection.collection(AIDataset.collection.collectionName);
|
|
16
|
+
|
|
17
|
+
const legacyHistory = (title: string): mongoose.mongo.Document => ({
|
|
18
|
+
_id: new mongoose.Types.ObjectId(),
|
|
19
|
+
created: new Date("2026-01-05T09:00:00.000Z"),
|
|
20
|
+
deleted: false,
|
|
21
|
+
prompts: [
|
|
22
|
+
{text: `Question about ${title}`, type: "user"},
|
|
23
|
+
{model: "gemini", text: `Answer about ${title}`, type: "assistant"},
|
|
24
|
+
],
|
|
25
|
+
title,
|
|
26
|
+
updated: new Date("2026-01-05T09:01:00.000Z"),
|
|
27
|
+
userId,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const legacyDataset = (historyId: unknown): mongoose.mongo.Document => ({
|
|
31
|
+
_id: new mongoose.Types.ObjectId(),
|
|
32
|
+
columns: [{name: "count", type: "number"}],
|
|
33
|
+
created: new Date("2026-01-05T09:02:00.000Z"),
|
|
34
|
+
deleted: false,
|
|
35
|
+
historyId,
|
|
36
|
+
rowCount: 1,
|
|
37
|
+
rows: [[1]],
|
|
38
|
+
updated: new Date("2026-01-05T09:02:00.000Z"),
|
|
39
|
+
userId,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
interface Seeded {
|
|
43
|
+
histories: mongoose.mongo.Document[];
|
|
44
|
+
stringHistoryId: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Five ObjectId histories (two batches at batchSize 2, plus one), a String one, and refs. */
|
|
48
|
+
const seed = async (): Promise<Seeded> => {
|
|
49
|
+
const histories = ["a", "b", "c", "d", "e"].map(legacyHistory);
|
|
50
|
+
const stringHistoryId = "already-a-string-id";
|
|
51
|
+
await historiesCollection().insertMany([
|
|
52
|
+
...histories,
|
|
53
|
+
{...legacyHistory("string"), _id: stringHistoryId},
|
|
54
|
+
]);
|
|
55
|
+
await datasetsCollection().insertMany([
|
|
56
|
+
legacyDataset(histories[0]._id),
|
|
57
|
+
legacyDataset(histories[0]._id),
|
|
58
|
+
legacyDataset(histories[3]._id),
|
|
59
|
+
// A ref whose history was deleted long ago still becomes a String.
|
|
60
|
+
legacyDataset(new mongoose.Types.ObjectId()),
|
|
61
|
+
legacyDataset(stringHistoryId),
|
|
62
|
+
]);
|
|
63
|
+
return {histories, stringHistoryId};
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const idTypes = async (collection: mongoose.mongo.Collection, field: string): Promise<string[]> => {
|
|
67
|
+
const rows = await collection
|
|
68
|
+
.aggregate<{type: string}>([{$project: {_id: 0, type: {$type: `$${field}`}}}])
|
|
69
|
+
.toArray();
|
|
70
|
+
return [...new Set(rows.map((row) => row.type))].sort();
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const migrationContext = (dryRun: boolean, logs: string[] = []): MigrationContext => ({
|
|
74
|
+
addLog: async (_level, message): Promise<void> => {
|
|
75
|
+
logs.push(message);
|
|
76
|
+
},
|
|
77
|
+
dryRun,
|
|
78
|
+
logger: {debug: () => {}, error: () => {}, info: () => {}, warn: () => {}},
|
|
79
|
+
mongoose,
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe("gpt-history-string-ids migration", () => {
|
|
83
|
+
beforeEach(async () => {
|
|
84
|
+
await historiesCollection().deleteMany({});
|
|
85
|
+
await datasetsCollection().deleteMany({});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
afterEach(async () => {
|
|
89
|
+
await historiesCollection().deleteMany({});
|
|
90
|
+
await datasetsCollection().deleteMany({});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("moves ObjectId histories to their hex string ids, rewrites refs, and is a no-op the second time", async () => {
|
|
94
|
+
const {histories, stringHistoryId} = await seed();
|
|
95
|
+
const historyCount = await historiesCollection().countDocuments();
|
|
96
|
+
const datasetCount = await datasetsCollection().countDocuments();
|
|
97
|
+
|
|
98
|
+
const first = await migrateGptHistoryStringIds({
|
|
99
|
+
batchSize: 2,
|
|
100
|
+
connection: mongoose.connection,
|
|
101
|
+
dryRun: false,
|
|
102
|
+
});
|
|
103
|
+
const second = await migrateGptHistoryStringIds({
|
|
104
|
+
batchSize: 2,
|
|
105
|
+
connection: mongoose.connection,
|
|
106
|
+
dryRun: false,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(first).toEqual({datasetsUpdated: 4, historiesMigrated: 5});
|
|
110
|
+
expect(second).toEqual({datasetsUpdated: 0, historiesMigrated: 0});
|
|
111
|
+
expect(await historiesCollection().countDocuments()).toBe(historyCount);
|
|
112
|
+
expect(await datasetsCollection().countDocuments()).toBe(datasetCount);
|
|
113
|
+
expect(await idTypes(historiesCollection(), "_id")).toEqual(["string"]);
|
|
114
|
+
expect(await idTypes(datasetsCollection(), "historyId")).toEqual(["string"]);
|
|
115
|
+
|
|
116
|
+
const ids = (await historiesCollection().find({}).toArray()).map((doc) => doc._id).sort();
|
|
117
|
+
expect(ids).toEqual([...histories.map((doc) => doc._id.toHexString()), stringHistoryId].sort());
|
|
118
|
+
const firstHex = histories[0]._id.toHexString();
|
|
119
|
+
expect(await datasetsCollection().countDocuments({historyId: firstHex})).toBe(2);
|
|
120
|
+
expect(
|
|
121
|
+
await datasetsCollection().countDocuments({historyId: histories[3]._id.toHexString()})
|
|
122
|
+
).toBe(1);
|
|
123
|
+
expect(await datasetsCollection().countDocuments({historyId: stringHistoryId})).toBe(1);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("keeps every stored field, so the moved history loads through the model", async () => {
|
|
127
|
+
const {histories} = await seed();
|
|
128
|
+
|
|
129
|
+
await migrateGptHistoryStringIds({connection: mongoose.connection, dryRun: false});
|
|
130
|
+
|
|
131
|
+
const hexId = histories[1]._id.toHexString();
|
|
132
|
+
const history = await GptHistory.findById(hexId);
|
|
133
|
+
expect(history?._id).toBe(hexId);
|
|
134
|
+
expect(history?.title).toBe("b");
|
|
135
|
+
expect(history?.userId.toString()).toBe(userId.toString());
|
|
136
|
+
expect(history?.prompts.map((prompt) => prompt.text)).toEqual([
|
|
137
|
+
"Question about b",
|
|
138
|
+
"Answer about b",
|
|
139
|
+
]);
|
|
140
|
+
const datasets = await AIDataset.find({historyId: histories[0]._id.toHexString()});
|
|
141
|
+
expect(datasets).toHaveLength(2);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it("finishes a run that stopped after copying a history but before deleting the original", async () => {
|
|
145
|
+
const {histories} = await seed();
|
|
146
|
+
const interrupted = histories[2];
|
|
147
|
+
await historiesCollection().insertOne({...interrupted, _id: interrupted._id.toHexString()});
|
|
148
|
+
|
|
149
|
+
const result = await migrateGptHistoryStringIds({
|
|
150
|
+
connection: mongoose.connection,
|
|
151
|
+
dryRun: false,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
expect(result.historiesMigrated).toBe(5);
|
|
155
|
+
expect(await historiesCollection().countDocuments()).toBe(6);
|
|
156
|
+
expect(await idTypes(historiesCollection(), "_id")).toEqual(["string"]);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("rewrites refs left as ObjectIds when a run stopped after moving their history", async () => {
|
|
160
|
+
const orphanedHistory = new mongoose.Types.ObjectId();
|
|
161
|
+
const movedHistory: mongoose.mongo.Document = {
|
|
162
|
+
...legacyHistory("moved"),
|
|
163
|
+
_id: orphanedHistory.toHexString(),
|
|
164
|
+
};
|
|
165
|
+
await historiesCollection().insertOne(movedHistory);
|
|
166
|
+
await datasetsCollection().insertOne(legacyDataset(orphanedHistory));
|
|
167
|
+
|
|
168
|
+
const result = await migrateGptHistoryStringIds({
|
|
169
|
+
connection: mongoose.connection,
|
|
170
|
+
dryRun: false,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
expect(result).toEqual({datasetsUpdated: 1, historiesMigrated: 0});
|
|
174
|
+
expect(
|
|
175
|
+
await datasetsCollection().countDocuments({historyId: orphanedHistory.toHexString()})
|
|
176
|
+
).toBe(1);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it("only counts in a dry run", async () => {
|
|
180
|
+
await seed();
|
|
181
|
+
|
|
182
|
+
const result = await migrateGptHistoryStringIds({
|
|
183
|
+
connection: mongoose.connection,
|
|
184
|
+
dryRun: true,
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
expect(result).toEqual({datasetsUpdated: 4, historiesMigrated: 5});
|
|
188
|
+
expect(await idTypes(historiesCollection(), "_id")).toEqual(["objectId", "string"]);
|
|
189
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(5);
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
it("runs as a migration up through the migration context", async () => {
|
|
193
|
+
await seed();
|
|
194
|
+
|
|
195
|
+
const logs: string[] = [];
|
|
196
|
+
await gptHistoryStringIdsMigration.up(migrationContext(true, logs));
|
|
197
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(5);
|
|
198
|
+
|
|
199
|
+
await gptHistoryStringIdsMigration.up(migrationContext(false, logs));
|
|
200
|
+
await gptHistoryStringIdsMigration.up(migrationContext(false, logs));
|
|
201
|
+
expect(gptHistoryStringIdsMigration.slug).toBe("gpt-history-string-ids");
|
|
202
|
+
expect(logs).toEqual([
|
|
203
|
+
"[gpt-history-string-ids] would move 5 histories and 4 dataset refs",
|
|
204
|
+
"[gpt-history-string-ids] moved 5 histories and 4 dataset refs",
|
|
205
|
+
"[gpt-history-string-ids] moved 0 histories and 0 dataset refs",
|
|
206
|
+
]);
|
|
207
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(0);
|
|
208
|
+
expect(await datasetsCollection().countDocuments({historyId: {$type: "objectId"}})).toBe(0);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it("stops between batches when the run is cancelled", async () => {
|
|
212
|
+
await seed();
|
|
213
|
+
let checks = 0;
|
|
214
|
+
const checkCancellation = async (): Promise<void> => {
|
|
215
|
+
checks += 1;
|
|
216
|
+
if (checks > 1) {
|
|
217
|
+
throw new Error("cancelled");
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
await expect(
|
|
222
|
+
migrateGptHistoryStringIds({
|
|
223
|
+
batchSize: 2,
|
|
224
|
+
checkCancellation,
|
|
225
|
+
connection: mongoose.connection,
|
|
226
|
+
dryRun: false,
|
|
227
|
+
})
|
|
228
|
+
).rejects.toThrow("cancelled");
|
|
229
|
+
expect(await historiesCollection().countDocuments({_id: {$type: "objectId"}})).toBe(3);
|
|
230
|
+
});
|
|
231
|
+
});
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Data migration for Terreno 58: GptHistory became a synced collection with String `_id`s.
|
|
3
|
+
* Histories saved before 58 have ObjectId `_id`s, which sync clients and the String-typed
|
|
4
|
+
* schema cannot address. This migration re-inserts each one with `_id` set to the same id's
|
|
5
|
+
* hex string, so URLs and client caches that hold the hex id keep working, rewrites
|
|
6
|
+
* `AIDataset.historyId` refs to that hex string, and then deletes the ObjectId originals.
|
|
7
|
+
*
|
|
8
|
+
* It writes through the native driver: `syncPlugin` forbids the hard deletes and
|
|
9
|
+
* multi-document writes a move needs, and the copies keep every stored field as-is.
|
|
10
|
+
* Running it again is a no-op.
|
|
11
|
+
*/
|
|
12
|
+
import type {MigrationContext, MigrationLogger} from "@terreno/api";
|
|
13
|
+
import type mongoose from "mongoose";
|
|
14
|
+
|
|
15
|
+
import {AIDataset} from "../models/aiDataset";
|
|
16
|
+
import {GptHistory} from "../models/gptHistory";
|
|
17
|
+
|
|
18
|
+
/** Slug for the timestamped file an app adds to its `migrations/` directory. */
|
|
19
|
+
export const GPT_HISTORY_STRING_IDS_MIGRATION = "gpt-history-string-ids";
|
|
20
|
+
|
|
21
|
+
const DEFAULT_BATCH_SIZE = 500;
|
|
22
|
+
|
|
23
|
+
const DUPLICATE_KEY_CODE = 11000;
|
|
24
|
+
|
|
25
|
+
/** Only ObjectId `_id`s still need a move; String ids are already migrated or new. */
|
|
26
|
+
const OBJECT_ID_FILTER = {_id: {$type: "objectId" as const}};
|
|
27
|
+
|
|
28
|
+
/** Server-side ObjectId → hex string, so refs move without reading each dataset. */
|
|
29
|
+
const HISTORY_ID_TO_STRING = [{$set: {historyId: {$toString: "$historyId"}}}];
|
|
30
|
+
|
|
31
|
+
export interface GptHistoryStringIdsResult {
|
|
32
|
+
/** AIDataset rows whose ObjectId `historyId` was rewritten (or would be, in a dry run). */
|
|
33
|
+
datasetsUpdated: number;
|
|
34
|
+
/** Histories moved to a String `_id` (or that would move, in a dry run). */
|
|
35
|
+
historiesMigrated: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface WriteErrorLike {
|
|
39
|
+
code?: number;
|
|
40
|
+
writeErrors?: Array<{code?: number}>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** An insert that failed only because the hex-id copy exists from an interrupted earlier run. */
|
|
44
|
+
const isOnlyDuplicateKeyError = (error: unknown): boolean => {
|
|
45
|
+
const writeError = error as WriteErrorLike;
|
|
46
|
+
if (writeError.writeErrors?.length) {
|
|
47
|
+
return writeError.writeErrors.every((item) => item.code === DUPLICATE_KEY_CODE);
|
|
48
|
+
}
|
|
49
|
+
return writeError.code === DUPLICATE_KEY_CODE;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const insertCopies = async ({
|
|
53
|
+
collection,
|
|
54
|
+
copies,
|
|
55
|
+
}: {
|
|
56
|
+
collection: mongoose.mongo.Collection;
|
|
57
|
+
copies: mongoose.mongo.Document[];
|
|
58
|
+
}): Promise<void> => {
|
|
59
|
+
try {
|
|
60
|
+
await collection.insertMany(copies, {ordered: false});
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (!isOnlyDuplicateKeyError(error)) {
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Moves every ObjectId-id GptHistory to its hex-string id in batches and rewrites the
|
|
70
|
+
* AIDataset refs. In a dry run it only counts what it would change.
|
|
71
|
+
*/
|
|
72
|
+
export const migrateGptHistoryStringIds = async ({
|
|
73
|
+
batchSize = DEFAULT_BATCH_SIZE,
|
|
74
|
+
checkCancellation,
|
|
75
|
+
connection,
|
|
76
|
+
dryRun,
|
|
77
|
+
logger,
|
|
78
|
+
}: {
|
|
79
|
+
batchSize?: number;
|
|
80
|
+
checkCancellation?: () => Promise<void>;
|
|
81
|
+
connection: mongoose.Connection;
|
|
82
|
+
dryRun: boolean;
|
|
83
|
+
logger?: MigrationLogger;
|
|
84
|
+
}): Promise<GptHistoryStringIdsResult> => {
|
|
85
|
+
const histories = connection.collection(GptHistory.collection.collectionName);
|
|
86
|
+
const datasets = connection.collection(AIDataset.collection.collectionName);
|
|
87
|
+
const datasetFilter = {historyId: {$type: "objectId" as const}};
|
|
88
|
+
|
|
89
|
+
if (dryRun) {
|
|
90
|
+
const result = {
|
|
91
|
+
datasetsUpdated: await datasets.countDocuments(datasetFilter),
|
|
92
|
+
historiesMigrated: await histories.countDocuments(OBJECT_ID_FILTER),
|
|
93
|
+
};
|
|
94
|
+
logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] dry run`, result);
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let historiesMigrated = 0;
|
|
99
|
+
let datasetsUpdated = 0;
|
|
100
|
+
while (true) {
|
|
101
|
+
await checkCancellation?.();
|
|
102
|
+
const batch = await histories.find(OBJECT_ID_FILTER).limit(batchSize).toArray();
|
|
103
|
+
if (batch.length === 0) {
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
const originalIds = batch.map((doc) => doc._id);
|
|
107
|
+
// Copy first and delete second, so an interrupted run never loses a history: the next
|
|
108
|
+
// run finds the original again and the copy's duplicate key is ignored.
|
|
109
|
+
await insertCopies({
|
|
110
|
+
collection: histories,
|
|
111
|
+
copies: batch.map((doc) => ({...doc, _id: String(doc._id)})),
|
|
112
|
+
});
|
|
113
|
+
const moved = await datasets.updateMany({historyId: {$in: originalIds}}, HISTORY_ID_TO_STRING);
|
|
114
|
+
datasetsUpdated += moved.modifiedCount;
|
|
115
|
+
await histories.deleteMany({_id: {$in: originalIds}});
|
|
116
|
+
historiesMigrated += batch.length;
|
|
117
|
+
logger?.info(
|
|
118
|
+
`[${GPT_HISTORY_STRING_IDS_MIGRATION}] moved ${historiesMigrated} histories to String ids`
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Refs whose history was moved by an interrupted run, or never existed, still hold ObjectIds.
|
|
123
|
+
const remaining = await datasets.updateMany(datasetFilter, HISTORY_ID_TO_STRING);
|
|
124
|
+
datasetsUpdated += remaining.modifiedCount;
|
|
125
|
+
const result = {datasetsUpdated, historiesMigrated};
|
|
126
|
+
logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] done`, result);
|
|
127
|
+
return result;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Ready-made `up` for an app's `migrations/<YYYYMMDDHHmmss>-gpt-history-string-ids.ts` file.
|
|
132
|
+
* Run it before deploying Terreno 58 frontends, which address histories by String id. It has
|
|
133
|
+
* no `down`: new histories get UUIDs, which have no ObjectId to go back to.
|
|
134
|
+
*/
|
|
135
|
+
export const gptHistoryStringIdsMigration = {
|
|
136
|
+
slug: GPT_HISTORY_STRING_IDS_MIGRATION,
|
|
137
|
+
up: async (ctx: MigrationContext): Promise<void> => {
|
|
138
|
+
const result = await migrateGptHistoryStringIds({
|
|
139
|
+
checkCancellation: ctx.checkCancellation,
|
|
140
|
+
connection: ctx.mongoose.connection,
|
|
141
|
+
dryRun: ctx.dryRun,
|
|
142
|
+
logger: ctx.logger,
|
|
143
|
+
});
|
|
144
|
+
await ctx.addLog?.(
|
|
145
|
+
"info",
|
|
146
|
+
`[${GPT_HISTORY_STRING_IDS_MIGRATION}] ${ctx.dryRun ? "would move" : "moved"} ` +
|
|
147
|
+
`${result.historiesMigrated} histories and ${result.datasetsUpdated} dataset refs`
|
|
148
|
+
);
|
|
149
|
+
},
|
|
150
|
+
};
|
|
@@ -12,7 +12,7 @@ describe("AIDataset model", () => {
|
|
|
12
12
|
const userId = new mongoose.Types.ObjectId();
|
|
13
13
|
const dataset = await AIDataset.create({
|
|
14
14
|
columns: [{name: "count", type: "number"}],
|
|
15
|
-
historyId: new mongoose.Types.ObjectId(),
|
|
15
|
+
historyId: new mongoose.Types.ObjectId().toString(),
|
|
16
16
|
rowCount: 1,
|
|
17
17
|
rows: [[3]],
|
|
18
18
|
userId,
|
package/src/models/aiDataset.ts
CHANGED
|
@@ -32,11 +32,11 @@ const aiDatasetSchema: AIDatasetSchema = new mongoose.Schema(
|
|
|
32
32
|
type: Date,
|
|
33
33
|
},
|
|
34
34
|
historyId: {
|
|
35
|
-
description: "The conversation whose tool stored these rows",
|
|
35
|
+
description: "The conversation whose tool stored these rows (a GptHistory String id)",
|
|
36
36
|
index: true,
|
|
37
37
|
ref: "GptHistory",
|
|
38
38
|
required: true,
|
|
39
|
-
type:
|
|
39
|
+
type: String,
|
|
40
40
|
},
|
|
41
41
|
rowCount: {
|
|
42
42
|
description: "How many rows were stored",
|