@verdant-web/server 1.8.13 → 1.8.14-next.1

package/src/__snapshots__/migrations.test.ts.snap

@@ -0,0 +1,55 @@
+// Vitest Snapshot v1
+
+exports[`the v5->v6 migration > rewrites legacy oids in all ops and baselines 1`] = `
+[
+  {
+    "data": "{\\"op\\":\\"set\\",\\"value\\":{\\"@@type\\":\\"ref\\",\\"id\\":\\"weirds/b:ghi\\"}}",
+    "libraryId": "test-1",
+    "oid": "weirds/b",
+    "replicaId": "foo",
+    "serverOrder": 1,
+    "timestamp": 1,
+  },
+  {
+    "data": "{\\"op\\":\\"list-push\\",\\"value\\":{\\"@@type\\":\\"ref\\",\\"id\\":\\"weirds/b:abc\\"}}",
+    "libraryId": "test-1",
+    "oid": "weirds/b:ghi",
+    "replicaId": "foo",
+    "serverOrder": 2,
+    "timestamp": 1,
+  },
+  {
+    "data": "{\\"op\\":\\"list-remove\\",\\"value\\":{\\"@@type\\":\\"ref\\",\\"id\\":\\"weirds/b:abc\\"}}",
+    "libraryId": "test-1",
+    "oid": "weirds/b:ghi",
+    "replicaId": "foo",
+    "serverOrder": 3,
+    "timestamp": 2,
+  },
+  {
+    "data": "{\\"op\\":\\"set\\",\\"value\\":{\\"@@type\\":\\"ref\\",\\"id\\":\\"weirds/b:ghi\\"}}",
+    "libraryId": "test-2",
+    "oid": "weirds/b",
+    "replicaId": "bar",
+    "serverOrder": 1,
+    "timestamp": "1a",
+  },
+]
+`;
+
+exports[`the v5->v6 migration > rewrites legacy oids in all ops and baselines 2`] = `
+[
+  {
+    "libraryId": "test-1",
+    "oid": "weirds/b:ghi",
+    "snapshot": "{\\"@@type\\":\\"ref\\",\\"id\\":\\"weirds/b:abc\\"}",
+    "timestamp": "1",
+  },
+  {
+    "libraryId": "test-2",
+    "oid": "weirds/b:ghi",
+    "snapshot": "{\\"@@type\\":\\"ref\\",\\"id\\":\\"weirds/b:abc\\"}",
+    "timestamp": "1a",
+  },
+]
+`;

package/src/migrations/{v0.sql.ts → v0.ts}

@@ -1,4 +1,4 @@
-export default `
+export const sql = `
 CREATE TABLE IF NOT EXISTS ReplicaInfo (
   id TEXT PRIMARY KEY NOT NULL,
   libraryId TEXT NOT NULL,

package/src/migrations/{v1.sql.ts → v1.ts}

@@ -1,4 +1,4 @@
-export default `
+export const sql = `
 ALTER TABLE
   ReplicaInfo RENAME TO ReplicaInfoOld;
 

package/src/migrations/{v2.sql.ts → v2.ts}

@@ -1,6 +1,6 @@
 // fixes primary key for document baselines to include libraryId
 
-export default `
+export const sql = `
   ALTER TABLE
     DocumentBaseline RENAME TO DocumentBaselineOld;
 

package/src/migrations/{v3.sql.ts → v3.ts}

@@ -1,6 +1,6 @@
 // add the FileMetadata table
 
-export default `
+export const sql = `
   CREATE TABLE IF NOT EXISTS FileMetadata (
     libraryId TEXT NOT NULL,
     fileId TEXT NOT NULL,

package/src/migrations/{v4.sql.ts → v4.ts}

@@ -1,6 +1,6 @@
 // add server order to replicas and operations
 
-export default `
+export const sql = `
   ALTER TABLE ReplicaInfo ADD COLUMN ackedServerOrder INTEGER DEFAULT 0 NOT NULL;
   ALTER TABLE OperationHistory ADD COLUMN serverOrder INTEGER DEFAULT 0 NOT NULL;
 `;

package/src/migrations/v5.ts

@@ -0,0 +1,49 @@
+import {
+	convertLegacyOid,
+	replaceLegacyOidsInJsonString,
+} from '@verdant-web/common';
+import SQL from 'better-sqlite3';
+
+// read all operations and write them back without legacy IDs.
+export const procedure = (db: SQL.Database) => {
+	const operations = db
+		.prepare(/* sql */ `SELECT * from OperationHistory;`)
+		.all();
+	for (const op of operations) {
+		db.prepare(
+			/* sql */ `
+    UPDATE OperationHistory
+      SET oid = ?,
+          data = ?
+      WHERE oid = ? AND timestamp = ? AND libraryId = ?;
+    `,
+		).run(
+			convertLegacyOid(op.oid),
+			// data is already a string, so just leave it and do the conversion on the string
+			op.data ? replaceLegacyOidsInJsonString(op.data) : op.data,
+			op.oid,
+			op.timestamp,
+			op.libraryId,
+		);
+	}
+	const baselines = db
+		.prepare(/* sql */ `SELECT * from DocumentBaseline;`)
+		.all();
+	for (const baseline of baselines) {
+		db.prepare(
+			/* sql */ `
+    UPDATE DocumentBaseline
+      SET oid = ?,
+          snapshot = ?
+      WHERE oid = ? AND libraryId = ?;
+    `,
+		).run(
+			convertLegacyOid(baseline.oid),
+			baseline.snapshot
+				? replaceLegacyOidsInJsonString(baseline.snapshot)
+				: baseline.snapshot,
+			baseline.oid,
+			baseline.libraryId,
+		);
+	}
+};

package/src/migrations.test.ts

@@ -0,0 +1,133 @@
+import { describe, it, expect, afterAll, beforeAll } from 'vitest';
+import create from 'better-sqlite3';
+import { migrations } from './migrations.js';
+import fs from 'fs';
+
+describe('the v5->v6 migration', () => {
+	let dbFile: string;
+	let db: create.Database;
+	beforeAll(() => {
+		dbFile = `v5-migration-test-${Math.random()}.sqlite`;
+		db = create(dbFile);
+	});
+	afterAll(() => {
+		db.close();
+		fs.rmSync(dbFile);
+	});
+
+	it('rewrites legacy oids in all ops and baselines', () => {
+		// manually migrate to v4 before starting and seed the database with some
+		// operations and baselines.
+		migrations(db, 5);
+		// manually insert some operations and baselines
+		db.transaction(() => {
+			[
+				{
+					libraryId: 'test-1',
+					replicaId: 'foo',
+					oid: 'weirds/b',
+					data: {
+						op: 'set',
+						value: { '@@type': 'ref', id: 'weirds/b.objectMap.one:ghi' },
+					},
+					timestamp: '1',
+					serverOrder: 1,
+				},
+				{
+					libraryId: 'test-1',
+					replicaId: 'foo',
+					oid: 'weirds/b.list:ghi',
+					data: {
+						op: 'list-push',
+						value: { '@@type': 'ref', id: 'weirds/b.list.#:abc' },
+					},
+					timestamp: '1',
+					serverOrder: 2,
+				},
+				{
+					libraryId: 'test-1',
+					replicaId: 'foo',
+					oid: 'weirds/b.list:ghi',
+					data: {
+						op: 'list-remove',
+						value: { '@@type': 'ref', id: 'weirds/b.list.#:abc' },
+					},
+					timestamp: '2',
+					serverOrder: 3,
+				},
+				{
+					libraryId: 'test-2',
+					replicaId: 'bar',
+					oid: 'weirds/b',
+					data: {
+						op: 'set',
+						value: { '@@type': 'ref', id: 'weirds/b.objectMap.one:ghi' },
+					},
+					timestamp: '1a',
+					serverOrder: 1,
+				},
+			].forEach(
+				({ libraryId, replicaId, oid, data, timestamp, serverOrder }) => {
+					db.prepare(
+						/* sql */ `
+        INSERT INTO OperationHistory (libraryId, replicaId, oid, data, timestamp, serverOrder)
+        VALUES (?, ?, ?, ?, ?, ?);
+      `,
+					).run(
+						libraryId,
+						replicaId,
+						oid,
+						JSON.stringify(data),
+						timestamp,
+						serverOrder,
+					);
+				},
+			);
+			[
+				{
+					libraryId: 'test-1',
+					oid: 'weirds/b.objectMap.one:ghi',
+					snapshot: { '@@type': 'ref', id: 'weirds/b.list.#:abc' },
+					timestamp: '1',
+				},
+				{
+					libraryId: 'test-2',
+					oid: 'weirds/b.objectMap.one:ghi',
+					snapshot: { '@@type': 'ref', id: 'weirds/b.list.#:abc' },
+					timestamp: '1a',
+				},
+			].forEach(({ libraryId, oid, snapshot, timestamp }) => {
+				db.prepare(
+					/* sql */ `
+        INSERT INTO DocumentBaseline (libraryId, oid, snapshot, timestamp)
+        VALUES (?, ?, ?, ?);
+      `,
+				).run(libraryId, oid, JSON.stringify(snapshot), timestamp);
+			});
+		})();
+
+		migrations(db);
+
+		// check that the operations and baselines have been rewritten
+		db.transaction(() => {
+			const operations = db
+				.prepare(
+					/* sql */ `
+      SELECT * from OperationHistory;
+    `,
+				)
+				.all();
+			expect(operations).toMatchSnapshot();
+			const baselines = db
+				.prepare(
+					/* sql */ `
+      SELECT * from DocumentBaseline;
+    `,
+				)
+				.all();
+			expect(baselines).toMatchSnapshot();
+		})();
+
+		db.close();
+	});
+});

package/src/migrations.ts

@@ -1,13 +1,17 @@
 import type { Database } from 'better-sqlite3';
-import v0Sql from './migrations/v0.sql.js';
-import v1Sql from './migrations/v1.sql.js';
-import v2Sql from './migrations/v2.sql.js';
-import v3Sql from './migrations/v3.sql.js';
-import v4Sql from './migrations/v4.sql.js';
-
-const allMigrations = [v0Sql, v1Sql, v2Sql, v3Sql, v4Sql];
-
-export function migrations(db: Database) {
+import * as v0 from './migrations/v0.js';
+import * as v1 from './migrations/v1.js';
+import * as v2 from './migrations/v2.js';
+import * as v3 from './migrations/v3.js';
+import * as v4 from './migrations/v4.js';
+import * as v5 from './migrations/v5.js';
+
+export const allMigrations: {
+	sql?: string;
+	procedure?: (db: Database) => void;
+}[] = [v0, v1, v2, v3, v4, v5];
+
+export function migrations(db: Database, upTo: number = allMigrations.length) {
 	// create the versions table if it doesn't exist
 	db.prepare(
 		`
@@ -27,24 +31,29 @@ export function migrations(db: Database) {
 		.get() as { version: number } | undefined;
 	const version = versionResult?.version ?? 0;
 
-	const sqlToRun = allMigrations.slice(version);
+	const toRun = allMigrations.slice(version, upTo);
 
-	if (sqlToRun.length === 0) return;
+	if (toRun.length === 0) {
+		console.info('No migrations to run');
+		return;
+	}
 
-	console.info(
-		'Running migrations from ',
-		version,
-		'to',
-		allMigrations.length - 1,
-	);
+	console.info('Running migrations from ', version, 'to', upTo);
 
 	const run = db.transaction(() => {
-		sqlToRun.forEach((sql) => db.exec(sql));
+		for (const { sql, procedure } of toRun) {
+			if (sql) {
+				db.exec(sql);
+			}
+			if (procedure) {
+				procedure(db);
+			}
+		}
 		db.prepare(
 			`
 			INSERT INTO versions (version) VALUES (?);
 		`,
-		).run(allMigrations.length);
+		).run(upTo);
 	});
 
 	run();