@mengine/storage 0.1.0

package/dist/pg.js

@@ -0,0 +1,241 @@
+import { n as BaseDocStorage, t as AutoReconnectConnection } from "./auto-reconnect-0HwaV1v8.js";
+import { and, asc, eq, inArray, sql } from "drizzle-orm";
+import { drizzle } from "drizzle-orm/node-postgres";
+import pg from "pg";
+import { customType, integer, pgTable, primaryKey, text, timestamp } from "drizzle-orm/pg-core";
+//#region src/impls/pg/schema.ts
+/**
+* Postgres bytea maps to a Node `Buffer` on the wire; the storage contract works in `Uint8Array`,
+* so we convert in both directions instead of leaking `Buffer` past this boundary.
+*/
+const bytea = customType({
+	dataType() {
+		return "bytea";
+	},
+	fromDriver(value) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	},
+	toDriver(value) {
+		return Buffer.from(value);
+	}
+});
+const docSnapshots = pgTable("doc_snapshots", {
+	docId: text("doc_id").primaryKey(),
+	data: bytea("data").notNull(),
+	createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+	updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow()
+});
+const docUpdates = pgTable("doc_updates", {
+	docId: text("doc_id").notNull(),
+	seq: integer("seq").notNull(),
+	data: bytea("data").notNull(),
+	createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow()
+}, (table) => [primaryKey({ columns: [table.docId, table.seq] })]);
+const schema = {
+	docSnapshots,
+	docUpdates
+};
+const migrations = [[
+	`CREATE TABLE IF NOT EXISTS "doc_snapshots" (
+      "doc_id" text PRIMARY KEY NOT NULL,
+      "data" bytea NOT NULL,
+      "created_at" timestamp with time zone DEFAULT now() NOT NULL,
+      "updated_at" timestamp with time zone DEFAULT now() NOT NULL
+    )`,
+	`CREATE TABLE IF NOT EXISTS "doc_updates" (
+      "doc_id" text NOT NULL,
+      "seq" integer NOT NULL,
+      "data" bytea NOT NULL,
+      "created_at" timestamp with time zone DEFAULT now() NOT NULL,
+      PRIMARY KEY ("doc_id", "seq")
+    )`,
+	`ALTER TABLE "doc_updates" ALTER COLUMN "seq" DROP IDENTITY IF EXISTS`,
+	`DO $$
+    DECLARE
+      current_pk name;
+      current_pk_columns text[];
+    BEGIN
+      SELECT c.conname, array_agg(a.attname ORDER BY pk_key.ordinality)
+      INTO current_pk, current_pk_columns
+      FROM pg_constraint c
+      JOIN unnest(c.conkey) WITH ORDINALITY AS pk_key(attnum, ordinality) ON true
+      JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = pk_key.attnum
+      WHERE c.conrelid = 'doc_updates'::regclass AND c.contype = 'p'
+      GROUP BY c.conname;
+
+      IF current_pk_columns IS DISTINCT FROM ARRAY['doc_id', 'seq']::text[] THEN
+        IF current_pk IS NOT NULL THEN
+          EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', 'doc_updates', current_pk);
+        END IF;
+
+        ALTER TABLE "doc_updates" ADD PRIMARY KEY ("doc_id", "seq");
+      END IF;
+    END $$`
+]];
+/**
+* Run all pending DDL inside a single transaction.
+*
+* The package owns the doc tables, so it creates them on connect rather than relying on a
+* consumer-side migration runner.
+*/
+const migrate = async (pool) => {
+	const client = await pool.connect();
+	try {
+		await client.query("begin");
+		for (const step of migrations) for (const statement of step) await client.query(statement);
+		await client.query("commit");
+	} catch (error) {
+		await client.query("rollback");
+		throw error;
+	} finally {
+		client.release();
+	}
+};
+const migrator = {
+	version: migrations.length,
+	migrate
+};
+//#endregion
+//#region src/impls/pg/connection.ts
+var PgConnection = class extends AutoReconnectConnection {
+	pgOptions;
+	ownsPool = false;
+	constructor(pgOptions) {
+		super();
+		this.pgOptions = pgOptions;
+	}
+	async doConnect() {
+		let pool = this.pgOptions.pool;
+		if (!pool) {
+			if (!this.pgOptions.connectionString) throw new Error("PgConnection requires either a pool or a connectionString.");
+			pool = new pg.Pool({ connectionString: this.pgOptions.connectionString });
+			this.ownsPool = true;
+		}
+		await migrator.migrate(pool);
+		return {
+			pool,
+			db: drizzle(pool, { schema })
+		};
+	}
+	doDisconnect(conn) {
+		if (this.ownsPool) conn.pool.end().catch((error) => {
+			console.error("failed to close pg pool", error);
+		});
+	}
+};
+//#endregion
+//#region src/impls/pg/index.ts
+const toSnapshotRecord = (row) => {
+	if (!row) return null;
+	return {
+		docId: row.docId,
+		data: row.data,
+		createdAt: row.createdAt,
+		updatedAt: row.updatedAt
+	};
+};
+const toUpdateRecord = (row) => ({
+	docId: row.docId,
+	seq: row.seq,
+	data: row.data,
+	createdAt: row.createdAt
+});
+var PgDocStorage = class extends BaseDocStorage {
+	static identifier = "PgDocStorage";
+	connection = new PgConnection(this.options);
+	get db() {
+		return this.connection.inner.db;
+	}
+	async getDoc(docId) {
+		if (this.isReadonly) {
+			const [snapshotRow] = await this.db.select().from(docSnapshots).where(eq(docSnapshots.docId, docId)).limit(1);
+			const updateRows = await this.db.select().from(docUpdates).where(eq(docUpdates.docId, docId)).orderBy(asc(docUpdates.seq));
+			const snapshot = toSnapshotRecord(snapshotRow);
+			const updates = updateRows.map(toUpdateRecord);
+			return updates.length ? this.mergeUpdates(snapshot, updates) : snapshot;
+		}
+		return await this.db.transaction(async (tx) => {
+			await tx.execute(sql`select doc_id from ${docSnapshots} where doc_id = ${docId} for update`);
+			const [snapshotRow] = await tx.select().from(docSnapshots).where(eq(docSnapshots.docId, docId)).limit(1);
+			const updateRows = await tx.select().from(docUpdates).where(eq(docUpdates.docId, docId)).orderBy(asc(docUpdates.seq));
+			const snapshot = toSnapshotRecord(snapshotRow);
+			const updates = updateRows.map(toUpdateRecord);
+			if (!updates.length) return snapshot;
+			const nextSnapshot = this.mergeUpdates(snapshot, updates);
+			await tx.insert(docSnapshots).values({
+				docId: nextSnapshot.docId,
+				data: nextSnapshot.data,
+				createdAt: nextSnapshot.createdAt,
+				updatedAt: nextSnapshot.updatedAt
+			}).onConflictDoUpdate({
+				target: docSnapshots.docId,
+				set: {
+					data: nextSnapshot.data,
+					updatedAt: nextSnapshot.updatedAt
+				}
+			});
+			await tx.delete(docUpdates).where(and(eq(docUpdates.docId, docId), inArray(docUpdates.seq, updates.map((update) => update.seq))));
+			return nextSnapshot;
+		});
+	}
+	async pushDocUpdate(update, _origin) {
+		const now = /* @__PURE__ */ new Date();
+		await this.db.transaction(async (tx) => {
+			const [snapshot] = await tx.select({ docId: docSnapshots.docId }).from(docSnapshots).where(eq(docSnapshots.docId, update.docId)).for("update");
+			if (!snapshot) {
+				if ((await tx.insert(docSnapshots).values({
+					docId: update.docId,
+					data: update.data,
+					createdAt: now,
+					updatedAt: now
+				}).onConflictDoNothing().returning({ docId: docSnapshots.docId })).length) return;
+				await tx.select({ docId: docSnapshots.docId }).from(docSnapshots).where(eq(docSnapshots.docId, update.docId)).for("update");
+			}
+			const [{ seq }] = await tx.select({ seq: sql`coalesce(max(${docUpdates.seq}), 0) + 1` }).from(docUpdates).where(eq(docUpdates.docId, update.docId));
+			await tx.insert(docUpdates).values({
+				docId: update.docId,
+				seq,
+				data: update.data,
+				createdAt: now
+			});
+		});
+	}
+	async deleteDoc(docId) {
+		await this.db.transaction(async (tx) => {
+			await tx.delete(docUpdates).where(eq(docUpdates.docId, docId));
+			await tx.delete(docSnapshots).where(eq(docSnapshots.docId, docId));
+		});
+	}
+	async getDocSnapshot(docId) {
+		const [snapshot] = await this.db.select().from(docSnapshots).where(eq(docSnapshots.docId, docId)).limit(1);
+		return toSnapshotRecord(snapshot);
+	}
+	async setDocSnapshot(snapshot) {
+		await this.db.insert(docSnapshots).values({
+			docId: snapshot.docId,
+			data: snapshot.data,
+			createdAt: snapshot.createdAt,
+			updatedAt: snapshot.updatedAt
+		}).onConflictDoUpdate({
+			target: docSnapshots.docId,
+			set: {
+				data: snapshot.data,
+				updatedAt: snapshot.updatedAt
+			}
+		});
+		return true;
+	}
+	async getDocUpdates(docId) {
+		return (await this.db.select().from(docUpdates).where(eq(docUpdates.docId, docId)).orderBy(asc(docUpdates.seq))).map(toUpdateRecord);
+	}
+	async markUpdatesMerged(docId, updates) {
+		if (!updates.length) return 0;
+		await this.db.delete(docUpdates).where(and(eq(docUpdates.docId, docId), inArray(docUpdates.seq, updates.map((update) => update.seq))));
+		return updates.length;
+	}
+	subscribeDocUpdate(_callback) {
+		return () => {};
+	}
+};
+//#endregion
+export { PgConnection, PgDocStorage, docSnapshots, docUpdates, schema };

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@mengine/storage",
+  "version": "0.1.0",
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/one2x-ai/medeo-engine.git",
+    "directory": "packages/storage"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./idb": "./src/impls/idb/index.ts",
+    "./pg": "./src/impls/pg/index.ts",
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "exports": {
+      ".": "./dist/index.js",
+      "./idb": "./dist/idb.js",
+      "./pg": "./dist/pg.js",
+      "./package.json": "./package.json"
+    },
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "scripts": {
+    "build": "vp pack",
+    "dev": "vp pack --watch",
+    "test": "vp test",
+    "prepublishOnly": "vp pack"
+  },
+  "dependencies": {
+    "@mengine/utils": "workspace:*",
+    "drizzle-orm": "catalog:",
+    "lodash-es": "4.18.1",
+    "loro-crdt": "catalog:"
+  },
+  "devDependencies": {
+    "@testcontainers/postgresql": "catalog:",
+    "@types/lodash-es": "catalog:",
+    "@types/node": "catalog:",
+    "@types/pg": "catalog:",
+    "@typescript/native-preview": "catalog:",
+    "idb": "catalog:",
+    "pg": "catalog:",
+    "typescript": "catalog:",
+    "vite-plugin-wasm": "catalog:",
+    "vite-plus": "catalog:"
+  },
+  "peerDependencies": {
+    "idb": "catalog:",
+    "pg": "catalog:"
+  }
+}