@bitcobblers/wod-wiki-storage 0.35.2117 → 0.35.2120

package/dist/sql/index.cjs

@@ -124,7 +124,7 @@ function decodeRowJson(text2, binaryField) {
 }
 
 // src/schema.ts
-var DB_VERSION = 29;
+var DB_VERSION = 30;
 var COLUMN_TYPES = {
   notes: { date: "number", createdAt: "number" },
   page: { createdAt: "number" },
@@ -588,7 +588,7 @@ async function getRow(conn, def, key) {
 }
 async function getRows(conn, def, keys) {
   const found = /* @__PURE__ */ new Map();
-  for (const part of chunks(keys, 500)) {
+  for (const part of chunks(keys, 50)) {
     const params = [];
     const conditions = part.map((key) => {
       const parts = toParts(key);
@@ -611,7 +611,7 @@ async function listRowsFromIndexMany(conn, def, indexName, keys) {
     return out;
   }
   const found = /* @__PURE__ */ new Map();
-  for (const part of chunks(keys, 500)) {
+  for (const part of chunks(keys, 50)) {
     const params = [];
     const conditions = part.map((key) => {
       const parts = toParts(key);
@@ -905,25 +905,41 @@ async function initializeStore(db) {
     await tx.exec(
       `CREATE TABLE IF NOT EXISTS "${SCHEMA_TABLE}" ("key" TEXT PRIMARY KEY, "value" TEXT NOT NULL)`
     );
+    const ignored = (name) => name.startsWith("_wodwiki_") || name.startsWith("sqlite_");
     const storeTables = new Set(STORE_DEFS.map((d) => d.name));
+    for (const def of STORE_DEFS) {
+      const multi = def.indexes.find((i) => i.multiEntry);
+      if (multi) storeTables.add(junctionTable(def, multi));
+    }
     const tableNames = new Set(
-      (await tx.query(tableListSql(db.dialect))).map((r) => String(Object.values(r)[0]))
+      (await tx.query(tableListSql(db.dialect))).map((r) => String(Object.values(r)[0])).filter((n) => !ignored(n))
     );
     const guardRows = await tx.query(`SELECT "value" FROM "${SCHEMA_TABLE}" WHERE "key" = ?`, [SCHEMA_KEY]);
     let stored;
     if (guardRows[0]) {
       const guard = JSON.parse(String(guardRows[0].value));
-      if (typeof guard.version !== "number") {
+      if (typeof guard.version !== "number" || !Number.isSafeInteger(guard.version) || guard.version < 0) {
         throw new StorageError(500, `corrupt ${SCHEMA_TABLE} guard row`, "bad_schema_guard");
       }
       stored = guard.version;
-      if (stored !== DB_VERSION) {
+      if (stored > DB_VERSION) {
         throw new StorageError(
           500,
-          `database schema v${stored} is incompatible with v${DB_VERSION}; recreation required using an explicitly disposable database`,
+          `database schema v${stored} is newer than this server supports (v${DB_VERSION}); refusing to downgrade`,
           "schema_incompatible"
         );
       }
+      if (stored < DB_VERSION) {
+        const missing = [...storeTables].filter((name) => !tableNames.has(name));
+        const unexpected = [...tableNames].filter((name) => name !== SCHEMA_TABLE && !storeTables.has(name));
+        if (missing.length > 0 || unexpected.length > 0) {
+          throw new StorageError(
+            500,
+            `database schema v${stored} is incompatible with v${DB_VERSION} (missing tables: ${missing.join(", ") || "none"}; unrecognized tables: ${unexpected.join(", ") || "none"}); recreation required using an explicitly disposable database`,
+            "schema_incompatible"
+          );
+        }
+      }
     } else if ([...tableNames].some((name) => storeTables.has(name))) {
       throw new StorageError(
         500,
@@ -1970,28 +1986,323 @@ var BrowserTursoStorage = class extends SqlStorage {
   }
 };
 
+// src/sql/sync.ts
+var SYNC_OPS_TABLE = "_wodwiki_sync_ops";
+var SYNC_STATE_TABLE = "_wodwiki_sync_state";
+var SYNC_ARCHIVE_TABLE = "_wodwiki_sync_ops_archive";
+var BINARY_FIELD2 = { attachments: "data" };
+var STORE_NAMES = Object.fromEntries(STORE_DEFS.map((d) => [d.name, true]));
+var STATE_KEY = "meta";
+function isSafeInt(value) {
+  return typeof value === "number" && Number.isSafeInteger(value);
+}
+function isScalar(value) {
+  return typeof value === "string" || typeof value === "number" && Number.isFinite(value);
+}
+function assertKey(key, what) {
+  if (isScalar(key)) return;
+  if (Array.isArray(key) && key.length > 0 && key.every(isScalar)) return;
+  throw new StorageError(400, `${what} must be a string, finite number, or non-empty scalar tuple`, "bad_sync_key");
+}
+function validateMutation(raw) {
+  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new StorageError(400, "sync mutation must be an object", "bad_sync_mutation");
+  }
+  const mutation = raw;
+  if (mutation.op === "wipe") return { op: "wipe" };
+  if (typeof mutation.store !== "string" || !STORE_NAMES[mutation.store]) {
+    throw new StorageError(400, `sync mutation has unknown store ${String(mutation.store)}`, "bad_sync_mutation");
+  }
+  const store = mutation.store;
+  if (mutation.op === "clear") return { op: "clear", store };
+  if (mutation.op === "delete") {
+    assertKey(mutation.key, "sync delete key");
+    return { op: "delete", store, key: mutation.key };
+  }
+  if (mutation.op === "put") {
+    assertKey(mutation.key, "sync put key");
+    if (mutation.value === null || typeof mutation.value !== "object" || Array.isArray(mutation.value)) {
+      throw new StorageError(400, "sync put value must be an object row", "bad_sync_mutation");
+    }
+    return { op: "put", store, key: mutation.key, value: mutation.value };
+  }
+  throw new StorageError(400, `sync mutation has unknown op ${String(mutation.op)}`, "bad_sync_mutation");
+}
+function validateEntry(entry) {
+  if (typeof entry.id !== "string" || entry.id.length === 0 || entry.id.length > 200) {
+    throw new StorageError(400, "sync entry id must be a non-empty string", "bad_sync_entry");
+  }
+  if (!isSafeInt(entry.at) || entry.at < 0) {
+    throw new StorageError(400, "sync entry at must be a non-negative safe integer", "bad_sync_entry");
+  }
+  if (!Array.isArray(entry.ops) || entry.ops.length === 0) {
+    throw new StorageError(400, "sync entry ops must be a non-empty array", "bad_sync_entry");
+  }
+  for (const op of entry.ops) validateMutation(op);
+}
+async function runStatements(tx, statements) {
+  if (statements.length === 0) return [];
+  if (tx.batch) return tx.batch(statements);
+  const results = [];
+  for (const statement of statements) {
+    results.push(await tx.query(statement.sql, statement.params ?? []));
+  }
+  return results;
+}
+async function tableExists(conn, dialect, name) {
+  const sql = dialect === "postgres" ? `SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1` : `SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`;
+  return (await conn.query(sql, [name])).length > 0;
+}
+async function readSyncState(tx) {
+  const rows = await tx.query(`SELECT "value" FROM "${SYNC_STATE_TABLE}" WHERE "key" = ?`, [STATE_KEY]);
+  if (!rows[0]) throw new StorageError(409, "sync target is not initialized", "sync_not_initialized");
+  let parsed;
+  try {
+    parsed = JSON.parse(String(rows[0].value));
+  } catch {
+    throw new StorageError(400, "malformed sync state row", "bad_sync_state");
+  }
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new StorageError(400, "malformed sync state row", "bad_sync_state");
+  }
+  const state = parsed;
+  if (!isSafeInt(state.schema) || state.schema < 0 || !isSafeInt(state.rev) || state.rev < 0) {
+    throw new StorageError(400, "sync state schema/rev must be non-negative safe integers", "bad_sync_state");
+  }
+  return { schema: state.schema, rev: state.rev };
+}
+async function probeSyncTarget(db) {
+  if (await tableExists(db, db.dialect, SYNC_STATE_TABLE)) {
+    const state = await readSyncState(db);
+    return { kind: "current", schemaVersion: state.schema, rev: state.rev };
+  }
+  if (await tableExists(db, db.dialect, SCHEMA_TABLE)) {
+    const rows = await db.query(`SELECT "value" FROM "${SCHEMA_TABLE}" WHERE "key" = 'schema'`);
+    if (rows[0]) {
+      const guard = JSON.parse(String(rows[0].value));
+      if (typeof guard.version === "number" && Number.isSafeInteger(guard.version) && guard.version >= 0) {
+        return { kind: "legacy", schemaVersion: guard.version };
+      }
+    }
+    return { kind: "legacy", schemaVersion: 0 };
+  }
+  const others = await db.query(
+    db.dialect === "postgres" ? `SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'` : `SELECT name FROM sqlite_master WHERE type = 'table'`
+  );
+  const internal = (name) => name.startsWith("_wodwiki_") || name.startsWith("sqlite_");
+  return others.some((row) => !internal(String(Object.values(row)[0]))) ? { kind: "unknown" } : "empty";
+}
+async function initSyncTarget(db, schemaVersion) {
+  await db.transaction(async (tx) => {
+    await tx.exec(`CREATE TABLE IF NOT EXISTS "${SYNC_STATE_TABLE}" ("key" TEXT PRIMARY KEY, "value" TEXT NOT NULL)`);
+    await tx.exec(
+      `CREATE TABLE IF NOT EXISTS "${SYNC_OPS_TABLE}" ("op_id" TEXT PRIMARY KEY, "rev" INTEGER NOT NULL, "idx" INTEGER NOT NULL, "at" INTEGER NOT NULL, "data" TEXT NOT NULL)`
+    );
+    await tx.exec(
+      `CREATE INDEX IF NOT EXISTS "${SYNC_OPS_TABLE}_rev" ON "${SYNC_OPS_TABLE}" ("rev", "idx")`
+    );
+    await tx.exec(
+      `INSERT INTO "${SYNC_STATE_TABLE}" ("key", "value") VALUES (?, ?) ON CONFLICT ("key") DO NOTHING`,
+      [STATE_KEY, JSON.stringify({ schema: schemaVersion, rev: 0 })]
+    );
+  });
+}
+function encodeMutation(mutation) {
+  if (mutation.op !== "put") return JSON.stringify(mutation);
+  const encoded = encodeRowJson(mutation.value, BINARY_FIELD2[mutation.store]);
+  return `{"key":${JSON.stringify(mutation.key)},"op":"put","store":${JSON.stringify(mutation.store)},"value":${encoded}}`;
+}
+function decodeMutation(data) {
+  if (typeof data !== "string") {
+    throw new StorageError(400, "sync op data must be a string", "bad_sync_mutation");
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(data);
+  } catch {
+    throw new StorageError(400, "sync op data is not valid JSON", "bad_sync_mutation");
+  }
+  const mutation = validateMutation(parsed);
+  if (mutation.op !== "put") return mutation;
+  return {
+    op: "put",
+    store: mutation.store,
+    key: mutation.key,
+    value: decodeRowJson(JSON.stringify(mutation.value), BINARY_FIELD2[mutation.store])
+  };
+}
+async function applySyncBatch(db, entries, opts = {}) {
+  if (entries.length === 0) return 0;
+  for (const entry of entries) validateEntry(entry);
+  return db.transaction(async (tx) => {
+    const state = await readSyncState(tx);
+    if (opts.expectSchema !== void 0 && state.schema !== opts.expectSchema) {
+      throw new StorageError(
+        409,
+        `sync target schema changed from v${opts.expectSchema} to v${state.schema} between probe and write \u2014 retry the cycle`,
+        "sync_schema_conflict"
+      );
+    }
+    const firstIds = entries.map((entry) => `${entry.id}:0`);
+    const present = /* @__PURE__ */ new Set();
+    for (let i = 0; i < firstIds.length; i += 100) {
+      const chunk = firstIds.slice(i, i + 100);
+      const rows = await tx.query(
+        `SELECT "op_id" FROM "${SYNC_OPS_TABLE}" WHERE "op_id" IN (${chunk.map(() => "?").join(", ")})`,
+        chunk
+      );
+      for (const row of rows) present.add(String(row.op_id));
+    }
+    const statements = [];
+    let rev = state.rev;
+    let committed = 0;
+    for (const entry of entries) {
+      if (present.has(`${entry.id}:0`)) continue;
+      rev += 1;
+      committed += 1;
+      entry.ops.forEach((mutation, idx) => {
+        statements.push({
+          sql: `INSERT INTO "${SYNC_OPS_TABLE}" ("op_id", "rev", "idx", "at", "data") VALUES (?, ?, ?, ?, ?) ON CONFLICT ("op_id") DO NOTHING`,
+          params: [`${entry.id}:${idx}`, rev, idx, entry.at, encodeMutation(mutation)]
+        });
+      });
+    }
+    const nextSchema = opts.toSchemaVersion ?? state.schema;
+    statements.push({
+      sql: `UPDATE "${SYNC_STATE_TABLE}" SET "value" = ? WHERE "key" = ?`,
+      params: [JSON.stringify({ schema: nextSchema, rev }), STATE_KEY]
+    });
+    await runStatements(tx, statements);
+    return committed;
+  });
+}
+async function fetchSyncOps(db, afterRev, limit) {
+  if (!isSafeInt(afterRev) || afterRev < 0) {
+    throw new StorageError(400, "fetchSyncOps cursor must be a non-negative safe integer", "bad_sync_cursor");
+  }
+  const bounded = Math.min(Math.max(1, Math.floor(limit)), 4294967295);
+  const revRows = await db.query(
+    `SELECT DISTINCT "rev" FROM (SELECT "rev" FROM "${SYNC_OPS_TABLE}" WHERE "rev" > ? ORDER BY "rev" ASC LIMIT ?)`,
+    [afterRev, bounded]
+  );
+  const revs = revRows.map((row) => {
+    const rev = Number(row.rev);
+    if (!Number.isSafeInteger(rev) || rev < 0) {
+      throw new StorageError(400, "sync op rev must be a non-negative safe integer", "bad_sync_row");
+    }
+    return rev;
+  });
+  if (revs.length === 0) return [];
+  const rows = await db.query(
+    `SELECT "op_id", "rev", "idx", "at", "data" FROM "${SYNC_OPS_TABLE}" WHERE "rev" IN (${revs.map(() => "?").join(", ")}) ORDER BY "rev", "idx"`,
+    revs
+  );
+  return rows.map((row, at) => {
+    const opId = row.op_id;
+    const rev = Number(row.rev);
+    const idx = Number(row.idx);
+    const atMs = Number(row.at);
+    if (typeof opId !== "string" || opId.length === 0 || !Number.isSafeInteger(rev) || rev < 0 || !Number.isSafeInteger(idx) || idx < 0 || !Number.isSafeInteger(atMs) || atMs < 0) {
+      throw new StorageError(400, `sync op row ${at} has malformed shape`, "bad_sync_row");
+    }
+    return { opId, rev, idx, at: atMs, mutation: decodeMutation(row.data) };
+  });
+}
+async function replaceSyncLog(db, entries, toSchemaVersion) {
+  if (entries.length === 0) throw new StorageError(400, "replacement log must contain the converted snapshot", "bad_sync_entry");
+  for (const entry of entries) validateEntry(entry);
+  await db.transaction(async (tx) => {
+    await readSyncState(tx);
+    await tx.exec(
+      `CREATE TABLE IF NOT EXISTS "${SYNC_ARCHIVE_TABLE}" ("op_id" TEXT PRIMARY KEY, "rev" INTEGER NOT NULL, "idx" INTEGER NOT NULL, "at" INTEGER NOT NULL, "data" TEXT NOT NULL)`
+    );
+    await tx.exec(`INSERT INTO "${SYNC_ARCHIVE_TABLE}" ("op_id", "rev", "idx", "at", "data") SELECT "op_id", "rev", "idx", "at", "data" FROM "${SYNC_OPS_TABLE}"`);
+    await tx.exec(`DELETE FROM "${SYNC_OPS_TABLE}"`);
+    const statements = [];
+    let rev = 0;
+    for (const entry of entries) {
+      rev += 1;
+      entry.ops.forEach((mutation, idx) => {
+        statements.push({
+          sql: `INSERT INTO "${SYNC_OPS_TABLE}" ("op_id", "rev", "idx", "at", "data") VALUES (?, ?, ?, ?, ?) ON CONFLICT ("op_id") DO NOTHING`,
+          params: [`${entry.id}:${idx}`, rev, idx, entry.at, encodeMutation(mutation)]
+        });
+      });
+    }
+    statements.push({
+      sql: `UPDATE "${SYNC_STATE_TABLE}" SET "value" = ? WHERE "key" = ?`,
+      params: [JSON.stringify({ schema: toSchemaVersion, rev }), STATE_KEY]
+    });
+    await runStatements(tx, statements);
+  });
+}
+async function readRawSyncOps(db) {
+  const rows = await db.query(`SELECT "rev", "idx", "data" FROM "${SYNC_OPS_TABLE}" ORDER BY "rev", "idx"`);
+  return rows.map((row) => {
+    const rev = Number(row.rev);
+    const idx = Number(row.idx);
+    if (!Number.isSafeInteger(rev) || rev < 0 || !Number.isSafeInteger(idx) || idx < 0 || typeof row.data !== "string") {
+      throw new StorageError(400, "sync op row has malformed shape", "bad_sync_row");
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(row.data);
+    } catch {
+      throw new StorageError(400, "sync op data is not valid JSON", "bad_sync_row");
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new StorageError(400, "sync op data must be an object", "bad_sync_row");
+    }
+    const mutation = parsed;
+    if (typeof mutation.op !== "string" || typeof mutation.store !== "string") {
+      throw new StorageError(400, "sync op must carry string op/store", "bad_sync_row");
+    }
+    return {
+      op: mutation.op,
+      store: mutation.store,
+      key: mutation.key,
+      value: mutation.value === void 0 || mutation.value === null ? void 0 : decodeRowJson(JSON.stringify(mutation.value), BINARY_FIELD2[mutation.store])
+    };
+  });
+}
+async function readLegacyRows(db, table) {
+  if (!await tableExists(db, db.dialect, table)) return [];
+  const rows = await db.query(`SELECT "data" FROM "${table}"`);
+  return rows.map((row) => decodeRowJson(String(row.data), BINARY_FIELD2[table]));
+}
+
 exports.BrowserTursoStorage = BrowserTursoStorage;
 exports.MAX_BODY = MAX_BODY;
 exports.SCHEMA_TABLE = SCHEMA_TABLE;
+exports.SYNC_ARCHIVE_TABLE = SYNC_ARCHIVE_TABLE;
+exports.SYNC_OPS_TABLE = SYNC_OPS_TABLE;
+exports.SYNC_STATE_TABLE = SYNC_STATE_TABLE;
 exports.SqlStorage = SqlStorage;
 exports.StorageError = StorageError;
+exports.applySyncBatch = applySyncBatch;
 exports.clearRows = clearRows;
 exports.countRows = countRows;
 exports.decodeRowJson = decodeRowJson;
 exports.deleteRows = deleteRows;
 exports.encodeRowJson = encodeRowJson;
+exports.fetchSyncOps = fetchSyncOps;
 exports.getRow = getRow;
 exports.getRows = getRows;
 exports.handleDomainQuery = handleDomainQuery;
+exports.initSyncTarget = initSyncTarget;
 exports.initializeDomain = initializeDomain;
 exports.initializeStore = initializeStore;
 exports.libsqlDatabase = libsqlDatabase;
 exports.listRows = listRows;
 exports.listRowsFromIndexMany = listRowsFromIndexMany;
+exports.probeSyncTarget = probeSyncTarget;
 exports.putRow = putRow;
 exports.putRows = putRows;
+exports.readLegacyRows = readLegacyRows;
+exports.readRawSyncOps = readRawSyncOps;
 exports.rebuildProjections = rebuildProjections;
 exports.refreshDomain = refreshDomain;
+exports.replaceSyncLog = replaceSyncLog;
 exports.storeDefinition = storeDefinition;
 exports.validateTursoUrl = validateTursoUrl;
 exports.wipeDomain = wipeDomain;