@lika85456/s3qlite 0.1.0

package/cjs/src/cdc/extract.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLatestChangeId = exports.extractCDC = void 0;
+const effect_1 = require("effect");
+const query = `
+	SELECT
+		change_id AS changeId,
+		change_time AS changeTime,
+		change_txn_id AS changeTxnId,
+		change_type AS changeType,
+		table_name AS tableName,
+		id,
+		before,
+		after,
+		updates
+	FROM turso_cdc
+	WHERE change_id > ?
+		AND change_type != 2
+	ORDER BY change_id ASC
+`;
+const latestChangeIdQuery = `
+	SELECT COALESCE(MAX(change_id), 0) AS changeId
+	FROM turso_cdc
+`;
+/**
+ * Extracts CDC rows from the Turso database after a given change_id exclusively.
+ */
+const extractCDC = (tursoConnection, afterId) => effect_1.Effect.tryPromise(() => tursoConnection.all(query, afterId)).pipe(effect_1.Effect.orDie, effect_1.Effect.tap((r) => effect_1.Effect.logDebug(`extractCDC: extracted ${r.length} rows after change_id ${afterId}`).pipe(effect_1.Effect.orDie)));
+exports.extractCDC = extractCDC;
+const getLatestChangeId = (tursoConnection) => effect_1.Effect.tryPromise({
+    try: async () => {
+        const [row] = (await tursoConnection.all(latestChangeIdQuery));
+        return row?.changeId ?? 0;
+    },
+    catch: (error) => new Error(`Failed to read latest CDC change id: ${String(error)}`),
+});
+exports.getLatestChangeId = getLatestChangeId;
+//# sourceMappingURL=extract.js.map

package/cjs/src/cdc/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../src/cdc/extract.ts"],"names":[],"mappings":";;;AACA,mCAAgC;AAIhC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;CAeb,CAAC;AAEF,MAAM,mBAAmB,GAAG;;;CAG3B,CAAC;AAEF;;GAEG;AACI,MAAM,UAAU,GAAG,CACzB,eAAyB,EACzB,OAAe,EACoB,EAAE,CACrC,eAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAsB,CAAC,CAAC,IAAI,CACrF,eAAM,CAAC,KAAK,EACZ,eAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAChB,eAAM,CAAC,QAAQ,CACd,yBAAyB,CAAC,CAAC,MAAM,yBAAyB,OAAO,EAAE,CACnE,CAAC,IAAI,CAAC,eAAM,CAAC,KAAK,CAAC,CACpB,CACD,CAAC;AAXU,QAAA,UAAU,cAWpB;AAEI,MAAM,iBAAiB,GAAG,CAAC,eAAyB,EAAgC,EAAE,CAC5F,eAAM,CAAC,UAAU,CAAC;IACjB,GAAG,EAAE,KAAK,IAAI,EAAE;QACf,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAG1D,CAAC;QACJ,OAAO,GAAG,EAAE,QAAQ,IAAI,CAAC,CAAC;IAC3B,CAAC;IACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,wCAAwC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;CACpF,CAAC,CAAC;AAVS,QAAA,iBAAiB,qBAU1B","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport { Effect } from \"effect\";\n\nimport type { CDCRow } from \"./types\";\n\nconst query = `\n\tSELECT\n\t\tchange_id AS changeId,\n\t\tchange_time AS changeTime,\n\t\tchange_txn_id AS changeTxnId,\n\t\tchange_type AS changeType,\n\t\ttable_name AS tableName,\n\t\tid,\n\t\tbefore,\n\t\tafter,\n\t\tupdates\n\tFROM turso_cdc\n\tWHERE change_id > ?\n\t\tAND change_type != 2\n\tORDER BY change_id ASC\n`;\n\nconst latestChangeIdQuery = `\n\tSELECT COALESCE(MAX(change_id), 0) AS changeId\n\tFROM turso_cdc\n`;\n\n/**\n * Extracts CDC rows from the Turso database after a given change_id exclusively.\n */\nexport const extractCDC = (\n\ttursoConnection: Database,\n\tafterId: number,\n): Effect.Effect<readonly CDCRow[]> =>\n\tEffect.tryPromise(() => tursoConnection.all(query, afterId) as Promise<CDCRow[]>).pipe(\n\t\tEffect.orDie,\n\t\tEffect.tap((r) =>\n\t\t\tEffect.logDebug(\n\t\t\t\t`extractCDC: extracted ${r.length} rows after change_id ${afterId}`,\n\t\t\t).pipe(Effect.orDie),\n\t\t),\n\t);\n\nexport const getLatestChangeId = (tursoConnection: Database): Effect.Effect<number, Error> =>\n\tEffect.tryPromise({\n\t\ttry: async () => {\n\t\t\tconst [row] = (await tursoConnection.all(latestChangeIdQuery)) as {\n\t\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- SQLite aggregate MAX returns NULL when the CDC table is empty.\n\t\t\t\tchangeId: number | null;\n\t\t\t}[];\n\t\t\treturn row?.changeId ?? 0;\n\t\t},\n\t\tcatch: (error) => new Error(`Failed to read latest CDC change id: ${String(error)}`),\n\t});"]}

package/cjs/src/cdc/protobuf.js

@@ -0,0 +1,135 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.deserializeCDC = exports.serializeCDC = void 0;
+const protobufjs_1 = __importDefault(require("protobufjs"));
+const schema = protobufjs_1.default.parse(`
+syntax = "proto3";
+
+message CdcValue {
+	oneof kind {
+		bool null_value = 1;
+		double number_value = 2;
+		string string_value = 3;
+		bytes bytes_value = 4;
+	}
+}
+
+message CdcRow {
+	sint64 change_id = 1;
+	sint64 change_time = 2;
+	sint64 change_txn_id = 3;
+	sint32 change_type = 4;
+	string table_name = 5;
+	CdcValue id = 6;
+	bytes before = 7;
+	bytes after = 8;
+	bytes updates = 9;
+}
+
+message CdcBatch {
+	repeated CdcRow changes = 1;
+}
+`).root;
+const batchType = schema.lookupType("CdcBatch");
+const encodeValue = (value) => {
+    // oxlint-disable-next-line local-rules/no-null-undefined-option -- CDC payloads must preserve SQL NULL values.
+    if (value === null) {
+        return { kind: "nullValue", nullValue: true };
+    }
+    if (typeof value === "number") {
+        return { kind: "numberValue", numberValue: value };
+    }
+    if (typeof value === "string") {
+        return { kind: "stringValue", stringValue: value };
+    }
+    return { kind: "bytesValue", bytesValue: value };
+};
+// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs omits absent optional fields as undefined on decode.
+const decodeValue = (value) => {
+    if (!value) {
+        // oxlint-disable-next-line local-rules/no-null-undefined-option -- missing protobuf field maps back to SQL NULL.
+        return null;
+    }
+    switch (value.kind) {
+        case "nullValue":
+            // oxlint-disable-next-line local-rules/no-null-undefined-option -- protobuf null arm maps to SQL NULL.
+            return null;
+        case "numberValue":
+            return value.numberValue;
+        case "stringValue":
+            return value.stringValue;
+        case "bytesValue":
+            return Uint8Array.from(value.bytesValue);
+    }
+};
+const encodeRow = (row) => ({
+    changeId: row.changeId,
+    changeTime: row.changeTime,
+    changeTxnId: row.changeTxnId,
+    changeType: row.changeType,
+    // oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.
+    ...(row.tableName === null ? {} : { tableName: row.tableName }),
+    // oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.
+    ...(row.id === null ? {} : { id: encodeValue(row.id) }),
+    // oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.
+    ...(row.before === null ? {} : { before: row.before }),
+    // oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.
+    ...(row.after === null ? {} : { after: row.after }),
+    // oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.
+    ...(row.updates === null ? {} : { updates: row.updates }),
+});
+const decodeRow = (row) => {
+    const changeType = row.changeType;
+    if (changeType === -1) {
+        return {
+            changeId: Number(row.changeId),
+            changeTime: Number(row.changeTime),
+            changeTxnId: Number(row.changeTxnId),
+            changeType,
+            tableName: row.tableName || "",
+            id: decodeValue(row.id),
+            before: row.before ? Uint8Array.from(row.before) : new Uint8Array(),
+            // oxlint-disable-next-line local-rules/no-null-undefined-option -- Delete CDC rows have no post-delete image.
+            after: null,
+            // oxlint-disable-next-line local-rules/no-null-undefined-option -- Delete CDC rows have no updates mask.
+            updates: null,
+        };
+    }
+    if (changeType === 0) {
+        return {
+            changeId: Number(row.changeId),
+            changeTime: Number(row.changeTime),
+            changeTxnId: Number(row.changeTxnId),
+            changeType,
+            tableName: row.tableName || "",
+            id: decodeValue(row.id),
+            before: row.before ? Uint8Array.from(row.before) : new Uint8Array(),
+            after: row.after ? Uint8Array.from(row.after) : new Uint8Array(),
+            updates: row.updates ? Uint8Array.from(row.updates) : new Uint8Array(),
+        };
+    }
+    if (changeType === 1) {
+        return {
+            changeId: Number(row.changeId),
+            changeTime: Number(row.changeTime),
+            changeTxnId: Number(row.changeTxnId),
+            changeType,
+            tableName: row.tableName || "",
+            id: decodeValue(row.id),
+            // oxlint-disable-next-line local-rules/no-null-undefined-option -- Insert CDC rows have no previous image.
+            before: null,
+            after: row.after ? Uint8Array.from(row.after) : new Uint8Array(),
+            // oxlint-disable-next-line local-rules/no-null-undefined-option -- Insert CDC rows have no updates mask.
+            updates: null,
+        };
+    }
+    throw new Error(`Unsupported CDC change type ${String(row.changeType)}`);
+};
+const serializeCDC = (changes) => batchType.encode({ changes: changes.map(encodeRow) }).finish();
+exports.serializeCDC = serializeCDC;
+const deserializeCDC = (bytes) => batchType.decode(bytes).changes.map(decodeRow);
+exports.deserializeCDC = deserializeCDC;
+//# sourceMappingURL=protobuf.js.map

package/cjs/src/cdc/protobuf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"protobuf.js","sourceRoot":"","sources":["../../../../src/cdc/protobuf.ts"],"names":[],"mappings":";;;;;;AAAA,4DAAkC;AAIlC,MAAM,MAAM,GAAG,oBAAQ,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2B7B,CAAC,CAAC,IAAI,CAAC;AAER,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,UAAU,CAAkB,CAAC;AAwBjE,MAAM,WAAW,GAAG,CAAC,KAAe,EAAgB,EAAE;IACrD,+GAA+G;IAC/G,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACpD,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAClD,CAAC,CAAC;AAEF,mIAAmI;AACnI,MAAM,WAAW,GAAG,CAAC,KAA+B,EAAY,EAAE;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,iHAAiH;QACjH,OAAO,IAAI,CAAC;IACb,CAAC;IACD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,WAAW;YACf,uGAAuG;YACvG,OAAO,IAAI,CAAC;QACb,KAAK,aAAa;YACjB,OAAO,KAAK,CAAC,WAAW,CAAC;QAC1B,KAAK,aAAa;YACjB,OAAO,KAAK,CAAC,WAAW,CAAC;QAC1B,KAAK,YAAY;YAChB,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,GAAW,EAAc,EAAE,CAAC,CAAC;IAC/C,QAAQ,EAAE,GAAG,CAAC,QAAQ;IACtB,UAAU,EAAE,GAAG,CAAC,UAAU;IAC1B,WAAW,EAAE,GAAG,CAAC,WAAW;IAC5B,UAAU,EAAE,GAAG,CAAC,UAAU;IAC1B,0IAA0I;IAC1I,GAAG,CAAC,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC;IAC/D,0IAA0I;IAC1I,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;IACvD,0IAA0I;IAC1I,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IACtD,0IAA0I;IAC1I,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IACnD,0IAA0I;IAC1I,GAAG,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;CACzD,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,CAAC,GAAe,EAAU,EAAE;IAC7C,MAAM,UAAU,GAAG,GAAG,CAAC,UAAkC,CAAC;IAC1D,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACvB,OAAO;YACN,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YAClC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACpC,UAAU;YACV,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE;YACnE,8GAA8G;YAC9G,KAAK,EAAE,IAAI;YACX,yGAAyG;YACzG,OAAO,EAAE,IAAI;SACb,CAAC;IACH,CAAC;IACD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACN,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YAClC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACpC,UAAU;YACV,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE;YACnE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE;YAChE,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE;SACtE,CAAC;IACH,CAAC;IACD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACN,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YAClC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACpC,UAAU;YACV,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,2GAA2G;YAC3G,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,EAAE;YAChE,yGAAyG;YACzG,OAAO,EAAE,IAAI;SACb,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEK,MAAM,YAAY,GAAG,CAAC,OAA0B,EAAc,EAAE,CACtE,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAkB,CAAC,CAAC,MAAM,EAAE,CAAC;AADnE,QAAA,YAAY,gBACuD;AAEzE,MAAM,cAAc,GAAG,CAAC,KAAiB,EAAqB,EAAE,CACrE,SAAS,CAAC,MAAM,CAAC,KAAK,CAA6B,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AADhE,QAAA,cAAc,kBACkD","sourcesContent":["import protobuf from \"protobufjs\";\n\nimport type { CDCRow, CDCValue } from \"./types\";\n\nconst schema = protobuf.parse(`\nsyntax = \"proto3\";\n\nmessage CdcValue {\n\toneof kind {\n\t\tbool null_value = 1;\n\t\tdouble number_value = 2;\n\t\tstring string_value = 3;\n\t\tbytes bytes_value = 4;\n\t}\n}\n\nmessage CdcRow {\n\tsint64 change_id = 1;\n\tsint64 change_time = 2;\n\tsint64 change_txn_id = 3;\n\tsint32 change_type = 4;\n\tstring table_name = 5;\n\tCdcValue id = 6;\n\tbytes before = 7;\n\tbytes after = 8;\n\tbytes updates = 9;\n}\n\nmessage CdcBatch {\n\trepeated CdcRow changes = 1;\n}\n`).root;\n\nconst batchType = schema.lookupType(\"CdcBatch\") as protobuf.Type;\n\ntype ValueMessage =\n\t| { kind: \"nullValue\"; nullValue: boolean }\n\t| { kind: \"numberValue\"; numberValue: number }\n\t| { kind: \"stringValue\"; stringValue: string }\n\t| { kind: \"bytesValue\"; bytesValue: Uint8Array };\n\ntype RowMessage = {\n\tchangeId: number | bigint;\n\tchangeTime: number | bigint;\n\tchangeTxnId: number | bigint;\n\tchangeType: number;\n\ttableName?: string;\n\tid?: ValueMessage;\n\tbefore?: Uint8Array;\n\tafter?: Uint8Array;\n\tupdates?: Uint8Array;\n};\n\ntype BatchMessage = {\n\tchanges: readonly RowMessage[];\n};\n\nconst encodeValue = (value: CDCValue): ValueMessage => {\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- CDC payloads must preserve SQL NULL values.\n\tif (value === null) {\n\t\treturn { kind: \"nullValue\", nullValue: true };\n\t}\n\tif (typeof value === \"number\") {\n\t\treturn { kind: \"numberValue\", numberValue: value };\n\t}\n\tif (typeof value === \"string\") {\n\t\treturn { kind: \"stringValue\", stringValue: value };\n\t}\n\treturn { kind: \"bytesValue\", bytesValue: value };\n};\n\n// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs omits absent optional fields as undefined on decode.\nconst decodeValue = (value: ValueMessage | undefined): CDCValue => {\n\tif (!value) {\n\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- missing protobuf field maps back to SQL NULL.\n\t\treturn null;\n\t}\n\tswitch (value.kind) {\n\t\tcase \"nullValue\":\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobuf null arm maps to SQL NULL.\n\t\t\treturn null;\n\t\tcase \"numberValue\":\n\t\t\treturn value.numberValue;\n\t\tcase \"stringValue\":\n\t\t\treturn value.stringValue;\n\t\tcase \"bytesValue\":\n\t\t\treturn Uint8Array.from(value.bytesValue);\n\t}\n};\n\nconst encodeRow = (row: CDCRow): RowMessage => ({\n\tchangeId: row.changeId,\n\tchangeTime: row.changeTime,\n\tchangeTxnId: row.changeTxnId,\n\tchangeType: row.changeType,\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.\n\t...(row.tableName === null ? {} : { tableName: row.tableName }),\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.\n\t...(row.id === null ? {} : { id: encodeValue(row.id) }),\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.\n\t...(row.before === null ? {} : { before: row.before }),\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.\n\t...(row.after === null ? {} : { after: row.after }),\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- protobufjs optional fields are omitted with undefined during encoding.\n\t...(row.updates === null ? {} : { updates: row.updates }),\n});\n\nconst decodeRow = (row: RowMessage): CDCRow => {\n\tconst changeType = row.changeType as CDCRow[\"changeType\"];\n\tif (changeType === -1) {\n\t\treturn {\n\t\t\tchangeId: Number(row.changeId),\n\t\t\tchangeTime: Number(row.changeTime),\n\t\t\tchangeTxnId: Number(row.changeTxnId),\n\t\t\tchangeType,\n\t\t\ttableName: row.tableName || \"\",\n\t\t\tid: decodeValue(row.id),\n\t\t\tbefore: row.before ? Uint8Array.from(row.before) : new Uint8Array(),\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Delete CDC rows have no post-delete image.\n\t\t\tafter: null,\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Delete CDC rows have no updates mask.\n\t\t\tupdates: null,\n\t\t};\n\t}\n\tif (changeType === 0) {\n\t\treturn {\n\t\t\tchangeId: Number(row.changeId),\n\t\t\tchangeTime: Number(row.changeTime),\n\t\t\tchangeTxnId: Number(row.changeTxnId),\n\t\t\tchangeType,\n\t\t\ttableName: row.tableName || \"\",\n\t\t\tid: decodeValue(row.id),\n\t\t\tbefore: row.before ? Uint8Array.from(row.before) : new Uint8Array(),\n\t\t\tafter: row.after ? Uint8Array.from(row.after) : new Uint8Array(),\n\t\t\tupdates: row.updates ? Uint8Array.from(row.updates) : new Uint8Array(),\n\t\t};\n\t}\n\tif (changeType === 1) {\n\t\treturn {\n\t\t\tchangeId: Number(row.changeId),\n\t\t\tchangeTime: Number(row.changeTime),\n\t\t\tchangeTxnId: Number(row.changeTxnId),\n\t\t\tchangeType,\n\t\t\ttableName: row.tableName || \"\",\n\t\t\tid: decodeValue(row.id),\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Insert CDC rows have no previous image.\n\t\t\tbefore: null,\n\t\t\tafter: row.after ? Uint8Array.from(row.after) : new Uint8Array(),\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Insert CDC rows have no updates mask.\n\t\t\tupdates: null,\n\t\t};\n\t}\n\tthrow new Error(`Unsupported CDC change type ${String(row.changeType)}`);\n};\n\nexport const serializeCDC = (changes: readonly CDCRow[]): Uint8Array =>\n\tbatchType.encode({ changes: changes.map(encodeRow) } as BatchMessage).finish();\n\nexport const deserializeCDC = (bytes: Uint8Array): readonly CDCRow[] =>\n\t(batchType.decode(bytes) as unknown as BatchMessage).changes.map(decodeRow);"]}

package/cjs/src/cdc/testUtils.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyBatch = exports.compareDatabases = exports.snapshotDatabase = exports.compareSnapshots = void 0;
+const effect_1 = require("effect");
+const compareSnapshots = (left, right) => JSON.stringify(left) === JSON.stringify(right);
+exports.compareSnapshots = compareSnapshots;
+const snapshotDatabase = (database) => effect_1.Effect.gen(function* () {
+    const schema = yield* effect_1.Effect.tryPromise(() => database.all([
+        "SELECT type, name, tbl_name AS tblName, sql",
+        "FROM sqlite_schema",
+        "WHERE name NOT LIKE 'sqlite_%' AND name NOT LIKE 'turso_%'",
+        "ORDER BY type, name",
+    ].join(" ")));
+    const tableSnapshots = yield* effect_1.Effect.forEach(schema.filter((entry) => entry.type === "table"), (entry) => effect_1.Effect.gen(function* () {
+        const rows = yield* effect_1.Effect.tryPromise(() => database.all(`SELECT * FROM "${entry.name.replaceAll('"', '""')}"`));
+        const normalize = (row) => JSON.stringify(Object.fromEntries(Object.entries(row).map(([key, value]) => [
+            key,
+            value instanceof Uint8Array
+                ? { blob: Array.from(value) }
+                : value,
+        ])));
+        return [
+            entry.name,
+            [...rows].sort((left, right) => normalize(left).localeCompare(normalize(right))),
+        ];
+    }), { concurrency: 1 });
+    return {
+        schema: schema.map((entry) => ({
+            ...entry,
+            sql: entry.sql
+                ? entry.sql.replace(/^(CREATE\s+(?:UNIQUE\s+)?(?:TABLE|INDEX|TRIGGER|VIEW|MATERIALIZED\s+VIEW)\s+)IF\s+NOT\s+EXISTS\s+/i, "$1")
+                : // oxlint-disable-next-line local-rules/no-null-undefined-option -- snapshotting must preserve sqlite_schema NULL SQL entries.
+                    null,
+        })),
+        tables: Object.fromEntries(tableSnapshots),
+    };
+});
+exports.snapshotDatabase = snapshotDatabase;
+const compareDatabases = (left, right) => effect_1.Effect.all([(0, exports.snapshotDatabase)(left), (0, exports.snapshotDatabase)(right)], { concurrency: 1 }).pipe(effect_1.Effect.map(([leftSnapshot, rightSnapshot]) => (0, exports.compareSnapshots)(leftSnapshot, rightSnapshot)));
+exports.compareDatabases = compareDatabases;
+const applyBatch = (database, batch) => batch.transactional
+    ? effect_1.Effect.gen(function* () {
+        yield* effect_1.Effect.tryPromise(() => database.exec("BEGIN IMMEDIATE")).pipe(effect_1.Effect.asVoid);
+        yield* effect_1.Effect.forEach(batch.statements, (statement) => effect_1.Effect.tryPromise(() => database.exec(statement)).pipe(effect_1.Effect.asVoid), { concurrency: 1, discard: true });
+        yield* effect_1.Effect.tryPromise(() => database.exec("COMMIT")).pipe(effect_1.Effect.asVoid);
+    }).pipe(effect_1.Effect.catchAll((error) => effect_1.Effect.tryPromise(() => database.exec("ROLLBACK")).pipe(effect_1.Effect.asVoid, effect_1.Effect.catchAll(() => effect_1.Effect.void), effect_1.Effect.andThen(effect_1.Effect.fail(error)))))
+    : effect_1.Effect.forEach(batch.statements, (statement) => effect_1.Effect.tryPromise(() => database.exec(statement)).pipe(effect_1.Effect.asVoid), { concurrency: 1, discard: true });
+exports.applyBatch = applyBatch;
+//# sourceMappingURL=testUtils.js.map

package/cjs/src/cdc/testUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testUtils.js","sourceRoot":"","sources":["../../../../src/cdc/testUtils.ts"],"names":[],"mappings":";;;AACA,mCAAgC;AAezB,MAAM,gBAAgB,GAAG,CAAC,IAAsB,EAAE,KAAuB,EAAW,EAAE,CAC5F,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AADnC,QAAA,gBAAgB,oBACmB;AAEzC,MAAM,gBAAgB,GAAG,CAAC,QAAkB,EAA0C,EAAE,CAC9F,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,UAAU,CACtC,GAAG,EAAE,CACJ,QAAQ,CAAC,GAAG,CACX;QACC,6CAA6C;QAC7C,oBAAoB;QACpB,4DAA4D;QAC5D,qBAAqB;KACrB,CAAC,IAAI,CAAC,GAAG,CAAC,CACwB,CACrC,CAAC;IACF,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,OAAO,CAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,EAChD,CAAC,KAAK,EAAE,EAAE,CACT,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACnB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,UAAU,CACpC,GAAG,EAAE,CACJ,QAAQ,CAAC,GAAG,CACX,kBAAkB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CACN,CACjD,CAAC;QACF,MAAM,SAAS,GAAG,CAAC,GAA4B,EAAU,EAAE,CAC1D,IAAI,CAAC,SAAS,CACb,MAAM,CAAC,WAAW,CACjB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YACzC,GAAG;YACH,KAAK,YAAY,UAAU;gBAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAC7B,CAAC,CAAC,KAAK;SACR,CAAC,CACF,CACD,CAAC;QACH,OAAO;YACN,KAAK,CAAC,IAAI;YACV,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC9B,SAAS,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAC/C;SACQ,CAAC;IACZ,CAAC,CAAC,EACH,EAAE,WAAW,EAAE,CAAC,EAAE,CAClB,CAAC;IACF,OAAO;QACN,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC9B,GAAG,KAAK;YACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACb,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CACjB,oGAAoG,EACpG,IAAI,CACJ;gBACF,CAAC,CAAC,8HAA8H;oBAC/H,IAAI;SACN,CAAC,CAAC;QACH,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;KAC1C,CAAC;AACH,CAAC,CAAC,CAAC;AAxDS,QAAA,gBAAgB,oBAwDzB;AAEG,MAAM,gBAAgB,GAAG,CAAC,IAAc,EAAE,KAAe,EAAiC,EAAE,CAClG,eAAM,CAAC,GAAG,CAAC,CAAC,IAAA,wBAAgB,EAAC,IAAI,CAAC,EAAE,IAAA,wBAAgB,EAAC,KAAK,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CACrF,eAAM,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,aAAa,CAAC,EAAE,EAAE,CAC5C,IAAA,wBAAgB,EAAC,YAAY,EAAE,aAAa,CAAC,CAC7C,CACD,CAAC;AALU,QAAA,gBAAgB,oBAK1B;AAEI,MAAM,UAAU,GAAG,CACzB,QAAkB,EAClB,KAGC,EAC4B,EAAE,CAC/B,KAAK,CAAC,aAAa;IAClB,CAAC,CAAC,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpB,KAAK,CAAC,CAAC,eAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CACpE,eAAM,CAAC,MAAM,CACb,CAAC;QACF,KAAK,CAAC,CAAC,eAAM,CAAC,OAAO,CACpB,KAAK,CAAC,UAAU,EAChB,CAAC,SAAS,EAAE,EAAE,CACb,eAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,MAAM,CAAC,EACtE,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CACjC,CAAC;QACF,KAAK,CAAC,CAAC,eAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,MAAM,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC,IAAI,CACN,eAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACzB,eAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CACtD,eAAM,CAAC,MAAM,EACb,eAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,eAAM,CAAC,IAAI,CAAC,EAClC,eAAM,CAAC,OAAO,CAAC,eAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAClC,CACD,CACD;IACF,CAAC,CAAC,eAAM,CAAC,OAAO,CACd,KAAK,CAAC,UAAU,EAChB,CAAC,SAAS,EAAE,EAAE,CACb,eAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,MAAM,CAAC,EACtE,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CACjC,CAAC;AAjCQ,QAAA,UAAU,cAiClB","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport { Effect } from \"effect\";\n\nexport type SchemaEntry = {\n\ttype: string;\n\tname: string;\n\ttblName: string;\n\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- sqlite_schema.sql is NULL for implicit indexes and similar internal objects.\n\tsql: string | null;\n};\n\nexport type DatabaseSnapshot = {\n\tschema: readonly SchemaEntry[];\n\ttables: Readonly<Record<string, readonly Record<string, unknown>[]>>;\n};\n\nexport const compareSnapshots = (left: DatabaseSnapshot, right: DatabaseSnapshot): boolean =>\n\tJSON.stringify(left) === JSON.stringify(right);\n\nexport const snapshotDatabase = (database: Database): Effect.Effect<DatabaseSnapshot, Error> =>\n\tEffect.gen(function* () {\n\t\tconst schema = yield* Effect.tryPromise(\n\t\t\t() =>\n\t\t\t\tdatabase.all(\n\t\t\t\t\t[\n\t\t\t\t\t\t\"SELECT type, name, tbl_name AS tblName, sql\",\n\t\t\t\t\t\t\"FROM sqlite_schema\",\n\t\t\t\t\t\t\"WHERE name NOT LIKE 'sqlite_%' AND name NOT LIKE 'turso_%'\",\n\t\t\t\t\t\t\"ORDER BY type, name\",\n\t\t\t\t\t].join(\" \"),\n\t\t\t\t) as Promise<readonly SchemaEntry[]>,\n\t\t);\n\t\tconst tableSnapshots = yield* Effect.forEach(\n\t\t\tschema.filter((entry) => entry.type === \"table\"),\n\t\t\t(entry) =>\n\t\t\t\tEffect.gen(function* () {\n\t\t\t\t\tconst rows = yield* Effect.tryPromise(\n\t\t\t\t\t\t() =>\n\t\t\t\t\t\t\tdatabase.all(\n\t\t\t\t\t\t\t\t`SELECT * FROM \"${entry.name.replaceAll('\"', '\"\"')}\"`,\n\t\t\t\t\t\t\t) as Promise<readonly Record<string, unknown>[]>,\n\t\t\t\t\t);\n\t\t\t\t\tconst normalize = (row: Record<string, unknown>): string =>\n\t\t\t\t\t\tJSON.stringify(\n\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\tObject.entries(row).map(([key, value]) => [\n\t\t\t\t\t\t\t\t\tkey,\n\t\t\t\t\t\t\t\t\tvalue instanceof Uint8Array\n\t\t\t\t\t\t\t\t\t\t? { blob: Array.from(value) }\n\t\t\t\t\t\t\t\t\t\t: value,\n\t\t\t\t\t\t\t\t]),\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t);\n\t\t\t\t\treturn [\n\t\t\t\t\t\tentry.name,\n\t\t\t\t\t\t[...rows].sort((left, right) =>\n\t\t\t\t\t\t\tnormalize(left).localeCompare(normalize(right)),\n\t\t\t\t\t\t),\n\t\t\t\t\t] as const;\n\t\t\t\t}),\n\t\t\t{ concurrency: 1 },\n\t\t);\n\t\treturn {\n\t\t\tschema: schema.map((entry) => ({\n\t\t\t\t...entry,\n\t\t\t\tsql: entry.sql\n\t\t\t\t\t? entry.sql.replace(\n\t\t\t\t\t\t\t/^(CREATE\\s+(?:UNIQUE\\s+)?(?:TABLE|INDEX|TRIGGER|VIEW|MATERIALIZED\\s+VIEW)\\s+)IF\\s+NOT\\s+EXISTS\\s+/i,\n\t\t\t\t\t\t\t\"$1\",\n\t\t\t\t\t\t)\n\t\t\t\t\t: // oxlint-disable-next-line local-rules/no-null-undefined-option -- snapshotting must preserve sqlite_schema NULL SQL entries.\n\t\t\t\t\t\tnull,\n\t\t\t})),\n\t\t\ttables: Object.fromEntries(tableSnapshots),\n\t\t};\n\t});\n\nexport const compareDatabases = (left: Database, right: Database): Effect.Effect<boolean, Error> =>\n\tEffect.all([snapshotDatabase(left), snapshotDatabase(right)], { concurrency: 1 }).pipe(\n\t\tEffect.map(([leftSnapshot, rightSnapshot]) =>\n\t\t\tcompareSnapshots(leftSnapshot, rightSnapshot),\n\t\t),\n\t);\n\nexport const applyBatch = (\n\tdatabase: Database,\n\tbatch: {\n\t\treadonly transactional: boolean;\n\t\treadonly statements: readonly string[];\n\t},\n): Effect.Effect<void, Error> =>\n\tbatch.transactional\n\t\t? Effect.gen(function* () {\n\t\t\t\tyield* Effect.tryPromise(() => database.exec(\"BEGIN IMMEDIATE\")).pipe(\n\t\t\t\t\tEffect.asVoid,\n\t\t\t\t);\n\t\t\t\tyield* Effect.forEach(\n\t\t\t\t\tbatch.statements,\n\t\t\t\t\t(statement) =>\n\t\t\t\t\t\tEffect.tryPromise(() => database.exec(statement)).pipe(Effect.asVoid),\n\t\t\t\t\t{ concurrency: 1, discard: true },\n\t\t\t\t);\n\t\t\t\tyield* Effect.tryPromise(() => database.exec(\"COMMIT\")).pipe(Effect.asVoid);\n\t\t\t}).pipe(\n\t\t\t\tEffect.catchAll((error) =>\n\t\t\t\t\tEffect.tryPromise(() => database.exec(\"ROLLBACK\")).pipe(\n\t\t\t\t\t\tEffect.asVoid,\n\t\t\t\t\t\tEffect.catchAll(() => Effect.void),\n\t\t\t\t\t\tEffect.andThen(Effect.fail(error)),\n\t\t\t\t\t),\n\t\t\t\t),\n\t\t\t)\n\t\t: Effect.forEach(\n\t\t\t\tbatch.statements,\n\t\t\t\t(statement) =>\n\t\t\t\t\tEffect.tryPromise(() => database.exec(statement)).pipe(Effect.asVoid),\n\t\t\t\t{ concurrency: 1, discard: true },\n\t\t\t);"]}

package/cjs/src/cdc/truncate.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.truncate = void 0;
+const effect_1 = require("effect");
+const truncate = (db) => effect_1.Effect.promise(() => db.run("PRAGMA wal_checkpoint(TRUNCATE);"));
+exports.truncate = truncate;
+//# sourceMappingURL=truncate.js.map

package/cjs/src/cdc/truncate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"truncate.js","sourceRoot":"","sources":["../../../../src/cdc/truncate.ts"],"names":[],"mappings":";;;AACA,mCAAgC;AAEzB,MAAM,QAAQ,GAAG,CAAC,EAAY,EAAE,EAAE,CACxC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC,CAAC;AADrD,QAAA,QAAQ,YAC6C","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport { Effect } from \"effect\";\n\nexport const truncate = (db: Database) =>\n\tEffect.promise(() => db.run(\"PRAGMA wal_checkpoint(TRUNCATE);\"));"]}

package/cjs/src/cdc/types.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CDCChangeType = void 0;
+exports.CDCChangeType = {
+    Delete: -1,
+    Update: 0,
+    Insert: 1,
+    // Commit: 2,
+};
+//# sourceMappingURL=types.js.map

package/cjs/src/cdc/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/cdc/types.ts"],"names":[],"mappings":";;;AAGa,QAAA,aAAa,GAAG;IAC5B,MAAM,EAAE,CAAC,CAAC;IACV,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,aAAa;CACJ,CAAC","sourcesContent":["// oxlint-disable-next-line local-rules/no-null-undefined-option -- SQLite CDC rows must preserve SQL NULL values.\nexport type CDCValue = number | string | Uint8Array | null;\n\nexport const CDCChangeType = {\n\tDelete: -1,\n\tUpdate: 0,\n\tInsert: 1,\n\t// Commit: 2,\n} as const;\n\nexport type CDCChangeType = (typeof CDCChangeType)[keyof typeof CDCChangeType];\n\ntype CDCBase = {\n\tchangeId: number;\n\tchangeTime: number;\n\tchangeTxnId: number;\n};\n\nexport type CDCRow =\n\t| (CDCBase & {\n\t\t\tchangeType: typeof CDCChangeType.Insert;\n\t\t\ttableName: string;\n\t\t\tid: CDCValue;\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Insert CDC rows have no previous record image.\n\t\t\tbefore: Uint8Array | null;\n\t\t\tafter: Uint8Array;\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Insert CDC rows have no updates mask.\n\t\t\tupdates: Uint8Array | null;\n\t  })\n\t| (CDCBase & {\n\t\t\tchangeType: typeof CDCChangeType.Update;\n\t\t\ttableName: string;\n\t\t\tid: CDCValue;\n\t\t\tbefore: Uint8Array;\n\t\t\tafter: Uint8Array;\n\t\t\tupdates: Uint8Array;\n\t  })\n\t| (CDCBase & {\n\t\t\tchangeType: typeof CDCChangeType.Delete;\n\t\t\ttableName: string;\n\t\t\tid: CDCValue;\n\t\t\tbefore: Uint8Array;\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Delete CDC rows have no post-delete record image.\n\t\t\tafter: Uint8Array | null;\n\t\t\t// oxlint-disable-next-line local-rules/no-null-undefined-option -- Delete CDC rows have no updates mask.\n\t\t\tupdates: Uint8Array | null;\n\t  });"]}

package/cjs/src/cdc/withoutCDC.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withoutCDC = void 0;
+const effect_1 = require("effect");
+const withoutCDC = (database, effect) => effect_1.Effect.acquireUseRelease(effect_1.Effect.promise(() => database.exec("PRAGMA capture_data_changes_conn('off')")), () => effect, () => effect_1.Effect.promise(() => database.exec("PRAGMA capture_data_changes_conn('full')")));
+exports.withoutCDC = withoutCDC;
+//# sourceMappingURL=withoutCDC.js.map

package/cjs/src/cdc/withoutCDC.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"withoutCDC.js","sourceRoot":"","sources":["../../../../src/cdc/withoutCDC.ts"],"names":[],"mappings":";;;AACA,mCAAgC;AAEzB,MAAM,UAAU,GAAG,CACzB,QAAkB,EAClB,MAA8B,EACL,EAAE,CAC3B,eAAM,CAAC,iBAAiB,CACvB,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC,EAC9E,GAAG,EAAE,CAAC,MAAM,EACZ,GAAG,EAAE,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC,CACrF,CAAC;AARU,QAAA,UAAU,cAQpB","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport { Effect } from \"effect\";\n\nexport const withoutCDC = <A, E, R>(\n\tdatabase: Database,\n\teffect: Effect.Effect<A, E, R>,\n): Effect.Effect<A, E, R> =>\n\tEffect.acquireUseRelease(\n\t\tEffect.promise(() => database.exec(\"PRAGMA capture_data_changes_conn('off')\")),\n\t\t() => effect,\n\t\t() => Effect.promise(() => database.exec(\"PRAGMA capture_data_changes_conn('full')\")),\n\t);"]}

package/cjs/src/connection.js

@@ -0,0 +1,128 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.connect = exports.initializeContexts = void 0;
+const effect_1 = require("effect");
+const batches_1 = require("./batches");
+const extract_1 = require("./cdc/extract");
+const truncate_1 = require("./cdc/truncate");
+const contexts_1 = require("./contexts");
+const fileKV_1 = require("./kv/fileKV");
+const s3KV_1 = require("./kv/s3KV");
+const syncFiles_1 = require("./kv/syncFiles");
+const pull_1 = require("./pull");
+const push_1 = require("./push");
+const storage_1 = require("./storage");
+const wrapDatabase_1 = require("./wrapDatabase");
+const initializeContexts = (connectionOptions) => effect_1.Effect.gen(function* () {
+    const s3kv = yield* (0, s3KV_1.makeS3KV)(connectionOptions.bucket);
+    const filekv = yield* (0, fileKV_1.makeFileKV)(connectionOptions.localDirectory ?? "./.s3qlite/");
+    return effect_1.Context.empty().pipe(effect_1.Context.add(storage_1.RemoteKV, s3kv), effect_1.Context.add(storage_1.LocalKV, filekv));
+});
+exports.initializeContexts = initializeContexts;
+const connect = (dbName, options) => effect_1.Effect.gen(function* () {
+    const remote = yield* storage_1.RemoteKV;
+    const local = yield* storage_1.LocalKV;
+    const { localHead, remoteHead } = yield* effect_1.Effect.all({
+        localHead: (0, storage_1.getJson)(local, (0, storage_1.headKey)(dbName)),
+        remoteHead: (0, storage_1.getJson)(remote, (0, storage_1.headKey)(dbName)),
+        // localDb: local.get(dbKey(dbName)),
+    });
+    if (effect_1.Option.isNone(localHead) && effect_1.Option.isNone(remoteHead)) {
+        const snapshotId = crypto.randomUUID();
+        // connection is scoped, will die when the effect is done
+        const connection = yield* local.connect((0, storage_1.dbKey)(dbName));
+        yield* (0, truncate_1.truncate)(connection);
+        yield* effect_1.Effect.promise(() => connection.close());
+        const head = {
+            snapshots: [
+                {
+                    id: snapshotId,
+                    batchIdsApplied: [],
+                },
+            ],
+            batches: [],
+        };
+        // TODO: this should be atomic, and the head(s) probably updated only after snapshot is written.
+        const { remoteResult } = yield* effect_1.Effect.all({
+            remoteResult: remote.set((0, storage_1.headKey)(dbName), (0, storage_1.encodeJson)(head)),
+            snapshot: local.clone((0, storage_1.dbKey)(dbName), (0, storage_1.snapshotKey)(snapshotId)).pipe(effect_1.Effect.flatMap(() => local.readStream((0, storage_1.snapshotKey)(snapshotId))), effect_1.Effect.flatMap(effect_1.Option.match({
+                onNone: () => effect_1.Effect.die(new Error("Snapshot not found after clone")),
+                onSome: (stream) => remote.writeStream((0, storage_1.snapshotKey)(snapshotId), stream),
+            }))),
+        });
+        yield* local.set((0, storage_1.headKey)(dbName), (0, storage_1.encodeJson)({
+            head,
+            remoteEtag: remoteResult.etag,
+            lastSyncedLocalChangeId: 0,
+        }));
+    }
+    if (effect_1.Option.isSome(remoteHead) && effect_1.Option.isNone(localHead)) {
+        const head = remoteHead.value.value;
+        const latestSnapshot = head.snapshots.at(-1);
+        if (!latestSnapshot) {
+            return yield* effect_1.Effect.die(new Error("Remote head has no snapshots"));
+        }
+        const lastAppliedBatchId = latestSnapshot.batchIdsApplied.at(-1);
+        const batchStartIndex = lastAppliedBatchId
+            ? head.batches.findIndex((b) => b.id === lastAppliedBatchId) + 1
+            : 0;
+        const batchesToDownload = head.batches.slice(batchStartIndex);
+        yield* (0, syncFiles_1.pullFiles)([
+            (0, storage_1.snapshotKey)(latestSnapshot.id),
+            ...batchesToDownload.map((batch) => (0, storage_1.batchKey)(batch.id)),
+        ]);
+        yield* local.clone((0, storage_1.snapshotKey)(latestSnapshot.id), (0, storage_1.dbKey)(dbName));
+        const db = yield* local.connect((0, storage_1.dbKey)(dbName));
+        for (const batch of batchesToDownload) {
+            yield* (0, batches_1.applyBatch)(db, batch);
+        }
+        const lastSyncedLocalChangeId = yield* (0, extract_1.getLatestChangeId)(db);
+        yield* (0, truncate_1.truncate)(db);
+        yield* effect_1.Effect.promise(() => db.close());
+        const lastBatchId = batchesToDownload.at(-1)?.id;
+        if (lastBatchId) {
+            yield* local.clone((0, storage_1.dbKey)(dbName), (0, storage_1.baseKey)(latestSnapshot.id, lastBatchId));
+        }
+        yield* local.set((0, storage_1.headKey)(dbName), (0, storage_1.encodeJson)({
+            head,
+            remoteEtag: remoteHead.value.etag,
+            lastSyncedLocalChangeId,
+        }));
+    }
+    if (effect_1.Option.isNone(remoteHead) && effect_1.Option.isSome(localHead)) {
+        return yield* effect_1.Effect.die(new Error("Local head exists but remote head does not. This is an unexpected state."));
+    }
+    if (effect_1.Option.isSome(remoteHead) && effect_1.Option.isSome(localHead)) {
+        // do nothing?
+    }
+    const connectionConfigValue = {
+        bucket: options.bucket,
+        dbName,
+        livePath: (0, storage_1.dbKey)(dbName),
+        localDirectory: options.localDirectory ?? "./.s3qlite/",
+        localHeadPath: (0, storage_1.headKey)(dbName),
+        options,
+    };
+    const connection = yield* local.connect((0, storage_1.dbKey)(dbName));
+    let currentConnection = connection;
+    const connectionRef = yield* effect_1.Ref.make(connection);
+    const connectionStateValue = {
+        getConnection: effect_1.Ref.get(connectionRef),
+        setConnection: (conn) => effect_1.Effect.sync(() => {
+            currentConnection = conn;
+        }).pipe(effect_1.Effect.flatMap(() => effect_1.Ref.set(connectionRef, conn))),
+    };
+    const ctx = effect_1.Context.empty().pipe(effect_1.Context.add(contexts_1.ConnectionConfig, connectionConfigValue), effect_1.Context.add(contexts_1.ConnectionState, connectionStateValue), effect_1.Context.add(storage_1.LocalKV, local), effect_1.Context.add(storage_1.RemoteKV, remote));
+    const wrapper = (0, wrapDatabase_1.wrapDatabase)(() => currentConnection, (call) => call());
+    wrapper.pull = (() => (0, pull_1.pull)().pipe(effect_1.Effect.provide(ctx), effect_1.Effect.asVoid));
+    wrapper.push = () => (0, push_1.push)().pipe(effect_1.Effect.provide(ctx), effect_1.Effect.asVoid);
+    wrapper.sync = (() => effect_1.Effect.gen(function* () {
+        yield* (0, pull_1.pull)();
+        yield* (0, push_1.push)();
+    }).pipe(effect_1.Effect.provide(ctx)));
+    wrapper.fork = () => effect_1.Effect.die(new Error("Not implemented"));
+    wrapper.checkpoint = () => effect_1.Effect.die(new Error("Not implemented"));
+    return wrapper;
+});
+exports.connect = connect;
+//# sourceMappingURL=connection.js.map

package/cjs/src/connection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../../src/connection.ts"],"names":[],"mappings":";;;AAAA,mCAAsD;AAEtD,uCAAuC;AACvC,2CAAkD;AAClD,6CAA0C;AAC1C,yCAA+D;AAC/D,wCAAyC;AACzC,oCAAqC;AACrC,8CAA2C;AAC3C,iCAA8B;AAC9B,iCAA8B;AAC9B,uCAUmB;AAEnB,iDAA8C;AAEvC,MAAM,kBAAkB,GAAG,CAAC,iBAAoC,EAAE,EAAE,CAC1E,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,IAAA,eAAQ,EAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,IAAA,mBAAU,EAAC,iBAAiB,CAAC,cAAc,IAAI,aAAa,CAAC,CAAC;IACpF,OAAO,gBAAO,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,gBAAO,CAAC,GAAG,CAAC,kBAAQ,EAAE,IAAI,CAAC,EAAE,gBAAO,CAAC,GAAG,CAAC,iBAAO,EAAE,MAAM,CAAC,CAAC,CAAC;AACxF,CAAC,CAAC,CAAC;AALS,QAAA,kBAAkB,sBAK3B;AAEG,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,OAA0B,EAAE,EAAE,CACrE,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,kBAAQ,CAAC;IAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,iBAAO,CAAC;IAE7B,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CAAC;QACnD,SAAS,EAAE,IAAA,iBAAO,EAAa,KAAK,EAAE,IAAA,iBAAO,EAAC,MAAM,CAAC,CAAC;QACtD,UAAU,EAAE,IAAA,iBAAO,EAAO,MAAM,EAAE,IAAA,iBAAO,EAAC,MAAM,CAAC,CAAC;QAClD,qCAAqC;KACrC,CAAC,CAAC;IAEH,IAAI,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,eAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAEvC,yDAAyD;QACzD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,CAAC,CAAC;QACvD,KAAK,CAAC,CAAC,IAAA,mBAAQ,EAAC,UAAU,CAAC,CAAC;QAC5B,KAAK,CAAC,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAS;YAClB,SAAS,EAAE;gBACV;oBACC,EAAE,EAAE,UAAU;oBACd,eAAe,EAAE,EAAE;iBACnB;aACD;YACD,OAAO,EAAE,EAAE;SACX,CAAC;QAEF,gGAAgG;QAChG,MAAM,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CAAC;YAC1C,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,IAAA,iBAAO,EAAC,MAAM,CAAC,EAAE,IAAA,oBAAU,EAAC,IAAI,CAAC,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,EAAE,IAAA,qBAAW,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CACjE,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,IAAA,qBAAW,EAAC,UAAU,CAAC,CAAC,CAAC,EAC/D,eAAM,CAAC,OAAO,CACb,eAAM,CAAC,KAAK,CAAC;gBACZ,MAAM,EAAE,GAAG,EAAE,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;gBACrE,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,IAAA,qBAAW,EAAC,UAAU,CAAC,EAAE,MAAM,CAAC;aACvE,CAAC,CACF,CACD;SACD,CAAC,CAAC;QAEH,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CACf,IAAA,iBAAO,EAAC,MAAM,CAAC,EACf,IAAA,oBAAU,EAAa;YACtB,IAAI;YACJ,UAAU,EAAE,YAAY,CAAC,IAAI;YAC7B,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;IACH,CAAC;IAED,IAAI,eAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC;QACpC,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7C,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,eAAe,GAAG,kBAAkB;YACzC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,kBAAkB,CAAC,GAAG,CAAC;YAChE,CAAC,CAAC,CAAC,CAAC;QACL,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAE9D,KAAK,CAAC,CAAC,IAAA,qBAAS,EAAC;YAChB,IAAA,qBAAW,EAAC,cAAc,CAAC,EAAE,CAAC;YAC9B,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,kBAAQ,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACvD,CAAC,CAAC;QAEH,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAA,qBAAW,EAAC,cAAc,CAAC,EAAE,CAAC,EAAE,IAAA,eAAK,EAAC,MAAM,CAAC,CAAC,CAAC;QAElE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,CAAC,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;YACvC,KAAK,CAAC,CAAC,IAAA,oBAAU,EAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,uBAAuB,GAAG,KAAK,CAAC,CAAC,IAAA,2BAAiB,EAAC,EAAE,CAAC,CAAC;QAC7D,KAAK,CAAC,CAAC,IAAA,mBAAQ,EAAC,EAAE,CAAC,CAAC;QACpB,KAAK,CAAC,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;QAExC,MAAM,WAAW,GAAG,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,WAAW,EAAE,CAAC;YACjB,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,EAAE,IAAA,iBAAO,EAAC,cAAc,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CACf,IAAA,iBAAO,EAAC,MAAM,CAAC,EACf,IAAA,oBAAU,EAAa;YACtB,IAAI;YACJ,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI;YACjC,uBAAuB;SACvB,CAAC,CACF,CAAC;IACH,CAAC;IAED,IAAI,eAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,OAAO,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CACvB,IAAI,KAAK,CACR,0EAA0E,CAC1E,CACD,CAAC;IACH,CAAC;IAED,IAAI,eAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,cAAc;IACf,CAAC;IAED,MAAM,qBAAqB,GAAG;QAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;QACN,QAAQ,EAAE,IAAA,eAAK,EAAC,MAAM,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,aAAa;QACvD,aAAa,EAAE,IAAA,iBAAO,EAAC,MAAM,CAAC;QAC9B,OAAO;KACP,CAAC;IAEF,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,CAAC,CAAC;IAEvD,IAAI,iBAAiB,GAAG,UAAU,CAAC;IAEnC,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,YAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,oBAAoB,GAAG;QAC5B,aAAa,EAAE,YAAG,CAAC,GAAG,CAAC,aAAa,CAAC;QACrC,aAAa,EAAE,CAAC,IAAuB,EAAE,EAAE,CAC1C,eAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,iBAAiB,GAAG,IAAI,CAAC;QAC1B,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAG,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;KAC5D,CAAC;IAEF,MAAM,GAAG,GAAG,gBAAO,CAAC,KAAK,EAAE,CAAC,IAAI,CAC/B,gBAAO,CAAC,GAAG,CAAC,2BAAgB,EAAE,qBAAqB,CAAC,EACpD,gBAAO,CAAC,GAAG,CAAC,0BAAe,EAAE,oBAAoB,CAAC,EAClD,gBAAO,CAAC,GAAG,CAAC,iBAAO,EAAE,KAAK,CAAC,EAC3B,gBAAO,CAAC,GAAG,CAAC,kBAAQ,EAAE,MAAM,CAAC,CAC7B,CAAC;IAEF,MAAM,OAAO,GAAG,IAAA,2BAAY,EAC3B,GAAG,EAAE,CAAC,iBAAiB,EACvB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CACc,CAAC;IAEhC,OAAO,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,CACpB,IAAA,WAAI,GAAE,CAAC,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,eAAM,CAAC,MAAM,CAAC,CAAwB,CAAC;IACzE,OAAO,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC,IAAA,WAAI,GAAE,CAAC,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,eAAM,CAAC,MAAM,CAAC,CAAC;IACrE,OAAO,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,CACpB,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACnB,KAAK,CAAC,CAAC,IAAA,WAAI,GAAE,CAAC;QACd,KAAK,CAAC,CAAC,IAAA,WAAI,GAAE,CAAC;IACf,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAwB,CAAC;IACtD,OAAO,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC9D,OAAO,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAEpE,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC,CAAC;AA5JS,QAAA,OAAO,WA4JhB","sourcesContent":["import { Context, Effect, Option, Ref } from \"effect\";\n\nimport { applyBatch } from \"./batches\";\nimport { getLatestChangeId } from \"./cdc/extract\";\nimport { truncate } from \"./cdc/truncate\";\nimport { ConnectionConfig, ConnectionState } from \"./contexts\";\nimport { makeFileKV } from \"./kv/fileKV\";\nimport { makeS3KV } from \"./kv/s3KV\";\nimport { pullFiles } from \"./kv/syncFiles\";\nimport { pull } from \"./pull\";\nimport { push } from \"./push\";\nimport {\n\tLocalKV,\n\tRemoteKV,\n\tbaseKey,\n\tbatchKey,\n\tdbKey,\n\tencodeJson,\n\tgetJson,\n\theadKey,\n\tsnapshotKey,\n} from \"./storage\";\nimport type { ConnectionOptions, Head, S3qliteDatabase, StoredHead } from \"./types\";\nimport { wrapDatabase } from \"./wrapDatabase\";\n\nexport const initializeContexts = (connectionOptions: ConnectionOptions) =>\n\tEffect.gen(function* () {\n\t\tconst s3kv = yield* makeS3KV(connectionOptions.bucket);\n\t\tconst filekv = yield* makeFileKV(connectionOptions.localDirectory ?? \"./.s3qlite/\");\n\t\treturn Context.empty().pipe(Context.add(RemoteKV, s3kv), Context.add(LocalKV, filekv));\n\t});\n\nexport const connect = (dbName: string, options: ConnectionOptions) =>\n\tEffect.gen(function* () {\n\t\tconst remote = yield* RemoteKV;\n\t\tconst local = yield* LocalKV;\n\n\t\tconst { localHead, remoteHead } = yield* Effect.all({\n\t\t\tlocalHead: getJson<StoredHead>(local, headKey(dbName)),\n\t\t\tremoteHead: getJson<Head>(remote, headKey(dbName)),\n\t\t\t// localDb: local.get(dbKey(dbName)),\n\t\t});\n\n\t\tif (Option.isNone(localHead) && Option.isNone(remoteHead)) {\n\t\t\tconst snapshotId = crypto.randomUUID();\n\n\t\t\t// connection is scoped, will die when the effect is done\n\t\t\tconst connection = yield* local.connect(dbKey(dbName));\n\t\t\tyield* truncate(connection);\n\t\t\tyield* Effect.promise(() => connection.close());\n\n\t\t\tconst head: Head = {\n\t\t\t\tsnapshots: [\n\t\t\t\t\t{\n\t\t\t\t\t\tid: snapshotId,\n\t\t\t\t\t\tbatchIdsApplied: [],\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t\tbatches: [],\n\t\t\t};\n\n\t\t\t// TODO: this should be atomic, and the head(s) probably updated only after snapshot is written.\n\t\t\tconst { remoteResult } = yield* Effect.all({\n\t\t\t\tremoteResult: remote.set(headKey(dbName), encodeJson(head)),\n\t\t\t\tsnapshot: local.clone(dbKey(dbName), snapshotKey(snapshotId)).pipe(\n\t\t\t\t\tEffect.flatMap(() => local.readStream(snapshotKey(snapshotId))),\n\t\t\t\t\tEffect.flatMap(\n\t\t\t\t\t\tOption.match({\n\t\t\t\t\t\t\tonNone: () => Effect.die(new Error(\"Snapshot not found after clone\")),\n\t\t\t\t\t\t\tonSome: (stream) => remote.writeStream(snapshotKey(snapshotId), stream),\n\t\t\t\t\t\t}),\n\t\t\t\t\t),\n\t\t\t\t),\n\t\t\t});\n\n\t\t\tyield* local.set(\n\t\t\t\theadKey(dbName),\n\t\t\t\tencodeJson<StoredHead>({\n\t\t\t\t\thead,\n\t\t\t\t\tremoteEtag: remoteResult.etag,\n\t\t\t\t\tlastSyncedLocalChangeId: 0,\n\t\t\t\t}),\n\t\t\t);\n\t\t}\n\n\t\tif (Option.isSome(remoteHead) && Option.isNone(localHead)) {\n\t\t\tconst head = remoteHead.value.value;\n\t\t\tconst latestSnapshot = head.snapshots.at(-1);\n\n\t\t\tif (!latestSnapshot) {\n\t\t\t\treturn yield* Effect.die(new Error(\"Remote head has no snapshots\"));\n\t\t\t}\n\n\t\t\tconst lastAppliedBatchId = latestSnapshot.batchIdsApplied.at(-1);\n\t\t\tconst batchStartIndex = lastAppliedBatchId\n\t\t\t\t? head.batches.findIndex((b) => b.id === lastAppliedBatchId) + 1\n\t\t\t\t: 0;\n\t\t\tconst batchesToDownload = head.batches.slice(batchStartIndex);\n\n\t\t\tyield* pullFiles([\n\t\t\t\tsnapshotKey(latestSnapshot.id),\n\t\t\t\t...batchesToDownload.map((batch) => batchKey(batch.id)),\n\t\t\t]);\n\n\t\t\tyield* local.clone(snapshotKey(latestSnapshot.id), dbKey(dbName));\n\n\t\t\tconst db = yield* local.connect(dbKey(dbName));\n\t\t\tfor (const batch of batchesToDownload) {\n\t\t\t\tyield* applyBatch(db, batch);\n\t\t\t}\n\t\t\tconst lastSyncedLocalChangeId = yield* getLatestChangeId(db);\n\t\t\tyield* truncate(db);\n\t\t\tyield* Effect.promise(() => db.close());\n\n\t\t\tconst lastBatchId = batchesToDownload.at(-1)?.id;\n\t\t\tif (lastBatchId) {\n\t\t\t\tyield* local.clone(dbKey(dbName), baseKey(latestSnapshot.id, lastBatchId));\n\t\t\t}\n\n\t\t\tyield* local.set(\n\t\t\t\theadKey(dbName),\n\t\t\t\tencodeJson<StoredHead>({\n\t\t\t\t\thead,\n\t\t\t\t\tremoteEtag: remoteHead.value.etag,\n\t\t\t\t\tlastSyncedLocalChangeId,\n\t\t\t\t}),\n\t\t\t);\n\t\t}\n\n\t\tif (Option.isNone(remoteHead) && Option.isSome(localHead)) {\n\t\t\treturn yield* Effect.die(\n\t\t\t\tnew Error(\n\t\t\t\t\t\"Local head exists but remote head does not. This is an unexpected state.\",\n\t\t\t\t),\n\t\t\t);\n\t\t}\n\n\t\tif (Option.isSome(remoteHead) && Option.isSome(localHead)) {\n\t\t\t// do nothing?\n\t\t}\n\n\t\tconst connectionConfigValue = {\n\t\t\tbucket: options.bucket,\n\t\t\tdbName,\n\t\t\tlivePath: dbKey(dbName),\n\t\t\tlocalDirectory: options.localDirectory ?? \"./.s3qlite/\",\n\t\t\tlocalHeadPath: headKey(dbName),\n\t\t\toptions,\n\t\t};\n\n\t\tconst connection = yield* local.connect(dbKey(dbName));\n\n\t\tlet currentConnection = connection;\n\n\t\tconst connectionRef = yield* Ref.make(connection);\n\n\t\tconst connectionStateValue = {\n\t\t\tgetConnection: Ref.get(connectionRef),\n\t\t\tsetConnection: (conn: typeof connection) =>\n\t\t\t\tEffect.sync(() => {\n\t\t\t\t\tcurrentConnection = conn;\n\t\t\t\t}).pipe(Effect.flatMap(() => Ref.set(connectionRef, conn))),\n\t\t};\n\n\t\tconst ctx = Context.empty().pipe(\n\t\t\tContext.add(ConnectionConfig, connectionConfigValue),\n\t\t\tContext.add(ConnectionState, connectionStateValue),\n\t\t\tContext.add(LocalKV, local),\n\t\t\tContext.add(RemoteKV, remote),\n\t\t);\n\n\t\tconst wrapper = wrapDatabase(\n\t\t\t() => currentConnection,\n\t\t\t(call) => call(),\n\t\t) as unknown as S3qliteDatabase;\n\n\t\twrapper.pull = (() =>\n\t\t\tpull().pipe(Effect.provide(ctx), Effect.asVoid)) as typeof wrapper.pull;\n\t\twrapper.push = () => push().pipe(Effect.provide(ctx), Effect.asVoid);\n\t\twrapper.sync = (() =>\n\t\t\tEffect.gen(function* () {\n\t\t\t\tyield* pull();\n\t\t\t\tyield* push();\n\t\t\t}).pipe(Effect.provide(ctx))) as typeof wrapper.sync;\n\t\twrapper.fork = () => Effect.die(new Error(\"Not implemented\"));\n\t\twrapper.checkpoint = () => Effect.die(new Error(\"Not implemented\"));\n\n\t\treturn wrapper;\n\t});"]}

package/cjs/src/contexts.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConnectionState = exports.ConnectionConfig = void 0;
+const effect_1 = require("effect");
+class ConnectionConfig extends effect_1.Context.Tag("s3qlite/ConnectionConfig")() {
+}
+exports.ConnectionConfig = ConnectionConfig;
+class ConnectionState extends effect_1.Context.Tag("s3qlite/ConnectionState")() {
+}
+exports.ConnectionState = ConnectionState;
+//# sourceMappingURL=contexts.js.map

package/cjs/src/contexts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contexts.js","sourceRoot":"","sources":["../../../src/contexts.ts"],"names":[],"mappings":";;;AACA,mCAAiC;AAcjC,MAAa,gBAAiB,SAAQ,gBAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,EAG1E;CAAG;AAHN,4CAGM;AAEN,MAAa,eAAgB,SAAQ,gBAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,EAMxE;CAAG;AANN,0CAMM","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport { Context } from \"effect\";\nimport type { Effect } from \"effect\";\n\nimport type { ConnectionOptions } from \"./types\";\n\nexport type ConnectionConfigValue = {\n\treadonly bucket: string;\n\treadonly dbName: string;\n\treadonly livePath: string;\n\treadonly localDirectory: string;\n\treadonly localHeadPath: string;\n\treadonly options: ConnectionOptions;\n};\n\nexport class ConnectionConfig extends Context.Tag(\"s3qlite/ConnectionConfig\")<\n\tConnectionConfig,\n\tConnectionConfigValue\n>() {}\n\nexport class ConnectionState extends Context.Tag(\"s3qlite/ConnectionState\")<\n\tConnectionState,\n\t{\n\t\treadonly getConnection: Effect.Effect<Database>;\n\t\treadonly setConnection: (connection: Database) => Effect.Effect<void>;\n\t}\n>() {}"]}

package/cjs/src/index.js

@@ -0,0 +1,30 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.connect = exports.initContexts = void 0;
+const effect_1 = require("effect");
+const connection_1 = require("./connection");
+Object.defineProperty(exports, "initContexts", { enumerable: true, get: function () { return connection_1.initializeContexts; } });
+__exportStar(require("./cdc/apply"), exports);
+__exportStar(require("./cdc/extract"), exports);
+__exportStar(require("./cdc/types"), exports);
+__exportStar(require("./kv/fileKV"), exports);
+__exportStar(require("./kv/kv"), exports);
+__exportStar(require("./kv/memoryKV"), exports);
+__exportStar(require("./kv/s3KV"), exports);
+const connect = (dbName, connectionOptions) => (0, connection_1.initializeContexts)(connectionOptions).pipe(effect_1.Effect.flatMap((ctx) => (0, connection_1.connect)(dbName, connectionOptions).pipe(effect_1.Effect.provide(ctx))));
+exports.connect = connect;
+//# sourceMappingURL=index.js.map

package/cjs/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,mCAAgC;AAEhC,6CAA8E;AAW/C,6FAXM,+BAAkB,OAWZ;AAR3C,8CAA4B;AAC5B,gDAA8B;AAC9B,8CAA4B;AAC5B,8CAA4B;AAC5B,0CAAwB;AACxB,gDAA8B;AAC9B,4CAA0B;AAInB,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,iBAAoC,EAAE,EAAE,CAC/E,IAAA,+BAAkB,EAAC,iBAAiB,CAAC,CAAC,IAAI,CACzC,eAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACtB,IAAA,oBAAe,EAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CACpE,CACD,CAAC;AALU,QAAA,OAAO,WAKjB","sourcesContent":["import { Effect } from \"effect\";\n\nimport { connect as connectInternal, initializeContexts } from \"./connection\";\nimport type { ConnectionOptions } from \"./types\";\n\nexport * from \"./cdc/apply\";\nexport * from \"./cdc/extract\";\nexport * from \"./cdc/types\";\nexport * from \"./kv/fileKV\";\nexport * from \"./kv/kv\";\nexport * from \"./kv/memoryKV\";\nexport * from \"./kv/s3KV\";\n\nexport { initializeContexts as initContexts };\n\nexport const connect = (dbName: string, connectionOptions: ConnectionOptions) =>\n\tinitializeContexts(connectionOptions).pipe(\n\t\tEffect.flatMap((ctx) =>\n\t\t\tconnectInternal(dbName, connectionOptions).pipe(Effect.provide(ctx)),\n\t\t),\n\t);"]}