@lika85456/s3qlite 0.1.0

package/cjs/src/pull.js

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pull = void 0;
+const effect_1 = require("effect");
+const uuid_1 = require("uuid");
+const batches_1 = require("./batches");
+const extract_1 = require("./cdc/extract");
+const truncate_1 = require("./cdc/truncate");
+const contexts_1 = require("./contexts");
+const syncFiles_1 = require("./kv/syncFiles");
+const storage_1 = require("./storage");
+const pull = () => effect_1.Effect.gen(function* () {
+    const localKV = yield* storage_1.LocalKV;
+    const remoteKV = yield* storage_1.RemoteKV;
+    const { dbName } = yield* contexts_1.ConnectionConfig;
+    const state = yield* contexts_1.ConnectionState;
+    const tursoConnection = { current: yield* state.getConnection };
+    yield* effect_1.Effect.logDebug("s3qlite.pull.start").pipe(effect_1.Effect.annotateLogs({ dbName }));
+    const localHead = yield* (0, storage_1.getJson)(localKV, (0, storage_1.headKey)(dbName)).pipe(effect_1.Effect.flatMap(effect_1.Option.match({
+        onNone: () => effect_1.Effect.die(new Error("Expected local stored head to exist")),
+        onSome: effect_1.Effect.succeed,
+    })));
+    const remoteHeadOption = yield* remoteKV.getIfChanged((0, storage_1.headKey)(dbName), localHead.value.remoteEtag);
+    // no changes detected, skip pull
+    if (effect_1.Option.isNone(remoteHeadOption)) {
+        yield* effect_1.Effect.logDebug("s3qlite.pull.skip").pipe(effect_1.Effect.annotateLogs({ dbName }));
+        return { newLocalBatch: effect_1.Option.none() };
+    }
+    const remoteHead = yield* (0, storage_1.decodeJson)(remoteHeadOption.value.value).pipe(effect_1.Effect.orDieWith((e) => new Error(`Failed to decode remote head: ${String(e)}. Value to decode: ${remoteHeadOption.value.value}`)));
+    const remoteBatchesToApply = remoteHead.batches.slice(localHead.value.head.batches.length);
+    // Instead of rollbacking the whole database we are keeping a CoW copy of the snapshot+all applied batches up to a certain batch id.
+    const lastLocalSnapshotId = localHead.value.head.snapshots.at(-1)?.id;
+    const lastLocalBatchId = localHead.value.head.batches.at(-1)?.id;
+    if (!lastLocalBatchId || !lastLocalSnapshotId) {
+        return yield* effect_1.Effect.die(new Error("Local head is in an invalid state, missing batches or snapshots")).pipe(effect_1.Effect.annotateLogs({ localHead: localHead, remoteHead }));
+    }
+    const baseSnapshotKey = (0, storage_1.baseKey)(lastLocalSnapshotId, lastLocalBatchId);
+    if (!(yield* localKV.exists(baseSnapshotKey))) {
+        return yield* effect_1.Effect.die(new Error("Base snapshot is missing!"));
+    }
+    const { newLocalBatch } = yield* effect_1.Effect.scoped(effect_1.Effect.gen(function* () {
+        const workingCopyKey = `${(0, uuid_1.v4)()}.working-copy`;
+        yield* effect_1.Effect.addFinalizer(() => localKV.delete(workingCopyKey)).pipe(effect_1.Effect.exit);
+        const { newLocalBatch, workingConnection } = yield* effect_1.Effect.all({
+            newLocalBatch: (0, batches_1.extractBatch)(tursoConnection.current, localHead.value.lastSyncedLocalChangeId),
+            syncFiles: (0, syncFiles_1.pullFiles)(remoteBatchesToApply.map((batch) => (0, storage_1.batchKey)(batch.id))),
+            workingConnection: localKV
+                .clone(baseSnapshotKey, workingCopyKey)
+                .pipe(effect_1.Effect.flatMap(() => localKV.connect(workingCopyKey))),
+        }, {
+            concurrency: "unbounded",
+        });
+        // maybe it would be faster to check the sizes of the batches, and if its not bigger than a few megabytes it could just be all loaded into memory in parallel, and then applied at once, as one transaction
+        for (const batch of remoteBatchesToApply) {
+            yield* (0, batches_1.applyBatch)(workingConnection, batch);
+        }
+        const nextLastRemoteAppliedChangeId = yield* (0, extract_1.getLatestChangeId)(workingConnection);
+        const newHead = structuredClone(remoteHead);
+        const lastRemoteSnapshotId = newHead.snapshots.at(-1)?.id;
+        const lastRemoteBatchId = newHead.batches.at(-1)?.id;
+        if (!lastRemoteBatchId || !lastRemoteSnapshotId) {
+            return yield* effect_1.Effect.die(new Error("Local head is in an invalid state, missing batches or snapshots")).pipe(effect_1.Effect.annotateLogs({ newHead }));
+        }
+        yield* (0, truncate_1.truncate)(workingConnection);
+        yield* localKV.delete((0, storage_1.walKey)(workingCopyKey.split(".")[0]));
+        yield* localKV.clone(workingCopyKey, (0, storage_1.baseKey)(lastRemoteSnapshotId, lastRemoteBatchId));
+        const allPendingBatches = [
+            ...(localHead.value.pendingBatches ?? []),
+            ...(effect_1.Option.isSome(newLocalBatch)
+                ? [
+                    {
+                        id: newLocalBatch.value.batch.id,
+                        lastChangeId: newLocalBatch.value.lastLocalChangeId,
+                    },
+                ]
+                : []),
+        ];
+        for (const pb of allPendingBatches) {
+            yield* (0, batches_1.applyBatch)(workingConnection, { id: pb.id });
+        }
+        yield* localKV.set((0, storage_1.headKey)(dbName), (0, storage_1.encodeJson)({
+            head: newHead,
+            remoteEtag: remoteHeadOption.value.etag,
+            lastSyncedLocalChangeId: nextLastRemoteAppliedChangeId,
+            ...(allPendingBatches.length > 0
+                ? { pendingBatches: allPendingBatches }
+                : {}),
+        }));
+        const oldConnection = yield* state.getConnection;
+        yield* effect_1.Effect.promise(() => oldConnection.close());
+        yield* localKV.delete((0, storage_1.walKey)(dbName));
+        yield* localKV.delete((0, storage_1.shmKey)(dbName));
+        yield* (0, truncate_1.truncate)(workingConnection);
+        yield* localKV.clone(workingCopyKey, (0, storage_1.dbKey)(dbName));
+        return { newLocalBatch };
+    }));
+    yield* state.setConnection(yield* localKV.connect((0, storage_1.dbKey)(dbName)));
+    return { newLocalBatch };
+});
+exports.pull = pull;
+//# sourceMappingURL=pull.js.map

package/cjs/src/pull.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/pull.ts"],"names":[],"mappings":";;;AACA,mCAAwC;AACxC,+BAA0B;AAG1B,uCAAqD;AACrD,2CAAkD;AAClD,6CAA0C;AAC1C,yCAA+D;AAC/D,8CAA2C;AAC3C,uCAYmB;AAGZ,MAAM,IAAI,GAAG,GAIlB,EAAE,CACH,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,iBAAO,CAAC;IAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,kBAAQ,CAAC;IACjC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,2BAAgB,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,0BAAe,CAAC;IACrC,MAAM,eAAe,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAEhE,KAAK,CAAC,CAAC,eAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,IAAA,iBAAO,EAAa,OAAO,EAAE,IAAA,iBAAO,EAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAC1E,eAAM,CAAC,OAAO,CACb,eAAM,CAAC,KAAK,CAAC;QACZ,MAAM,EAAE,GAAG,EAAE,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC1E,MAAM,EAAE,eAAM,CAAC,OAAO;KACtB,CAAC,CACF,CACD,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY,CACpD,IAAA,iBAAO,EAAC,MAAM,CAAC,EACf,SAAS,CAAC,KAAK,CAAC,UAAU,CAC1B,CAAC;IAEF,iCAAiC;IACjC,IAAI,eAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,CAAC,eAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAClF,OAAO,EAAE,aAAa,EAAE,eAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,IAAA,oBAAU,EAAO,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAC5E,eAAM,CAAC,SAAS,CACf,CAAC,CAAC,EAAE,EAAE,CACL,IAAI,KAAK,CACR,iCAAiC,MAAM,CAAC,CAAC,CAAC,sBAAsB,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,CAC9F,CACF,CACD,CAAC;IAEF,MAAM,oBAAoB,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3F,oIAAoI;IACpI,MAAM,mBAAmB,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACtE,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAEjE,IAAI,CAAC,gBAAgB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CACvB,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAC5E,CAAC,IAAI,CAAC,eAAM,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,eAAe,GAAG,IAAA,iBAAO,EAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,MAAM,CAC7C,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACnB,MAAM,cAAc,GAAG,GAAG,IAAA,SAAE,GAAE,eAAe,CAAC;QAE9C,KAAK,CAAC,CAAC,eAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,IAAI,CAAC,CAAC;QAEnF,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CAC7D;YACC,aAAa,EAAE,IAAA,sBAAY,EAC1B,eAAe,CAAC,OAAO,EACvB,SAAS,CAAC,KAAK,CAAC,uBAAuB,CACvC;YACD,SAAS,EAAE,IAAA,qBAAS,EACnB,oBAAoB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,kBAAQ,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CACvD;YACD,iBAAiB,EAAE,OAAO;iBACxB,KAAK,CAAC,eAAe,EAAE,cAAc,CAAC;iBACtC,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;SAC7D,EACD;YACC,WAAW,EAAE,WAAW;SACxB,CACD,CAAC;QAEF,2MAA2M;QAC3M,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;YAC1C,KAAK,CAAC,CAAC,IAAA,oBAAU,EAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,6BAA6B,GAAG,KAAK,CAAC,CAAC,IAAA,2BAAiB,EAAC,iBAAiB,CAAC,CAAC;QAElF,MAAM,OAAO,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,oBAAoB,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1D,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAErD,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACjD,OAAO,KAAK,CAAC,CAAC,eAAM,CAAC,GAAG,CACvB,IAAI,KAAK,CACR,iEAAiE,CACjE,CACD,CAAC,IAAI,CAAC,eAAM,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,KAAK,CAAC,CAAC,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,CAAC;QACnC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,gBAAM,EAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5D,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CACnB,cAAc,EACd,IAAA,iBAAO,EAAC,oBAAoB,EAAE,iBAAiB,CAAC,CAChD,CAAC;QAEF,MAAM,iBAAiB,GAAG;YACzB,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;YACzC,GAAG,CAAC,eAAM,CAAC,MAAM,CAAC,aAAa,CAAC;gBAC/B,CAAC,CAAC;oBACA;wBACC,EAAE,EAAE,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;wBAChC,YAAY,EAAE,aAAa,CAAC,KAAK,CAAC,iBAAiB;qBACnD;iBACD;gBACF,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACpC,KAAK,CAAC,CAAC,IAAA,oBAAU,EAAC,iBAAiB,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CACjB,IAAA,iBAAO,EAAC,MAAM,CAAC,EACf,IAAA,oBAAU,EAAa;YACtB,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,gBAAgB,CAAC,KAAK,CAAC,IAAI;YACvC,uBAAuB,EAAE,6BAA6B;YACtD,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;gBAC/B,CAAC,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE;gBACvC,CAAC,CAAC,EAAE,CAAC;SACN,CAAC,CACF,CAAC;QAEF,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;QACjD,KAAK,CAAC,CAAC,eAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;QACnD,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,CAAC,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,CAAC;QACnC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,IAAA,eAAK,EAAC,MAAM,CAAC,CAAC,CAAC;QAEpD,OAAO,EAAE,aAAa,EAAE,CAAC;IAC1B,CAAC,CAAC,CACF,CAAC;IAEF,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAElE,OAAO,EAAE,aAAa,EAAE,CAAC;AAC1B,CAAC,CAAC,CAAC;AAzJS,QAAA,IAAI,QAyJb","sourcesContent":["import type { Scope } from \"effect\";\nimport { Effect, Option } from \"effect\";\nimport { v4 } from \"uuid\";\n\nimport type { ExtractedBatch } from \"./batches\";\nimport { applyBatch, extractBatch } from \"./batches\";\nimport { getLatestChangeId } from \"./cdc/extract\";\nimport { truncate } from \"./cdc/truncate\";\nimport { ConnectionConfig, ConnectionState } from \"./contexts\";\nimport { pullFiles } from \"./kv/syncFiles\";\nimport {\n\tLocalKV,\n\tRemoteKV,\n\tbaseKey,\n\tbatchKey,\n\tdbKey,\n\tdecodeJson,\n\tencodeJson,\n\tgetJson,\n\theadKey,\n\tshmKey,\n\twalKey,\n} from \"./storage\";\nimport type { Head, StoredHead } from \"./types\";\n\nexport const pull = (): Effect.Effect<\n\t{ newLocalBatch: Option.Option<ExtractedBatch> },\n\tError,\n\tConnectionConfig | ConnectionState | LocalKV | RemoteKV | Scope.Scope\n> =>\n\tEffect.gen(function* () {\n\t\tconst localKV = yield* LocalKV;\n\t\tconst remoteKV = yield* RemoteKV;\n\t\tconst { dbName } = yield* ConnectionConfig;\n\t\tconst state = yield* ConnectionState;\n\t\tconst tursoConnection = { current: yield* state.getConnection };\n\n\t\tyield* Effect.logDebug(\"s3qlite.pull.start\").pipe(Effect.annotateLogs({ dbName }));\n\n\t\tconst localHead = yield* getJson<StoredHead>(localKV, headKey(dbName)).pipe(\n\t\t\tEffect.flatMap(\n\t\t\t\tOption.match({\n\t\t\t\t\tonNone: () => Effect.die(new Error(\"Expected local stored head to exist\")),\n\t\t\t\t\tonSome: Effect.succeed,\n\t\t\t\t}),\n\t\t\t),\n\t\t);\n\n\t\tconst remoteHeadOption = yield* remoteKV.getIfChanged(\n\t\t\theadKey(dbName),\n\t\t\tlocalHead.value.remoteEtag,\n\t\t);\n\n\t\t// no changes detected, skip pull\n\t\tif (Option.isNone(remoteHeadOption)) {\n\t\t\tyield* Effect.logDebug(\"s3qlite.pull.skip\").pipe(Effect.annotateLogs({ dbName }));\n\t\t\treturn { newLocalBatch: Option.none() };\n\t\t}\n\n\t\tconst remoteHead = yield* decodeJson<Head>(remoteHeadOption.value.value).pipe(\n\t\t\tEffect.orDieWith(\n\t\t\t\t(e) =>\n\t\t\t\t\tnew Error(\n\t\t\t\t\t\t`Failed to decode remote head: ${String(e)}. Value to decode: ${remoteHeadOption.value.value}`,\n\t\t\t\t\t),\n\t\t\t),\n\t\t);\n\n\t\tconst remoteBatchesToApply = remoteHead.batches.slice(localHead.value.head.batches.length);\n\t\t// Instead of rollbacking the whole database we are keeping a CoW copy of the snapshot+all applied batches up to a certain batch id.\n\t\tconst lastLocalSnapshotId = localHead.value.head.snapshots.at(-1)?.id;\n\t\tconst lastLocalBatchId = localHead.value.head.batches.at(-1)?.id;\n\n\t\tif (!lastLocalBatchId || !lastLocalSnapshotId) {\n\t\t\treturn yield* Effect.die(\n\t\t\t\tnew Error(\"Local head is in an invalid state, missing batches or snapshots\"),\n\t\t\t).pipe(Effect.annotateLogs({ localHead: localHead, remoteHead }));\n\t\t}\n\n\t\tconst baseSnapshotKey = baseKey(lastLocalSnapshotId, lastLocalBatchId);\n\t\tif (!(yield* localKV.exists(baseSnapshotKey))) {\n\t\t\treturn yield* Effect.die(new Error(\"Base snapshot is missing!\"));\n\t\t}\n\n\t\tconst { newLocalBatch } = yield* Effect.scoped(\n\t\t\tEffect.gen(function* () {\n\t\t\t\tconst workingCopyKey = `${v4()}.working-copy`;\n\n\t\t\t\tyield* Effect.addFinalizer(() => localKV.delete(workingCopyKey)).pipe(Effect.exit);\n\n\t\t\t\tconst { newLocalBatch, workingConnection } = yield* Effect.all(\n\t\t\t\t\t{\n\t\t\t\t\t\tnewLocalBatch: extractBatch(\n\t\t\t\t\t\t\ttursoConnection.current,\n\t\t\t\t\t\t\tlocalHead.value.lastSyncedLocalChangeId,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tsyncFiles: pullFiles(\n\t\t\t\t\t\t\tremoteBatchesToApply.map((batch) => batchKey(batch.id)),\n\t\t\t\t\t\t),\n\t\t\t\t\t\tworkingConnection: localKV\n\t\t\t\t\t\t\t.clone(baseSnapshotKey, workingCopyKey)\n\t\t\t\t\t\t\t.pipe(Effect.flatMap(() => localKV.connect(workingCopyKey))),\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tconcurrency: \"unbounded\",\n\t\t\t\t\t},\n\t\t\t\t);\n\n\t\t\t\t// maybe it would be faster to check the sizes of the batches, and if its not bigger than a few megabytes it could just be all loaded into memory in parallel, and then applied at once, as one transaction\n\t\t\t\tfor (const batch of remoteBatchesToApply) {\n\t\t\t\t\tyield* applyBatch(workingConnection, batch);\n\t\t\t\t}\n\n\t\t\t\tconst nextLastRemoteAppliedChangeId = yield* getLatestChangeId(workingConnection);\n\n\t\t\t\tconst newHead = structuredClone(remoteHead);\n\n\t\t\t\tconst lastRemoteSnapshotId = newHead.snapshots.at(-1)?.id;\n\t\t\t\tconst lastRemoteBatchId = newHead.batches.at(-1)?.id;\n\n\t\t\t\tif (!lastRemoteBatchId || !lastRemoteSnapshotId) {\n\t\t\t\t\treturn yield* Effect.die(\n\t\t\t\t\t\tnew Error(\n\t\t\t\t\t\t\t\"Local head is in an invalid state, missing batches or snapshots\",\n\t\t\t\t\t\t),\n\t\t\t\t\t).pipe(Effect.annotateLogs({ newHead }));\n\t\t\t\t}\n\n\t\t\t\tyield* truncate(workingConnection);\n\t\t\t\tyield* localKV.delete(walKey(workingCopyKey.split(\".\")[0]));\n\n\t\t\t\tyield* localKV.clone(\n\t\t\t\t\tworkingCopyKey,\n\t\t\t\t\tbaseKey(lastRemoteSnapshotId, lastRemoteBatchId),\n\t\t\t\t);\n\n\t\t\t\tconst allPendingBatches = [\n\t\t\t\t\t...(localHead.value.pendingBatches ?? []),\n\t\t\t\t\t...(Option.isSome(newLocalBatch)\n\t\t\t\t\t\t? [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tid: newLocalBatch.value.batch.id,\n\t\t\t\t\t\t\t\t\tlastChangeId: newLocalBatch.value.lastLocalChangeId,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t: []),\n\t\t\t\t];\n\n\t\t\t\tfor (const pb of allPendingBatches) {\n\t\t\t\t\tyield* applyBatch(workingConnection, { id: pb.id });\n\t\t\t\t}\n\n\t\t\t\tyield* localKV.set(\n\t\t\t\t\theadKey(dbName),\n\t\t\t\t\tencodeJson<StoredHead>({\n\t\t\t\t\t\thead: newHead,\n\t\t\t\t\t\tremoteEtag: remoteHeadOption.value.etag,\n\t\t\t\t\t\tlastSyncedLocalChangeId: nextLastRemoteAppliedChangeId,\n\t\t\t\t\t\t...(allPendingBatches.length > 0\n\t\t\t\t\t\t\t? { pendingBatches: allPendingBatches }\n\t\t\t\t\t\t\t: {}),\n\t\t\t\t\t}),\n\t\t\t\t);\n\n\t\t\t\tconst oldConnection = yield* state.getConnection;\n\t\t\t\tyield* Effect.promise(() => oldConnection.close());\n\t\t\t\tyield* localKV.delete(walKey(dbName));\n\t\t\t\tyield* localKV.delete(shmKey(dbName));\n\t\t\t\tyield* truncate(workingConnection);\n\t\t\t\tyield* localKV.clone(workingCopyKey, dbKey(dbName));\n\n\t\t\t\treturn { newLocalBatch };\n\t\t\t}),\n\t\t);\n\n\t\tyield* state.setConnection(yield* localKV.connect(dbKey(dbName)));\n\n\t\treturn { newLocalBatch };\n\t});"]}

package/cjs/src/push.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.push = void 0;
+const effect_1 = require("effect");
+const batches_1 = require("./batches");
+const extract_1 = require("./cdc/extract");
+const truncate_1 = require("./cdc/truncate");
+const contexts_1 = require("./contexts");
+const syncFiles_1 = require("./kv/syncFiles");
+const storage_1 = require("./storage");
+const push = () => effect_1.Effect.gen(function* () {
+    const localKV = yield* storage_1.LocalKV;
+    const remoteKV = yield* storage_1.RemoteKV;
+    const { dbName } = yield* contexts_1.ConnectionConfig;
+    const state = yield* contexts_1.ConnectionState;
+    const tursoConnection = { current: yield* state.getConnection };
+    const localHead = yield* (0, storage_1.getJson)(localKV, (0, storage_1.headKey)(dbName)).pipe(effect_1.Effect.orDie, effect_1.Effect.flatMap(effect_1.Option.match({
+        onNone: () => effect_1.Effect.die(new Error("Expected local stored head to exist")),
+        onSome: effect_1.Effect.succeed,
+    })));
+    const pendingBatches = [];
+    if (localHead.value.pendingBatches?.length) {
+        pendingBatches.push(...localHead.value.pendingBatches.map((pb) => ({
+            batch: { id: pb.id },
+            lastChangeId: pb.lastChangeId,
+        })));
+    }
+    const extracted = yield* (0, batches_1.extractBatch)(tursoConnection.current, localHead.value.lastSyncedLocalChangeId);
+    if (effect_1.Option.isSome(extracted)) {
+        pendingBatches.push({
+            batch: extracted.value.batch,
+            lastChangeId: extracted.value.lastLocalChangeId,
+        });
+    }
+    if (pendingBatches.length === 0) {
+        return;
+    }
+    yield* (0, syncFiles_1.pushFiles)(pendingBatches.map((pb) => (0, storage_1.batchKey)(pb.batch.id))).pipe(effect_1.Effect.orDie);
+    const newHead = {
+        snapshots: localHead.value.head.snapshots,
+        batches: localHead.value.head.batches.concat(pendingBatches.map((pb) => pb.batch)),
+    };
+    const result = yield* remoteKV.cas((0, storage_1.headKey)(dbName), (0, storage_1.encodeJson)(newHead), localHead.value.remoteEtag);
+    const lastSyncedLocalChangeId = yield* (0, extract_1.getLatestChangeId)(tursoConnection.current).pipe(effect_1.Effect.orDie);
+    yield* localKV.set((0, storage_1.headKey)(dbName), (0, storage_1.encodeJson)({
+        head: newHead,
+        remoteEtag: result.etag,
+        lastSyncedLocalChangeId,
+    }));
+    const lastSnapshotId = newHead.snapshots.at(-1)?.id;
+    const lastBatchId = newHead.batches.at(-1)?.id;
+    if (lastSnapshotId && lastBatchId) {
+        yield* (0, truncate_1.truncate)(tursoConnection.current);
+        yield* localKV.clone((0, storage_1.dbKey)(dbName), (0, storage_1.baseKey)(lastSnapshotId, lastBatchId));
+    }
+});
+exports.push = push;
+//# sourceMappingURL=push.js.map

package/cjs/src/push.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"push.js","sourceRoot":"","sources":["../../../src/push.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,uCAAyC;AACzC,2CAAkD;AAClD,6CAA0C;AAC1C,yCAA+D;AAC/D,8CAA2C;AAC3C,uCASmB;AAGZ,MAAM,IAAI,GAAG,GAAG,EAAE,CACxB,eAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,iBAAO,CAAC;IAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,kBAAQ,CAAC;IACjC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,2BAAgB,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,0BAAe,CAAC;IACrC,MAAM,eAAe,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAEhE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,IAAA,iBAAO,EAAa,OAAO,EAAE,IAAA,iBAAO,EAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAC1E,eAAM,CAAC,KAAK,EACZ,eAAM,CAAC,OAAO,CACb,eAAM,CAAC,KAAK,CAAC;QACZ,MAAM,EAAE,GAAG,EAAE,CAAC,eAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC1E,MAAM,EAAE,eAAM,CAAC,OAAO;KACtB,CAAC,CACF,CACD,CAAC;IAEF,MAAM,cAAc,GAA6C,EAAE,CAAC;IAEpE,IAAI,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;QAC5C,cAAc,CAAC,IAAI,CAClB,GAAG,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;YACpB,YAAY,EAAE,EAAE,CAAC,YAAY;SAC7B,CAAC,CAAC,CACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,IAAA,sBAAY,EACpC,eAAe,CAAC,OAAO,EACvB,SAAS,CAAC,KAAK,CAAC,uBAAuB,CACvC,CAAC;IAEF,IAAI,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,cAAc,CAAC,IAAI,CAAC;YACnB,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,KAAK;YAC5B,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB;SAC/C,CAAC,CAAC;IACJ,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO;IACR,CAAC;IAED,KAAK,CAAC,CAAC,IAAA,qBAAS,EAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAA,kBAAQ,EAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAM,CAAC,KAAK,CAAC,CAAC;IAEvF,MAAM,OAAO,GAAS;QACrB,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS;QACzC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;KAClF,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CACjC,IAAA,iBAAO,EAAC,MAAM,CAAC,EACf,IAAA,oBAAU,EAAC,OAAO,CAAC,EACnB,SAAS,CAAC,KAAK,CAAC,UAAU,CAC1B,CAAC;IAEF,MAAM,uBAAuB,GAAG,KAAK,CAAC,CAAC,IAAA,2BAAiB,EAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,CACrF,eAAM,CAAC,KAAK,CACZ,CAAC;IAEF,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CACjB,IAAA,iBAAO,EAAC,MAAM,CAAC,EACf,IAAA,oBAAU,EAAa;QACtB,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,uBAAuB;KACvB,CAAC,CACF,CAAC;IAEF,MAAM,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACpD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC/C,IAAI,cAAc,IAAI,WAAW,EAAE,CAAC;QACnC,KAAK,CAAC,CAAC,IAAA,mBAAQ,EAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACzC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAA,eAAK,EAAC,MAAM,CAAC,EAAE,IAAA,iBAAO,EAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAC3E,CAAC;AACF,CAAC,CAAC,CAAC;AA7ES,QAAA,IAAI,QA6Eb","sourcesContent":["import { Effect, Option } from \"effect\";\n\nimport { extractBatch } from \"./batches\";\nimport { getLatestChangeId } from \"./cdc/extract\";\nimport { truncate } from \"./cdc/truncate\";\nimport { ConnectionConfig, ConnectionState } from \"./contexts\";\nimport { pushFiles } from \"./kv/syncFiles\";\nimport {\n\tLocalKV,\n\tRemoteKV,\n\tbaseKey,\n\tbatchKey,\n\tdbKey,\n\tencodeJson,\n\tgetJson,\n\theadKey,\n} from \"./storage\";\nimport type { Batch, Head, StoredHead } from \"./types\";\n\nexport const push = () =>\n\tEffect.gen(function* () {\n\t\tconst localKV = yield* LocalKV;\n\t\tconst remoteKV = yield* RemoteKV;\n\t\tconst { dbName } = yield* ConnectionConfig;\n\t\tconst state = yield* ConnectionState;\n\t\tconst tursoConnection = { current: yield* state.getConnection };\n\n\t\tconst localHead = yield* getJson<StoredHead>(localKV, headKey(dbName)).pipe(\n\t\t\tEffect.orDie,\n\t\t\tEffect.flatMap(\n\t\t\t\tOption.match({\n\t\t\t\t\tonNone: () => Effect.die(new Error(\"Expected local stored head to exist\")),\n\t\t\t\t\tonSome: Effect.succeed,\n\t\t\t\t}),\n\t\t\t),\n\t\t);\n\n\t\tconst pendingBatches: { batch: Batch; lastChangeId: number }[] = [];\n\n\t\tif (localHead.value.pendingBatches?.length) {\n\t\t\tpendingBatches.push(\n\t\t\t\t...localHead.value.pendingBatches.map((pb) => ({\n\t\t\t\t\tbatch: { id: pb.id },\n\t\t\t\t\tlastChangeId: pb.lastChangeId,\n\t\t\t\t})),\n\t\t\t);\n\t\t}\n\n\t\tconst extracted = yield* extractBatch(\n\t\t\ttursoConnection.current,\n\t\t\tlocalHead.value.lastSyncedLocalChangeId,\n\t\t);\n\n\t\tif (Option.isSome(extracted)) {\n\t\t\tpendingBatches.push({\n\t\t\t\tbatch: extracted.value.batch,\n\t\t\t\tlastChangeId: extracted.value.lastLocalChangeId,\n\t\t\t});\n\t\t}\n\n\t\tif (pendingBatches.length === 0) {\n\t\t\treturn;\n\t\t}\n\n\t\tyield* pushFiles(pendingBatches.map((pb) => batchKey(pb.batch.id))).pipe(Effect.orDie);\n\n\t\tconst newHead: Head = {\n\t\t\tsnapshots: localHead.value.head.snapshots,\n\t\t\tbatches: localHead.value.head.batches.concat(pendingBatches.map((pb) => pb.batch)),\n\t\t};\n\n\t\tconst result = yield* remoteKV.cas(\n\t\t\theadKey(dbName),\n\t\t\tencodeJson(newHead),\n\t\t\tlocalHead.value.remoteEtag,\n\t\t);\n\n\t\tconst lastSyncedLocalChangeId = yield* getLatestChangeId(tursoConnection.current).pipe(\n\t\t\tEffect.orDie,\n\t\t);\n\n\t\tyield* localKV.set(\n\t\t\theadKey(dbName),\n\t\t\tencodeJson<StoredHead>({\n\t\t\t\thead: newHead,\n\t\t\t\tremoteEtag: result.etag,\n\t\t\t\tlastSyncedLocalChangeId,\n\t\t\t}),\n\t\t);\n\n\t\tconst lastSnapshotId = newHead.snapshots.at(-1)?.id;\n\t\tconst lastBatchId = newHead.batches.at(-1)?.id;\n\t\tif (lastSnapshotId && lastBatchId) {\n\t\t\tyield* truncate(tursoConnection.current);\n\t\t\tyield* localKV.clone(dbKey(dbName), baseKey(lastSnapshotId, lastBatchId));\n\t\t}\n\t});"]}

package/cjs/src/storage.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getJson = exports.decodeJson = exports.encodeJson = exports.baseKey = exports.shmKey = exports.walKey = exports.dbKey = exports.snapshotKey = exports.batchKey = exports.headKey = exports.RemoteKV = exports.LocalKV = void 0;
+const effect_1 = require("effect");
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+class LocalKV extends effect_1.Context.Tag("s3qlite/LocalKV")() {
+}
+exports.LocalKV = LocalKV;
+class RemoteKV extends effect_1.Context.Tag("s3qlite/RemoteKV")() {
+}
+exports.RemoteKV = RemoteKV;
+const headKey = (dbName) => `${dbName}.json`;
+exports.headKey = headKey;
+const batchKey = (batchId) => `${batchId}.batch`;
+exports.batchKey = batchKey;
+const snapshotKey = (snapshotId) => `${snapshotId}.snapshot`;
+exports.snapshotKey = snapshotKey;
+const dbKey = (dbName) => `${dbName}.db`;
+exports.dbKey = dbKey;
+const walKey = (dbName) => `${dbName}.db-wal`;
+exports.walKey = walKey;
+const shmKey = (dbName) => `${dbName}.db-shm`;
+exports.shmKey = shmKey;
+const baseKey = (snapshotId, batchId) => `${snapshotId}-${batchId}.base`;
+exports.baseKey = baseKey;
+const encodeJson = (value) => textEncoder.encode(JSON.stringify(value));
+exports.encodeJson = encodeJson;
+const decodeJson = (value) => effect_1.Effect.try({
+    try: () => JSON.parse(textDecoder.decode(value)),
+    catch: (error) => error instanceof SyntaxError
+        ? error
+        : new SyntaxError(`Failed to parse "${value}": ${String(error)}`),
+});
+exports.decodeJson = decodeJson;
+const getJson = (kv, key) => kv.get(key).pipe(effect_1.Effect.flatMap(effect_1.Option.match({
+    onNone: () => effect_1.Effect.succeed(effect_1.Option.none()),
+    onSome: ({ etag, value }) => (0, exports.decodeJson)(value).pipe(effect_1.Effect.map((decoded) => effect_1.Option.some({ etag, value: decoded }))),
+})));
+exports.getJson = getJson;
+//# sourceMappingURL=storage.js.map

package/cjs/src/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../../src/storage.ts"],"names":[],"mappings":";;;AAAA,mCAAiD;AAIjD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AACtC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AAEtC,MAAa,OAAQ,SAAQ,gBAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAGxD;CAAG;AAHN,0BAGM;AAEN,MAAa,QAAS,SAAQ,gBAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAgB;CAAG;AAAhF,4BAAgF;AAEzE,MAAM,OAAO,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC;AAAvD,QAAA,OAAO,WAAgD;AAE7D,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAU,EAAE,CAAC,GAAG,OAAO,QAAQ,CAAC;AAA3D,QAAA,QAAQ,YAAmD;AAEjE,MAAM,WAAW,GAAG,CAAC,UAAkB,EAAU,EAAE,CAAC,GAAG,UAAU,WAAW,CAAC;AAAvE,QAAA,WAAW,eAA4D;AAE7E,MAAM,KAAK,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,KAAK,CAAC;AAAnD,QAAA,KAAK,SAA8C;AAEzD,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,SAAS,CAAC;AAAxD,QAAA,MAAM,UAAkD;AAE9D,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,SAAS,CAAC;AAAxD,QAAA,MAAM,UAAkD;AAE9D,MAAM,OAAO,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CACtE,GAAG,UAAU,IAAI,OAAO,OAAO,CAAC;AADpB,QAAA,OAAO,WACa;AAE1B,MAAM,UAAU,GAAG,CAAI,KAAQ,EAAc,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAApF,QAAA,UAAU,cAA0E;AAE1F,MAAM,UAAU,GAAG,CAAI,KAAiB,EAAiC,EAAE,CACjF,eAAM,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAM;IACrD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAChB,KAAK,YAAY,WAAW;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,WAAW,CAAC,oBAAoB,KAAK,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;CACnE,CAAC,CAAC;AAPS,QAAA,UAAU,cAOnB;AAEG,MAAM,OAAO,GAAG,CACtB,EAAM,EACN,GAAW,EAC6D,EAAE,CAC1E,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CACf,eAAM,CAAC,OAAO,CACb,eAAM,CAAC,KAAK,CAAC;IACZ,MAAM,EAAE,GAAG,EAAE,CAAC,eAAM,CAAC,OAAO,CAAC,eAAM,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAC3B,IAAA,kBAAU,EAAI,KAAK,CAAC,CAAC,IAAI,CACxB,eAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,eAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAC9D;CACF,CAAC,CACF,CACD,CAAC;AAdU,QAAA,OAAO,WAcjB","sourcesContent":["import { Context, Effect, Option } from \"effect\";\n\nimport type { CloneTrait, ConnectDatabaseTrait, KV } from \"./kv/kv\";\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\n\nexport class LocalKV extends Context.Tag(\"s3qlite/LocalKV\")<\n\tLocalKV,\n\tKV & CloneTrait & ConnectDatabaseTrait\n>() {}\n\nexport class RemoteKV extends Context.Tag(\"s3qlite/RemoteKV\")<RemoteKV, KV>() {}\n\nexport const headKey = (dbName: string): string => `${dbName}.json`;\n\nexport const batchKey = (batchId: string): string => `${batchId}.batch`;\n\nexport const snapshotKey = (snapshotId: string): string => `${snapshotId}.snapshot`;\n\nexport const dbKey = (dbName: string): string => `${dbName}.db`;\n\nexport const walKey = (dbName: string): string => `${dbName}.db-wal`;\n\nexport const shmKey = (dbName: string): string => `${dbName}.db-shm`;\n\nexport const baseKey = (snapshotId: string, batchId: string): string =>\n\t`${snapshotId}-${batchId}.base`;\n\nexport const encodeJson = <A>(value: A): Uint8Array => textEncoder.encode(JSON.stringify(value));\n\nexport const decodeJson = <A>(value: Uint8Array): Effect.Effect<A, SyntaxError> =>\n\tEffect.try({\n\t\ttry: () => JSON.parse(textDecoder.decode(value)) as A,\n\t\tcatch: (error) =>\n\t\t\terror instanceof SyntaxError\n\t\t\t\t? error\n\t\t\t\t: new SyntaxError(`Failed to parse \"${value}\": ${String(error)}`),\n\t});\n\nexport const getJson = <A>(\n\tkv: KV,\n\tkey: string,\n): Effect.Effect<Option.Option<{ value: A; etag: string }>, SyntaxError> =>\n\tkv.get(key).pipe(\n\t\tEffect.flatMap(\n\t\t\tOption.match({\n\t\t\t\tonNone: () => Effect.succeed(Option.none()),\n\t\t\t\tonSome: ({ etag, value }) =>\n\t\t\t\t\tdecodeJson<A>(value).pipe(\n\t\t\t\t\t\tEffect.map((decoded) => Option.some({ etag, value: decoded })),\n\t\t\t\t\t),\n\t\t\t}),\n\t\t),\n\t);"]}

package/cjs/src/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/cjs/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/types.ts"],"names":[],"mappings":"","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport type { Effect } from \"effect\";\n\nexport type Snapshot = {\n\tid: string;\n\n\t// thanks to this we can do full history\n\tbaseSnapshotId?: string;\n\tbatchIdsApplied: string[];\n};\n\nexport type Batch = {\n\tid: string;\n};\n\nexport type Head = {\n\tsnapshots: Snapshot[];\n\tbatches: Batch[];\n};\n\nexport type PendingBatch = {\n\tid: string;\n\tlastChangeId: number;\n};\n\nexport type StoredHead = {\n\thead: Head;\n\n\t/**\n\t * The latest etag of the remote head this instance has been synced to.\n\t */\n\tremoteEtag: string;\n\n\tlastSyncedLocalChangeId: number;\n\n\tpendingBatches?: PendingBatch[];\n};\n\n// export type ConflictResolutionMode = \"fail\" | \"preferRemote\" | \"preferLocal\";\n//\n// export type ConflictOperation = \"pull\" | \"push\" | \"sync\";\n//\n// export type ConflictResolutionContext = {\n// \treadonly cause: Error;\n// \treadonly localHead: StoredHead;\n// \treadonly operation: ConflictOperation;\n// \treadonly remoteHead: Head;\n// };\n//\n// export type ConflictResolver = (\n// \tcontext: ConflictResolutionContext,\n// ) => Effect.Effect<ConflictResolutionMode, Error>;\n//\n// type StrongConsistencyOptions = {\n// \tmode: \"strong\";\n// };\n//\n// type EventualConsistencyOptions = {\n// \tmode: \"eventual\";\n// \tsync?: \"manual\" | { everyMs: number };\n// };\n\nexport type ConnectionOptions = {\n\tbucket: string;\n\t// conflictResolution?: ConflictResolutionMode | ConflictResolver;\n\tlocalDirectory?: string;\n};\n// } & (StrongConsistencyOptions | EventualConsistencyOptions);\n\nexport type S3qliteDatabase = Database & {\n\tpull: () => Effect.Effect<void, Error>;\n\tpush: () => Effect.Effect<void, Error>;\n\tsync: () => Effect.Effect<void, Error>;\n\tfork: (dbName: string) => Effect.Effect<void, Error>;\n\tcheckpoint: () => Effect.Effect<void, Error>;\n};"]}

package/cjs/src/wrapDatabase.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wrapDatabase = void 0;
+const asyncDatabaseMethods = new Set([
+    "all",
+    "close",
+    "exec",
+    "get",
+    "iterate",
+    "pragma",
+    "run",
+]);
+const asyncStatementMethods = new Set(["all", "close", "get", "run"]);
+const syncStatementMethods = new Set(["bind", "pluck", "raw", "safeIntegers"]);
+const wrapTransaction = (transaction, wrap) => new Proxy(transaction, {
+    apply: (target, thisArg, args) => wrap(() => Reflect.apply(target, thisArg, args), "transaction", args),
+    get: (target, property, receiver) => {
+        const value = Reflect.get(target, property, receiver);
+        if (typeof value !== "function") {
+            return value;
+        }
+        return wrapTransaction(value, wrap);
+    },
+});
+const wrapStatement = (statement, wrap) => {
+    let proxy;
+    proxy = new Proxy(statement, {
+        get: (target, property, receiver) => {
+            const value = Reflect.get(target, property, receiver);
+            if (typeof value !== "function") {
+                return value;
+            }
+            if (syncStatementMethods.has(property)) {
+                return (...args) => {
+                    Reflect.apply(value, target, args);
+                    return proxy;
+                };
+            }
+            if (property === "iterate") {
+                return (...args) => (async function* () {
+                    const iterable = wrap(() => Reflect.apply(value, target, args), property, args);
+                    for await (const item of iterable) {
+                        yield item;
+                    }
+                })();
+            }
+            if (asyncStatementMethods.has(property)) {
+                return (...args) => wrap(() => Reflect.apply(value, target, args), property, args);
+            }
+            return (...args) => Reflect.apply(value, target, args);
+        },
+    });
+    return proxy;
+};
+const wrapDatabase = (getTarget, wrap) => {
+    return new Proxy({}, {
+        get: (wrapper, property, receiver) => {
+            if (Reflect.has(wrapper, property)) {
+                return Reflect.get(wrapper, property, receiver);
+            }
+            const original = getTarget();
+            const value = Reflect.get(original, property, original);
+            if (typeof value !== "function") {
+                return value;
+            }
+            if (property === "prepare") {
+                return (...args) => wrapStatement(Reflect.apply(value, getTarget(), args), wrap);
+            }
+            if (property === "transaction") {
+                return (...args) => wrapTransaction(Reflect.apply(value, getTarget(), args), wrap);
+            }
+            if (asyncDatabaseMethods.has(property)) {
+                return (...args) => wrap(() => Reflect.apply(value, getTarget(), args), property, args);
+            }
+            return (...args) => Reflect.apply(value, getTarget(), args);
+        },
+    });
+};
+exports.wrapDatabase = wrapDatabase;
+//# sourceMappingURL=wrapDatabase.js.map

package/cjs/src/wrapDatabase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wrapDatabase.js","sourceRoot":"","sources":["../../../src/wrapDatabase.ts"],"names":[],"mappings":";;;AAAA,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAc;IACjD,KAAK;IACL,OAAO;IACP,MAAM;IACN,KAAK;IACL,SAAS;IACT,QAAQ;IACR,KAAK;CACL,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAEnF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAc,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;AAE5F,MAAM,eAAe,GAAG,CACvB,WAAc,EACd,IAA4E,EACxE,EAAE,CACN,IAAI,KAAK,CAAC,WAAW,EAAE;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAChC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEtD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,eAAe,CAAC,KAAiD,EAAE,IAAI,CAAC,CAAC;IACjF,CAAC;CACD,CAAM,CAAC;AAET,MAAM,aAAa,GAAG,CACrB,SAAY,EACZ,IAA4E,EACxE,EAAE;IACN,IAAI,KAAQ,CAAC;IAEb,KAAK,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE;QAC5B,GAAG,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAEtD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC;YACd,CAAC;YAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE;oBAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;oBACnC,OAAO,KAAK,CAAC;gBACd,CAAC,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,CAAC,KAAK,SAAS,CAAC;oBACf,MAAM,QAAQ,GAAG,IAAI,CACpB,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EACxC,QAAQ,EACR,IAAI,CACsB,CAAC;oBAE5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;wBACnC,MAAM,IAAI,CAAC;oBACZ,CAAC;gBACF,CAAC,CAAC,EAAE,CAAC;YACP,CAAC;YAED,IAAI,qBAAqB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACzC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACjE,CAAC;YAED,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACnE,CAAC;KACD,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEK,MAAM,YAAY,GAAG,CAC3B,SAAkB,EAClB,IAA4E,EACxE,EAAE;IACN,OAAO,IAAI,KAAK,CAAC,EAAO,EAAE;QACzB,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;YACpC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACpC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,CAAC;YAED,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAExD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC;YACd,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAAW,EAAE,IAAI,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,eAAe,CACd,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAE1B,EACZ,IAAI,CACJ,CAAC;YACJ,CAAC;YAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACtE,CAAC;YAED,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAAC,CAAC;QACxE,CAAC;KACD,CAAC,CAAC;AACJ,CAAC,CAAC;AAxCW,QAAA,YAAY,gBAwCvB","sourcesContent":["const asyncDatabaseMethods = new Set<PropertyKey>([\n\t\"all\",\n\t\"close\",\n\t\"exec\",\n\t\"get\",\n\t\"iterate\",\n\t\"pragma\",\n\t\"run\",\n]);\n\nconst asyncStatementMethods = new Set<PropertyKey>([\"all\", \"close\", \"get\", \"run\"]);\n\nconst syncStatementMethods = new Set<PropertyKey>([\"bind\", \"pluck\", \"raw\", \"safeIntegers\"]);\n\nconst wrapTransaction = <T extends (...args: readonly unknown[]) => unknown>(\n\ttransaction: T,\n\twrap: <R>(call: () => R, method: PropertyKey, args: readonly unknown[]) => R,\n): T =>\n\tnew Proxy(transaction, {\n\t\tapply: (target, thisArg, args) =>\n\t\t\twrap(() => Reflect.apply(target, thisArg, args), \"transaction\", args),\n\t\tget: (target, property, receiver) => {\n\t\t\tconst value = Reflect.get(target, property, receiver);\n\n\t\t\tif (typeof value !== \"function\") {\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t\treturn wrapTransaction(value as (...args: readonly unknown[]) => unknown, wrap);\n\t\t},\n\t}) as T;\n\nconst wrapStatement = <T extends object>(\n\tstatement: T,\n\twrap: <R>(call: () => R, method: PropertyKey, args: readonly unknown[]) => R,\n): T => {\n\tlet proxy: T;\n\n\tproxy = new Proxy(statement, {\n\t\tget: (target, property, receiver) => {\n\t\t\tconst value = Reflect.get(target, property, receiver);\n\n\t\t\tif (typeof value !== \"function\") {\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t\tif (syncStatementMethods.has(property)) {\n\t\t\t\treturn (...args: unknown[]) => {\n\t\t\t\t\tReflect.apply(value, target, args);\n\t\t\t\t\treturn proxy;\n\t\t\t\t};\n\t\t\t}\n\n\t\t\tif (property === \"iterate\") {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\t(async function* () {\n\t\t\t\t\t\tconst iterable = wrap(\n\t\t\t\t\t\t\t() => Reflect.apply(value, target, args),\n\t\t\t\t\t\t\tproperty,\n\t\t\t\t\t\t\targs,\n\t\t\t\t\t\t) as AsyncIterable<unknown>;\n\n\t\t\t\t\t\tfor await (const item of iterable) {\n\t\t\t\t\t\t\tyield item;\n\t\t\t\t\t\t}\n\t\t\t\t\t})();\n\t\t\t}\n\n\t\t\tif (asyncStatementMethods.has(property)) {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrap(() => Reflect.apply(value, target, args), property, args);\n\t\t\t}\n\n\t\t\treturn (...args: unknown[]) => Reflect.apply(value, target, args);\n\t\t},\n\t});\n\n\treturn proxy;\n};\n\nexport const wrapDatabase = <T extends object>(\n\tgetTarget: () => T,\n\twrap: <R>(call: () => R, method: PropertyKey, args: readonly unknown[]) => R,\n): T => {\n\treturn new Proxy({} as T, {\n\t\tget: (wrapper, property, receiver) => {\n\t\t\tif (Reflect.has(wrapper, property)) {\n\t\t\t\treturn Reflect.get(wrapper, property, receiver);\n\t\t\t}\n\n\t\t\tconst original = getTarget();\n\t\t\tconst value = Reflect.get(original, property, original);\n\n\t\t\tif (typeof value !== \"function\") {\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t\tif (property === \"prepare\") {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrapStatement(Reflect.apply(value, getTarget(), args) as object, wrap);\n\t\t\t}\n\n\t\t\tif (property === \"transaction\") {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrapTransaction(\n\t\t\t\t\t\tReflect.apply(value, getTarget(), args) as (\n\t\t\t\t\t\t\t...args: readonly unknown[]\n\t\t\t\t\t\t) => unknown,\n\t\t\t\t\t\twrap,\n\t\t\t\t\t);\n\t\t\t}\n\n\t\t\tif (asyncDatabaseMethods.has(property)) {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrap(() => Reflect.apply(value, getTarget(), args), property, args);\n\t\t\t}\n\n\t\t\treturn (...args: unknown[]) => Reflect.apply(value, getTarget(), args);\n\t\t},\n\t});\n};"]}

package/dts/effect.d.ts

@@ -0,0 +1 @@
+export * from "./src/index.js";

package/dts/index.d.ts

@@ -0,0 +1,16 @@
+import { S3 } from "@effect-aws/client-s3";
+import type { Database } from "@tursodatabase/database";
+import type { ConnectionOptions } from "./src/types.js";
+type PromiseMethods = {
+    pull: () => Promise<void>;
+    push: () => Promise<void>;
+    sync: () => Promise<void>;
+    fork: (dbName: string) => Promise<void>;
+    checkpoint: () => Promise<void>;
+};
+export type PromiseS3qliteDatabase = Database & PromiseMethods;
+export type PromiseConnectionOptions = ConnectionOptions & {
+    s3?: Parameters<typeof S3.layer>[0];
+};
+export declare const connect: (dbName: string, { s3, ...connectionOptions }: PromiseConnectionOptions) => Promise<PromiseS3qliteDatabase>;
+export * from "./src/index.js";

package/dts/src/batches.d.ts

@@ -0,0 +1,10 @@
+import type { Database } from "@tursodatabase/database";
+import { Effect, Option } from "effect";
+import { LocalKV } from "./storage.js";
+import type { Batch } from "./types.js";
+export declare const applyBatch: (database: Database, batch: Batch) => Effect.Effect<void, never, LocalKV>;
+export type ExtractedBatch = {
+    batch: Batch;
+    lastLocalChangeId: number;
+};
+export declare const extractBatch: (database: Database, lastChangeId: number) => Effect.Effect<Option.Option<ExtractedBatch>, never, LocalKV>;

package/dts/src/cdc/apply.d.ts

@@ -0,0 +1,14 @@
+import type { Database } from "@tursodatabase/database";
+import { Effect } from "effect";
+import type { CDCRow } from "./types.js";
+declare const ReplayError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ReplayError";
+} & Readonly<A>;
+export declare class ReplayError extends ReplayError_base<{
+    message: string;
+    cause: unknown;
+}> {
+}
+export declare const applyCDC: (database: Database, change: CDCRow) => Effect.Effect<void, Error>;
+export declare const replayCDC: (database: Database, changes: readonly CDCRow[]) => Effect.Effect<void, ReplayError>;
+export {};

package/dts/src/cdc/extract.d.ts

@@ -0,0 +1,8 @@
+import type { Database } from "@tursodatabase/database";
+import { Effect } from "effect";
+import type { CDCRow } from "./types.js";
+/**
+ * Extracts CDC rows from the Turso database after a given change_id exclusively.
+ */
+export declare const extractCDC: (tursoConnection: Database, afterId: number) => Effect.Effect<readonly CDCRow[]>;
+export declare const getLatestChangeId: (tursoConnection: Database) => Effect.Effect<number, Error>;

package/dts/src/cdc/protobuf.d.ts

@@ -0,0 +1,3 @@
+import type { CDCRow } from "./types.js";
+export declare const serializeCDC: (changes: readonly CDCRow[]) => Uint8Array;
+export declare const deserializeCDC: (bytes: Uint8Array) => readonly CDCRow[];

package/dts/src/cdc/testUtils.d.ts

@@ -0,0 +1,19 @@
+import type { Database } from "@tursodatabase/database";
+import { Effect } from "effect";
+export type SchemaEntry = {
+    type: string;
+    name: string;
+    tblName: string;
+    sql: string | null;
+};
+export type DatabaseSnapshot = {
+    schema: readonly SchemaEntry[];
+    tables: Readonly<Record<string, readonly Record<string, unknown>[]>>;
+};
+export declare const compareSnapshots: (left: DatabaseSnapshot, right: DatabaseSnapshot) => boolean;
+export declare const snapshotDatabase: (database: Database) => Effect.Effect<DatabaseSnapshot, Error>;
+export declare const compareDatabases: (left: Database, right: Database) => Effect.Effect<boolean, Error>;
+export declare const applyBatch: (database: Database, batch: {
+    readonly transactional: boolean;
+    readonly statements: readonly string[];
+}) => Effect.Effect<void, Error>;

package/dts/src/cdc/truncate.d.ts

@@ -0,0 +1,6 @@
+import type { Database } from "@tursodatabase/database";
+import { Effect } from "effect";
+export declare const truncate: (db: Database) => Effect.Effect<{
+    changes: number;
+    lastInsertRowid: number;
+}, never, never>;

package/dts/src/cdc/types.d.ts

@@ -0,0 +1,35 @@
+export type CDCValue = number | string | Uint8Array | null;
+export declare const CDCChangeType: {
+    readonly Delete: -1;
+    readonly Update: 0;
+    readonly Insert: 1;
+};
+export type CDCChangeType = (typeof CDCChangeType)[keyof typeof CDCChangeType];
+type CDCBase = {
+    changeId: number;
+    changeTime: number;
+    changeTxnId: number;
+};
+export type CDCRow = (CDCBase & {
+    changeType: typeof CDCChangeType.Insert;
+    tableName: string;
+    id: CDCValue;
+    before: Uint8Array | null;
+    after: Uint8Array;
+    updates: Uint8Array | null;
+}) | (CDCBase & {
+    changeType: typeof CDCChangeType.Update;
+    tableName: string;
+    id: CDCValue;
+    before: Uint8Array;
+    after: Uint8Array;
+    updates: Uint8Array;
+}) | (CDCBase & {
+    changeType: typeof CDCChangeType.Delete;
+    tableName: string;
+    id: CDCValue;
+    before: Uint8Array;
+    after: Uint8Array | null;
+    updates: Uint8Array | null;
+});
+export {};

package/dts/src/cdc/withoutCDC.d.ts

@@ -0,0 +1,3 @@
+import type { Database } from "@tursodatabase/database";
+import { Effect } from "effect";
+export declare const withoutCDC: <A, E, R>(database: Database, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;

package/dts/src/connection.d.ts

@@ -0,0 +1,5 @@
+import { Context, Effect } from "effect";
+import { LocalKV, RemoteKV } from "./storage.js";
+import type { ConnectionOptions, S3qliteDatabase } from "./types.js";
+export declare const initializeContexts: (connectionOptions: ConnectionOptions) => Effect.Effect<Context.Context<LocalKV | RemoteKV>, never, import("effect/Scope").Scope | import("@effect/platform/FileSystem").FileSystem | import("@effect-aws/client-s3").S3Service>;
+export declare const connect: (dbName: string, options: ConnectionOptions) => Effect.Effect<S3qliteDatabase, Error, import("effect/Scope").Scope | LocalKV | RemoteKV>;

package/dts/src/contexts.d.ts

@@ -0,0 +1,22 @@
+import type { Database } from "@tursodatabase/database";
+import { Context } from "effect";
+import type { Effect } from "effect";
+import type { ConnectionOptions } from "./types.js";
+export type ConnectionConfigValue = {
+    readonly bucket: string;
+    readonly dbName: string;
+    readonly livePath: string;
+    readonly localDirectory: string;
+    readonly localHeadPath: string;
+    readonly options: ConnectionOptions;
+};
+declare const ConnectionConfig_base: Context.TagClass<ConnectionConfig, "s3qlite/ConnectionConfig", ConnectionConfigValue>;
+export declare class ConnectionConfig extends ConnectionConfig_base {
+}
+declare const ConnectionState_base: Context.TagClass<ConnectionState, "s3qlite/ConnectionState", {
+    readonly getConnection: Effect.Effect<Database>;
+    readonly setConnection: (connection: Database) => Effect.Effect<void>;
+}>;
+export declare class ConnectionState extends ConnectionState_base {
+}
+export {};

package/dts/src/index.d.ts

@@ -0,0 +1,12 @@
+import { Effect } from "effect";
+import { initializeContexts } from "./connection.js";
+import type { ConnectionOptions } from "./types.js";
+export * from "./cdc/apply.js";
+export * from "./cdc/extract.js";
+export * from "./cdc/types.js";
+export * from "./kv/fileKV.js";
+export * from "./kv/kv.js";
+export * from "./kv/memoryKV.js";
+export * from "./kv/s3KV.js";
+export { initializeContexts as initContexts };
+export declare const connect: (dbName: string, connectionOptions: ConnectionOptions) => Effect.Effect<import("./types.js").S3qliteDatabase, Error, import("effect/Scope").Scope | import("@effect/platform/FileSystem").FileSystem | import("@effect-aws/client-s3").S3Service>;

package/dts/src/kv/fileKV.d.ts

@@ -0,0 +1,5 @@
+import { FileSystem } from "@effect/platform/FileSystem";
+import type { Scope } from "effect";
+import { Effect } from "effect";
+import type { CloneTrait, ConnectDatabaseTrait, KV } from "./kv.js";
+export declare const makeFileKV: (rootDirectory: string) => Effect.Effect<KV & CloneTrait & ConnectDatabaseTrait, never, FileSystem | Scope.Scope>;

package/dts/src/kv/kv.d.ts

@@ -0,0 +1,42 @@
+import type { Database } from "@tursodatabase/database";
+import type { Effect, Option, Scope, Stream } from "effect";
+declare const ConflictError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ConflictError";
+} & Readonly<A>;
+export declare class ConflictError extends ConflictError_base<{
+    readonly key: string;
+}> {
+}
+export type KV = {
+    get(key: string): Effect.Effect<Option.Option<{
+        value: Uint8Array;
+        etag: string;
+    }>>;
+    /** Optimization for S3 calls, so there isn't head+get, but only one request*/
+    getIfChanged(key: string, etag: string): Effect.Effect<Option.Option<{
+        value: Uint8Array;
+        etag: string;
+    }>>;
+    exists(key: string): Effect.Effect<boolean>;
+    set(key: string, value: Uint8Array): Effect.Effect<{
+        etag: string;
+    }>;
+    cas(key: string, value: Uint8Array, etag: string): Effect.Effect<{
+        etag: string;
+    }, ConflictError>;
+    delete(key: string): Effect.Effect<void>;
+    readStream(key: string): Effect.Effect<Option.Option<Stream.Stream<Uint8Array>>>;
+    writeStream(key: string, content: Stream.Stream<Uint8Array>): Effect.Effect<{
+        etag: string;
+    }>;
+};
+export type CloneTrait = {
+    clone(from: string, to: string): Effect.Effect<void>;
+};
+export type ConnectDatabaseTrait = {
+    /**
+     * Creates connection with CDC full enabled
+     */
+    connect(key: string): Effect.Effect<Database, Error, Scope.Scope>;
+};
+export {};

package/dts/src/kv/memoryKV.d.ts

@@ -0,0 +1,5 @@
+import { FileSystem } from "@effect/platform/FileSystem";
+import { Effect } from "effect";
+import type { Scope } from "effect";
+import type { CloneTrait, ConnectDatabaseTrait, KV } from "./kv.js";
+export declare const makeMemoryKV: () => Effect.Effect<KV & CloneTrait & ConnectDatabaseTrait, never, FileSystem | Scope.Scope>;

package/dts/src/kv/s3KV.d.ts

@@ -0,0 +1,4 @@
+import type { S3Service } from "@effect-aws/client-s3";
+import { Effect } from "effect";
+import type { KV } from "./kv.js";
+export declare const makeS3KV: (bucket: string) => Effect.Effect<KV, never, S3Service>;

package/dts/src/kv/syncFiles.d.ts

@@ -0,0 +1,4 @@
+import { Effect } from "effect";
+import { LocalKV, RemoteKV } from "../storage.js";
+export declare const pullFiles: (keys: string[]) => Effect.Effect<undefined[], Error, LocalKV | RemoteKV>;
+export declare const pushFiles: (keys: string[]) => Effect.Effect<undefined[], Error, LocalKV | RemoteKV>;

package/dts/src/pull.d.ts

@@ -0,0 +1,8 @@
+import type { Scope } from "effect";
+import { Effect, Option } from "effect";
+import type { ExtractedBatch } from "./batches.js";
+import { ConnectionConfig, ConnectionState } from "./contexts.js";
+import { LocalKV, RemoteKV } from "./storage.js";
+export declare const pull: () => Effect.Effect<{
+    newLocalBatch: Option.Option<ExtractedBatch>;
+}, Error, ConnectionConfig | ConnectionState | LocalKV | RemoteKV | Scope.Scope>;

package/dts/src/push.d.ts

@@ -0,0 +1,4 @@
+import { Effect } from "effect";
+import { ConnectionConfig, ConnectionState } from "./contexts.js";
+import { LocalKV, RemoteKV } from "./storage.js";
+export declare const push: () => Effect.Effect<void, import(".").ConflictError, LocalKV | RemoteKV | ConnectionConfig | ConnectionState>;