@lika85456/s3qlite 0.1.0

package/esm/src/pull.js

@@ -0,0 +1,97 @@
+import { Effect, Option } from "effect";
+import { v4 } from "uuid";
+import { applyBatch, extractBatch } from "./batches.js";
+import { getLatestChangeId } from "./cdc/extract.js";
+import { truncate } from "./cdc/truncate.js";
+import { ConnectionConfig, ConnectionState } from "./contexts.js";
+import { pullFiles } from "./kv/syncFiles.js";
+import { LocalKV, RemoteKV, baseKey, batchKey, dbKey, decodeJson, encodeJson, getJson, headKey, shmKey, walKey, } from "./storage.js";
+export const pull = () => Effect.gen(function* () {
+    const localKV = yield* LocalKV;
+    const remoteKV = yield* RemoteKV;
+    const { dbName } = yield* ConnectionConfig;
+    const state = yield* ConnectionState;
+    const tursoConnection = { current: yield* state.getConnection };
+    yield* Effect.logDebug("s3qlite.pull.start").pipe(Effect.annotateLogs({ dbName }));
+    const localHead = yield* getJson(localKV, headKey(dbName)).pipe(Effect.flatMap(Option.match({
+        onNone: () => Effect.die(new Error("Expected local stored head to exist")),
+        onSome: Effect.succeed,
+    })));
+    const remoteHeadOption = yield* remoteKV.getIfChanged(headKey(dbName), localHead.value.remoteEtag);
+    // no changes detected, skip pull
+    if (Option.isNone(remoteHeadOption)) {
+        yield* Effect.logDebug("s3qlite.pull.skip").pipe(Effect.annotateLogs({ dbName }));
+        return { newLocalBatch: Option.none() };
+    }
+    const remoteHead = yield* decodeJson(remoteHeadOption.value.value).pipe(Effect.orDieWith((e) => new Error(`Failed to decode remote head: ${String(e)}. Value to decode: ${remoteHeadOption.value.value}`)));
+    const remoteBatchesToApply = remoteHead.batches.slice(localHead.value.head.batches.length);
+    // Instead of rollbacking the whole database we are keeping a CoW copy of the snapshot+all applied batches up to a certain batch id.
+    const lastLocalSnapshotId = localHead.value.head.snapshots.at(-1)?.id;
+    const lastLocalBatchId = localHead.value.head.batches.at(-1)?.id;
+    if (!lastLocalBatchId || !lastLocalSnapshotId) {
+        return yield* Effect.die(new Error("Local head is in an invalid state, missing batches or snapshots")).pipe(Effect.annotateLogs({ localHead: localHead, remoteHead }));
+    }
+    const baseSnapshotKey = baseKey(lastLocalSnapshotId, lastLocalBatchId);
+    if (!(yield* localKV.exists(baseSnapshotKey))) {
+        return yield* Effect.die(new Error("Base snapshot is missing!"));
+    }
+    const { newLocalBatch } = yield* Effect.scoped(Effect.gen(function* () {
+        const workingCopyKey = `${v4()}.working-copy`;
+        yield* Effect.addFinalizer(() => localKV.delete(workingCopyKey)).pipe(Effect.exit);
+        const { newLocalBatch, workingConnection } = yield* Effect.all({
+            newLocalBatch: extractBatch(tursoConnection.current, localHead.value.lastSyncedLocalChangeId),
+            syncFiles: pullFiles(remoteBatchesToApply.map((batch) => batchKey(batch.id))),
+            workingConnection: localKV
+                .clone(baseSnapshotKey, workingCopyKey)
+                .pipe(Effect.flatMap(() => localKV.connect(workingCopyKey))),
+        }, {
+            concurrency: "unbounded",
+        });
+        // maybe it would be faster to check the sizes of the batches, and if its not bigger than a few megabytes it could just be all loaded into memory in parallel, and then applied at once, as one transaction
+        for (const batch of remoteBatchesToApply) {
+            yield* applyBatch(workingConnection, batch);
+        }
+        const nextLastRemoteAppliedChangeId = yield* getLatestChangeId(workingConnection);
+        const newHead = structuredClone(remoteHead);
+        const lastRemoteSnapshotId = newHead.snapshots.at(-1)?.id;
+        const lastRemoteBatchId = newHead.batches.at(-1)?.id;
+        if (!lastRemoteBatchId || !lastRemoteSnapshotId) {
+            return yield* Effect.die(new Error("Local head is in an invalid state, missing batches or snapshots")).pipe(Effect.annotateLogs({ newHead }));
+        }
+        yield* truncate(workingConnection);
+        yield* localKV.delete(walKey(workingCopyKey.split(".")[0]));
+        yield* localKV.clone(workingCopyKey, baseKey(lastRemoteSnapshotId, lastRemoteBatchId));
+        const allPendingBatches = [
+            ...(localHead.value.pendingBatches ?? []),
+            ...(Option.isSome(newLocalBatch)
+                ? [
+                    {
+                        id: newLocalBatch.value.batch.id,
+                        lastChangeId: newLocalBatch.value.lastLocalChangeId,
+                    },
+                ]
+                : []),
+        ];
+        for (const pb of allPendingBatches) {
+            yield* applyBatch(workingConnection, { id: pb.id });
+        }
+        yield* localKV.set(headKey(dbName), encodeJson({
+            head: newHead,
+            remoteEtag: remoteHeadOption.value.etag,
+            lastSyncedLocalChangeId: nextLastRemoteAppliedChangeId,
+            ...(allPendingBatches.length > 0
+                ? { pendingBatches: allPendingBatches }
+                : {}),
+        }));
+        const oldConnection = yield* state.getConnection;
+        yield* Effect.promise(() => oldConnection.close());
+        yield* localKV.delete(walKey(dbName));
+        yield* localKV.delete(shmKey(dbName));
+        yield* truncate(workingConnection);
+        yield* localKV.clone(workingCopyKey, dbKey(dbName));
+        return { newLocalBatch };
+    }));
+    yield* state.setConnection(yield* localKV.connect(dbKey(dbName)));
+    return { newLocalBatch };
+});
+//# sourceMappingURL=pull.js.map

package/esm/src/pull.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/pull.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EACN,OAAO,EACP,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,KAAK,EACL,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,MAAM,EACN,MAAM,GACN,MAAM,WAAW,CAAC;AAGnB,MAAM,CAAC,MAAM,IAAI,GAAG,GAIlB,EAAE,CACH,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC;IAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC;IACjC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,gBAAgB,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC;IACrC,MAAM,eAAe,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAEhE,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,OAAO,CAAa,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAC1E,MAAM,CAAC,OAAO,CACb,MAAM,CAAC,KAAK,CAAC;QACZ,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC1E,MAAM,EAAE,MAAM,CAAC,OAAO;KACtB,CAAC,CACF,CACD,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY,CACpD,OAAO,CAAC,MAAM,CAAC,EACf,SAAS,CAAC,KAAK,CAAC,UAAU,CAC1B,CAAC;IAEF,iCAAiC;IACjC,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAClF,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,UAAU,CAAO,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAC5E,MAAM,CAAC,SAAS,CACf,CAAC,CAAC,EAAE,EAAE,CACL,IAAI,KAAK,CACR,iCAAiC,MAAM,CAAC,CAAC,CAAC,sBAAsB,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,CAC9F,CACF,CACD,CAAC;IAEF,MAAM,oBAAoB,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3F,oIAAoI;IACpI,MAAM,mBAAmB,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACtE,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAEjE,IAAI,CAAC,gBAAgB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CACvB,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAC5E,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC;IACvE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAC7C,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACnB,MAAM,cAAc,GAAG,GAAG,EAAE,EAAE,eAAe,CAAC;QAE9C,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnF,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAC7D;YACC,aAAa,EAAE,YAAY,CAC1B,eAAe,CAAC,OAAO,EACvB,SAAS,CAAC,KAAK,CAAC,uBAAuB,CACvC;YACD,SAAS,EAAE,SAAS,CACnB,oBAAoB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CACvD;YACD,iBAAiB,EAAE,OAAO;iBACxB,KAAK,CAAC,eAAe,EAAE,cAAc,CAAC;iBACtC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;SAC7D,EACD;YACC,WAAW,EAAE,WAAW;SACxB,CACD,CAAC;QAEF,2MAA2M;QAC3M,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;YAC1C,KAAK,CAAC,CAAC,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,6BAA6B,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;QAElF,MAAM,OAAO,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,oBAAoB,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1D,MAAM,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAErD,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACjD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CACvB,IAAI,KAAK,CACR,iEAAiE,CACjE,CACD,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,KAAK,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACnC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5D,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CACnB,cAAc,EACd,OAAO,CAAC,oBAAoB,EAAE,iBAAiB,CAAC,CAChD,CAAC;QAEF,MAAM,iBAAiB,GAAG;YACzB,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;YACzC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;gBAC/B,CAAC,CAAC;oBACA;wBACC,EAAE,EAAE,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;wBAChC,YAAY,EAAE,aAAa,CAAC,KAAK,CAAC,iBAAiB;qBACnD;iBACD;gBACF,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACpC,KAAK,CAAC,CAAC,UAAU,CAAC,iBAAiB,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CACjB,OAAO,CAAC,MAAM,CAAC,EACf,UAAU,CAAa;YACtB,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,gBAAgB,CAAC,KAAK,CAAC,IAAI;YACvC,uBAAuB,EAAE,6BAA6B;YACtD,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;gBAC/B,CAAC,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE;gBACvC,CAAC,CAAC,EAAE,CAAC;SACN,CAAC,CACF,CAAC;QAEF,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;QACjD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC;QACnD,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACnC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpD,OAAO,EAAE,aAAa,EAAE,CAAC;IAC1B,CAAC,CAAC,CACF,CAAC;IAEF,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAElE,OAAO,EAAE,aAAa,EAAE,CAAC;AAC1B,CAAC,CAAC,CAAC","sourcesContent":["import type { Scope } from \"effect\";\nimport { Effect, Option } from \"effect\";\nimport { v4 } from \"uuid\";\n\nimport type { ExtractedBatch } from \"./batches\";\nimport { applyBatch, extractBatch } from \"./batches\";\nimport { getLatestChangeId } from \"./cdc/extract\";\nimport { truncate } from \"./cdc/truncate\";\nimport { ConnectionConfig, ConnectionState } from \"./contexts\";\nimport { pullFiles } from \"./kv/syncFiles\";\nimport {\n\tLocalKV,\n\tRemoteKV,\n\tbaseKey,\n\tbatchKey,\n\tdbKey,\n\tdecodeJson,\n\tencodeJson,\n\tgetJson,\n\theadKey,\n\tshmKey,\n\twalKey,\n} from \"./storage\";\nimport type { Head, StoredHead } from \"./types\";\n\nexport const pull = (): Effect.Effect<\n\t{ newLocalBatch: Option.Option<ExtractedBatch> },\n\tError,\n\tConnectionConfig | ConnectionState | LocalKV | RemoteKV | Scope.Scope\n> =>\n\tEffect.gen(function* () {\n\t\tconst localKV = yield* LocalKV;\n\t\tconst remoteKV = yield* RemoteKV;\n\t\tconst { dbName } = yield* ConnectionConfig;\n\t\tconst state = yield* ConnectionState;\n\t\tconst tursoConnection = { current: yield* state.getConnection };\n\n\t\tyield* Effect.logDebug(\"s3qlite.pull.start\").pipe(Effect.annotateLogs({ dbName }));\n\n\t\tconst localHead = yield* getJson<StoredHead>(localKV, headKey(dbName)).pipe(\n\t\t\tEffect.flatMap(\n\t\t\t\tOption.match({\n\t\t\t\t\tonNone: () => Effect.die(new Error(\"Expected local stored head to exist\")),\n\t\t\t\t\tonSome: Effect.succeed,\n\t\t\t\t}),\n\t\t\t),\n\t\t);\n\n\t\tconst remoteHeadOption = yield* remoteKV.getIfChanged(\n\t\t\theadKey(dbName),\n\t\t\tlocalHead.value.remoteEtag,\n\t\t);\n\n\t\t// no changes detected, skip pull\n\t\tif (Option.isNone(remoteHeadOption)) {\n\t\t\tyield* Effect.logDebug(\"s3qlite.pull.skip\").pipe(Effect.annotateLogs({ dbName }));\n\t\t\treturn { newLocalBatch: Option.none() };\n\t\t}\n\n\t\tconst remoteHead = yield* decodeJson<Head>(remoteHeadOption.value.value).pipe(\n\t\t\tEffect.orDieWith(\n\t\t\t\t(e) =>\n\t\t\t\t\tnew Error(\n\t\t\t\t\t\t`Failed to decode remote head: ${String(e)}. Value to decode: ${remoteHeadOption.value.value}`,\n\t\t\t\t\t),\n\t\t\t),\n\t\t);\n\n\t\tconst remoteBatchesToApply = remoteHead.batches.slice(localHead.value.head.batches.length);\n\t\t// Instead of rollbacking the whole database we are keeping a CoW copy of the snapshot+all applied batches up to a certain batch id.\n\t\tconst lastLocalSnapshotId = localHead.value.head.snapshots.at(-1)?.id;\n\t\tconst lastLocalBatchId = localHead.value.head.batches.at(-1)?.id;\n\n\t\tif (!lastLocalBatchId || !lastLocalSnapshotId) {\n\t\t\treturn yield* Effect.die(\n\t\t\t\tnew Error(\"Local head is in an invalid state, missing batches or snapshots\"),\n\t\t\t).pipe(Effect.annotateLogs({ localHead: localHead, remoteHead }));\n\t\t}\n\n\t\tconst baseSnapshotKey = baseKey(lastLocalSnapshotId, lastLocalBatchId);\n\t\tif (!(yield* localKV.exists(baseSnapshotKey))) {\n\t\t\treturn yield* Effect.die(new Error(\"Base snapshot is missing!\"));\n\t\t}\n\n\t\tconst { newLocalBatch } = yield* Effect.scoped(\n\t\t\tEffect.gen(function* () {\n\t\t\t\tconst workingCopyKey = `${v4()}.working-copy`;\n\n\t\t\t\tyield* Effect.addFinalizer(() => localKV.delete(workingCopyKey)).pipe(Effect.exit);\n\n\t\t\t\tconst { newLocalBatch, workingConnection } = yield* Effect.all(\n\t\t\t\t\t{\n\t\t\t\t\t\tnewLocalBatch: extractBatch(\n\t\t\t\t\t\t\ttursoConnection.current,\n\t\t\t\t\t\t\tlocalHead.value.lastSyncedLocalChangeId,\n\t\t\t\t\t\t),\n\t\t\t\t\t\tsyncFiles: pullFiles(\n\t\t\t\t\t\t\tremoteBatchesToApply.map((batch) => batchKey(batch.id)),\n\t\t\t\t\t\t),\n\t\t\t\t\t\tworkingConnection: localKV\n\t\t\t\t\t\t\t.clone(baseSnapshotKey, workingCopyKey)\n\t\t\t\t\t\t\t.pipe(Effect.flatMap(() => localKV.connect(workingCopyKey))),\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tconcurrency: \"unbounded\",\n\t\t\t\t\t},\n\t\t\t\t);\n\n\t\t\t\t// maybe it would be faster to check the sizes of the batches, and if its not bigger than a few megabytes it could just be all loaded into memory in parallel, and then applied at once, as one transaction\n\t\t\t\tfor (const batch of remoteBatchesToApply) {\n\t\t\t\t\tyield* applyBatch(workingConnection, batch);\n\t\t\t\t}\n\n\t\t\t\tconst nextLastRemoteAppliedChangeId = yield* getLatestChangeId(workingConnection);\n\n\t\t\t\tconst newHead = structuredClone(remoteHead);\n\n\t\t\t\tconst lastRemoteSnapshotId = newHead.snapshots.at(-1)?.id;\n\t\t\t\tconst lastRemoteBatchId = newHead.batches.at(-1)?.id;\n\n\t\t\t\tif (!lastRemoteBatchId || !lastRemoteSnapshotId) {\n\t\t\t\t\treturn yield* Effect.die(\n\t\t\t\t\t\tnew Error(\n\t\t\t\t\t\t\t\"Local head is in an invalid state, missing batches or snapshots\",\n\t\t\t\t\t\t),\n\t\t\t\t\t).pipe(Effect.annotateLogs({ newHead }));\n\t\t\t\t}\n\n\t\t\t\tyield* truncate(workingConnection);\n\t\t\t\tyield* localKV.delete(walKey(workingCopyKey.split(\".\")[0]));\n\n\t\t\t\tyield* localKV.clone(\n\t\t\t\t\tworkingCopyKey,\n\t\t\t\t\tbaseKey(lastRemoteSnapshotId, lastRemoteBatchId),\n\t\t\t\t);\n\n\t\t\t\tconst allPendingBatches = [\n\t\t\t\t\t...(localHead.value.pendingBatches ?? []),\n\t\t\t\t\t...(Option.isSome(newLocalBatch)\n\t\t\t\t\t\t? [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tid: newLocalBatch.value.batch.id,\n\t\t\t\t\t\t\t\t\tlastChangeId: newLocalBatch.value.lastLocalChangeId,\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t: []),\n\t\t\t\t];\n\n\t\t\t\tfor (const pb of allPendingBatches) {\n\t\t\t\t\tyield* applyBatch(workingConnection, { id: pb.id });\n\t\t\t\t}\n\n\t\t\t\tyield* localKV.set(\n\t\t\t\t\theadKey(dbName),\n\t\t\t\t\tencodeJson<StoredHead>({\n\t\t\t\t\t\thead: newHead,\n\t\t\t\t\t\tremoteEtag: remoteHeadOption.value.etag,\n\t\t\t\t\t\tlastSyncedLocalChangeId: nextLastRemoteAppliedChangeId,\n\t\t\t\t\t\t...(allPendingBatches.length > 0\n\t\t\t\t\t\t\t? { pendingBatches: allPendingBatches }\n\t\t\t\t\t\t\t: {}),\n\t\t\t\t\t}),\n\t\t\t\t);\n\n\t\t\t\tconst oldConnection = yield* state.getConnection;\n\t\t\t\tyield* Effect.promise(() => oldConnection.close());\n\t\t\t\tyield* localKV.delete(walKey(dbName));\n\t\t\t\tyield* localKV.delete(shmKey(dbName));\n\t\t\t\tyield* truncate(workingConnection);\n\t\t\t\tyield* localKV.clone(workingCopyKey, dbKey(dbName));\n\n\t\t\t\treturn { newLocalBatch };\n\t\t\t}),\n\t\t);\n\n\t\tyield* state.setConnection(yield* localKV.connect(dbKey(dbName)));\n\n\t\treturn { newLocalBatch };\n\t});"]}

package/esm/src/push.js

@@ -0,0 +1,54 @@
+import { Effect, Option } from "effect";
+import { extractBatch } from "./batches.js";
+import { getLatestChangeId } from "./cdc/extract.js";
+import { truncate } from "./cdc/truncate.js";
+import { ConnectionConfig, ConnectionState } from "./contexts.js";
+import { pushFiles } from "./kv/syncFiles.js";
+import { LocalKV, RemoteKV, baseKey, batchKey, dbKey, encodeJson, getJson, headKey, } from "./storage.js";
+export const push = () => Effect.gen(function* () {
+    const localKV = yield* LocalKV;
+    const remoteKV = yield* RemoteKV;
+    const { dbName } = yield* ConnectionConfig;
+    const state = yield* ConnectionState;
+    const tursoConnection = { current: yield* state.getConnection };
+    const localHead = yield* getJson(localKV, headKey(dbName)).pipe(Effect.orDie, Effect.flatMap(Option.match({
+        onNone: () => Effect.die(new Error("Expected local stored head to exist")),
+        onSome: Effect.succeed,
+    })));
+    const pendingBatches = [];
+    if (localHead.value.pendingBatches?.length) {
+        pendingBatches.push(...localHead.value.pendingBatches.map((pb) => ({
+            batch: { id: pb.id },
+            lastChangeId: pb.lastChangeId,
+        })));
+    }
+    const extracted = yield* extractBatch(tursoConnection.current, localHead.value.lastSyncedLocalChangeId);
+    if (Option.isSome(extracted)) {
+        pendingBatches.push({
+            batch: extracted.value.batch,
+            lastChangeId: extracted.value.lastLocalChangeId,
+        });
+    }
+    if (pendingBatches.length === 0) {
+        return;
+    }
+    yield* pushFiles(pendingBatches.map((pb) => batchKey(pb.batch.id))).pipe(Effect.orDie);
+    const newHead = {
+        snapshots: localHead.value.head.snapshots,
+        batches: localHead.value.head.batches.concat(pendingBatches.map((pb) => pb.batch)),
+    };
+    const result = yield* remoteKV.cas(headKey(dbName), encodeJson(newHead), localHead.value.remoteEtag);
+    const lastSyncedLocalChangeId = yield* getLatestChangeId(tursoConnection.current).pipe(Effect.orDie);
+    yield* localKV.set(headKey(dbName), encodeJson({
+        head: newHead,
+        remoteEtag: result.etag,
+        lastSyncedLocalChangeId,
+    }));
+    const lastSnapshotId = newHead.snapshots.at(-1)?.id;
+    const lastBatchId = newHead.batches.at(-1)?.id;
+    if (lastSnapshotId && lastBatchId) {
+        yield* truncate(tursoConnection.current);
+        yield* localKV.clone(dbKey(dbName), baseKey(lastSnapshotId, lastBatchId));
+    }
+});
+//# sourceMappingURL=push.js.map

package/esm/src/push.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"push.js","sourceRoot":"","sources":["../../../src/push.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EACN,OAAO,EACP,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,KAAK,EACL,UAAU,EACV,OAAO,EACP,OAAO,GACP,MAAM,WAAW,CAAC;AAGnB,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,EAAE,CACxB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC;IAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC;IACjC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,gBAAgB,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC;IACrC,MAAM,eAAe,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAEhE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,OAAO,CAAa,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAC1E,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,CACb,MAAM,CAAC,KAAK,CAAC;QACZ,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC1E,MAAM,EAAE,MAAM,CAAC,OAAO;KACtB,CAAC,CACF,CACD,CAAC;IAEF,MAAM,cAAc,GAA6C,EAAE,CAAC;IAEpE,IAAI,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;QAC5C,cAAc,CAAC,IAAI,CAClB,GAAG,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;YACpB,YAAY,EAAE,EAAE,CAAC,YAAY;SAC7B,CAAC,CAAC,CACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,YAAY,CACpC,eAAe,CAAC,OAAO,EACvB,SAAS,CAAC,KAAK,CAAC,uBAAuB,CACvC,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,cAAc,CAAC,IAAI,CAAC;YACnB,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,KAAK;YAC5B,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,iBAAiB;SAC/C,CAAC,CAAC;IACJ,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO;IACR,CAAC;IAED,KAAK,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEvF,MAAM,OAAO,GAAS;QACrB,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS;QACzC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;KAClF,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CACjC,OAAO,CAAC,MAAM,CAAC,EACf,UAAU,CAAC,OAAO,CAAC,EACnB,SAAS,CAAC,KAAK,CAAC,UAAU,CAC1B,CAAC;IAEF,MAAM,uBAAuB,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,CACrF,MAAM,CAAC,KAAK,CACZ,CAAC;IAEF,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CACjB,OAAO,CAAC,MAAM,CAAC,EACf,UAAU,CAAa;QACtB,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,uBAAuB;KACvB,CAAC,CACF,CAAC;IAEF,MAAM,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACpD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC/C,IAAI,cAAc,IAAI,WAAW,EAAE,CAAC;QACnC,KAAK,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACzC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAC3E,CAAC;AACF,CAAC,CAAC,CAAC","sourcesContent":["import { Effect, Option } from \"effect\";\n\nimport { extractBatch } from \"./batches\";\nimport { getLatestChangeId } from \"./cdc/extract\";\nimport { truncate } from \"./cdc/truncate\";\nimport { ConnectionConfig, ConnectionState } from \"./contexts\";\nimport { pushFiles } from \"./kv/syncFiles\";\nimport {\n\tLocalKV,\n\tRemoteKV,\n\tbaseKey,\n\tbatchKey,\n\tdbKey,\n\tencodeJson,\n\tgetJson,\n\theadKey,\n} from \"./storage\";\nimport type { Batch, Head, StoredHead } from \"./types\";\n\nexport const push = () =>\n\tEffect.gen(function* () {\n\t\tconst localKV = yield* LocalKV;\n\t\tconst remoteKV = yield* RemoteKV;\n\t\tconst { dbName } = yield* ConnectionConfig;\n\t\tconst state = yield* ConnectionState;\n\t\tconst tursoConnection = { current: yield* state.getConnection };\n\n\t\tconst localHead = yield* getJson<StoredHead>(localKV, headKey(dbName)).pipe(\n\t\t\tEffect.orDie,\n\t\t\tEffect.flatMap(\n\t\t\t\tOption.match({\n\t\t\t\t\tonNone: () => Effect.die(new Error(\"Expected local stored head to exist\")),\n\t\t\t\t\tonSome: Effect.succeed,\n\t\t\t\t}),\n\t\t\t),\n\t\t);\n\n\t\tconst pendingBatches: { batch: Batch; lastChangeId: number }[] = [];\n\n\t\tif (localHead.value.pendingBatches?.length) {\n\t\t\tpendingBatches.push(\n\t\t\t\t...localHead.value.pendingBatches.map((pb) => ({\n\t\t\t\t\tbatch: { id: pb.id },\n\t\t\t\t\tlastChangeId: pb.lastChangeId,\n\t\t\t\t})),\n\t\t\t);\n\t\t}\n\n\t\tconst extracted = yield* extractBatch(\n\t\t\ttursoConnection.current,\n\t\t\tlocalHead.value.lastSyncedLocalChangeId,\n\t\t);\n\n\t\tif (Option.isSome(extracted)) {\n\t\t\tpendingBatches.push({\n\t\t\t\tbatch: extracted.value.batch,\n\t\t\t\tlastChangeId: extracted.value.lastLocalChangeId,\n\t\t\t});\n\t\t}\n\n\t\tif (pendingBatches.length === 0) {\n\t\t\treturn;\n\t\t}\n\n\t\tyield* pushFiles(pendingBatches.map((pb) => batchKey(pb.batch.id))).pipe(Effect.orDie);\n\n\t\tconst newHead: Head = {\n\t\t\tsnapshots: localHead.value.head.snapshots,\n\t\t\tbatches: localHead.value.head.batches.concat(pendingBatches.map((pb) => pb.batch)),\n\t\t};\n\n\t\tconst result = yield* remoteKV.cas(\n\t\t\theadKey(dbName),\n\t\t\tencodeJson(newHead),\n\t\t\tlocalHead.value.remoteEtag,\n\t\t);\n\n\t\tconst lastSyncedLocalChangeId = yield* getLatestChangeId(tursoConnection.current).pipe(\n\t\t\tEffect.orDie,\n\t\t);\n\n\t\tyield* localKV.set(\n\t\t\theadKey(dbName),\n\t\t\tencodeJson<StoredHead>({\n\t\t\t\thead: newHead,\n\t\t\t\tremoteEtag: result.etag,\n\t\t\t\tlastSyncedLocalChangeId,\n\t\t\t}),\n\t\t);\n\n\t\tconst lastSnapshotId = newHead.snapshots.at(-1)?.id;\n\t\tconst lastBatchId = newHead.batches.at(-1)?.id;\n\t\tif (lastSnapshotId && lastBatchId) {\n\t\t\tyield* truncate(tursoConnection.current);\n\t\t\tyield* localKV.clone(dbKey(dbName), baseKey(lastSnapshotId, lastBatchId));\n\t\t}\n\t});"]}

package/esm/src/storage.js

@@ -0,0 +1,26 @@
+import { Context, Effect, Option } from "effect";
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+export class LocalKV extends Context.Tag("s3qlite/LocalKV")() {
+}
+export class RemoteKV extends Context.Tag("s3qlite/RemoteKV")() {
+}
+export const headKey = (dbName) => `${dbName}.json`;
+export const batchKey = (batchId) => `${batchId}.batch`;
+export const snapshotKey = (snapshotId) => `${snapshotId}.snapshot`;
+export const dbKey = (dbName) => `${dbName}.db`;
+export const walKey = (dbName) => `${dbName}.db-wal`;
+export const shmKey = (dbName) => `${dbName}.db-shm`;
+export const baseKey = (snapshotId, batchId) => `${snapshotId}-${batchId}.base`;
+export const encodeJson = (value) => textEncoder.encode(JSON.stringify(value));
+export const decodeJson = (value) => Effect.try({
+    try: () => JSON.parse(textDecoder.decode(value)),
+    catch: (error) => error instanceof SyntaxError
+        ? error
+        : new SyntaxError(`Failed to parse "${value}": ${String(error)}`),
+});
+export const getJson = (kv, key) => kv.get(key).pipe(Effect.flatMap(Option.match({
+    onNone: () => Effect.succeed(Option.none()),
+    onSome: ({ etag, value }) => decodeJson(value).pipe(Effect.map((decoded) => Option.some({ etag, value: decoded }))),
+})));
+//# sourceMappingURL=storage.js.map

package/esm/src/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../../src/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAIjD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AACtC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AAEtC,MAAM,OAAO,OAAQ,SAAQ,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAGxD;CAAG;AAEN,MAAM,OAAO,QAAS,SAAQ,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAgB;CAAG;AAEhF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC;AAEpE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAU,EAAE,CAAC,GAAG,OAAO,QAAQ,CAAC;AAExE,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAkB,EAAU,EAAE,CAAC,GAAG,UAAU,WAAW,CAAC;AAEpF,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,KAAK,CAAC;AAEhE,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,SAAS,CAAC;AAErE,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,GAAG,MAAM,SAAS,CAAC;AAErE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CACtE,GAAG,UAAU,IAAI,OAAO,OAAO,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAI,KAAQ,EAAc,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAEjG,MAAM,CAAC,MAAM,UAAU,GAAG,CAAI,KAAiB,EAAiC,EAAE,CACjF,MAAM,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAM;IACrD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAChB,KAAK,YAAY,WAAW;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,WAAW,CAAC,oBAAoB,KAAK,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;CACnE,CAAC,CAAC;AAEJ,MAAM,CAAC,MAAM,OAAO,GAAG,CACtB,EAAM,EACN,GAAW,EAC6D,EAAE,CAC1E,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CACf,MAAM,CAAC,OAAO,CACb,MAAM,CAAC,KAAK,CAAC;IACZ,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAC3B,UAAU,CAAI,KAAK,CAAC,CAAC,IAAI,CACxB,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAC9D;CACF,CAAC,CACF,CACD,CAAC","sourcesContent":["import { Context, Effect, Option } from \"effect\";\n\nimport type { CloneTrait, ConnectDatabaseTrait, KV } from \"./kv/kv\";\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\n\nexport class LocalKV extends Context.Tag(\"s3qlite/LocalKV\")<\n\tLocalKV,\n\tKV & CloneTrait & ConnectDatabaseTrait\n>() {}\n\nexport class RemoteKV extends Context.Tag(\"s3qlite/RemoteKV\")<RemoteKV, KV>() {}\n\nexport const headKey = (dbName: string): string => `${dbName}.json`;\n\nexport const batchKey = (batchId: string): string => `${batchId}.batch`;\n\nexport const snapshotKey = (snapshotId: string): string => `${snapshotId}.snapshot`;\n\nexport const dbKey = (dbName: string): string => `${dbName}.db`;\n\nexport const walKey = (dbName: string): string => `${dbName}.db-wal`;\n\nexport const shmKey = (dbName: string): string => `${dbName}.db-shm`;\n\nexport const baseKey = (snapshotId: string, batchId: string): string =>\n\t`${snapshotId}-${batchId}.base`;\n\nexport const encodeJson = <A>(value: A): Uint8Array => textEncoder.encode(JSON.stringify(value));\n\nexport const decodeJson = <A>(value: Uint8Array): Effect.Effect<A, SyntaxError> =>\n\tEffect.try({\n\t\ttry: () => JSON.parse(textDecoder.decode(value)) as A,\n\t\tcatch: (error) =>\n\t\t\terror instanceof SyntaxError\n\t\t\t\t? error\n\t\t\t\t: new SyntaxError(`Failed to parse \"${value}\": ${String(error)}`),\n\t});\n\nexport const getJson = <A>(\n\tkv: KV,\n\tkey: string,\n): Effect.Effect<Option.Option<{ value: A; etag: string }>, SyntaxError> =>\n\tkv.get(key).pipe(\n\t\tEffect.flatMap(\n\t\t\tOption.match({\n\t\t\t\tonNone: () => Effect.succeed(Option.none()),\n\t\t\t\tonSome: ({ etag, value }) =>\n\t\t\t\t\tdecodeJson<A>(value).pipe(\n\t\t\t\t\t\tEffect.map((decoded) => Option.some({ etag, value: decoded })),\n\t\t\t\t\t),\n\t\t\t}),\n\t\t),\n\t);"]}

package/esm/src/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/esm/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/types.ts"],"names":[],"mappings":"","sourcesContent":["import type { Database } from \"@tursodatabase/database\";\nimport type { Effect } from \"effect\";\n\nexport type Snapshot = {\n\tid: string;\n\n\t// thanks to this we can do full history\n\tbaseSnapshotId?: string;\n\tbatchIdsApplied: string[];\n};\n\nexport type Batch = {\n\tid: string;\n};\n\nexport type Head = {\n\tsnapshots: Snapshot[];\n\tbatches: Batch[];\n};\n\nexport type PendingBatch = {\n\tid: string;\n\tlastChangeId: number;\n};\n\nexport type StoredHead = {\n\thead: Head;\n\n\t/**\n\t * The latest etag of the remote head this instance has been synced to.\n\t */\n\tremoteEtag: string;\n\n\tlastSyncedLocalChangeId: number;\n\n\tpendingBatches?: PendingBatch[];\n};\n\n// export type ConflictResolutionMode = \"fail\" | \"preferRemote\" | \"preferLocal\";\n//\n// export type ConflictOperation = \"pull\" | \"push\" | \"sync\";\n//\n// export type ConflictResolutionContext = {\n// \treadonly cause: Error;\n// \treadonly localHead: StoredHead;\n// \treadonly operation: ConflictOperation;\n// \treadonly remoteHead: Head;\n// };\n//\n// export type ConflictResolver = (\n// \tcontext: ConflictResolutionContext,\n// ) => Effect.Effect<ConflictResolutionMode, Error>;\n//\n// type StrongConsistencyOptions = {\n// \tmode: \"strong\";\n// };\n//\n// type EventualConsistencyOptions = {\n// \tmode: \"eventual\";\n// \tsync?: \"manual\" | { everyMs: number };\n// };\n\nexport type ConnectionOptions = {\n\tbucket: string;\n\t// conflictResolution?: ConflictResolutionMode | ConflictResolver;\n\tlocalDirectory?: string;\n};\n// } & (StrongConsistencyOptions | EventualConsistencyOptions);\n\nexport type S3qliteDatabase = Database & {\n\tpull: () => Effect.Effect<void, Error>;\n\tpush: () => Effect.Effect<void, Error>;\n\tsync: () => Effect.Effect<void, Error>;\n\tfork: (dbName: string) => Effect.Effect<void, Error>;\n\tcheckpoint: () => Effect.Effect<void, Error>;\n};"]}

package/esm/src/wrapDatabase.js

@@ -0,0 +1,76 @@
+const asyncDatabaseMethods = new Set([
+    "all",
+    "close",
+    "exec",
+    "get",
+    "iterate",
+    "pragma",
+    "run",
+]);
+const asyncStatementMethods = new Set(["all", "close", "get", "run"]);
+const syncStatementMethods = new Set(["bind", "pluck", "raw", "safeIntegers"]);
+const wrapTransaction = (transaction, wrap) => new Proxy(transaction, {
+    apply: (target, thisArg, args) => wrap(() => Reflect.apply(target, thisArg, args), "transaction", args),
+    get: (target, property, receiver) => {
+        const value = Reflect.get(target, property, receiver);
+        if (typeof value !== "function") {
+            return value;
+        }
+        return wrapTransaction(value, wrap);
+    },
+});
+const wrapStatement = (statement, wrap) => {
+    let proxy;
+    proxy = new Proxy(statement, {
+        get: (target, property, receiver) => {
+            const value = Reflect.get(target, property, receiver);
+            if (typeof value !== "function") {
+                return value;
+            }
+            if (syncStatementMethods.has(property)) {
+                return (...args) => {
+                    Reflect.apply(value, target, args);
+                    return proxy;
+                };
+            }
+            if (property === "iterate") {
+                return (...args) => (async function* () {
+                    const iterable = wrap(() => Reflect.apply(value, target, args), property, args);
+                    for await (const item of iterable) {
+                        yield item;
+                    }
+                })();
+            }
+            if (asyncStatementMethods.has(property)) {
+                return (...args) => wrap(() => Reflect.apply(value, target, args), property, args);
+            }
+            return (...args) => Reflect.apply(value, target, args);
+        },
+    });
+    return proxy;
+};
+export const wrapDatabase = (getTarget, wrap) => {
+    return new Proxy({}, {
+        get: (wrapper, property, receiver) => {
+            if (Reflect.has(wrapper, property)) {
+                return Reflect.get(wrapper, property, receiver);
+            }
+            const original = getTarget();
+            const value = Reflect.get(original, property, original);
+            if (typeof value !== "function") {
+                return value;
+            }
+            if (property === "prepare") {
+                return (...args) => wrapStatement(Reflect.apply(value, getTarget(), args), wrap);
+            }
+            if (property === "transaction") {
+                return (...args) => wrapTransaction(Reflect.apply(value, getTarget(), args), wrap);
+            }
+            if (asyncDatabaseMethods.has(property)) {
+                return (...args) => wrap(() => Reflect.apply(value, getTarget(), args), property, args);
+            }
+            return (...args) => Reflect.apply(value, getTarget(), args);
+        },
+    });
+};
+//# sourceMappingURL=wrapDatabase.js.map

package/esm/src/wrapDatabase.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wrapDatabase.js","sourceRoot":"","sources":["../../../src/wrapDatabase.ts"],"names":[],"mappings":"AAAA,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAc;IACjD,KAAK;IACL,OAAO;IACP,MAAM;IACN,KAAK;IACL,SAAS;IACT,QAAQ;IACR,KAAK;CACL,CAAC,CAAC;AAEH,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAEnF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAc,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;AAE5F,MAAM,eAAe,GAAG,CACvB,WAAc,EACd,IAA4E,EACxE,EAAE,CACN,IAAI,KAAK,CAAC,WAAW,EAAE;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,CAChC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEtD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,eAAe,CAAC,KAAiD,EAAE,IAAI,CAAC,CAAC;IACjF,CAAC;CACD,CAAM,CAAC;AAET,MAAM,aAAa,GAAG,CACrB,SAAY,EACZ,IAA4E,EACxE,EAAE;IACN,IAAI,KAAQ,CAAC;IAEb,KAAK,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE;QAC5B,GAAG,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAEtD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC;YACd,CAAC;YAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE;oBAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;oBACnC,OAAO,KAAK,CAAC;gBACd,CAAC,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,CAAC,KAAK,SAAS,CAAC;oBACf,MAAM,QAAQ,GAAG,IAAI,CACpB,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EACxC,QAAQ,EACR,IAAI,CACsB,CAAC;oBAE5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;wBACnC,MAAM,IAAI,CAAC;oBACZ,CAAC;gBACF,CAAC,CAAC,EAAE,CAAC;YACP,CAAC;YAED,IAAI,qBAAqB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACzC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACjE,CAAC;YAED,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACnE,CAAC;KACD,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAC3B,SAAkB,EAClB,IAA4E,EACxE,EAAE;IACN,OAAO,IAAI,KAAK,CAAC,EAAO,EAAE;QACzB,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;YACpC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACpC,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,CAAC;YAED,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAExD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC;YACd,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAAW,EAAE,IAAI,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,eAAe,CACd,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAE1B,EACZ,IAAI,CACJ,CAAC;YACJ,CAAC;YAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAC7B,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACtE,CAAC;YAED,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,CAAC,CAAC;QACxE,CAAC;KACD,CAAC,CAAC;AACJ,CAAC,CAAC","sourcesContent":["const asyncDatabaseMethods = new Set<PropertyKey>([\n\t\"all\",\n\t\"close\",\n\t\"exec\",\n\t\"get\",\n\t\"iterate\",\n\t\"pragma\",\n\t\"run\",\n]);\n\nconst asyncStatementMethods = new Set<PropertyKey>([\"all\", \"close\", \"get\", \"run\"]);\n\nconst syncStatementMethods = new Set<PropertyKey>([\"bind\", \"pluck\", \"raw\", \"safeIntegers\"]);\n\nconst wrapTransaction = <T extends (...args: readonly unknown[]) => unknown>(\n\ttransaction: T,\n\twrap: <R>(call: () => R, method: PropertyKey, args: readonly unknown[]) => R,\n): T =>\n\tnew Proxy(transaction, {\n\t\tapply: (target, thisArg, args) =>\n\t\t\twrap(() => Reflect.apply(target, thisArg, args), \"transaction\", args),\n\t\tget: (target, property, receiver) => {\n\t\t\tconst value = Reflect.get(target, property, receiver);\n\n\t\t\tif (typeof value !== \"function\") {\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t\treturn wrapTransaction(value as (...args: readonly unknown[]) => unknown, wrap);\n\t\t},\n\t}) as T;\n\nconst wrapStatement = <T extends object>(\n\tstatement: T,\n\twrap: <R>(call: () => R, method: PropertyKey, args: readonly unknown[]) => R,\n): T => {\n\tlet proxy: T;\n\n\tproxy = new Proxy(statement, {\n\t\tget: (target, property, receiver) => {\n\t\t\tconst value = Reflect.get(target, property, receiver);\n\n\t\t\tif (typeof value !== \"function\") {\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t\tif (syncStatementMethods.has(property)) {\n\t\t\t\treturn (...args: unknown[]) => {\n\t\t\t\t\tReflect.apply(value, target, args);\n\t\t\t\t\treturn proxy;\n\t\t\t\t};\n\t\t\t}\n\n\t\t\tif (property === \"iterate\") {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\t(async function* () {\n\t\t\t\t\t\tconst iterable = wrap(\n\t\t\t\t\t\t\t() => Reflect.apply(value, target, args),\n\t\t\t\t\t\t\tproperty,\n\t\t\t\t\t\t\targs,\n\t\t\t\t\t\t) as AsyncIterable<unknown>;\n\n\t\t\t\t\t\tfor await (const item of iterable) {\n\t\t\t\t\t\t\tyield item;\n\t\t\t\t\t\t}\n\t\t\t\t\t})();\n\t\t\t}\n\n\t\t\tif (asyncStatementMethods.has(property)) {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrap(() => Reflect.apply(value, target, args), property, args);\n\t\t\t}\n\n\t\t\treturn (...args: unknown[]) => Reflect.apply(value, target, args);\n\t\t},\n\t});\n\n\treturn proxy;\n};\n\nexport const wrapDatabase = <T extends object>(\n\tgetTarget: () => T,\n\twrap: <R>(call: () => R, method: PropertyKey, args: readonly unknown[]) => R,\n): T => {\n\treturn new Proxy({} as T, {\n\t\tget: (wrapper, property, receiver) => {\n\t\t\tif (Reflect.has(wrapper, property)) {\n\t\t\t\treturn Reflect.get(wrapper, property, receiver);\n\t\t\t}\n\n\t\t\tconst original = getTarget();\n\t\t\tconst value = Reflect.get(original, property, original);\n\n\t\t\tif (typeof value !== \"function\") {\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t\tif (property === \"prepare\") {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrapStatement(Reflect.apply(value, getTarget(), args) as object, wrap);\n\t\t\t}\n\n\t\t\tif (property === \"transaction\") {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrapTransaction(\n\t\t\t\t\t\tReflect.apply(value, getTarget(), args) as (\n\t\t\t\t\t\t\t...args: readonly unknown[]\n\t\t\t\t\t\t) => unknown,\n\t\t\t\t\t\twrap,\n\t\t\t\t\t);\n\t\t\t}\n\n\t\t\tif (asyncDatabaseMethods.has(property)) {\n\t\t\t\treturn (...args: unknown[]) =>\n\t\t\t\t\twrap(() => Reflect.apply(value, getTarget(), args), property, args);\n\t\t\t}\n\n\t\t\treturn (...args: unknown[]) => Reflect.apply(value, getTarget(), args);\n\t\t},\n\t});\n};"]}

package/package.json

@@ -0,0 +1,35 @@
+{
+	"name": "@lika85456/s3qlite",
+	"version": "0.1.0",
+	"type": "module",
+	"sideEffects": false,
+	"dependencies": {
+		"@effect-aws/client-s3": "^1.11.0",
+		"@effect/platform": "0.96.0",
+		"@effect/platform-bun": "^0.89.0",
+		"@effect/platform-node": "^0.106.0",
+		"@tursodatabase/database": "^0.6.0",
+		"effect": "^3.21.2",
+		"protobufjs": "^8.4.2",
+		"uuid": "^14.0.0"
+	},
+	"peerDependencies": {
+		"zod": "^4.0.0"
+	},
+	"main": "./cjs/index.js",
+	"module": "./esm/index.js",
+	"types": "./dts/index.d.ts",
+	"exports": {
+		"./package.json": "./package.json",
+		".": {
+			"types": "./dts/index.d.ts",
+			"import": "./esm/index.js",
+			"default": "./cjs/index.js"
+		},
+		"./effect": {
+			"types": "./dts/effect.d.ts",
+			"import": "./esm/effect.js",
+			"default": "./cjs/effect.js"
+		}
+	}
+}