@livestore/common 0.3.0-dev.2 → 0.3.0-dev.21

package/dist/index.js

@@ -8,11 +8,11 @@ export * from './rehydrate-from-mutationlog.js';
 export * from './query-info.js';
 export * from './derived-mutations.js';
 export * from './sync/index.js';
-export * as Devtools from './devtools/index.js';
+export * as Devtools from './devtools/mod.js';
 export * from './debug-info.js';
 export * from './bounded-collections.js';
 export * from './version.js';
 export * from './query-builder/mod.js';
-export * from './sync/syncstate.js';
+export * as SyncState from './sync/syncstate.js';
 export * from './otel.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAA;AACzC,cAAc,WAAW,CAAA;AACzB,cAAc,oBAAoB,CAAA;AAClC,cAAc,mCAAmC,CAAA;AACjD,cAAc,eAAe,CAAA;AAC7B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,iCAAiC,CAAA;AAC/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,iBAAiB,CAAA;AAC/B,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAA;AAC/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,0BAA0B,CAAA;AACxC,cAAc,cAAc,CAAA;AAC5B,cAAc,wBAAwB,CAAA;AACtC,cAAc,qBAAqB,CAAA;AACnC,cAAc,WAAW,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAA;AACzC,cAAc,WAAW,CAAA;AACzB,cAAc,oBAAoB,CAAA;AAClC,cAAc,mCAAmC,CAAA;AACjD,cAAc,eAAe,CAAA;AAC7B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,iCAAiC,CAAA;AAC/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AACtC,cAAc,iBAAiB,CAAA;AAC/B,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAA;AAC7C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,0BAA0B,CAAA;AACxC,cAAc,cAAc,CAAA;AAC5B,cAAc,wBAAwB,CAAA;AACtC,OAAO,KAAK,SAAS,MAAM,qBAAqB,CAAA;AAChD,cAAc,WAAW,CAAA"}

package/dist/init-singleton-tables.d.ts

@@ -1,4 +1,4 @@
-import type { SynchronousDatabase } from './adapter-types.js';
+import type { SqliteDb } from './adapter-types.js';
 import type { LiveStoreSchema } from './schema/mod.js';
-export declare const initializeSingletonTables: (schema: LiveStoreSchema, db: SynchronousDatabase) => void;
+export declare const initializeSingletonTables: (schema: LiveStoreSchema, db: SqliteDb) => void;
 //# sourceMappingURL=init-singleton-tables.d.ts.map

package/dist/init-singleton-tables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"init-singleton-tables.d.ts","sourceRoot":"","sources":["../src/init-singleton-tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAItD,eAAO,MAAM,yBAAyB,WAAY,eAAe,MAAM,mBAAmB,SAkBzF,CAAA"}
+{"version":3,"file":"init-singleton-tables.d.ts","sourceRoot":"","sources":["../src/init-singleton-tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAItD,eAAO,MAAM,yBAAyB,GAAI,QAAQ,eAAe,EAAE,IAAI,QAAQ,SAkB9E,CAAA"}

package/dist/init-singleton-tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"init-singleton-tables.js","sourceRoot":"","sources":["../src/init-singleton-tables.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,EAAE,iBAAiB,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAElD,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,MAAuB,EAAE,EAAuB,EAAE,EAAE;IAC5F,KAAK,MAAM,CAAC,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,aAAa,GAAG,uBAAuB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;YAElE,MAAM,kBAAkB,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,IAAI,CAAC,CAAA;YAChE,MAAM,YAAY,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAE5E,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAA;YACzC,MAAM,WAAW,GAAG,GAAG,CAAA,eAAe,SAAS,KAAK,kBAAkB,CAAC,IAAI,CACzE,IAAI,CACL,YAAY,YAAY,mCAAmC,SAAS,0BAA0B,CAAA;YAE/F,MAAM,UAAU,GAAG,iBAAiB,CAAC,EAAE,GAAG,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,WAAW,CAAC,CAAA;YAExF,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;AACH,CAAC,CAAA"}
+{"version":3,"file":"init-singleton-tables.js","sourceRoot":"","sources":["../src/init-singleton-tables.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,EAAE,iBAAiB,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAElD,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,MAAuB,EAAE,EAAY,EAAE,EAAE;IACjF,KAAK,MAAM,CAAC,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,aAAa,GAAG,uBAAuB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;YAElE,MAAM,kBAAkB,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,IAAI,CAAC,CAAA;YAChE,MAAM,YAAY,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAE5E,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAA;YACzC,MAAM,WAAW,GAAG,GAAG,CAAA,eAAe,SAAS,KAAK,kBAAkB,CAAC,IAAI,CACzE,IAAI,CACL,YAAY,YAAY,mCAAmC,SAAS,0BAA0B,CAAA;YAE/F,MAAM,UAAU,GAAG,iBAAiB,CAAC,EAAE,GAAG,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,WAAW,CAAC,CAAA;YAExF,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;AACH,CAAC,CAAA"}

package/dist/leader-thread/LeaderSyncProcessor.d.ts

@@ -0,0 +1,39 @@
+import type { Scope } from '@livestore/utils/effect';
+import { Effect } from '@livestore/utils/effect';
+import type { SqliteDb } from '../adapter-types.js';
+import { UnexpectedError } from '../adapter-types.js';
+import type { LiveStoreSchema } from '../schema/mod.js';
+import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.js';
+export declare const BACKEND_PUSH_BATCH_SIZE = 50;
+/**
+ * The LeaderSyncProcessor manages synchronization of mutations between
+ * the local state and the sync backend, ensuring efficient and orderly processing.
+ *
+ * In the LeaderSyncProcessor, pulling always has precedence over pushing.
+ *
+ * Responsibilities:
+ * - Queueing incoming local mutations in a localPushMailbox.
+ * - Broadcasting mutations to client sessions via pull queues.
+ * - Pushing mutations to the sync backend.
+ *
+ * Notes:
+ *
+ * local push processing:
+ * - localPushMailbox:
+ *   - Maintains events in ascending order.
+ *   - Uses `Deferred` objects to resolve/reject events based on application success.
+ * - Processes events from the mailbox, applying mutations in batches.
+ * - Controlled by a `Latch` to manage execution flow.
+ * - The latch closes on pull receipt and re-opens post-pull completion.
+ * - Processes up to `maxBatchSize` events per cycle.
+ *
+ */
+export declare const makeLeaderSyncProcessor: ({ schema, dbMissing, dbMutationLog, clientId, initialBlockingSyncContext, }: {
+    schema: LiveStoreSchema;
+    /** Only used to know whether we can safely query dbMutationLog during setup execution */
+    dbMissing: boolean;
+    dbMutationLog: SqliteDb;
+    clientId: string;
+    initialBlockingSyncContext: InitialBlockingSyncContext;
+}) => Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope>;
+//# sourceMappingURL=LeaderSyncProcessor.d.ts.map

package/dist/leader-thread/LeaderSyncProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LeaderSyncProcessor.d.ts","sourceRoot":"","sources":["../../src/leader-thread/LeaderSyncProcessor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,KAAK,EAAU,MAAM,yBAAyB,CAAA;AACxE,OAAO,EAGL,MAAM,EAUP,MAAM,yBAAyB,CAAA;AAGhC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,KAAK,EAAE,eAAe,EAA2B,MAAM,kBAAkB,CAAA;AAgBhF,OAAO,KAAK,EAAE,0BAA0B,EAAmB,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAGlG,eAAO,MAAM,uBAAuB,KAAK,CAAA;AAQzC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,uBAAuB,GAAI,6EAMrC;IACD,MAAM,EAAE,eAAe,CAAA;IACvB,yFAAyF;IACzF,SAAS,EAAE,OAAO,CAAA;IAClB,aAAa,EAAE,QAAQ,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,0BAA0B,EAAE,0BAA0B,CAAA;CACvD,KAAG,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,CAqO/D,CAAA"}

package/dist/leader-thread/LeaderSyncProcessor.js

@@ -0,0 +1,527 @@
+import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
+import { BucketQueue, Deferred, Effect, Exit, FiberHandle, Option, OtelTracer, ReadonlyArray, Schema, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
+import { UnexpectedError } from '../adapter-types.js';
+import { EventId, getMutationDef, MUTATION_LOG_META_TABLE, MutationEvent, mutationLogMetaTable, SESSION_CHANGESET_META_TABLE, } from '../schema/mod.js';
+import { updateRows } from '../sql-queries/index.js';
+import { LeaderAheadError } from '../sync/sync.js';
+import * as SyncState from '../sync/syncstate.js';
+import { sql } from '../util.js';
+import { makeApplyMutation } from './apply-mutation.js';
+import { execSql } from './connection.js';
+import { getBackendHeadFromDb, getClientHeadFromDb, getMutationEventsSince, updateBackendHead } from './mutationlog.js';
+import { LeaderThreadCtx } from './types.js';
+export const BACKEND_PUSH_BATCH_SIZE = 50;
+/**
+ * The LeaderSyncProcessor manages synchronization of mutations between
+ * the local state and the sync backend, ensuring efficient and orderly processing.
+ *
+ * In the LeaderSyncProcessor, pulling always has precedence over pushing.
+ *
+ * Responsibilities:
+ * - Queueing incoming local mutations in a localPushMailbox.
+ * - Broadcasting mutations to client sessions via pull queues.
+ * - Pushing mutations to the sync backend.
+ *
+ * Notes:
+ *
+ * local push processing:
+ * - localPushMailbox:
+ *   - Maintains events in ascending order.
+ *   - Uses `Deferred` objects to resolve/reject events based on application success.
+ * - Processes events from the mailbox, applying mutations in batches.
+ * - Controlled by a `Latch` to manage execution flow.
+ * - The latch closes on pull receipt and re-opens post-pull completion.
+ * - Processes up to `maxBatchSize` events per cycle.
+ *
+ */
+export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clientId, initialBlockingSyncContext, }) => Effect.gen(function* () {
+    const syncBackendQueue = yield* BucketQueue.make();
+    const syncStateSref = yield* SubscriptionRef.make(undefined);
+    const isLocalEvent = (mutationEventEncoded) => {
+        const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation);
+        return mutationDef.options.clientOnly;
+    };
+    /**
+     * Tracks generations of queued local push events.
+     * If a batch is rejected, all subsequent push queue items with the same generation are also rejected,
+     * even if they would be valid on their own.
+     */
+    const currentLocalPushGenerationRef = { current: 0 };
+    // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
+    const ctxRef = {
+        current: undefined,
+    };
+    const localPushesQueue = yield* BucketQueue.make();
+    const localPushesLatch = yield* Effect.makeLatch(true);
+    const pullLatch = yield* Effect.makeLatch(true);
+    const push = (newEvents, options) => Effect.gen(function* () {
+        // TODO validate batch
+        if (newEvents.length === 0)
+            return;
+        // if (options.generation < currentLocalPushGenerationRef.current) {
+        //   debugger
+        //   // We can safely drop this batch as it's from a previous push generation
+        //   return
+        // }
+        if (clientId === 'client-b') {
+            // console.log(
+            //   'push from client session',
+            //   newEvents.map((item) => item.toJSON()),
+            // )
+        }
+        const waitForProcessing = options?.waitForProcessing ?? false;
+        const generation = currentLocalPushGenerationRef.current;
+        if (waitForProcessing) {
+            const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
+            const items = newEvents.map((mutationEventEncoded, i) => [mutationEventEncoded, deferreds[i], generation]);
+            yield* BucketQueue.offerAll(localPushesQueue, items);
+            yield* Effect.all(deferreds);
+        }
+        else {
+            const items = newEvents.map((mutationEventEncoded) => [mutationEventEncoded, undefined, generation]);
+            yield* BucketQueue.offerAll(localPushesQueue, items);
+        }
+    }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:local-push', {
+        attributes: {
+            batchSize: newEvents.length,
+            batch: TRACE_VERBOSE ? newEvents : undefined,
+        },
+        links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
+    }));
+    const pushPartial = ({ mutationEvent: partialMutationEvent, clientId, sessionId, }) => Effect.gen(function* () {
+        const syncState = yield* syncStateSref;
+        if (syncState === undefined)
+            return shouldNeverHappen('Not initialized');
+        const mutationDef = getMutationDef(schema, partialMutationEvent.mutation);
+        const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
+            ...partialMutationEvent,
+            clientId,
+            sessionId,
+            ...EventId.nextPair(syncState.localHead, mutationDef.options.clientOnly),
+        });
+        yield* push([mutationEventEncoded]);
+    }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie));
+    // Starts various background loops
+    const boot = ({ dbReady }) => Effect.gen(function* () {
+        const span = yield* Effect.currentSpan.pipe(Effect.orDie);
+        const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
+        const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
+        ctxRef.current = {
+            otelSpan,
+            span,
+            devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+        };
+        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbMutationLog);
+        const initialLocalHead = dbMissing ? EventId.ROOT : getClientHeadFromDb(dbMutationLog);
+        if (initialBackendHead > initialLocalHead.global) {
+            return shouldNeverHappen(`During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`);
+        }
+        const pendingMutationEvents = yield* getMutationEventsSince({
+            global: initialBackendHead,
+            client: EventId.clientDefault,
+        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))));
+        const initialSyncState = new SyncState.SyncState({
+            pending: pendingMutationEvents,
+            // On the leader we don't need a rollback tail beyond `pending` items
+            rollbackTail: [],
+            upstreamHead: { global: initialBackendHead, client: EventId.clientDefault },
+            localHead: initialLocalHead,
+        });
+        /** State transitions need to happen atomically, so we use a Ref to track the state */
+        yield* SubscriptionRef.set(syncStateSref, initialSyncState);
+        // Rehydrate sync queue
+        if (pendingMutationEvents.length > 0) {
+            const filteredBatch = pendingMutationEvents
+                // Don't sync clientOnly mutations
+                .filter((mutationEventEncoded) => {
+                const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation);
+                return mutationDef.options.clientOnly === false;
+            });
+            yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch);
+        }
+        const shutdownOnError = (cause) => Effect.gen(function* () {
+            yield* shutdownChannel.send(UnexpectedError.make({ cause }));
+            yield* Effect.die(cause);
+        });
+        yield* backgroundApplyLocalPushes({
+            localPushesLatch,
+            localPushesQueue,
+            pullLatch,
+            syncStateSref,
+            syncBackendQueue,
+            schema,
+            isLocalEvent,
+            otelSpan,
+            currentLocalPushGenerationRef,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped);
+        const backendPushingFiberHandle = yield* FiberHandle.make();
+        yield* FiberHandle.run(backendPushingFiberHandle, backgroundBackendPushing({
+            dbReady,
+            syncBackendQueue,
+            otelSpan,
+            devtoolsLatch: ctxRef.current?.devtoolsLatch,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)));
+        yield* backgroundBackendPulling({
+            dbReady,
+            initialBackendHead,
+            isLocalEvent,
+            restartBackendPushing: (filteredRebasedPending) => Effect.gen(function* () {
+                // Stop current pushing fiber
+                yield* FiberHandle.clear(backendPushingFiberHandle);
+                // Reset the sync queue
+                yield* BucketQueue.clear(syncBackendQueue);
+                yield* BucketQueue.offerAll(syncBackendQueue, filteredRebasedPending);
+                // Restart pushing fiber
+                yield* FiberHandle.run(backendPushingFiberHandle, backgroundBackendPushing({
+                    dbReady,
+                    syncBackendQueue,
+                    otelSpan,
+                    devtoolsLatch: ctxRef.current?.devtoolsLatch,
+                }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)));
+            }),
+            syncStateSref,
+            localPushesLatch,
+            pullLatch,
+            otelSpan,
+            initialBlockingSyncContext,
+            devtoolsLatch: ctxRef.current?.devtoolsLatch,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped);
+        return { initialLeaderHead: initialLocalHead };
+    }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'));
+    return {
+        push,
+        pushPartial,
+        boot,
+        syncState: Subscribable.make({
+            get: Effect.gen(function* () {
+                const syncState = yield* syncStateSref;
+                if (syncState === undefined)
+                    return shouldNeverHappen('Not initialized');
+                return syncState;
+            }),
+            changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
+        }),
+    };
+});
+const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendQueue, schema, isLocalEvent, otelSpan, currentLocalPushGenerationRef, }) => Effect.gen(function* () {
+    const { connectedClientSessionPullQueues, clientId } = yield* LeaderThreadCtx;
+    const applyMutationItems = yield* makeApplyMutationItems;
+    while (true) {
+        // TODO make batch size configurable
+        const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, 10);
+        // Wait for the backend pulling to finish
+        yield* localPushesLatch.await;
+        // Prevent backend pull processing until this local push is finished
+        yield* pullLatch.close;
+        // Since the generation might have changed since enqueuing, we need to filter out items with older generation
+        // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
+        const filteredBatchItems = batchItems
+            .filter(([_1, _2, generation]) => generation === currentLocalPushGenerationRef.current)
+            .map(([mutationEventEncoded, deferred]) => [mutationEventEncoded, deferred]);
+        if (filteredBatchItems.length === 0) {
+            // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
+            // Allow the backend pulling to start
+            yield* pullLatch.open;
+            continue;
+        }
+        const [newEvents, deferreds] = ReadonlyArray.unzip(filteredBatchItems);
+        const syncState = yield* syncStateSref;
+        if (syncState === undefined)
+            return shouldNeverHappen('Not initialized');
+        const updateResult = SyncState.updateSyncState({
+            syncState,
+            payload: { _tag: 'local-push', newEvents },
+            isLocalEvent,
+            isEqualEvent: MutationEvent.isEqualEncoded,
+        });
+        switch (updateResult._tag) {
+            case 'unexpected-error': {
+                otelSpan?.addEvent('local-push:unexpected-error', {
+                    batchSize: newEvents.length,
+                    newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+                });
+                return yield* Effect.fail(updateResult.cause);
+            }
+            case 'rebase': {
+                return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
+            }
+            case 'reject': {
+                otelSpan?.addEvent('local-push:reject', {
+                    batchSize: newEvents.length,
+                    updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+                });
+                /*
+      
+                TODO: how to test this?
+                */
+                currentLocalPushGenerationRef.current++;
+                const nextGeneration = currentLocalPushGenerationRef.current;
+                const providedId = newEvents.at(0).id;
+                // All subsequent pushes with same generation should be rejected as well
+                // We're also handling the case where the localPushQueue already contains events
+                // from the next generation which we preserve in the queue
+                const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, (item) => item[2] >= nextGeneration);
+                if ((yield* BucketQueue.size(localPushesQueue)) > 0) {
+                    console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
+                    debugger;
+                }
+                const allDeferredsToReject = [
+                    ...deferreds,
+                    ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+                ].filter(isNotUndefined);
+                yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({
+                    minimumExpectedId: updateResult.expectedMinimumId,
+                    providedId,
+                    // nextGeneration,
+                })));
+                // Allow the backend pulling to start
+                yield* pullLatch.open;
+                // In this case we're skipping state update and down/upstream processing
+                // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+                continue;
+            }
+            case 'advance': {
+                break;
+            }
+            default: {
+                casesHandled(updateResult);
+            }
+        }
+        yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState);
+        if (clientId === 'client-b') {
+            // yield* Effect.log('offer upstream-advance due to local-push')
+            // debugger
+        }
+        yield* connectedClientSessionPullQueues.offer({
+            payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents },
+            remaining: 0,
+        });
+        otelSpan?.addEvent('local-push', {
+            batchSize: newEvents.length,
+            updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+        });
+        // Don't sync clientOnly mutations
+        const filteredBatch = updateResult.newEvents.filter((mutationEventEncoded) => {
+            const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation);
+            return mutationDef.options.clientOnly === false;
+        });
+        yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch);
+        yield* applyMutationItems({ batchItems: newEvents, deferreds });
+        // Allow the backend pulling to start
+        yield* pullLatch.open;
+    }
+});
+// TODO how to handle errors gracefully
+const makeApplyMutationItems = Effect.gen(function* () {
+    const leaderThreadCtx = yield* LeaderThreadCtx;
+    const { dbReadModel: db, dbMutationLog } = leaderThreadCtx;
+    const applyMutation = yield* makeApplyMutation;
+    return ({ batchItems, deferreds }) => Effect.gen(function* () {
+        db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
+        dbMutationLog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
+        yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
+            if (Exit.isSuccess(exit))
+                return;
+            // Rollback in case of an error
+            db.execute('ROLLBACK', undefined);
+            dbMutationLog.execute('ROLLBACK', undefined);
+        }));
+        for (let i = 0; i < batchItems.length; i++) {
+            yield* applyMutation(batchItems[i]);
+            if (deferreds?.[i] !== undefined) {
+                yield* Deferred.succeed(deferreds[i], void 0);
+            }
+        }
+        db.execute('COMMIT', undefined); // Commit the transaction
+        dbMutationLog.execute('COMMIT', undefined); // Commit the transaction
+    }).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems', {
+        attributes: { count: batchItems.length },
+    }), Effect.tapCauseLogPretty, UnexpectedError.mapToUnexpectedError);
+});
+const backgroundBackendPulling = ({ dbReady, initialBackendHead, isLocalEvent, restartBackendPushing, otelSpan, syncStateSref, localPushesLatch, pullLatch, devtoolsLatch, initialBlockingSyncContext, }) => Effect.gen(function* () {
+    const { syncBackend, dbReadModel: db, dbMutationLog, connectedClientSessionPullQueues, schema, clientId, } = yield* LeaderThreadCtx;
+    if (syncBackend === undefined)
+        return;
+    const cursorInfo = yield* getCursorInfo(initialBackendHead);
+    const applyMutationItems = yield* makeApplyMutationItems;
+    const onNewPullChunk = (newEvents, remaining) => Effect.gen(function* () {
+        if (newEvents.length === 0)
+            return;
+        if (devtoolsLatch !== undefined) {
+            yield* devtoolsLatch.await;
+        }
+        // Prevent more local pushes from being processed until this pull is finished
+        yield* localPushesLatch.close;
+        // Wait for pending local pushes to finish
+        yield* pullLatch.await;
+        const syncState = yield* syncStateSref;
+        if (syncState === undefined)
+            return shouldNeverHappen('Not initialized');
+        const trimRollbackUntil = newEvents.at(-1).id;
+        const updateResult = SyncState.updateSyncState({
+            syncState,
+            payload: { _tag: 'upstream-advance', newEvents, trimRollbackUntil },
+            isLocalEvent,
+            isEqualEvent: MutationEvent.isEqualEncoded,
+            ignoreLocalEvents: true,
+        });
+        if (updateResult._tag === 'reject') {
+            return shouldNeverHappen('The leader thread should never reject upstream advances');
+        }
+        else if (updateResult._tag === 'unexpected-error') {
+            otelSpan?.addEvent('backend-pull:unexpected-error', {
+                newEventsCount: newEvents.length,
+                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+            });
+            return yield* Effect.fail(updateResult.cause);
+        }
+        const newBackendHead = newEvents.at(-1).id;
+        updateBackendHead(dbMutationLog, newBackendHead);
+        if (updateResult._tag === 'rebase') {
+            otelSpan?.addEvent('backend-pull:rebase', {
+                newEventsCount: newEvents.length,
+                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+                rollbackCount: updateResult.eventsToRollback.length,
+                updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+            });
+            const filteredRebasedPending = updateResult.newSyncState.pending.filter((mutationEvent) => {
+                const mutationDef = getMutationDef(schema, mutationEvent.mutation);
+                return mutationDef.options.clientOnly === false;
+            });
+            yield* restartBackendPushing(filteredRebasedPending);
+            if (updateResult.eventsToRollback.length > 0) {
+                yield* rollback({ db, dbMutationLog, eventIdsToRollback: updateResult.eventsToRollback.map((_) => _.id) });
+            }
+            yield* connectedClientSessionPullQueues.offer({
+                payload: {
+                    _tag: 'upstream-rebase',
+                    newEvents: updateResult.newEvents,
+                    rollbackUntil: updateResult.eventsToRollback.at(0).id,
+                    trimRollbackUntil,
+                },
+                remaining,
+            });
+        }
+        else {
+            otelSpan?.addEvent('backend-pull:advance', {
+                newEventsCount: newEvents.length,
+                updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+            });
+            if (clientId === 'client-b') {
+                // yield* Effect.log('offer upstream-advance due to pull')
+            }
+            yield* connectedClientSessionPullQueues.offer({
+                payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents, trimRollbackUntil },
+                remaining,
+            });
+        }
+        trimChangesetRows(db, newBackendHead);
+        yield* applyMutationItems({ batchItems: updateResult.newEvents, deferreds: undefined });
+        yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState);
+        if (remaining === 0) {
+            // Allow local pushes to be processed again
+            yield* localPushesLatch.open;
+        }
+    });
+    yield* syncBackend.pull(cursorInfo).pipe(
+    // TODO only take from queue while connected
+    Stream.tap(({ batch, remaining }) => Effect.gen(function* () {
+        // yield* Effect.spanEvent('batch', {
+        //   attributes: {
+        //     batchSize: batch.length,
+        //     batch: TRACE_VERBOSE ? batch : undefined,
+        //   },
+        // })
+        // Wait for the db to be initially created
+        yield* dbReady;
+        // NOTE we only want to take process mutations when the sync backend is connected
+        // (e.g. needed for simulating being offline)
+        // TODO remove when there's a better way to handle this in stream above
+        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
+        yield* onNewPullChunk(batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)), remaining);
+        yield* initialBlockingSyncContext.update({ processed: batch.length, remaining });
+    })), Stream.runDrain, Effect.interruptible);
+}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pulling'));
+const rollback = ({ db, dbMutationLog, eventIdsToRollback, }) => Effect.gen(function* () {
+    const rollbackEvents = db
+        .select(sql `SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`)
+        .map((_) => ({ id: { global: _.idGlobal, client: _.idClient }, changeset: _.changeset, debug: _.debug }))
+        .toSorted((a, b) => EventId.compare(a.id, b.id));
+    // Apply changesets in reverse order
+    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
+        const { changeset } = rollbackEvents[i];
+        if (changeset !== null) {
+            db.makeChangeset(changeset).invert().apply();
+        }
+    }
+    const eventIdPairChunks = ReadonlyArray.chunksOf(100)(eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`));
+    // Delete the changeset rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+        db.execute(sql `DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`);
+    }
+    // Delete the mutation log rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+        dbMutationLog.execute(sql `DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`);
+    }
+}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:rollback', {
+    attributes: { count: eventIdsToRollback.length },
+}));
+const getCursorInfo = (remoteHead) => Effect.gen(function* () {
+    const { dbMutationLog } = yield* LeaderThreadCtx;
+    if (remoteHead === EventId.ROOT.global)
+        return Option.none();
+    const MutationlogQuerySchema = Schema.Struct({
+        syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
+    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head);
+    const syncMetadataOption = yield* Effect.sync(() => dbMutationLog.select(sql `SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idClient ASC LIMIT 1`)).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie);
+    return Option.some({
+        cursor: { global: remoteHead, client: EventId.clientDefault },
+        metadata: syncMetadataOption,
+    });
+}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }));
+const backgroundBackendPushing = ({ dbReady, syncBackendQueue, otelSpan, devtoolsLatch, }) => Effect.gen(function* () {
+    const { syncBackend, dbMutationLog } = yield* LeaderThreadCtx;
+    if (syncBackend === undefined)
+        return;
+    yield* dbReady;
+    while (true) {
+        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
+        // TODO make batch size configurable
+        const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, BACKEND_PUSH_BATCH_SIZE);
+        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
+        if (devtoolsLatch !== undefined) {
+            yield* devtoolsLatch.await;
+        }
+        otelSpan?.addEvent('backend-push', {
+            batchSize: queueItems.length,
+            batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
+        });
+        // TODO handle push errors (should only happen during concurrent pull+push)
+        const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either);
+        if (pushResult._tag === 'Left') {
+            if (LS_DEV) {
+                yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() });
+            }
+            otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() });
+            // wait for interrupt caused by background pulling which will then restart pushing
+            return yield* Effect.never;
+        }
+        const { metadata } = pushResult.right;
+        // TODO try to do this in a single query
+        for (let i = 0; i < queueItems.length; i++) {
+            const mutationEventEncoded = queueItems[i];
+            yield* execSql(dbMutationLog, ...updateRows({
+                tableName: MUTATION_LOG_META_TABLE,
+                columns: mutationLogMetaTable.sqliteDef.columns,
+                where: { idGlobal: mutationEventEncoded.id.global, idClient: mutationEventEncoded.id.client },
+                updateValues: { syncMetadataJson: metadata[i] },
+            }));
+        }
+    }
+}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'));
+const trimChangesetRows = (db, newHead) => {
+    // Since we're using the session changeset rows to query for the current head,
+    // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
+    db.execute(sql `DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`);
+};
+//# sourceMappingURL=LeaderSyncProcessor.js.map