@livestore/common 0.4.0-dev.9 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/ClientSessionLeaderThreadProxy.d.ts +27 -12
- package/dist/ClientSessionLeaderThreadProxy.d.ts.map +1 -1
- package/dist/ClientSessionLeaderThreadProxy.js.map +1 -1
- package/dist/WorkerTransportError.d.ts +11 -0
- package/dist/WorkerTransportError.d.ts.map +1 -0
- package/dist/WorkerTransportError.js +11 -0
- package/dist/WorkerTransportError.js.map +1 -0
- package/dist/adapter-types.d.ts +37 -7
- package/dist/adapter-types.d.ts.map +1 -1
- package/dist/adapter-types.js +27 -1
- package/dist/adapter-types.js.map +1 -1
- package/dist/bounded-collections.d.ts.map +1 -1
- package/dist/bounded-collections.js +6 -4
- package/dist/bounded-collections.js.map +1 -1
- package/dist/debug-info.d.ts.map +1 -1
- package/dist/debug-info.js +33 -6
- package/dist/debug-info.js.map +1 -1
- package/dist/devtools/devtools-compatibility.test.d.ts +2 -0
- package/dist/devtools/devtools-compatibility.test.d.ts.map +1 -0
- package/dist/devtools/devtools-compatibility.test.js +15 -0
- package/dist/devtools/devtools-compatibility.test.js.map +1 -0
- package/dist/devtools/devtools-messages-client-session.d.ts +55 -24
- package/dist/devtools/devtools-messages-client-session.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-client-session.js +22 -5
- package/dist/devtools/devtools-messages-client-session.js.map +1 -1
- package/dist/devtools/devtools-messages-common.d.ts +11 -14
- package/dist/devtools/devtools-messages-common.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-common.js +7 -9
- package/dist/devtools/devtools-messages-common.js.map +1 -1
- package/dist/devtools/devtools-messages-leader.d.ts +65 -30
- package/dist/devtools/devtools-messages-leader.d.ts.map +1 -1
- package/dist/devtools/devtools-messages-leader.js +29 -11
- package/dist/devtools/devtools-messages-leader.js.map +1 -1
- package/dist/devtools/devtools-sessioninfo.d.ts +14 -2
- package/dist/devtools/devtools-sessioninfo.d.ts.map +1 -1
- package/dist/devtools/devtools-sessioninfo.js +7 -4
- package/dist/devtools/devtools-sessioninfo.js.map +1 -1
- package/dist/devtools/mod.d.ts +13 -2
- package/dist/devtools/mod.d.ts.map +1 -1
- package/dist/devtools/mod.js +10 -3
- package/dist/devtools/mod.js.map +1 -1
- package/dist/errors.d.ts +48 -18
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +20 -12
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.d.ts +53 -6
- package/dist/leader-thread/LeaderSyncProcessor.d.ts.map +1 -1
- package/dist/leader-thread/LeaderSyncProcessor.js +325 -257
- package/dist/leader-thread/LeaderSyncProcessor.js.map +1 -1
- package/dist/leader-thread/RejectedPushError.d.ts +107 -0
- package/dist/leader-thread/RejectedPushError.d.ts.map +1 -0
- package/dist/leader-thread/RejectedPushError.js +78 -0
- package/dist/leader-thread/RejectedPushError.js.map +1 -0
- package/dist/leader-thread/connection.js +1 -1
- package/dist/leader-thread/connection.js.map +1 -1
- package/dist/leader-thread/eventlog.d.ts +19 -14
- package/dist/leader-thread/eventlog.d.ts.map +1 -1
- package/dist/leader-thread/eventlog.js +78 -18
- package/dist/leader-thread/eventlog.js.map +1 -1
- package/dist/leader-thread/leader-worker-devtools.d.ts +1 -2
- package/dist/leader-thread/leader-worker-devtools.d.ts.map +1 -1
- package/dist/leader-thread/leader-worker-devtools.js +90 -58
- package/dist/leader-thread/leader-worker-devtools.js.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.d.ts +15 -7
- package/dist/leader-thread/make-leader-thread-layer.d.ts.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.js +49 -17
- package/dist/leader-thread/make-leader-thread-layer.js.map +1 -1
- package/dist/leader-thread/make-leader-thread-layer.test.d.ts +2 -0
- package/dist/leader-thread/make-leader-thread-layer.test.d.ts.map +1 -0
- package/dist/leader-thread/make-leader-thread-layer.test.js +32 -0
- package/dist/leader-thread/make-leader-thread-layer.test.js.map +1 -0
- package/dist/leader-thread/materialize-event.d.ts +1 -1
- package/dist/leader-thread/materialize-event.d.ts.map +1 -1
- package/dist/leader-thread/materialize-event.js +28 -9
- package/dist/leader-thread/materialize-event.js.map +1 -1
- package/dist/leader-thread/mod.d.ts +1 -0
- package/dist/leader-thread/mod.d.ts.map +1 -1
- package/dist/leader-thread/mod.js +1 -0
- package/dist/leader-thread/mod.js.map +1 -1
- package/dist/leader-thread/recreate-db.d.ts +2 -2
- package/dist/leader-thread/recreate-db.d.ts.map +1 -1
- package/dist/leader-thread/recreate-db.js +6 -6
- package/dist/leader-thread/recreate-db.js.map +1 -1
- package/dist/leader-thread/shutdown-channel.d.ts +2 -2
- package/dist/leader-thread/shutdown-channel.d.ts.map +1 -1
- package/dist/leader-thread/shutdown-channel.js +2 -2
- package/dist/leader-thread/shutdown-channel.js.map +1 -1
- package/dist/leader-thread/stream-events.d.ts +56 -0
- package/dist/leader-thread/stream-events.d.ts.map +1 -0
- package/dist/leader-thread/stream-events.js +167 -0
- package/dist/leader-thread/stream-events.js.map +1 -0
- package/dist/leader-thread/types.d.ts +95 -17
- package/dist/leader-thread/types.d.ts.map +1 -1
- package/dist/leader-thread/types.js +13 -0
- package/dist/leader-thread/types.js.map +1 -1
- package/dist/logging.d.ts +40 -0
- package/dist/logging.d.ts.map +1 -0
- package/dist/logging.js +33 -0
- package/dist/logging.js.map +1 -0
- package/dist/make-client-session.d.ts +5 -3
- package/dist/make-client-session.d.ts.map +1 -1
- package/dist/make-client-session.js +7 -4
- package/dist/make-client-session.js.map +1 -1
- package/dist/materializer-helper.d.ts +6 -6
- package/dist/materializer-helper.d.ts.map +1 -1
- package/dist/materializer-helper.js +18 -8
- package/dist/materializer-helper.js.map +1 -1
- package/dist/otel.d.ts +2 -1
- package/dist/otel.d.ts.map +1 -1
- package/dist/otel.js +7 -2
- package/dist/otel.js.map +1 -1
- package/dist/rematerialize-from-eventlog.d.ts +3 -3
- package/dist/rematerialize-from-eventlog.d.ts.map +1 -1
- package/dist/rematerialize-from-eventlog.js +40 -29
- package/dist/rematerialize-from-eventlog.js.map +1 -1
- package/dist/schema/EventDef/define.d.ts +161 -0
- package/dist/schema/EventDef/define.d.ts.map +1 -0
- package/dist/schema/EventDef/define.js +140 -0
- package/dist/schema/EventDef/define.js.map +1 -0
- package/dist/schema/EventDef/deprecated.d.ts +99 -0
- package/dist/schema/EventDef/deprecated.d.ts.map +1 -0
- package/dist/schema/EventDef/deprecated.js +144 -0
- package/dist/schema/EventDef/deprecated.js.map +1 -0
- package/dist/schema/EventDef/deprecated.test.d.ts +2 -0
- package/dist/schema/EventDef/deprecated.test.d.ts.map +1 -0
- package/dist/schema/EventDef/deprecated.test.js +95 -0
- package/dist/schema/EventDef/deprecated.test.js.map +1 -0
- package/dist/schema/EventDef/event-def.d.ts +110 -0
- package/dist/schema/EventDef/event-def.d.ts.map +1 -0
- package/dist/schema/EventDef/event-def.js +2 -0
- package/dist/schema/EventDef/event-def.js.map +1 -0
- package/dist/schema/EventDef/facts.d.ts +118 -0
- package/dist/schema/EventDef/facts.d.ts.map +1 -0
- package/dist/schema/EventDef/facts.js +53 -0
- package/dist/schema/EventDef/facts.js.map +1 -0
- package/dist/schema/EventDef/materializer.d.ts +155 -0
- package/dist/schema/EventDef/materializer.d.ts.map +1 -0
- package/dist/schema/EventDef/materializer.js +83 -0
- package/dist/schema/EventDef/materializer.js.map +1 -0
- package/dist/schema/EventDef/mod.d.ts +6 -0
- package/dist/schema/EventDef/mod.d.ts.map +1 -0
- package/dist/schema/EventDef/mod.js +6 -0
- package/dist/schema/EventDef/mod.js.map +1 -0
- package/dist/schema/EventSequenceNumber/client.d.ts +136 -0
- package/dist/schema/EventSequenceNumber/client.d.ts.map +1 -0
- package/dist/schema/EventSequenceNumber/client.js +193 -0
- package/dist/schema/EventSequenceNumber/client.js.map +1 -0
- package/dist/schema/EventSequenceNumber/global.d.ts +15 -0
- package/dist/schema/EventSequenceNumber/global.d.ts.map +1 -0
- package/dist/schema/EventSequenceNumber/global.js +14 -0
- package/dist/schema/EventSequenceNumber/global.js.map +1 -0
- package/dist/schema/EventSequenceNumber/mod.d.ts +37 -0
- package/dist/schema/EventSequenceNumber/mod.d.ts.map +1 -0
- package/dist/schema/EventSequenceNumber/mod.js +37 -0
- package/dist/schema/EventSequenceNumber/mod.js.map +1 -0
- package/dist/schema/EventSequenceNumber.test.js +44 -44
- package/dist/schema/EventSequenceNumber.test.js.map +1 -1
- package/dist/schema/{LiveStoreEvent.d.ts → LiveStoreEvent/client.d.ts} +102 -111
- package/dist/schema/LiveStoreEvent/client.d.ts.map +1 -0
- package/dist/schema/LiveStoreEvent/client.js +176 -0
- package/dist/schema/LiveStoreEvent/client.js.map +1 -0
- package/dist/schema/LiveStoreEvent/client.test.d.ts +2 -0
- package/dist/schema/LiveStoreEvent/client.test.d.ts.map +1 -0
- package/dist/schema/LiveStoreEvent/client.test.js +111 -0
- package/dist/schema/LiveStoreEvent/client.test.js.map +1 -0
- package/dist/schema/LiveStoreEvent/for-event-def.d.ts +52 -0
- package/dist/schema/LiveStoreEvent/for-event-def.d.ts.map +1 -0
- package/dist/schema/LiveStoreEvent/for-event-def.js +2 -0
- package/dist/schema/LiveStoreEvent/for-event-def.js.map +1 -0
- package/dist/schema/LiveStoreEvent/global.d.ts +36 -0
- package/dist/schema/LiveStoreEvent/global.d.ts.map +1 -0
- package/dist/schema/LiveStoreEvent/global.js +31 -0
- package/dist/schema/LiveStoreEvent/global.js.map +1 -0
- package/dist/schema/LiveStoreEvent/input.d.ts +46 -0
- package/dist/schema/LiveStoreEvent/input.d.ts.map +1 -0
- package/dist/schema/LiveStoreEvent/input.js +26 -0
- package/dist/schema/LiveStoreEvent/input.js.map +1 -0
- package/dist/schema/LiveStoreEvent/mod.d.ts +5 -0
- package/dist/schema/LiveStoreEvent/mod.d.ts.map +1 -0
- package/dist/schema/LiveStoreEvent/mod.js +5 -0
- package/dist/schema/LiveStoreEvent/mod.js.map +1 -0
- package/dist/schema/events.d.ts +1 -1
- package/dist/schema/events.d.ts.map +1 -1
- package/dist/schema/events.js +1 -1
- package/dist/schema/events.js.map +1 -1
- package/dist/schema/mod.d.ts +6 -4
- package/dist/schema/mod.d.ts.map +1 -1
- package/dist/schema/mod.js +5 -4
- package/dist/schema/mod.js.map +1 -1
- package/dist/schema/schema.d.ts +16 -1
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/schema.js +32 -4
- package/dist/schema/schema.js.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.d.ts +2 -1
- package/dist/schema/state/sqlite/client-document-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.js +36 -15
- package/dist/schema/state/sqlite/client-document-def.js.map +1 -1
- package/dist/schema/state/sqlite/client-document-def.test.js +121 -2
- package/dist/schema/state/sqlite/client-document-def.test.js.map +1 -1
- package/dist/schema/state/sqlite/column-annotations.d.ts.map +1 -1
- package/dist/schema/state/sqlite/column-annotations.js +1 -1
- package/dist/schema/state/sqlite/column-annotations.js.map +1 -1
- package/dist/schema/state/sqlite/column-annotations.test.js +2 -2
- package/dist/schema/state/sqlite/column-annotations.test.js.map +1 -1
- package/dist/schema/state/sqlite/column-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/column-def.js +96 -47
- package/dist/schema/state/sqlite/column-def.js.map +1 -1
- package/dist/schema/state/sqlite/column-def.test.js +51 -12
- package/dist/schema/state/sqlite/column-def.test.js.map +1 -1
- package/dist/schema/state/sqlite/column-spec.d.ts.map +1 -1
- package/dist/schema/state/sqlite/column-spec.js +30 -12
- package/dist/schema/state/sqlite/column-spec.js.map +1 -1
- package/dist/schema/state/sqlite/column-spec.test.js +24 -15
- package/dist/schema/state/sqlite/column-spec.test.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/ast/sqlite.js +2 -2
- package/dist/schema/state/sqlite/db-schema/ast/sqlite.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/field-defs.d.ts +16 -10
- package/dist/schema/state/sqlite/db-schema/dsl/field-defs.d.ts.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/field-defs.js +15 -4
- package/dist/schema/state/sqlite/db-schema/dsl/field-defs.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/field-defs.test.js +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/field-defs.test.js.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.d.ts.map +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.js +1 -1
- package/dist/schema/state/sqlite/db-schema/dsl/mod.js.map +1 -1
- package/dist/schema/state/sqlite/mod.d.ts +2 -2
- package/dist/schema/state/sqlite/mod.d.ts.map +1 -1
- package/dist/schema/state/sqlite/mod.js +5 -7
- package/dist/schema/state/sqlite/mod.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/api.d.ts +51 -22
- package/dist/schema/state/sqlite/query-builder/api.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/astToSql.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/astToSql.js +99 -22
- package/dist/schema/state/sqlite/query-builder/astToSql.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.d.ts +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.d.ts.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.js +28 -15
- package/dist/schema/state/sqlite/query-builder/impl.js.map +1 -1
- package/dist/schema/state/sqlite/query-builder/impl.test.js +231 -93
- package/dist/schema/state/sqlite/query-builder/impl.test.js.map +1 -1
- package/dist/schema/state/sqlite/schema-helpers.d.ts +2 -2
- package/dist/schema/state/sqlite/schema-helpers.d.ts.map +1 -1
- package/dist/schema/state/sqlite/schema-helpers.js +24 -14
- package/dist/schema/state/sqlite/schema-helpers.js.map +1 -1
- package/dist/schema/state/sqlite/schema-helpers.test.d.ts +2 -0
- package/dist/schema/state/sqlite/schema-helpers.test.d.ts.map +1 -0
- package/dist/schema/state/sqlite/schema-helpers.test.js +36 -0
- package/dist/schema/state/sqlite/schema-helpers.test.js.map +1 -0
- package/dist/schema/state/sqlite/{system-tables.d.ts → system-tables/eventlog-tables.d.ts} +21 -450
- package/dist/schema/state/sqlite/system-tables/eventlog-tables.d.ts.map +1 -0
- package/dist/schema/state/sqlite/system-tables/eventlog-tables.js +54 -0
- package/dist/schema/state/sqlite/system-tables/eventlog-tables.js.map +1 -0
- package/dist/schema/state/sqlite/system-tables/mod.d.ts +3 -0
- package/dist/schema/state/sqlite/system-tables/mod.d.ts.map +1 -0
- package/dist/schema/state/sqlite/system-tables/mod.js +3 -0
- package/dist/schema/state/sqlite/system-tables/mod.js.map +1 -0
- package/dist/schema/state/sqlite/system-tables/state-tables.d.ts +456 -0
- package/dist/schema/state/sqlite/system-tables/state-tables.d.ts.map +1 -0
- package/dist/schema/state/sqlite/system-tables/state-tables.js +55 -0
- package/dist/schema/state/sqlite/system-tables/state-tables.js.map +1 -0
- package/dist/schema/state/sqlite/table-def.d.ts +5 -3
- package/dist/schema/state/sqlite/table-def.d.ts.map +1 -1
- package/dist/schema/state/sqlite/table-def.js +1 -1
- package/dist/schema/state/sqlite/table-def.js.map +1 -1
- package/dist/schema/state/sqlite/table-def.test.js +92 -3
- package/dist/schema/state/sqlite/table-def.test.js.map +1 -1
- package/dist/schema/unknown-events.d.ts +47 -0
- package/dist/schema/unknown-events.d.ts.map +1 -0
- package/dist/schema/unknown-events.js +69 -0
- package/dist/schema/unknown-events.js.map +1 -0
- package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.d.ts +2 -0
- package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.d.ts.map +1 -0
- package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.js +73 -0
- package/dist/schema-management/__tests__/migrations-autoincrement-quoting.test.js.map +1 -0
- package/dist/schema-management/common.js +2 -2
- package/dist/schema-management/common.js.map +1 -1
- package/dist/schema-management/migrations.d.ts +32 -2
- package/dist/schema-management/migrations.d.ts.map +1 -1
- package/dist/schema-management/migrations.js +38 -6
- package/dist/schema-management/migrations.js.map +1 -1
- package/dist/schema-management/validate-schema.d.ts +3 -3
- package/dist/schema-management/validate-schema.d.ts.map +1 -1
- package/dist/schema-management/validate-schema.js +2 -2
- package/dist/schema-management/validate-schema.js.map +1 -1
- package/dist/sql-queries/sql-queries.d.ts.map +1 -1
- package/dist/sql-queries/sql-queries.js +18 -6
- package/dist/sql-queries/sql-queries.js.map +1 -1
- package/dist/sql-queries/sql-query-builder.d.ts.map +1 -1
- package/dist/sql-queries/sql-query-builder.js.map +1 -1
- package/dist/sqlite-db-helper.js +3 -3
- package/dist/sqlite-db-helper.js.map +1 -1
- package/dist/sqlite-types.d.ts +5 -5
- package/dist/sqlite-types.d.ts.map +1 -1
- package/dist/sqlite-types.js.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.d.ts +12 -12
- package/dist/sync/ClientSessionSyncProcessor.d.ts.map +1 -1
- package/dist/sync/ClientSessionSyncProcessor.js +99 -114
- package/dist/sync/ClientSessionSyncProcessor.js.map +1 -1
- package/dist/sync/errors.d.ts +0 -33
- package/dist/sync/errors.d.ts.map +1 -1
- package/dist/sync/errors.js +5 -22
- package/dist/sync/errors.js.map +1 -1
- package/dist/sync/index.d.ts +2 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +2 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/mock-sync-backend.d.ts +10 -8
- package/dist/sync/mock-sync-backend.d.ts.map +1 -1
- package/dist/sync/mock-sync-backend.js +71 -69
- package/dist/sync/mock-sync-backend.js.map +1 -1
- package/dist/sync/next/compact-events.d.ts.map +1 -1
- package/dist/sync/next/compact-events.js +11 -12
- package/dist/sync/next/compact-events.js.map +1 -1
- package/dist/sync/next/facts.d.ts +5 -5
- package/dist/sync/next/facts.d.ts.map +1 -1
- package/dist/sync/next/facts.js +7 -8
- package/dist/sync/next/facts.js.map +1 -1
- package/dist/sync/next/history-dag-common.d.ts +54 -15
- package/dist/sync/next/history-dag-common.d.ts.map +1 -1
- package/dist/sync/next/history-dag-common.js +198 -9
- package/dist/sync/next/history-dag-common.js.map +1 -1
- package/dist/sync/next/history-dag.d.ts.map +1 -1
- package/dist/sync/next/history-dag.js +11 -11
- package/dist/sync/next/history-dag.js.map +1 -1
- package/dist/sync/next/rebase-events.d.ts +5 -5
- package/dist/sync/next/rebase-events.d.ts.map +1 -1
- package/dist/sync/next/rebase-events.js +6 -6
- package/dist/sync/next/rebase-events.js.map +1 -1
- package/dist/sync/next/test/compact-events.calculator.test.js +2 -2
- package/dist/sync/next/test/compact-events.calculator.test.js.map +1 -1
- package/dist/sync/next/test/compact-events.test.d.ts.map +1 -1
- package/dist/sync/next/test/compact-events.test.js +2 -2
- package/dist/sync/next/test/compact-events.test.js.map +1 -1
- package/dist/sync/next/test/event-fixtures.d.ts +2 -2
- package/dist/sync/next/test/event-fixtures.d.ts.map +1 -1
- package/dist/sync/next/test/event-fixtures.js +11 -11
- package/dist/sync/next/test/event-fixtures.js.map +1 -1
- package/dist/sync/sync-backend-kv.d.ts +3 -3
- package/dist/sync/sync-backend-kv.d.ts.map +1 -1
- package/dist/sync/sync-backend-kv.js +3 -3
- package/dist/sync/sync-backend-kv.js.map +1 -1
- package/dist/sync/sync-backend.d.ts +33 -13
- package/dist/sync/sync-backend.d.ts.map +1 -1
- package/dist/sync/sync-backend.js +38 -1
- package/dist/sync/sync-backend.js.map +1 -1
- package/dist/sync/sync.d.ts +23 -2
- package/dist/sync/sync.d.ts.map +1 -1
- package/dist/sync/syncstate.d.ts +55 -55
- package/dist/sync/syncstate.d.ts.map +1 -1
- package/dist/sync/syncstate.js +80 -98
- package/dist/sync/syncstate.js.map +1 -1
- package/dist/sync/syncstate.test.js +221 -132
- package/dist/sync/syncstate.test.js.map +1 -1
- package/dist/sync/transport-chunking.d.ts +36 -0
- package/dist/sync/transport-chunking.d.ts.map +1 -0
- package/dist/sync/transport-chunking.js +56 -0
- package/dist/sync/transport-chunking.js.map +1 -0
- package/dist/sync/validate-push-payload.d.ts +2 -2
- package/dist/sync/validate-push-payload.d.ts.map +1 -1
- package/dist/sync/validate-push-payload.js +4 -6
- package/dist/sync/validate-push-payload.js.map +1 -1
- package/dist/testing/event-factory.d.ts +3 -3
- package/dist/testing/event-factory.d.ts.map +1 -1
- package/dist/testing/event-factory.js +5 -7
- package/dist/testing/event-factory.js.map +1 -1
- package/dist/util.js +2 -2
- package/dist/util.js.map +1 -1
- package/dist/version.d.ts +24 -5
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +25 -8
- package/dist/version.js.map +1 -1
- package/package.json +67 -15
- package/src/ClientSessionLeaderThreadProxy.ts +27 -12
- package/src/WorkerTransportError.ts +12 -0
- package/src/adapter-types.ts +50 -7
- package/src/bounded-collections.ts +6 -5
- package/src/debug-info.ts +37 -6
- package/src/devtools/devtools-compatibility.test.ts +18 -0
- package/src/devtools/devtools-messages-client-session.ts +22 -4
- package/src/devtools/devtools-messages-common.ts +7 -12
- package/src/devtools/devtools-messages-leader.ts +29 -10
- package/src/devtools/devtools-sessioninfo.ts +8 -5
- package/src/devtools/mod.ts +11 -2
- package/src/errors.ts +32 -24
- package/src/index.ts +4 -1
- package/src/leader-thread/LeaderSyncProcessor.ts +523 -373
- package/src/leader-thread/RejectedPushError.ts +106 -0
- package/src/leader-thread/connection.ts +1 -1
- package/src/leader-thread/eventlog.ts +112 -39
- package/src/leader-thread/leader-worker-devtools.ts +201 -120
- package/src/leader-thread/make-leader-thread-layer.test.ts +44 -0
- package/src/leader-thread/make-leader-thread-layer.ts +125 -40
- package/src/leader-thread/materialize-event.ts +40 -10
- package/src/leader-thread/mod.ts +1 -0
- package/src/leader-thread/recreate-db.ts +7 -7
- package/src/leader-thread/shutdown-channel.ts +4 -8
- package/src/leader-thread/stream-events.ts +206 -0
- package/src/leader-thread/types.ts +68 -18
- package/src/logging.ts +62 -0
- package/src/make-client-session.ts +11 -5
- package/src/materializer-helper.ts +27 -16
- package/src/otel.ts +13 -2
- package/src/rematerialize-from-eventlog.ts +61 -51
- package/src/schema/EventDef/define.ts +217 -0
- package/src/schema/EventDef/deprecated.test.ts +129 -0
- package/src/schema/EventDef/deprecated.ts +175 -0
- package/src/schema/EventDef/event-def.ts +125 -0
- package/src/schema/EventDef/facts.ts +135 -0
- package/src/schema/EventDef/materializer.ts +172 -0
- package/src/schema/EventDef/mod.ts +5 -0
- package/src/schema/EventSequenceNumber/client.ts +257 -0
- package/src/schema/EventSequenceNumber/global.ts +19 -0
- package/src/schema/EventSequenceNumber/mod.ts +37 -0
- package/src/schema/EventSequenceNumber.test.ts +72 -53
- package/src/schema/LiveStoreEvent/client.test.ts +129 -0
- package/src/schema/LiveStoreEvent/client.ts +235 -0
- package/src/schema/LiveStoreEvent/for-event-def.ts +60 -0
- package/src/schema/LiveStoreEvent/global.ts +45 -0
- package/src/schema/LiveStoreEvent/input.ts +63 -0
- package/src/schema/LiveStoreEvent/mod.ts +4 -0
- package/src/schema/events.ts +1 -1
- package/src/schema/mod.ts +6 -4
- package/src/schema/schema.ts +46 -5
- package/src/schema/state/sqlite/client-document-def.test.ts +144 -5
- package/src/schema/state/sqlite/client-document-def.ts +47 -34
- package/src/schema/state/sqlite/column-annotations.test.ts +3 -2
- package/src/schema/state/sqlite/column-annotations.ts +2 -1
- package/src/schema/state/sqlite/column-def.test.ts +66 -12
- package/src/schema/state/sqlite/column-def.ts +119 -47
- package/src/schema/state/sqlite/column-spec.test.ts +32 -17
- package/src/schema/state/sqlite/column-spec.ts +37 -11
- package/src/schema/state/sqlite/db-schema/ast/sqlite.ts +2 -2
- package/src/schema/state/sqlite/db-schema/dsl/field-defs.test.ts +2 -1
- package/src/schema/state/sqlite/db-schema/dsl/field-defs.ts +41 -15
- package/src/schema/state/sqlite/db-schema/dsl/mod.ts +13 -19
- package/src/schema/state/sqlite/mod.ts +7 -8
- package/src/schema/state/sqlite/query-builder/api.ts +55 -17
- package/src/schema/state/sqlite/query-builder/astToSql.ts +110 -21
- package/src/schema/state/sqlite/query-builder/impl.test.ts +267 -93
- package/src/schema/state/sqlite/query-builder/impl.ts +26 -13
- package/src/schema/state/sqlite/schema-helpers.test.ts +44 -0
- package/src/schema/state/sqlite/schema-helpers.ts +30 -22
- package/src/schema/state/sqlite/system-tables/eventlog-tables.ts +64 -0
- package/src/schema/state/sqlite/system-tables/mod.ts +2 -0
- package/src/schema/state/sqlite/system-tables/state-tables.ts +69 -0
- package/src/schema/state/sqlite/table-def.test.ts +114 -3
- package/src/schema/state/sqlite/table-def.ts +16 -22
- package/src/schema/unknown-events.ts +131 -0
- package/src/schema-management/__tests__/migrations-autoincrement-quoting.test.ts +88 -0
- package/src/schema-management/common.ts +2 -2
- package/src/schema-management/migrations.ts +42 -9
- package/src/schema-management/validate-schema.ts +3 -3
- package/src/sql-queries/sql-queries.ts +18 -6
- package/src/sql-queries/sql-query-builder.ts +1 -0
- package/src/sqlite-db-helper.ts +3 -3
- package/src/sqlite-types.ts +6 -5
- package/src/sync/ClientSessionSyncProcessor.ts +152 -142
- package/src/sync/errors.ts +12 -24
- package/src/sync/index.ts +2 -0
- package/src/sync/mock-sync-backend.ts +146 -104
- package/src/sync/next/compact-events.ts +10 -11
- package/src/sync/next/facts.ts +13 -14
- package/src/sync/next/history-dag-common.ts +280 -26
- package/src/sync/next/history-dag.ts +17 -13
- package/src/sync/next/rebase-events.ts +12 -12
- package/src/sync/next/test/compact-events.calculator.test.ts +3 -2
- package/src/sync/next/test/compact-events.test.ts +4 -3
- package/src/sync/next/test/event-fixtures.ts +13 -13
- package/src/sync/sync-backend-kv.ts +4 -3
- package/src/sync/sync-backend.ts +66 -17
- package/src/sync/sync.ts +24 -2
- package/src/sync/syncstate.test.ts +583 -419
- package/src/sync/syncstate.ts +127 -122
- package/src/sync/transport-chunking.ts +90 -0
- package/src/sync/validate-push-payload.ts +6 -8
- package/src/testing/event-factory.ts +10 -12
- package/src/util.ts +2 -2
- package/src/version.ts +33 -8
- package/dist/schema/EventDef.d.ts +0 -126
- package/dist/schema/EventDef.d.ts.map +0 -1
- package/dist/schema/EventDef.js +0 -46
- package/dist/schema/EventDef.js.map +0 -1
- package/dist/schema/EventSequenceNumber.d.ts +0 -80
- package/dist/schema/EventSequenceNumber.d.ts.map +0 -1
- package/dist/schema/EventSequenceNumber.js +0 -139
- package/dist/schema/EventSequenceNumber.js.map +0 -1
- package/dist/schema/LiveStoreEvent.d.ts.map +0 -1
- package/dist/schema/LiveStoreEvent.js +0 -147
- package/dist/schema/LiveStoreEvent.js.map +0 -1
- package/dist/schema/state/sqlite/system-tables.d.ts.map +0 -1
- package/dist/schema/state/sqlite/system-tables.js +0 -81
- package/dist/schema/state/sqlite/system-tables.js.map +0 -1
- package/dist/sync/next/graphology.d.ts +0 -8
- package/dist/sync/next/graphology.d.ts.map +0 -1
- package/dist/sync/next/graphology.js +0 -30
- package/dist/sync/next/graphology.js.map +0 -1
- package/dist/sync/next/graphology_.d.ts +0 -3
- package/dist/sync/next/graphology_.d.ts.map +0 -1
- package/dist/sync/next/graphology_.js +0 -3
- package/dist/sync/next/graphology_.js.map +0 -1
- package/src/schema/EventDef.ts +0 -222
- package/src/schema/EventSequenceNumber.ts +0 -199
- package/src/schema/LiveStoreEvent.ts +0 -286
- package/src/schema/state/sqlite/system-tables.ts +0 -106
- package/src/sync/next/ambient.d.ts +0 -3
- package/src/sync/next/graphology.ts +0 -41
- package/src/sync/next/graphology_.ts +0 -2
|
@@ -1,14 +1,18 @@
|
|
|
1
|
-
import { casesHandled, isNotUndefined, LS_DEV,
|
|
2
|
-
import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option,
|
|
3
|
-
import {
|
|
1
|
+
import { casesHandled, isNotUndefined, LS_DEV, TRACE_VERBOSE } from '@livestore/utils';
|
|
2
|
+
import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, Queue, ReadonlyArray, Schedule, Schema, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
|
|
3
|
+
import { UnknownError } from "../adapter-types.js";
|
|
4
|
+
import { IntentionalShutdownCause } from "../errors.js";
|
|
4
5
|
import { makeMaterializerHash } from "../materializer-helper.js";
|
|
5
|
-
import { EventSequenceNumber,
|
|
6
|
-
import {
|
|
6
|
+
import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from "../schema/mod.js";
|
|
7
|
+
import { EVENTLOG_META_TABLE, SYNC_STATUS_TABLE } from "../schema/state/sqlite/system-tables/eventlog-tables.js";
|
|
8
|
+
import { isRejectedPushError, LeaderAheadError, NonMonotonicBatchError, StaleRebaseGenerationError } from "./RejectedPushError.js";
|
|
7
9
|
import * as SyncState from "../sync/syncstate.js";
|
|
8
10
|
import { sql } from "../util.js";
|
|
9
11
|
import * as Eventlog from "./eventlog.js";
|
|
10
12
|
import { rollback } from "./materialize-event.js";
|
|
11
13
|
import { LeaderThreadCtx } from "./types.js";
|
|
14
|
+
/** Serialize value to JSON string for trace attributes */
|
|
15
|
+
const jsonStringify = Schema.encodeSync(Schema.parseJson());
|
|
12
16
|
/**
|
|
13
17
|
* The LeaderSyncProcessor manages synchronization of events between
|
|
14
18
|
* the local state and the sync backend, ensuring efficient and orderly processing.
|
|
@@ -27,11 +31,11 @@ import { LeaderThreadCtx } from "./types.js";
|
|
|
27
31
|
* - Maintains events in ascending order.
|
|
28
32
|
* - Uses `Deferred` objects to resolve/reject events based on application success.
|
|
29
33
|
* - Processes events from the queue, applying events in batches.
|
|
30
|
-
* - Controlled by a `
|
|
31
|
-
* - The
|
|
34
|
+
* - Controlled by a mutex (`Semaphore(1)`) to ensure mutual exclusion between local push and backend pull processing.
|
|
35
|
+
* - The backend pull side acquires the mutex before processing and releases it on post-pull completion.
|
|
32
36
|
* - Processes up to `maxBatchSize` events per cycle.
|
|
33
37
|
*
|
|
34
|
-
* Currently we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
|
|
38
|
+
* Currently, we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
|
|
35
39
|
*
|
|
36
40
|
* Tricky concurrency scenarios:
|
|
37
41
|
* - Queued local push batches becoming invalid due to a prior local push item being rejected.
|
|
@@ -39,23 +43,20 @@ import { LeaderThreadCtx } from "./types.js";
|
|
|
39
43
|
*
|
|
40
44
|
* See ClientSessionSyncProcessor for how the leader and session sync processors are similar/different.
|
|
41
45
|
*/
|
|
42
|
-
export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, livePull, params, testing, }) => Effect.gen(function* () {
|
|
46
|
+
export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, onBackendIdMismatch, livePull, params, testing, }) => Effect.gen(function* () {
|
|
43
47
|
const syncBackendPushQueue = yield* BucketQueue.make();
|
|
44
|
-
const localPushBatchSize = params.localPushBatchSize ??
|
|
45
|
-
const backendPushBatchSize = params.backendPushBatchSize ??
|
|
48
|
+
const localPushBatchSize = params.localPushBatchSize ?? 10;
|
|
49
|
+
const backendPushBatchSize = params.backendPushBatchSize ?? 50;
|
|
46
50
|
const syncStateSref = yield* SubscriptionRef.make(undefined);
|
|
47
|
-
const isClientEvent = (eventEncoded) =>
|
|
48
|
-
const { eventDef } = getEventDef(schema, eventEncoded.name);
|
|
49
|
-
return eventDef.options.clientOnly;
|
|
50
|
-
};
|
|
51
|
+
const isClientEvent = (eventEncoded) => schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false;
|
|
51
52
|
const connectedClientSessionPullQueues = yield* makePullQueueSet;
|
|
52
53
|
// This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
|
|
53
54
|
const ctxRef = {
|
|
54
55
|
current: undefined,
|
|
55
56
|
};
|
|
56
57
|
const localPushesQueue = yield* BucketQueue.make();
|
|
57
|
-
|
|
58
|
-
const
|
|
58
|
+
// Ensures mutual exclusion between local push and backend pull processing.
|
|
59
|
+
const localPushBackendPullMutex = yield* Effect.makeSemaphore(1);
|
|
59
60
|
/**
|
|
60
61
|
* Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
|
|
61
62
|
* events from being pushed in a scenario like this:
|
|
@@ -65,9 +66,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
65
66
|
*
|
|
66
67
|
* Thus the purpose of the pushHeadRef is the guard the integrity of the local push queue
|
|
67
68
|
*/
|
|
68
|
-
const pushHeadRef = { current: EventSequenceNumber.ROOT };
|
|
69
|
+
const pushHeadRef = { current: EventSequenceNumber.Client.ROOT };
|
|
69
70
|
const advancePushHead = (eventNum) => {
|
|
70
|
-
pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum);
|
|
71
|
+
pushHeadRef.current = EventSequenceNumber.Client.max(pushHeadRef.current, eventNum);
|
|
71
72
|
};
|
|
72
73
|
// NOTE: New events are only pushed to sync backend after successful local push processing
|
|
73
74
|
const push = (newEvents, options) => Effect.gen(function* () {
|
|
@@ -77,7 +78,7 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
77
78
|
yield* validatePushBatch(newEvents, pushHeadRef.current);
|
|
78
79
|
advancePushHead(newEvents.at(-1).seqNum);
|
|
79
80
|
const waitForProcessing = options?.waitForProcessing ?? false;
|
|
80
|
-
if (waitForProcessing) {
|
|
81
|
+
if (waitForProcessing === true) {
|
|
81
82
|
const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
|
|
82
83
|
const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i]]);
|
|
83
84
|
yield* BucketQueue.offerAll(localPushesQueue, items);
|
|
@@ -90,34 +91,51 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
90
91
|
}).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
|
|
91
92
|
attributes: {
|
|
92
93
|
batchSize: newEvents.length,
|
|
93
|
-
batch: TRACE_VERBOSE ? newEvents : undefined,
|
|
94
|
+
batch: TRACE_VERBOSE === true ? newEvents : undefined,
|
|
94
95
|
},
|
|
95
|
-
links: ctxRef.current?.span
|
|
96
|
+
links: ctxRef.current?.span !== undefined
|
|
97
|
+
? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }]
|
|
98
|
+
: undefined,
|
|
96
99
|
}));
|
|
97
100
|
const pushPartial = ({ event: { name, args }, clientId, sessionId }) => Effect.gen(function* () {
|
|
98
|
-
const syncState = yield* syncStateSref;
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
|
|
102
|
+
const resolution = yield* resolveEventDef(schema, {
|
|
103
|
+
operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
|
|
104
|
+
event: {
|
|
105
|
+
name,
|
|
106
|
+
args,
|
|
107
|
+
clientId,
|
|
108
|
+
sessionId,
|
|
109
|
+
seqNum: syncState.localHead,
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
if (resolution._tag === 'unknown') {
|
|
113
|
+
// Ignore partial pushes for unrecognised events – they are still
|
|
114
|
+
// persisted server-side once a schema update ships.
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
const eventEncoded = new LiveStoreEvent.Client.EncodedWithMeta({
|
|
103
118
|
name,
|
|
104
119
|
args,
|
|
105
120
|
clientId,
|
|
106
121
|
sessionId,
|
|
107
|
-
...EventSequenceNumber.nextPair({
|
|
122
|
+
...EventSequenceNumber.Client.nextPair({
|
|
123
|
+
seqNum: syncState.localHead,
|
|
124
|
+
isClient: resolution.eventDef.options.clientOnly,
|
|
125
|
+
}),
|
|
108
126
|
});
|
|
109
127
|
yield* push([eventEncoded]);
|
|
110
|
-
}).pipe(
|
|
128
|
+
}).pipe(
|
|
129
|
+
// pushPartial constructs the event sequence number internally, so these errors should never happen.
|
|
130
|
+
Effect.catchIf(isRejectedPushError, Effect.die));
|
|
111
131
|
// Starts various background loops
|
|
112
132
|
const boot = Effect.gen(function* () {
|
|
113
133
|
const span = yield* Effect.currentSpan.pipe(Effect.orDie);
|
|
114
|
-
const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
|
|
115
134
|
const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
|
|
116
135
|
const runtime = yield* Effect.runtime();
|
|
117
136
|
ctxRef.current = {
|
|
118
|
-
otelSpan,
|
|
119
137
|
span,
|
|
120
|
-
devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
|
|
138
|
+
devtoolsLatch: devtools.enabled === true ? devtools.syncBackendLatch : undefined,
|
|
121
139
|
runtime,
|
|
122
140
|
};
|
|
123
141
|
/** State transitions need to happen atomically, so we use a Ref to track the state */
|
|
@@ -125,35 +143,34 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
125
143
|
// Rehydrate sync queue
|
|
126
144
|
if (initialSyncState.pending.length > 0) {
|
|
127
145
|
const globalPendingEvents = initialSyncState.pending
|
|
128
|
-
// Don't sync
|
|
146
|
+
// Don't sync client-local events
|
|
129
147
|
.filter((eventEncoded) => {
|
|
130
|
-
const
|
|
131
|
-
return eventDef.options.clientOnly === false;
|
|
148
|
+
const eventDef = schema.eventsDefsMap.get(eventEncoded.name);
|
|
149
|
+
return eventDef === undefined ? true : eventDef.options.clientOnly === false;
|
|
132
150
|
});
|
|
133
151
|
if (globalPendingEvents.length > 0) {
|
|
134
152
|
yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingEvents);
|
|
135
153
|
}
|
|
136
154
|
}
|
|
155
|
+
const handleBackendIdMismatchError = (error) => handleBackendIdMismatch({ error, onBackendIdMismatch, shutdownChannel });
|
|
137
156
|
const maybeShutdownOnError = (cause) => Effect.gen(function* () {
|
|
138
157
|
if (onError === 'ignore') {
|
|
139
|
-
if (LS_DEV) {
|
|
158
|
+
if (LS_DEV === true) {
|
|
140
159
|
yield* Effect.logDebug(`Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`, Cause.pretty(cause));
|
|
141
160
|
}
|
|
142
161
|
return;
|
|
143
162
|
}
|
|
144
|
-
const errorToSend = Cause.isFailType(cause) ? cause.error :
|
|
163
|
+
const errorToSend = Cause.isFailType(cause) === true ? cause.error : UnknownError.make({ cause });
|
|
145
164
|
yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie);
|
|
146
|
-
return yield* Effect.
|
|
165
|
+
return yield* Effect.failCause(cause).pipe(Effect.orDie);
|
|
147
166
|
});
|
|
148
167
|
yield* backgroundApplyLocalPushes({
|
|
149
|
-
|
|
168
|
+
localPushBackendPullMutex,
|
|
150
169
|
localPushesQueue,
|
|
151
|
-
pullLatch,
|
|
152
170
|
syncStateSref,
|
|
153
171
|
syncBackendPushQueue,
|
|
154
172
|
schema,
|
|
155
173
|
isClientEvent,
|
|
156
|
-
otelSpan,
|
|
157
174
|
connectedClientSessionPullQueues,
|
|
158
175
|
localPushBatchSize,
|
|
159
176
|
testing: {
|
|
@@ -163,10 +180,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
163
180
|
const backendPushingFiberHandle = yield* FiberHandle.make();
|
|
164
181
|
const backendPushingEffect = backgroundBackendPushing({
|
|
165
182
|
syncBackendPushQueue,
|
|
166
|
-
otelSpan,
|
|
167
183
|
devtoolsLatch: ctxRef.current?.devtoolsLatch,
|
|
168
184
|
backendPushBatchSize,
|
|
169
|
-
}).pipe(Effect.catchAllCause(maybeShutdownOnError));
|
|
185
|
+
}).pipe(Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchAllCause(maybeShutdownOnError));
|
|
170
186
|
yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
|
|
171
187
|
yield* backgroundBackendPulling({
|
|
172
188
|
isClientEvent,
|
|
@@ -180,19 +196,19 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
180
196
|
yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
|
|
181
197
|
}),
|
|
182
198
|
syncStateSref,
|
|
183
|
-
|
|
184
|
-
pullLatch,
|
|
199
|
+
localPushBackendPullMutex,
|
|
185
200
|
livePull,
|
|
186
201
|
dbState,
|
|
187
|
-
otelSpan,
|
|
188
202
|
initialBlockingSyncContext,
|
|
189
203
|
devtoolsLatch: ctxRef.current?.devtoolsLatch,
|
|
190
204
|
connectedClientSessionPullQueues,
|
|
191
205
|
advancePushHead,
|
|
192
206
|
}).pipe(Effect.retry({
|
|
193
|
-
//
|
|
194
|
-
while
|
|
195
|
-
|
|
207
|
+
// Retry pulling when we've lost connection to the sync backend
|
|
208
|
+
// We're using `until` with a refinement instead of `while` to narrow `IsOfflineError` out of the error type.
|
|
209
|
+
// See https://github.com/Effect-TS/effect/issues/6122
|
|
210
|
+
until: (error) => error._tag !== 'IsOfflineError',
|
|
211
|
+
}), Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchAllCause(maybeShutdownOnError),
|
|
196
212
|
// Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
|
|
197
213
|
// This might be a bug in Effect. Only seems to happen in the browser.
|
|
198
214
|
Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())), Effect.forkScoped);
|
|
@@ -216,17 +232,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
216
232
|
- full new state db snapshot in the "rebase" case
|
|
217
233
|
- downside: importing the snapshot is expensive
|
|
218
234
|
*/
|
|
219
|
-
const pullQueue = ({ cursor }) =>
|
|
220
|
-
const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized');
|
|
221
|
-
return connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime));
|
|
222
|
-
};
|
|
235
|
+
const pullQueue = ({ cursor }) => Effect.fromNullable(ctxRef.current?.runtime).pipe(Effect.orDieDebugger, Effect.flatMap((runtime) => connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime))));
|
|
223
236
|
const syncState = Subscribable.make({
|
|
224
|
-
get: Effect.
|
|
225
|
-
const syncState = yield* syncStateSref;
|
|
226
|
-
if (syncState === undefined)
|
|
227
|
-
return shouldNeverHappen('Not initialized');
|
|
228
|
-
return syncState;
|
|
229
|
-
}),
|
|
237
|
+
get: syncStateSref.pipe(Effect.flatMap(Effect.fromNullable), Effect.orDieDebugger),
|
|
230
238
|
changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
|
|
231
239
|
});
|
|
232
240
|
return {
|
|
@@ -238,100 +246,103 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
|
|
|
238
246
|
syncState,
|
|
239
247
|
};
|
|
240
248
|
});
|
|
241
|
-
const backgroundApplyLocalPushes = ({
|
|
249
|
+
const backgroundApplyLocalPushes = ({ localPushBackendPullMutex, localPushesQueue, syncStateSref, syncBackendPushQueue, schema, isClientEvent, connectedClientSessionPullQueues, localPushBatchSize, testing, }) => Effect.gen(function* () {
|
|
242
250
|
while (true) {
|
|
243
251
|
if (testing.delay !== undefined) {
|
|
244
252
|
yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'));
|
|
245
253
|
}
|
|
246
254
|
const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
|
|
247
|
-
//
|
|
248
|
-
yield*
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
if (newEvents.length === 0) {
|
|
259
|
-
// console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
|
|
260
|
-
// Allow the backend pulling to start
|
|
261
|
-
yield* pullLatch.open;
|
|
262
|
-
continue;
|
|
263
|
-
}
|
|
264
|
-
const mergeResult = SyncState.merge({
|
|
265
|
-
syncState,
|
|
266
|
-
payload: { _tag: 'local-push', newEvents },
|
|
267
|
-
isClientEvent,
|
|
268
|
-
isEqualEvent: LiveStoreEvent.isEqualEncoded,
|
|
269
|
-
});
|
|
270
|
-
switch (mergeResult._tag) {
|
|
271
|
-
case 'unexpected-error': {
|
|
272
|
-
otelSpan?.addEvent(`push:unexpected-error`, {
|
|
273
|
-
batchSize: newEvents.length,
|
|
274
|
-
newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
|
|
255
|
+
// Applies a batch of local pushes, guarded by the localPushBackendPullMutex to ensure mutual exclusion with backend pulling
|
|
256
|
+
yield* Effect.gen(function* () {
|
|
257
|
+
const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
|
|
258
|
+
const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
|
|
259
|
+
// Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
|
|
260
|
+
// It's important that we filter after acquiring the localPushBackendPullMutex, otherwise we might filter with the old generation
|
|
261
|
+
const [droppedItems, filteredItems] = ReadonlyArray.partition(batchItems, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration);
|
|
262
|
+
if (droppedItems.length > 0) {
|
|
263
|
+
yield* Effect.spanEvent(`push:drop-old-generation`, {
|
|
264
|
+
droppedCount: droppedItems.length,
|
|
265
|
+
currentRebaseGeneration,
|
|
275
266
|
});
|
|
276
|
-
|
|
267
|
+
/**
|
|
268
|
+
* Dropped pushes may still have a deferred awaiting completion.
|
|
269
|
+
* Fail it so the caller learns the leader advanced and resubmits with the updated generation.
|
|
270
|
+
*/
|
|
271
|
+
yield* Effect.forEach(droppedItems.filter((item) => item[1] !== undefined), ([eventEncoded, deferred]) => Deferred.fail(deferred, StaleRebaseGenerationError.make({
|
|
272
|
+
currentRebaseGeneration,
|
|
273
|
+
providedRebaseGeneration: eventEncoded.seqNum.rebaseGeneration,
|
|
274
|
+
sessionId: eventEncoded.sessionId,
|
|
275
|
+
})));
|
|
277
276
|
}
|
|
278
|
-
|
|
279
|
-
return
|
|
277
|
+
if (filteredItems.length === 0) {
|
|
278
|
+
return;
|
|
280
279
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
280
|
+
const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems);
|
|
281
|
+
yield* Effect.annotateCurrentSpan({
|
|
282
|
+
'batchSize': newEvents.length,
|
|
283
|
+
...(TRACE_VERBOSE === true ? { 'newEvents': jsonStringify(newEvents) } : {}),
|
|
284
|
+
});
|
|
285
|
+
const mergeResult = yield* SyncState.merge({
|
|
286
|
+
syncState,
|
|
287
|
+
payload: { _tag: 'local-push', newEvents },
|
|
288
|
+
isClientEvent,
|
|
289
|
+
isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
|
|
290
|
+
});
|
|
291
|
+
switch (mergeResult._tag) {
|
|
292
|
+
case 'rebase': {
|
|
293
|
+
return yield* Effect.dieDebugger('The leader thread should never have to rebase due to a local push');
|
|
294
|
+
}
|
|
295
|
+
case 'reject': {
|
|
296
|
+
yield* Effect.spanEvent(`push:reject`, {
|
|
297
|
+
batchSize: newEvents.length,
|
|
298
|
+
...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
|
|
299
|
+
});
|
|
300
|
+
// TODO: how to test this?
|
|
301
|
+
const nextRebaseGeneration = currentRebaseGeneration + 1;
|
|
302
|
+
const providedNum = newEvents.at(0).seqNum;
|
|
303
|
+
// All subsequent pushes with same generation should be rejected as well
|
|
304
|
+
// We're also handling the case where the localPushQueue already contains events
|
|
305
|
+
// from the next generation which we preserve in the queue
|
|
306
|
+
const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
|
|
307
|
+
// TODO we still need to better understand and handle this scenario
|
|
308
|
+
if (LS_DEV === true && (yield* BucketQueue.size(localPushesQueue)) > 0) {
|
|
309
|
+
console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
|
|
310
|
+
// oxlint-disable-next-line eslint(no-debugger) -- intentional breakpoint for unexpected queue state
|
|
311
|
+
debugger;
|
|
312
|
+
}
|
|
313
|
+
const allDeferredsToReject = [
|
|
314
|
+
...deferreds,
|
|
315
|
+
...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
|
|
316
|
+
].filter(isNotUndefined);
|
|
317
|
+
yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum, sessionId: newEvents.at(0).sessionId })));
|
|
318
|
+
// In this case we're skipping state update and down/upstream processing
|
|
319
|
+
// We've cleared the local push queue and are now waiting for new local pushes / backend pulls
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
case 'advance': {
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
default: {
|
|
326
|
+
casesHandled(mergeResult);
|
|
298
327
|
}
|
|
299
|
-
const allDeferredsToReject = [
|
|
300
|
-
...deferreds,
|
|
301
|
-
...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
|
|
302
|
-
].filter(isNotUndefined);
|
|
303
|
-
yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum })));
|
|
304
|
-
// Allow the backend pulling to start
|
|
305
|
-
yield* pullLatch.open;
|
|
306
|
-
// In this case we're skipping state update and down/upstream processing
|
|
307
|
-
// We've cleared the local push queue and are now waiting for new local pushes / backend pulls
|
|
308
|
-
continue;
|
|
309
|
-
}
|
|
310
|
-
case 'advance': {
|
|
311
|
-
break;
|
|
312
|
-
}
|
|
313
|
-
default: {
|
|
314
|
-
casesHandled(mergeResult);
|
|
315
328
|
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
// Allow the backend pulling to start
|
|
334
|
-
yield* pullLatch.open;
|
|
329
|
+
yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
|
|
330
|
+
yield* connectedClientSessionPullQueues.offer({
|
|
331
|
+
payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
|
|
332
|
+
leaderHead: mergeResult.newSyncState.localHead,
|
|
333
|
+
});
|
|
334
|
+
yield* Effect.spanEvent(`push:advance`, {
|
|
335
|
+
batchSize: newEvents.length,
|
|
336
|
+
...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
|
|
337
|
+
});
|
|
338
|
+
// Don't sync client-local events
|
|
339
|
+
const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
|
|
340
|
+
const eventDef = schema.eventsDefsMap.get(eventEncoded.name);
|
|
341
|
+
return eventDef === undefined ? true : eventDef.options.clientOnly === false;
|
|
342
|
+
});
|
|
343
|
+
yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
|
|
344
|
+
yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
|
|
345
|
+
}).pipe(localPushBackendPullMutex.withPermits(1));
|
|
335
346
|
}
|
|
336
347
|
});
|
|
337
348
|
// TODO how to handle errors gracefully
|
|
@@ -341,7 +352,7 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
|
|
|
341
352
|
db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
|
|
342
353
|
dbEventlog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
|
|
343
354
|
yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
|
|
344
|
-
if (Exit.isSuccess(exit))
|
|
355
|
+
if (Exit.isSuccess(exit) === true)
|
|
345
356
|
return;
|
|
346
357
|
// Rollback in case of an error
|
|
347
358
|
db.execute('ROLLBACK', undefined);
|
|
@@ -360,129 +371,134 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
|
|
|
360
371
|
}).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:LeaderSyncProcessor:materializeEventItems', {
|
|
361
372
|
attributes: { batchSize: batchItems.length },
|
|
362
373
|
}), Effect.tapCauseLogPretty);
|
|
363
|
-
const backgroundBackendPulling = ({ isClientEvent, restartBackendPushing,
|
|
374
|
+
const backgroundBackendPulling = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pulling')(function* ({ isClientEvent, restartBackendPushing, dbState, syncStateSref, localPushBackendPullMutex, livePull, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, advancePushHead, }) {
|
|
364
375
|
const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx;
|
|
365
376
|
if (syncBackend === undefined)
|
|
366
377
|
return;
|
|
367
|
-
|
|
368
|
-
|
|
378
|
+
let pullMutexHeld = false;
|
|
379
|
+
const releasePullMutexIfHeld = Effect.gen(function* () {
|
|
380
|
+
if (pullMutexHeld === false)
|
|
369
381
|
return;
|
|
382
|
+
pullMutexHeld = false;
|
|
383
|
+
yield* localPushBackendPullMutex.release(1);
|
|
384
|
+
});
|
|
385
|
+
const isPullPaginationComplete = (pageInfo) => pageInfo._tag === 'NoMore';
|
|
386
|
+
const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
|
|
370
387
|
if (devtoolsLatch !== undefined) {
|
|
371
388
|
yield* devtoolsLatch.await;
|
|
372
389
|
}
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
if (syncState === undefined)
|
|
379
|
-
return shouldNeverHappen('Not initialized');
|
|
380
|
-
const mergeResult = SyncState.merge({
|
|
381
|
-
syncState,
|
|
382
|
-
payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
|
|
383
|
-
isClientEvent,
|
|
384
|
-
isEqualEvent: LiveStoreEvent.isEqualEncoded,
|
|
385
|
-
ignoreClientEvents: true,
|
|
386
|
-
});
|
|
387
|
-
if (mergeResult._tag === 'reject') {
|
|
388
|
-
return shouldNeverHappen('The leader thread should never reject upstream advances');
|
|
390
|
+
if (newEvents.length === 0) {
|
|
391
|
+
if (isPullPaginationComplete(pageInfo) === true) {
|
|
392
|
+
yield* releasePullMutexIfHeld;
|
|
393
|
+
}
|
|
394
|
+
return;
|
|
389
395
|
}
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
});
|
|
395
|
-
return yield* new UnexpectedError({ cause: mergeResult.message });
|
|
396
|
+
// Prevent more local pushes from being processed until this pull pagination sequence is finished.
|
|
397
|
+
if (pullMutexHeld === false) {
|
|
398
|
+
yield* localPushBackendPullMutex.take(1);
|
|
399
|
+
pullMutexHeld = true;
|
|
396
400
|
}
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
|
|
403
|
-
rollbackCount: mergeResult.rollbackEvents.length,
|
|
404
|
-
mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
|
|
401
|
+
const chunkExit = yield* Effect.gen(function* () {
|
|
402
|
+
const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
|
|
403
|
+
yield* Effect.annotateCurrentSpan({
|
|
404
|
+
'merge.newEventsCount': newEvents.length,
|
|
405
|
+
...(TRACE_VERBOSE === true ? { 'merge.newEvents': jsonStringify(newEvents) } : {}),
|
|
405
406
|
});
|
|
406
|
-
const
|
|
407
|
-
|
|
408
|
-
|
|
407
|
+
const mergeResult = yield* SyncState.merge({
|
|
408
|
+
syncState,
|
|
409
|
+
payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
|
|
410
|
+
isClientEvent,
|
|
411
|
+
isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
|
|
412
|
+
ignoreClientEvents: true,
|
|
409
413
|
});
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
414
|
+
if (mergeResult._tag === 'reject') {
|
|
415
|
+
return yield* Effect.dieDebugger('The leader thread should never reject upstream advances');
|
|
416
|
+
}
|
|
417
|
+
const newBackendHead = newEvents.at(-1).seqNum;
|
|
418
|
+
Eventlog.updateBackendHead(dbEventlog, newBackendHead);
|
|
419
|
+
if (mergeResult._tag === 'rebase') {
|
|
420
|
+
yield* Effect.spanEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
|
|
421
|
+
newEventsCount: newEvents.length,
|
|
422
|
+
...(TRACE_VERBOSE === true ? { newEvents: jsonStringify(newEvents) } : {}),
|
|
423
|
+
rollbackCount: mergeResult.rollbackEvents.length,
|
|
424
|
+
...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
|
|
425
|
+
});
|
|
426
|
+
const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
|
|
427
|
+
const eventDef = schema.eventsDefsMap.get(event.name);
|
|
428
|
+
return eventDef === undefined ? true : eventDef.options.clientOnly === false;
|
|
429
|
+
});
|
|
430
|
+
yield* restartBackendPushing(globalRebasedPendingEvents);
|
|
431
|
+
if (mergeResult.rollbackEvents.length > 0) {
|
|
432
|
+
yield* rollback({
|
|
433
|
+
dbState: db,
|
|
434
|
+
dbEventlog,
|
|
435
|
+
eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
yield* connectedClientSessionPullQueues.offer({
|
|
439
|
+
payload: SyncState.payloadFromMergeResult(mergeResult),
|
|
440
|
+
leaderHead: mergeResult.newSyncState.localHead,
|
|
416
441
|
});
|
|
417
442
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
// `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
|
|
440
|
-
// `newEvents` instead which we filter via `mergeResult.confirmedEvents`
|
|
441
|
-
const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum)));
|
|
442
|
-
yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnexpectedError.mapToUnexpectedError);
|
|
443
|
+
else {
|
|
444
|
+
yield* Effect.spanEvent(`pull:advance`, {
|
|
445
|
+
newEventsCount: newEvents.length,
|
|
446
|
+
...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
|
|
447
|
+
});
|
|
448
|
+
// Ensure push fiber is active after advance by restarting with current pending (non-client) events
|
|
449
|
+
const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
|
|
450
|
+
const eventDef = schema.eventsDefsMap.get(event.name);
|
|
451
|
+
return eventDef === undefined ? true : eventDef.options.clientOnly === false;
|
|
452
|
+
});
|
|
453
|
+
yield* restartBackendPushing(globalPendingEvents);
|
|
454
|
+
yield* connectedClientSessionPullQueues.offer({
|
|
455
|
+
payload: SyncState.payloadFromMergeResult(mergeResult),
|
|
456
|
+
leaderHead: mergeResult.newSyncState.localHead,
|
|
457
|
+
});
|
|
458
|
+
if (mergeResult.confirmedEvents.length > 0) {
|
|
459
|
+
// `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
|
|
460
|
+
// `newEvents` instead which we filter via `mergeResult.confirmedEvents`
|
|
461
|
+
const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum)));
|
|
462
|
+
yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(Effect.orDieDebugger);
|
|
463
|
+
}
|
|
443
464
|
}
|
|
465
|
+
// Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
|
|
466
|
+
trimChangesetRows(db, newBackendHead);
|
|
467
|
+
advancePushHead(mergeResult.newSyncState.localHead);
|
|
468
|
+
yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
|
|
469
|
+
yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
|
|
470
|
+
}).pipe(Effect.exit);
|
|
471
|
+
if (Exit.isFailure(chunkExit) === true) {
|
|
472
|
+
yield* releasePullMutexIfHeld;
|
|
473
|
+
return yield* Effect.failCause(chunkExit.cause);
|
|
444
474
|
}
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
advancePushHead(mergeResult.newSyncState.localHead);
|
|
448
|
-
yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
|
|
449
|
-
yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
|
|
450
|
-
// Allow local pushes to be processed again
|
|
451
|
-
if (pageInfo._tag === 'NoMore') {
|
|
452
|
-
yield* localPushesLatch.open;
|
|
475
|
+
if (isPullPaginationComplete(pageInfo) === true) {
|
|
476
|
+
yield* releasePullMutexIfHeld;
|
|
453
477
|
}
|
|
454
478
|
});
|
|
455
|
-
const syncState = yield* syncStateSref;
|
|
456
|
-
if (syncState === undefined)
|
|
457
|
-
return shouldNeverHappen('Not initialized');
|
|
479
|
+
const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
|
|
458
480
|
const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global });
|
|
459
481
|
const hashMaterializerResult = makeMaterializerHash({ schema, dbState });
|
|
460
482
|
yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
|
|
461
483
|
// TODO only take from queue while connected
|
|
462
484
|
Stream.tap(({ batch, pageInfo }) => Effect.gen(function* () {
|
|
463
|
-
// yield* Effect.spanEvent('batch', {
|
|
464
|
-
// attributes: {
|
|
465
|
-
// batchSize: batch.length,
|
|
466
|
-
// batch: TRACE_VERBOSE ? batch : undefined,
|
|
467
|
-
// },
|
|
468
|
-
// })
|
|
469
485
|
// NOTE we only want to take process events when the sync backend is connected
|
|
470
486
|
// (e.g. needed for simulating being offline)
|
|
471
487
|
// TODO remove when there's a better way to handle this in stream above
|
|
472
488
|
yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
|
|
473
|
-
yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
|
|
489
|
+
yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.Client.EncodedWithMeta.fromGlobal(_.eventEncoded, {
|
|
474
490
|
syncMetadata: _.metadata,
|
|
475
491
|
// TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
|
|
476
492
|
// This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
|
|
477
|
-
materializerHashLeader: hashMaterializerResult(LiveStoreEvent.
|
|
493
|
+
materializerHashLeader: hashMaterializerResult(LiveStoreEvent.Global.toClientEncoded(_.eventEncoded)),
|
|
478
494
|
materializerHashSession: Option.none(),
|
|
479
495
|
})), pageInfo);
|
|
480
496
|
yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo });
|
|
481
|
-
})), Stream.runDrain, Effect.interruptible);
|
|
497
|
+
})), Stream.runDrain, Effect.interruptible, Effect.ensuring(releasePullMutexIfHeld));
|
|
482
498
|
// Should only ever happen when livePull is false
|
|
483
499
|
yield* Effect.logDebug('backend-pulling finished', { livePull });
|
|
484
|
-
})
|
|
485
|
-
const backgroundBackendPushing = ({ syncBackendPushQueue,
|
|
500
|
+
});
|
|
501
|
+
const backgroundBackendPushing = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pushing')(function* ({ syncBackendPushQueue, devtoolsLatch, backendPushBatchSize, }) {
|
|
486
502
|
const { syncBackend } = yield* LeaderThreadCtx;
|
|
487
503
|
if (syncBackend === undefined)
|
|
488
504
|
return;
|
|
@@ -493,45 +509,47 @@ const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatc
|
|
|
493
509
|
if (devtoolsLatch !== undefined) {
|
|
494
510
|
yield* devtoolsLatch.await;
|
|
495
511
|
}
|
|
496
|
-
|
|
512
|
+
yield* Effect.spanEvent('backend-push', {
|
|
497
513
|
batchSize: queueItems.length,
|
|
498
|
-
|
|
514
|
+
...(TRACE_VERBOSE === true ? { batch: jsonStringify(queueItems) } : {}),
|
|
499
515
|
});
|
|
500
516
|
// Push with declarative retry/backoff using Effect schedules
|
|
501
517
|
// - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
|
|
502
518
|
// - Delay clamped at 30s (continues retrying at 30s)
|
|
503
519
|
// - Resets automatically after successful push
|
|
504
520
|
// TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
|
|
505
|
-
// Only retry for transient UnexpectedError cases
|
|
506
|
-
const isRetryable = (err) => err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnexpectedError';
|
|
507
|
-
// Input: InvalidPushError | IsOfflineError, Output: Duration
|
|
508
|
-
const retrySchedule = Schedule.exponential(Duration.seconds(1)).pipe(Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
|
|
509
|
-
Schedule.compose(Schedule.elapsed), Schedule.whileInput(isRetryable));
|
|
510
521
|
yield* Effect.gen(function* () {
|
|
511
522
|
const iteration = yield* Schedule.CurrentIterationMetadata;
|
|
512
523
|
const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either);
|
|
513
524
|
const retries = iteration.recurrence;
|
|
514
525
|
if (retries > 0 && pushResult._tag === 'Right') {
|
|
515
|
-
|
|
526
|
+
yield* Effect.spanEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
|
|
516
527
|
}
|
|
517
528
|
if (pushResult._tag === 'Left') {
|
|
518
|
-
|
|
529
|
+
yield* Effect.spanEvent('backend-push-error', {
|
|
519
530
|
error: pushResult.left.toString(),
|
|
520
531
|
retries,
|
|
521
532
|
batchSize: queueItems.length,
|
|
522
533
|
});
|
|
523
534
|
const error = pushResult.left;
|
|
524
|
-
if (error._tag === '
|
|
525
|
-
(error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')) {
|
|
535
|
+
if (error._tag === 'ServerAheadError') {
|
|
526
536
|
// It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
|
|
527
537
|
yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error });
|
|
528
538
|
return yield* Effect.never;
|
|
529
539
|
}
|
|
530
540
|
return yield* error;
|
|
531
541
|
}
|
|
532
|
-
}).pipe(
|
|
542
|
+
}).pipe(
|
|
543
|
+
// Retry transient errors
|
|
544
|
+
Effect.retry({
|
|
545
|
+
schedule: Schedule.exponential(Duration.seconds(1)).pipe(Schedule.modifyDelay((_, delay) => Duration.min(delay, Duration.seconds(30))) // Cap delay at 30s intervals.
|
|
546
|
+
),
|
|
547
|
+
while: (error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError',
|
|
548
|
+
}),
|
|
549
|
+
// This is needed to narrow the Error type. Our retry policy runs indefinitely, but Effect.retry does not narrow the Error type.
|
|
550
|
+
Effect.catchIf((error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError', Effect.die));
|
|
533
551
|
}
|
|
534
|
-
}
|
|
552
|
+
}, Effect.interruptible);
|
|
535
553
|
const trimChangesetRows = (db, newHead) => {
|
|
536
554
|
// Since we're using the session changeset rows to query for the current head,
|
|
537
555
|
// we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
|
|
@@ -551,15 +569,15 @@ const makePullQueueSet = Effect.gen(function* () {
|
|
|
551
569
|
const queue = yield* Queue.unbounded().pipe(Effect.acquireRelease(Queue.shutdown));
|
|
552
570
|
yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)));
|
|
553
571
|
const payloadsSinceCursor = Array.from(cachedPayloads.entries())
|
|
554
|
-
.flatMap(([seqNumStr, payloads]) => payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.fromString(seqNumStr) })))
|
|
555
|
-
.filter(({ seqNum }) => EventSequenceNumber.isGreaterThan(seqNum, cursor))
|
|
556
|
-
.toSorted((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
|
|
572
|
+
.flatMap(([seqNumStr, payloads]) => payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.Client.fromString(seqNumStr) })))
|
|
573
|
+
.filter(({ seqNum }) => EventSequenceNumber.Client.isGreaterThan(seqNum, cursor))
|
|
574
|
+
.toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
|
|
557
575
|
.map(({ payload }) => {
|
|
558
576
|
if (payload._tag === 'upstream-advance') {
|
|
559
577
|
return {
|
|
560
578
|
payload: {
|
|
561
579
|
_tag: 'upstream-advance',
|
|
562
|
-
newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.isGreaterThanOrEqual(cursor, eventEncoded.seqNum)),
|
|
580
|
+
newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, eventEncoded.seqNum)),
|
|
563
581
|
},
|
|
564
582
|
};
|
|
565
583
|
}
|
|
@@ -599,8 +617,8 @@ const makePullQueueSet = Effect.gen(function* () {
|
|
|
599
617
|
return queue;
|
|
600
618
|
});
|
|
601
619
|
const offer = (item) => Effect.gen(function* () {
|
|
602
|
-
const seqNumStr = EventSequenceNumber.toString(item.leaderHead);
|
|
603
|
-
if (cachedPayloads.has(seqNumStr)) {
|
|
620
|
+
const seqNumStr = EventSequenceNumber.Client.toString(item.leaderHead);
|
|
621
|
+
if (cachedPayloads.has(seqNumStr) === true) {
|
|
604
622
|
cachedPayloads.get(seqNumStr).push(item.payload);
|
|
605
623
|
}
|
|
606
624
|
else {
|
|
@@ -620,22 +638,72 @@ const makePullQueueSet = Effect.gen(function* () {
|
|
|
620
638
|
offer,
|
|
621
639
|
};
|
|
622
640
|
});
|
|
641
|
+
/**
|
|
642
|
+
* Validate a client-provided batch before it is admitted to the leader queue.
|
|
643
|
+
* Ensures the numbers form a strictly increasing chain and that the first
|
|
644
|
+
* event sits ahead of the current push head.
|
|
645
|
+
*/
|
|
623
646
|
const validatePushBatch = (batch, pushHead) => Effect.gen(function* () {
|
|
624
647
|
if (batch.length === 0) {
|
|
625
648
|
return;
|
|
626
649
|
}
|
|
627
|
-
//
|
|
650
|
+
// Defensive check: callers should already provide a strictly increasing sequence
|
|
651
|
+
// of event numbers.
|
|
628
652
|
for (let i = 1; i < batch.length; i++) {
|
|
629
|
-
if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum)) {
|
|
630
|
-
|
|
653
|
+
if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum) === true) {
|
|
654
|
+
return yield* NonMonotonicBatchError.make({
|
|
655
|
+
precedingSeqNum: batch[i - 1].seqNum,
|
|
656
|
+
violatingSeqNum: batch[i].seqNum,
|
|
657
|
+
violationIndex: i,
|
|
658
|
+
sessionId: batch[i].sessionId,
|
|
659
|
+
});
|
|
631
660
|
}
|
|
632
661
|
}
|
|
633
|
-
//
|
|
634
|
-
if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0].seqNum)) {
|
|
662
|
+
// Reject stale batches whose first event is at or behind the leader's push head.
|
|
663
|
+
if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0].seqNum) === true) {
|
|
635
664
|
return yield* LeaderAheadError.make({
|
|
636
665
|
minimumExpectedNum: pushHead,
|
|
637
666
|
providedNum: batch[0].seqNum,
|
|
667
|
+
sessionId: batch[0].sessionId,
|
|
638
668
|
});
|
|
639
669
|
}
|
|
640
670
|
});
|
|
671
|
+
/**
|
|
672
|
+
* Handles a BackendIdMismatchError based on the configured behavior.
|
|
673
|
+
* This occurs when the sync backend has been reset and has a new identity.
|
|
674
|
+
*/
|
|
675
|
+
const handleBackendIdMismatch = Effect.fn('@livestore/common:LeaderSyncProcessor:handleBackendIdMismatch')(function* ({ error, onBackendIdMismatch, shutdownChannel, }) {
|
|
676
|
+
const { dbEventlog, dbState } = yield* LeaderThreadCtx;
|
|
677
|
+
if (onBackendIdMismatch === 'reset') {
|
|
678
|
+
yield* Effect.logWarning('Sync backend identity changed (backend was reset). Clearing local storage and shutting down.', error);
|
|
679
|
+
// Clear local databases so the client can start fresh on next boot
|
|
680
|
+
yield* clearLocalDatabases({ dbEventlog, dbState });
|
|
681
|
+
// Send shutdown signal with special reason
|
|
682
|
+
yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'backend-id-mismatch' })).pipe(Effect.orDie);
|
|
683
|
+
return yield* Effect.die(error);
|
|
684
|
+
}
|
|
685
|
+
if (onBackendIdMismatch === 'shutdown') {
|
|
686
|
+
yield* Effect.logWarning('Sync backend identity changed (backend was reset). Shutting down without clearing local storage.', error);
|
|
687
|
+
yield* shutdownChannel.send(error).pipe(Effect.orDie);
|
|
688
|
+
return yield* Effect.die(error);
|
|
689
|
+
}
|
|
690
|
+
// ignore mode
|
|
691
|
+
if (LS_DEV === true) {
|
|
692
|
+
yield* Effect.logDebug('Ignoring BackendIdMismatchError (sync backend was reset but client continues with stale data)', error);
|
|
693
|
+
}
|
|
694
|
+
});
|
|
695
|
+
/**
|
|
696
|
+
* Clears local databases (eventlog and state) so the client can start fresh on next boot.
|
|
697
|
+
* This is used when the sync backend identity has changed (i.e. backend was reset).
|
|
698
|
+
*/
|
|
699
|
+
const clearLocalDatabases = ({ dbEventlog, dbState }) => Effect.sync(() => {
|
|
700
|
+
// Clear eventlog tables
|
|
701
|
+
dbEventlog.execute(sql `DELETE FROM ${EVENTLOG_META_TABLE}`);
|
|
702
|
+
dbEventlog.execute(sql `DELETE FROM ${SYNC_STATUS_TABLE}`);
|
|
703
|
+
// Drop all state tables - they'll be recreated on next boot
|
|
704
|
+
const tables = dbState.select(sql `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`);
|
|
705
|
+
for (const { name } of tables) {
|
|
706
|
+
dbState.execute(`DROP TABLE IF EXISTS "${name}"`);
|
|
707
|
+
}
|
|
708
|
+
});
|
|
641
709
|
//# sourceMappingURL=LeaderSyncProcessor.js.map
|