@rocicorp/zero 0.26.0-canary.8 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/ast-to-zql/src/ast-to-zql.d.ts.map +1 -1
- package/out/ast-to-zql/src/ast-to-zql.js +16 -27
- package/out/ast-to-zql/src/ast-to-zql.js.map +1 -1
- package/out/otel/src/log-options.d.ts +2 -2
- package/out/replicache/src/bg-interval.d.ts.map +1 -1
- package/out/replicache/src/bg-interval.js +3 -0
- package/out/replicache/src/bg-interval.js.map +1 -1
- package/out/shared/src/arrays.js +1 -1
- package/out/shared/src/arrays.js.map +1 -1
- package/out/shared/src/browser-env.js +0 -4
- package/out/shared/src/browser-env.js.map +1 -1
- package/out/shared/src/btree-set.js +4 -1
- package/out/shared/src/btree-set.js.map +1 -1
- package/out/shared/src/options.js +1 -1
- package/out/shared/src/options.js.map +1 -1
- package/out/shared/src/queue.js +1 -1
- package/out/shared/src/queue.js.map +1 -1
- package/out/z2s/src/compiler.d.ts.map +1 -1
- package/out/z2s/src/compiler.js +13 -11
- package/out/z2s/src/compiler.js.map +1 -1
- package/out/zero/package.json.js +1 -1
- package/out/zero/src/react.js +1 -3
- package/out/zero/src/react.js.map +1 -1
- package/out/zero-cache/src/auth/read-authorizer.js +0 -7
- package/out/zero-cache/src/auth/read-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/network.d.ts +3 -2
- package/out/zero-cache/src/config/network.d.ts.map +1 -1
- package/out/zero-cache/src/config/network.js +9 -2
- package/out/zero-cache/src/config/network.js.map +1 -1
- package/out/zero-cache/src/config/server-context.d.ts +16 -0
- package/out/zero-cache/src/config/server-context.d.ts.map +1 -0
- package/out/zero-cache/src/config/server-context.js +32 -0
- package/out/zero-cache/src/config/server-context.js.map +1 -0
- package/out/zero-cache/src/config/zero-config.d.ts +3 -3
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +2 -6
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/migration.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration.js +40 -51
- package/out/zero-cache/src/db/migration.js.map +1 -1
- package/out/zero-cache/src/db/run-transaction.d.ts +17 -0
- package/out/zero-cache/src/db/run-transaction.d.ts.map +1 -0
- package/out/zero-cache/src/db/run-transaction.js +24 -0
- package/out/zero-cache/src/db/run-transaction.js.map +1 -0
- package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.js +3 -3
- package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
- package/out/zero-cache/src/scripts/decommission.d.ts +1 -1
- package/out/zero-cache/src/scripts/deploy-permissions.js +2 -1
- package/out/zero-cache/src/scripts/deploy-permissions.js.map +1 -1
- package/out/zero-cache/src/scripts/permissions.d.ts +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +6 -2
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.js +1 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +7 -3
- package/out/zero-cache/src/server/runner/run-worker.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts +1 -1
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js +4 -4
- package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +11 -0
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.d.ts +5 -2
- package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +6 -6
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts +6 -4
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +130 -43
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.js +2 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +4 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +35 -10
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +10 -0
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +6 -3
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +19 -10
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts +1 -0
- package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/data.js +4 -2
- package/out/zero-cache/src/services/change-source/protocol/current/data.js.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts +3 -0
- package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts +6 -4
- package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/protocol/current/status.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +30 -12
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +23 -3
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +1 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +12 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +11 -2
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +80 -42
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +2 -1
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +22 -17
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts +10 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/schema/replication-state.js +49 -9
- package/out/zero-cache/src/services/replicator/schema/replication-state.js.map +1 -1
- package/out/zero-cache/src/services/running-state.d.ts +1 -0
- package/out/zero-cache/src/services/running-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/running-state.js +3 -0
- package/out/zero-cache/src/services/running-state.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-purger.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-purger.js +32 -28
- package/out/zero-cache/src/services/view-syncer/cvr-purger.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js +329 -155
- package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +387 -345
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +68 -16
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/row-record-cache.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/row-record-cache.js +13 -8
- package/out/zero-cache/src/services/view-syncer/row-record-cache.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/tracer.d.ts +2 -0
- package/out/zero-cache/src/services/view-syncer/tracer.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/tracer.js +7 -0
- package/out/zero-cache/src/services/view-syncer/tracer.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +58 -43
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/pg.js +0 -4
- package/out/zero-cache/src/types/pg.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +3 -1
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +1 -1
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +7 -1
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +8 -2
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-client/src/client/options.d.ts +7 -7
- package/out/zero-client/src/client/options.d.ts.map +1 -1
- package/out/zero-client/src/client/options.js.map +1 -1
- package/out/zero-client/src/client/query-manager.js +1 -1
- package/out/zero-client/src/client/query-manager.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-poke-handler.d.ts +5 -5
- package/out/zero-client/src/client/zero-poke-handler.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-poke-handler.js +15 -17
- package/out/zero-client/src/client/zero-poke-handler.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts +6 -2
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +44 -8
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/mod.d.ts +1 -1
- package/out/zero-client/src/mod.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.d.ts +2 -9
- package/out/zero-protocol/src/ast.d.ts.map +1 -1
- package/out/zero-protocol/src/ast.js +15 -32
- package/out/zero-protocol/src/ast.js.map +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js +5 -2
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zero-react/src/mod.d.ts +0 -2
- package/out/zero-react/src/mod.d.ts.map +1 -1
- package/out/zero-react/src/use-query.d.ts +6 -6
- package/out/zero-react/src/use-query.d.ts.map +1 -1
- package/out/zero-react/src/use-query.js +9 -2
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-react/src/zero-provider.d.ts +5 -5
- package/out/zero-react/src/zero-provider.d.ts.map +1 -1
- package/out/zero-react/src/zero-provider.js.map +1 -1
- package/out/zero-solid/src/solid-view.d.ts +0 -42
- package/out/zero-solid/src/solid-view.d.ts.map +1 -1
- package/out/zero-solid/src/solid-view.js +1 -1
- package/out/zero-solid/src/solid-view.js.map +1 -1
- package/out/zero-solid/src/use-query.d.ts +4 -4
- package/out/zero-solid/src/use-query.d.ts.map +1 -1
- package/out/zero-solid/src/use-query.js.map +1 -1
- package/out/zero-solid/src/use-zero.d.ts +5 -5
- package/out/zero-solid/src/use-zero.d.ts.map +1 -1
- package/out/zero-solid/src/use-zero.js.map +1 -1
- package/out/zero-types/src/default-types.d.ts +2 -0
- package/out/zero-types/src/default-types.d.ts.map +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +6 -48
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/builder/filter.d.ts.map +1 -1
- package/out/zql/src/builder/filter.js +0 -1
- package/out/zql/src/builder/filter.js.map +1 -1
- package/out/zql/src/ivm/array-view.d.ts.map +1 -1
- package/out/zql/src/ivm/array-view.js +6 -57
- package/out/zql/src/ivm/array-view.js.map +1 -1
- package/out/zql/src/ivm/view-apply-change.d.ts +3 -50
- package/out/zql/src/ivm/view-apply-change.d.ts.map +1 -1
- package/out/zql/src/ivm/view-apply-change.js +105 -358
- package/out/zql/src/ivm/view-apply-change.js.map +1 -1
- package/out/zql/src/mutate/mutator-registry.d.ts +3 -3
- package/out/zql/src/mutate/mutator-registry.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator-registry.js.map +1 -1
- package/out/zql/src/planner/planner-builder.d.ts.map +1 -1
- package/out/zql/src/planner/planner-builder.js +1 -2
- package/out/zql/src/planner/planner-builder.js.map +1 -1
- package/out/zql/src/query/complete-ordering.js +0 -6
- package/out/zql/src/query/complete-ordering.js.map +1 -1
- package/out/zql/src/query/expression.d.ts +2 -19
- package/out/zql/src/query/expression.d.ts.map +1 -1
- package/out/zql/src/query/expression.js +6 -50
- package/out/zql/src/query/expression.js.map +1 -1
- package/out/zql/src/query/query-delegate-base.js +3 -1
- package/out/zql/src/query/query-delegate-base.js.map +1 -1
- package/out/zql/src/query/query-impl.d.ts.map +1 -1
- package/out/zql/src/query/query-impl.js +8 -12
- package/out/zql/src/query/query-impl.js.map +1 -1
- package/out/zql/src/query/query-internals.js.map +1 -1
- package/out/zql/src/query/query-registry.d.ts +3 -3
- package/out/zql/src/query/query-registry.d.ts.map +1 -1
- package/out/zql/src/query/query-registry.js.map +1 -1
- package/out/zql/src/query/query.d.ts +28 -5
- package/out/zql/src/query/query.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +0 -2
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/resolve-scalar-subqueries.d.ts +10 -2
- package/out/zqlite/src/resolve-scalar-subqueries.d.ts.map +1 -1
- package/out/zqlite/src/resolve-scalar-subqueries.js +41 -9
- package/out/zqlite/src/resolve-scalar-subqueries.js.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.d.ts.map +1 -1
- package/out/zqlite/src/sqlite-cost-model.js +0 -1
- package/out/zqlite/src/sqlite-cost-model.js.map +1 -1
- package/package.json +3 -5
- package/out/zero-cache/src/services/change-source/custom/sync-schema.d.ts +0 -4
- package/out/zero-cache/src/services/change-source/custom/sync-schema.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-source/custom/sync-schema.js +0 -14
- package/out/zero-cache/src/services/change-source/custom/sync-schema.js.map +0 -1
- package/out/zero-cache/src/services/change-source/pg/sync-schema.d.ts +0 -5
- package/out/zero-cache/src/services/change-source/pg/sync-schema.d.ts.map +0 -1
- package/out/zero-cache/src/services/change-source/pg/sync-schema.js +0 -14
- package/out/zero-cache/src/services/change-source/pg/sync-schema.js.map +0 -1
- package/out/zero-react/src/paging-reducer.d.ts +0 -61
- package/out/zero-react/src/paging-reducer.d.ts.map +0 -1
- package/out/zero-react/src/paging-reducer.js +0 -77
- package/out/zero-react/src/paging-reducer.js.map +0 -1
- package/out/zero-react/src/use-rows.d.ts +0 -39
- package/out/zero-react/src/use-rows.d.ts.map +0 -1
- package/out/zero-react/src/use-rows.js +0 -130
- package/out/zero-react/src/use-rows.js.map +0 -1
- package/out/zero-react/src/use-zero-virtualizer.d.ts +0 -122
- package/out/zero-react/src/use-zero-virtualizer.d.ts.map +0 -1
- package/out/zero-react/src/use-zero-virtualizer.js +0 -342
- package/out/zero-react/src/use-zero-virtualizer.js.map +0 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { jsonObjectSchema, stringify } from "../../../../../shared/src/bigint-json.js";
|
|
1
2
|
import { parse } from "../../../../../shared/src/valita.js";
|
|
2
3
|
import { CREATE_CHANGELOG_SCHEMA } from "./change-log.js";
|
|
3
4
|
import { CREATE_COLUMN_METADATA_TABLE } from "./column-metadata.js";
|
|
@@ -10,16 +11,23 @@ const CREATE_RUNTIME_EVENTS_TABLE = `
|
|
|
10
11
|
);
|
|
11
12
|
`;
|
|
12
13
|
const CREATE_REPLICATION_STATE_SCHEMA = (
|
|
13
|
-
// replicaVersion
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
// publications
|
|
18
|
-
//
|
|
14
|
+
// replicaVersion : A value identifying the version at which the initial sync happened, i.e.
|
|
15
|
+
// the version at which all rows were copied, and to `_0_version` was set.
|
|
16
|
+
// This value is used to distinguish data from other replicas (e.g. if a
|
|
17
|
+
// replica is reset or if there are ever multiple replicas).
|
|
18
|
+
// publications : JSON stringified array of publication names
|
|
19
|
+
// initialSyncContext : Metadata related to the context of when and how the replica was initially
|
|
20
|
+
// synced. This corresponds with the same column stored in upstream and is
|
|
21
|
+
// used for debugging replica version mismatches, which can arise from a number
|
|
22
|
+
// of misconfigurations, such as dueling replication-managers, or restores of
|
|
23
|
+
// stale litestream backups.
|
|
24
|
+
// lock : Auto-magic column for enforcing single-row semantics.
|
|
25
|
+
/*sql*/
|
|
19
26
|
`
|
|
20
27
|
CREATE TABLE "_zero.replicationConfig" (
|
|
21
28
|
replicaVersion TEXT NOT NULL,
|
|
22
29
|
publications TEXT NOT NULL,
|
|
30
|
+
initialSyncContext TEXT DEFAULT '{}',
|
|
23
31
|
lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)
|
|
24
32
|
);
|
|
25
33
|
|
|
@@ -38,19 +46,36 @@ const subscriptionStateSchema = object({
|
|
|
38
46
|
...s,
|
|
39
47
|
publications: parse(JSON.parse(s.publications), stringArray)
|
|
40
48
|
}));
|
|
49
|
+
const subscriptionStateAndContextSchema = object({
|
|
50
|
+
replicaVersion: string(),
|
|
51
|
+
publications: string(),
|
|
52
|
+
initialSyncContext: string(),
|
|
53
|
+
watermark: string()
|
|
54
|
+
}).map((s) => ({
|
|
55
|
+
...s,
|
|
56
|
+
publications: parse(JSON.parse(s.publications), stringArray),
|
|
57
|
+
initialSyncContext: parse(
|
|
58
|
+
JSON.parse(s.initialSyncContext),
|
|
59
|
+
jsonObjectSchema
|
|
60
|
+
)
|
|
61
|
+
}));
|
|
41
62
|
const replicationStateSchema = object({
|
|
42
63
|
stateVersion: string()
|
|
43
64
|
});
|
|
44
|
-
function initReplicationState(db, publications, watermark, createTables = true) {
|
|
65
|
+
function initReplicationState(db, publications, watermark, initialSyncContext = {}, createTables = true) {
|
|
45
66
|
if (createTables) {
|
|
46
67
|
createReplicationStateTables(db);
|
|
47
68
|
}
|
|
48
69
|
db.prepare(
|
|
49
70
|
`
|
|
50
71
|
INSERT INTO "_zero.replicationConfig"
|
|
51
|
-
(replicaVersion, publications) VALUES (?, ?)
|
|
72
|
+
(replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)
|
|
52
73
|
`
|
|
53
|
-
).run(
|
|
74
|
+
).run(
|
|
75
|
+
watermark,
|
|
76
|
+
JSON.stringify(publications.sort()),
|
|
77
|
+
stringify(initialSyncContext)
|
|
78
|
+
);
|
|
54
79
|
db.prepare(
|
|
55
80
|
`
|
|
56
81
|
INSERT INTO "_zero.replicationState" (stateVersion) VALUES (?)
|
|
@@ -81,6 +106,7 @@ function getAscendingEvents(db) {
|
|
|
81
106
|
}
|
|
82
107
|
function getSubscriptionState(db) {
|
|
83
108
|
const result = db.get(
|
|
109
|
+
/*sql*/
|
|
84
110
|
`
|
|
85
111
|
SELECT c.replicaVersion, c.publications, s.stateVersion as watermark
|
|
86
112
|
FROM "_zero.replicationConfig" as c
|
|
@@ -90,6 +116,19 @@ function getSubscriptionState(db) {
|
|
|
90
116
|
);
|
|
91
117
|
return parse(result, subscriptionStateSchema);
|
|
92
118
|
}
|
|
119
|
+
function getSubscriptionStateAndContext(db) {
|
|
120
|
+
const result = db.get(
|
|
121
|
+
/*sql*/
|
|
122
|
+
`
|
|
123
|
+
SELECT c.replicaVersion, c.publications, c.initialSyncContext,
|
|
124
|
+
s.stateVersion as watermark
|
|
125
|
+
FROM "_zero.replicationConfig" as c
|
|
126
|
+
JOIN "_zero.replicationState" as s
|
|
127
|
+
ON c.lock = s.lock
|
|
128
|
+
`
|
|
129
|
+
);
|
|
130
|
+
return parse(result, subscriptionStateAndContextSchema);
|
|
131
|
+
}
|
|
93
132
|
function updateReplicationWatermark(db, watermark) {
|
|
94
133
|
db.run(`UPDATE "_zero.replicationState" SET stateVersion=?`, watermark);
|
|
95
134
|
}
|
|
@@ -103,6 +142,7 @@ export {
|
|
|
103
142
|
getAscendingEvents,
|
|
104
143
|
getReplicationState,
|
|
105
144
|
getSubscriptionState,
|
|
145
|
+
getSubscriptionStateAndContext,
|
|
106
146
|
initReplicationState,
|
|
107
147
|
recordEvent,
|
|
108
148
|
updateReplicationWatermark
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-state.js","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/**\n * Replication metadata, used for incremental view maintenance and catchup.\n *\n * These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n */\n\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\nimport {CREATE_TABLE_METADATA_TABLE} from './table-metadata.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' | 'upgrade' | 'vacuum';\n\n// event : The RuntimeEvent. Only one row per event is tracked.\n// Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n// Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n CREATE TABLE \"_zero.runtimeEvents\" (\n event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n // replicaVersion
|
|
1
|
+
{"version":3,"file":"replication-state.js","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/**\n * Replication metadata, used for incremental view maintenance and catchup.\n *\n * These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n */\n\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\nimport {CREATE_TABLE_METADATA_TABLE} from './table-metadata.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' | 'upgrade' | 'vacuum';\n\n// event : The RuntimeEvent. Only one row per event is tracked.\n// Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n// Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n CREATE TABLE \"_zero.runtimeEvents\" (\n event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n // replicaVersion : A value identifying the version at which the initial sync happened, i.e.\n // the version at which all rows were copied, and to `_0_version` was set.\n // This value is used to distinguish data from other replicas (e.g. if a\n // replica is reset or if there are ever multiple replicas).\n // publications : JSON stringified array of publication names\n // initialSyncContext : Metadata related to the context of when and how the replica was initially\n // synced. This corresponds with the same column stored in upstream and is\n // used for debugging replica version mismatches, which can arise from a number\n // of misconfigurations, such as dueling replication-managers, or restores of\n // stale litestream backups.\n // lock : Auto-magic column for enforcing single-row semantics.\n /*sql*/ `\n CREATE TABLE \"_zero.replicationConfig\" (\n replicaVersion TEXT NOT NULL,\n publications TEXT NOT NULL,\n initialSyncContext TEXT DEFAULT '{}',\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n ` +\n // stateVersion : The latest version replicated from upstream, starting with the initial\n // `replicaVersion` and moving forward to each subsequent commit watermark\n // (e.g. corresponding to a Postgres LSN). Versions are represented as\n // lexicographically sortable watermarks (e.g. LexiVersions).\n //\n `\n CREATE TABLE \"_zero.replicationState\" (\n stateVersion TEXT NOT NULL,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n ` +\n CREATE_CHANGELOG_SCHEMA +\n CREATE_RUNTIME_EVENTS_TABLE +\n CREATE_COLUMN_METADATA_TABLE +\n CREATE_TABLE_METADATA_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n .object({\n replicaVersion: v.string(),\n publications: v.string(),\n watermark: v.string(),\n })\n .map(s => ({\n ...s,\n publications: v.parse(JSON.parse(s.publications), stringArray),\n }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst subscriptionStateAndContextSchema = v\n .object({\n replicaVersion: v.string(),\n publications: v.string(),\n initialSyncContext: v.string(),\n watermark: v.string(),\n })\n .map(s => ({\n ...s,\n publications: v.parse(JSON.parse(s.publications), stringArray),\n initialSyncContext: v.parse(\n JSON.parse(s.initialSyncContext),\n jsonObjectSchema,\n ),\n }));\n\nexport type SubscriptionStateAndContext = v.Infer<\n typeof subscriptionStateAndContextSchema\n>;\n\nconst replicationStateSchema = v.object({\n stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n db: Database,\n publications: string[],\n watermark: string,\n initialSyncContext: JSONObject = {},\n createTables = true,\n) {\n if (createTables) {\n createReplicationStateTables(db);\n }\n db.prepare(\n `\n INSERT INTO \"_zero.replicationConfig\" \n (replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)\n `,\n ).run(\n watermark,\n JSON.stringify(publications.sort()),\n stringify(initialSyncContext),\n );\n db.prepare(\n `\n INSERT INTO \"_zero.replicationState\" (stateVersion) VALUES (?)\n `,\n ).run(watermark);\n recordEvent(db, 'sync');\n}\n\n/**\n * Exposed as a separate function for the custom change source,\n * which needs the tables to be created in order to construct\n * ChangeProcessor before it knows the initial watermark.\n */\nexport function createReplicationStateTables(db: Database) {\n db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n db.prepare(\n `\n INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n `,\n ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n const result = db\n .prepare(\n `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n ORDER BY timestamp ASC\n `,\n )\n .all<{event: string; timestamp: string}>();\n return result.map(({event, timestamp}) => ({\n event,\n timestamp: new Date(timestamp + 'Z'),\n }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n const result = db.get(/*sql*/ `\n SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n FROM \"_zero.replicationConfig\" as c\n JOIN \"_zero.replicationState\" as s\n ON c.lock = s.lock\n `);\n return v.parse(result, subscriptionStateSchema);\n}\n\nexport function getSubscriptionStateAndContext(\n db: StatementRunner,\n): SubscriptionStateAndContext {\n const result = db.get(/*sql*/ `\n SELECT c.replicaVersion, c.publications, c.initialSyncContext,\n s.stateVersion as watermark\n FROM \"_zero.replicationConfig\" as c\n JOIN \"_zero.replicationState\" as s\n ON c.lock = s.lock\n `);\n return v.parse(result, subscriptionStateAndContextSchema);\n}\n\nexport function updateReplicationWatermark(\n db: StatementRunner,\n watermark: string,\n) {\n db.run(`UPDATE \"_zero.replicationState\" SET stateVersion=?`, watermark);\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n return v.parse(result, replicationStateSchema);\n}\n"],"names":["v.array","v.string","v.object","v.parse"],"mappings":";;;;;;AA4BO,MAAM,8BAA8B;AAAA;AAAA;AAAA;AAAA;AAAA;AAO3C,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAmBR,0BACA,8BACA,+BACA;AAAA;AAEF,MAAM,cAAcA,MAAQC,QAAU;AAEtC,MAAM,0BAA0BC,OACtB;AAAA,EACN,gBAAgBD,OAAE;AAAA,EAClB,cAAcA,OAAE;AAAA,EAChB,WAAWA,OAAE;AACf,CAAC,EACA,IAAI,CAAA,OAAM;AAAA,EACT,GAAG;AAAA,EACH,cAAcE,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;AAC/D,EAAE;AAIJ,MAAM,oCAAoCD,OAChC;AAAA,EACN,gBAAgBD,OAAE;AAAA,EAClB,cAAcA,OAAE;AAAA,EAChB,oBAAoBA,OAAE;AAAA,EACtB,WAAWA,OAAE;AACf,CAAC,EACA,IAAI,CAAA,OAAM;AAAA,EACT,GAAG;AAAA,EACH,cAAcE,MAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,WAAW;AAAA,EAC7D,oBAAoBA;AAAAA,IAClB,KAAK,MAAM,EAAE,kBAAkB;AAAA,IAC/B;AAAA,EAAA;AAEJ,EAAE;AAMJ,MAAM,yBAAyBD,OAAS;AAAA,EACtC,cAAcD,OAAE;AAClB,CAAC;AAIM,SAAS,qBACd,IACA,cACA,WACA,qBAAiC,CAAA,GACjC,eAAe,MACf;AACA,MAAI,cAAc;AAChB,iCAA6B,EAAE;AAAA,EACjC;AACA,KAAG;AAAA,IACD;AAAA;AAAA;AAAA;AAAA,EAAA,EAIA;AAAA,IACA;AAAA,IACA,KAAK,UAAU,aAAa,MAAM;AAAA,IAClC,UAAU,kBAAkB;AAAA,EAAA;AAE9B,KAAG;AAAA,IACD;AAAA;AAAA;AAAA,EAAA,EAGA,IAAI,SAAS;AACf,cAAY,IAAI,MAAM;AACxB;AAOO,SAAS,6BAA6B,IAAc;AACzD,KAAG,KAAK,+BAA+B;AACzC;AAEO,SAAS,YAAY,IAAc,OAAqB;AAC7D,KAAG;AAAA,IACD;AAAA;AAAA;AAAA,EAAA,EAGA,IAAI,KAAK;AACb;AAEO,SAAS,mBAAmB,IAAc;AAC/C,QAAM,SAAS,GACZ;AAAA,IACC;AAAA;AAAA;AAAA,EAAA,EAID,IAAA;AACH,SAAO,OAAO,IAAI,CAAC,EAAC,OAAO,iBAAgB;AAAA,IACzC;AAAA,IACA,WAAW,oBAAI,KAAK,YAAY,GAAG;AAAA,EAAA,EACnC;AACJ;AAEO,SAAS,qBAAqB,IAAwC;AAC3E,QAAM,SAAS,GAAG;AAAA;AAAA,IAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAAA;AAM9B,SAAOE,MAAQ,QAAQ,uBAAuB;AAChD;AAEO,SAAS,+BACd,IAC6B;AAC7B,QAAM,SAAS,GAAG;AAAA;AAAA,IAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAAA;AAO9B,SAAOA,MAAQ,QAAQ,iCAAiC;AAC1D;AAEO,SAAS,2BACd,IACA,WACA;AACA,KAAG,IAAI,sDAAsD,SAAS;AACxE;AAEO,SAAS,oBAAoB,IAAuC;AACzE,QAAM,SAAS,GAAG,IAAI,mDAAmD;AACzE,SAAOA,MAAQ,QAAQ,sBAAsB;AAC/C;"}
|
|
@@ -15,6 +15,7 @@ export type UnregisterFn = () => void;
|
|
|
15
15
|
export declare class RunningState {
|
|
16
16
|
#private;
|
|
17
17
|
constructor(serviceName: string, retryConfig?: RetryConfig, setTimeoutFn?: typeof setTimeout, sleeper?: typeof sleepWithAbort);
|
|
18
|
+
get signal(): AbortSignal;
|
|
18
19
|
/**
|
|
19
20
|
* Returns `true` until {@link stop()} has been called.
|
|
20
21
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"running-state.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/running-state.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AAG5D,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC;AAEtC;;GAEG;AACH,qBAAa,YAAY;;gBAarB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,WAAW,EACzB,YAAY,oBAAa,EACzB,OAAO,wBAAiB;IA+B1B;;;;;OAKG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,YAAY,CAAC,CAAC,EAAE,UAAU,GAAG,YAAY;IAMzC;;OAEG;IACH,UAAU,CAAC,KAAK,SAAS,OAAO,EAAE,EAChC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,EAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,IAAI,EAAE,KAAK;IAWhB;;;OAGG;IACG,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI;IAQzC;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1D;;;;OAIG;IACH,YAAY;CAGb;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,wBAAwB;CACtC"}
|
|
1
|
+
{"version":3,"file":"running-state.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/running-state.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AAG5D,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC;AAEtC;;GAEG;AACH,qBAAa,YAAY;;gBAarB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,WAAW,EACzB,YAAY,oBAAa,EACzB,OAAO,wBAAiB;IA+B1B,IAAI,MAAM,IAAI,WAAW,CAExB;IAED;;;;;OAKG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,YAAY,CAAC,CAAC,EAAE,UAAU,GAAG,YAAY;IAMzC;;OAEG;IACH,UAAU,CAAC,KAAK,SAAS,OAAO,EAAE,EAChC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,EAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,IAAI,EAAE,KAAK;IAWhB;;;OAGG;IACG,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI;IAQzC;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1D;;;;OAIG;IACH,YAAY;CAGb;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,wBAAwB;CACtC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"running-state.js","sources":["../../../../../zero-cache/src/services/running-state.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {sleepWithAbort} from '../../../shared/src/sleep.ts';\n\nconst DEFAULT_INITIAL_RETRY_DELAY_MS = 25;\nexport const DEFAULT_MAX_RETRY_DELAY_MS = 10000;\n\nexport type RetryConfig = {\n initialRetryDelay?: number;\n maxRetryDelay?: number;\n};\n\nexport interface Cancelable {\n cancel(): void;\n}\n\nexport type UnregisterFn = () => void;\n\n/**\n * Facilitates lifecycle control with exponential backoff.\n */\nexport class RunningState {\n readonly #serviceName: string;\n readonly #controller: AbortController;\n readonly #sleep: typeof sleepWithAbort;\n readonly #setTimeout: typeof setTimeout;\n readonly #stopped: Promise<void>;\n\n readonly #initialRetryDelay: number;\n readonly #maxRetryDelay: number;\n readonly #pendingTimeouts = new Set<NodeJS.Timeout>();\n #retryDelay: number;\n\n constructor(\n serviceName: string,\n retryConfig?: RetryConfig,\n setTimeoutFn = setTimeout,\n sleeper = sleepWithAbort,\n ) {\n const {\n initialRetryDelay = DEFAULT_INITIAL_RETRY_DELAY_MS,\n maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,\n } = retryConfig ?? {};\n\n this.#serviceName = serviceName;\n this.#initialRetryDelay = initialRetryDelay;\n this.#maxRetryDelay = maxRetryDelay;\n this.#retryDelay = initialRetryDelay;\n\n this.#controller = new AbortController();\n this.#sleep = sleeper;\n this.#setTimeout = setTimeoutFn;\n\n const {promise, resolve} = resolver();\n this.#stopped = promise;\n this.#controller.signal.addEventListener(\n 'abort',\n () => {\n resolve();\n for (const timeout of this.#pendingTimeouts) {\n clearTimeout(timeout);\n }\n this.#pendingTimeouts.clear();\n },\n {once: true},\n );\n }\n\n /**\n * Returns `true` until {@link stop()} has been called.\n *\n * This is usually called as part of the service's main loop\n * conditional to determine if the next iteration should execute.\n */\n shouldRun(): boolean {\n return !this.#controller.signal.aborted;\n }\n\n /**\n * Registers a Cancelable object to be invoked when {@link stop()} is called.\n * Returns a method to unregister the object.\n */\n cancelOnStop(c: Cancelable): UnregisterFn {\n const onStop = () => c.cancel();\n this.#controller.signal.addEventListener('abort', onStop, {once: true});\n return () => this.#controller.signal.removeEventListener('abort', onStop);\n }\n\n /**\n * Sets a Timeout that is automatically cancelled if the service is cancelled.\n */\n setTimeout<TArgs extends unknown[]>(\n fn: (...args: TArgs) => void,\n timeoutMs: number,\n ...args: TArgs\n ) {\n const timeout = this.#setTimeout(() => {\n clearTimeout(timeout);\n this.#pendingTimeouts.delete(timeout);\n return fn(...args);\n }, timeoutMs);\n\n this.#pendingTimeouts.add(timeout);\n }\n\n /**\n * Returns a promise that resolves after `ms` milliseconds or when\n * the service is stopped.\n */\n async sleep(ms: number): Promise<void> {\n await Promise.race(this.#sleep(ms, this.#controller.signal));\n }\n\n /**\n * Called to stop the service. After this is called, {@link shouldRun()}\n * will return `false` and the {@link stopped()} Promise will be resolved.\n */\n stop(lc: LogContext, err?: unknown): void {\n if (this.shouldRun()) {\n const log = !err || err instanceof AbortError ? 'info' : 'error';\n lc[log]?.(`stopping ${this.#serviceName}`, err ?? '');\n this.#controller.abort();\n }\n }\n\n /**\n * Returns a Promise that resolves when {@link stop()} is called.\n */\n stopped(): Promise<void> {\n return this.#stopped;\n }\n\n /**\n * Call in response to an error or unexpected termination in the main\n * loop of the service. The returned Promise will resolve after an\n * exponential delay, or once {@link stop()} is called.\n *\n * If the supplied `err` is an `AbortError`, the service will shut down.\n */\n async backoff(lc: LogContext, err: unknown): Promise<void> {\n const delay = this.#retryDelay;\n this.#retryDelay = Math.min(delay * 2, this.#maxRetryDelay);\n\n if (err instanceof AbortError || err instanceof UnrecoverableError) {\n this.stop(lc, err);\n } else if (this.shouldRun()) {\n // Use delay-based log level: higher delay means more retries\n const log: 'info' | 'warn' | 'error' =\n delay < 1000 ? 'info' : delay < 6500 ? 'warn' : 'error';\n\n lc[log]?.(`retrying ${this.#serviceName} in ${delay} ms`, err);\n await this.sleep(delay);\n }\n }\n\n /**\n * When using {@link backoff()}, this method should be called when the\n * implementation receives a healthy signal (e.g. a successful\n * response). This resets the delay used in {@link backoff()}.\n */\n resetBackoff() {\n this.#retryDelay = this.#initialRetryDelay;\n }\n}\n\n/**\n * Superclass for Errors that should bypass exponential backoff\n * and immediately shut down the server.\n */\nexport class UnrecoverableError extends Error {\n readonly name = 'UnrecoverableError';\n}\n"],"names":[],"mappings":";;;AAKA,MAAM,iCAAiC;AAChC,MAAM,6BAA6B;AAgBnC,MAAM,aAAa;AAAA,EACf;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,uCAAuB,IAAA;AAAA,EAChC;AAAA,EAEA,YACE,aACA,aACA,eAAe,YACf,UAAU,gBACV;AACA,UAAM;AAAA,MACJ,oBAAoB;AAAA,MACpB,gBAAgB;AAAA,IAAA,IACd,eAAe,CAAA;AAEnB,SAAK,eAAe;AACpB,SAAK,qBAAqB;AAC1B,SAAK,iBAAiB;AACtB,SAAK,cAAc;AAEnB,SAAK,cAAc,IAAI,gBAAA;AACvB,SAAK,SAAS;AACd,SAAK,cAAc;AAEnB,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,WAAW;AAChB,SAAK,YAAY,OAAO;AAAA,MACtB;AAAA,MACA,MAAM;AACJ,gBAAA;AACA,mBAAW,WAAW,KAAK,kBAAkB;AAC3C,uBAAa,OAAO;AAAA,QACtB;AACA,aAAK,iBAAiB,MAAA;AAAA,MACxB;AAAA,MACA,EAAC,MAAM,KAAA;AAAA,IAAI;AAAA,EAEf;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,YAAqB;AACnB,WAAO,CAAC,KAAK,YAAY,OAAO;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAa,GAA6B;AACxC,UAAM,SAAS,MAAM,EAAE,OAAA;AACvB,SAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ,EAAC,MAAM,MAAK;AACtE,WAAO,MAAM,KAAK,YAAY,OAAO,oBAAoB,SAAS,MAAM;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,WACE,IACA,cACG,MACH;AACA,UAAM,UAAU,KAAK,YAAY,MAAM;AACrC,mBAAa,OAAO;AACpB,WAAK,iBAAiB,OAAO,OAAO;AACpC,aAAO,GAAG,GAAG,IAAI;AAAA,IACnB,GAAG,SAAS;AAEZ,SAAK,iBAAiB,IAAI,OAAO;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,MAAM,IAA2B;AACrC,UAAM,QAAQ,KAAK,KAAK,OAAO,IAAI,KAAK,YAAY,MAAM,CAAC;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,KAAK,IAAgB,KAAqB;AACxC,QAAI,KAAK,aAAa;AACpB,YAAM,MAAM,CAAC,OAAO,eAAe,aAAa,SAAS;AACzD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,IAAI,OAAO,EAAE;AACpD,WAAK,YAAY,MAAA;AAAA,IACnB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAyB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,QAAQ,IAAgB,KAA6B;AACzD,UAAM,QAAQ,KAAK;AACnB,SAAK,cAAc,KAAK,IAAI,QAAQ,GAAG,KAAK,cAAc;AAE1D,QAAI,eAAe,cAAc,eAAe,oBAAoB;AAClE,WAAK,KAAK,IAAI,GAAG;AAAA,IACnB,WAAW,KAAK,aAAa;AAE3B,YAAM,MACJ,QAAQ,MAAO,SAAS,QAAQ,OAAO,SAAS;AAElD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,OAAO,KAAK,OAAO,GAAG;AAC7D,YAAM,KAAK,MAAM,KAAK;AAAA,IACxB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,eAAe;AACb,SAAK,cAAc,KAAK;AAAA,EAC1B;AACF;AAMO,MAAM,2BAA2B,MAAM;AAAA,EACnC,OAAO;AAClB;"}
|
|
1
|
+
{"version":3,"file":"running-state.js","sources":["../../../../../zero-cache/src/services/running-state.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {sleepWithAbort} from '../../../shared/src/sleep.ts';\n\nconst DEFAULT_INITIAL_RETRY_DELAY_MS = 25;\nexport const DEFAULT_MAX_RETRY_DELAY_MS = 10000;\n\nexport type RetryConfig = {\n initialRetryDelay?: number;\n maxRetryDelay?: number;\n};\n\nexport interface Cancelable {\n cancel(): void;\n}\n\nexport type UnregisterFn = () => void;\n\n/**\n * Facilitates lifecycle control with exponential backoff.\n */\nexport class RunningState {\n readonly #serviceName: string;\n readonly #controller: AbortController;\n readonly #sleep: typeof sleepWithAbort;\n readonly #setTimeout: typeof setTimeout;\n readonly #stopped: Promise<void>;\n\n readonly #initialRetryDelay: number;\n readonly #maxRetryDelay: number;\n readonly #pendingTimeouts = new Set<NodeJS.Timeout>();\n #retryDelay: number;\n\n constructor(\n serviceName: string,\n retryConfig?: RetryConfig,\n setTimeoutFn = setTimeout,\n sleeper = sleepWithAbort,\n ) {\n const {\n initialRetryDelay = DEFAULT_INITIAL_RETRY_DELAY_MS,\n maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,\n } = retryConfig ?? {};\n\n this.#serviceName = serviceName;\n this.#initialRetryDelay = initialRetryDelay;\n this.#maxRetryDelay = maxRetryDelay;\n this.#retryDelay = initialRetryDelay;\n\n this.#controller = new AbortController();\n this.#sleep = sleeper;\n this.#setTimeout = setTimeoutFn;\n\n const {promise, resolve} = resolver();\n this.#stopped = promise;\n this.#controller.signal.addEventListener(\n 'abort',\n () => {\n resolve();\n for (const timeout of this.#pendingTimeouts) {\n clearTimeout(timeout);\n }\n this.#pendingTimeouts.clear();\n },\n {once: true},\n );\n }\n\n get signal(): AbortSignal {\n return this.#controller.signal;\n }\n\n /**\n * Returns `true` until {@link stop()} has been called.\n *\n * This is usually called as part of the service's main loop\n * conditional to determine if the next iteration should execute.\n */\n shouldRun(): boolean {\n return !this.#controller.signal.aborted;\n }\n\n /**\n * Registers a Cancelable object to be invoked when {@link stop()} is called.\n * Returns a method to unregister the object.\n */\n cancelOnStop(c: Cancelable): UnregisterFn {\n const onStop = () => c.cancel();\n this.#controller.signal.addEventListener('abort', onStop, {once: true});\n return () => this.#controller.signal.removeEventListener('abort', onStop);\n }\n\n /**\n * Sets a Timeout that is automatically cancelled if the service is cancelled.\n */\n setTimeout<TArgs extends unknown[]>(\n fn: (...args: TArgs) => void,\n timeoutMs: number,\n ...args: TArgs\n ) {\n const timeout = this.#setTimeout(() => {\n clearTimeout(timeout);\n this.#pendingTimeouts.delete(timeout);\n return fn(...args);\n }, timeoutMs);\n\n this.#pendingTimeouts.add(timeout);\n }\n\n /**\n * Returns a promise that resolves after `ms` milliseconds or when\n * the service is stopped.\n */\n async sleep(ms: number): Promise<void> {\n await Promise.race(this.#sleep(ms, this.#controller.signal));\n }\n\n /**\n * Called to stop the service. After this is called, {@link shouldRun()}\n * will return `false` and the {@link stopped()} Promise will be resolved.\n */\n stop(lc: LogContext, err?: unknown): void {\n if (this.shouldRun()) {\n const log = !err || err instanceof AbortError ? 'info' : 'error';\n lc[log]?.(`stopping ${this.#serviceName}`, err ?? '');\n this.#controller.abort();\n }\n }\n\n /**\n * Returns a Promise that resolves when {@link stop()} is called.\n */\n stopped(): Promise<void> {\n return this.#stopped;\n }\n\n /**\n * Call in response to an error or unexpected termination in the main\n * loop of the service. The returned Promise will resolve after an\n * exponential delay, or once {@link stop()} is called.\n *\n * If the supplied `err` is an `AbortError`, the service will shut down.\n */\n async backoff(lc: LogContext, err: unknown): Promise<void> {\n const delay = this.#retryDelay;\n this.#retryDelay = Math.min(delay * 2, this.#maxRetryDelay);\n\n if (err instanceof AbortError || err instanceof UnrecoverableError) {\n this.stop(lc, err);\n } else if (this.shouldRun()) {\n // Use delay-based log level: higher delay means more retries\n const log: 'info' | 'warn' | 'error' =\n delay < 1000 ? 'info' : delay < 6500 ? 'warn' : 'error';\n\n lc[log]?.(`retrying ${this.#serviceName} in ${delay} ms`, err);\n await this.sleep(delay);\n }\n }\n\n /**\n * When using {@link backoff()}, this method should be called when the\n * implementation receives a healthy signal (e.g. a successful\n * response). This resets the delay used in {@link backoff()}.\n */\n resetBackoff() {\n this.#retryDelay = this.#initialRetryDelay;\n }\n}\n\n/**\n * Superclass for Errors that should bypass exponential backoff\n * and immediately shut down the server.\n */\nexport class UnrecoverableError extends Error {\n readonly name = 'UnrecoverableError';\n}\n"],"names":[],"mappings":";;;AAKA,MAAM,iCAAiC;AAChC,MAAM,6BAA6B;AAgBnC,MAAM,aAAa;AAAA,EACf;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,uCAAuB,IAAA;AAAA,EAChC;AAAA,EAEA,YACE,aACA,aACA,eAAe,YACf,UAAU,gBACV;AACA,UAAM;AAAA,MACJ,oBAAoB;AAAA,MACpB,gBAAgB;AAAA,IAAA,IACd,eAAe,CAAA;AAEnB,SAAK,eAAe;AACpB,SAAK,qBAAqB;AAC1B,SAAK,iBAAiB;AACtB,SAAK,cAAc;AAEnB,SAAK,cAAc,IAAI,gBAAA;AACvB,SAAK,SAAS;AACd,SAAK,cAAc;AAEnB,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,WAAW;AAChB,SAAK,YAAY,OAAO;AAAA,MACtB;AAAA,MACA,MAAM;AACJ,gBAAA;AACA,mBAAW,WAAW,KAAK,kBAAkB;AAC3C,uBAAa,OAAO;AAAA,QACtB;AACA,aAAK,iBAAiB,MAAA;AAAA,MACxB;AAAA,MACA,EAAC,MAAM,KAAA;AAAA,IAAI;AAAA,EAEf;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,YAAqB;AACnB,WAAO,CAAC,KAAK,YAAY,OAAO;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAa,GAA6B;AACxC,UAAM,SAAS,MAAM,EAAE,OAAA;AACvB,SAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ,EAAC,MAAM,MAAK;AACtE,WAAO,MAAM,KAAK,YAAY,OAAO,oBAAoB,SAAS,MAAM;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,WACE,IACA,cACG,MACH;AACA,UAAM,UAAU,KAAK,YAAY,MAAM;AACrC,mBAAa,OAAO;AACpB,WAAK,iBAAiB,OAAO,OAAO;AACpC,aAAO,GAAG,GAAG,IAAI;AAAA,IACnB,GAAG,SAAS;AAEZ,SAAK,iBAAiB,IAAI,OAAO;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,MAAM,IAA2B;AACrC,UAAM,QAAQ,KAAK,KAAK,OAAO,IAAI,KAAK,YAAY,MAAM,CAAC;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,KAAK,IAAgB,KAAqB;AACxC,QAAI,KAAK,aAAa;AACpB,YAAM,MAAM,CAAC,OAAO,eAAe,aAAa,SAAS;AACzD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,IAAI,OAAO,EAAE;AACpD,WAAK,YAAY,MAAA;AAAA,IACnB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAyB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,QAAQ,IAAgB,KAA6B;AACzD,UAAM,QAAQ,KAAK;AACnB,SAAK,cAAc,KAAK,IAAI,QAAQ,GAAG,KAAK,cAAc;AAE1D,QAAI,eAAe,cAAc,eAAe,oBAAoB;AAClE,WAAK,KAAK,IAAI,GAAG;AAAA,IACnB,WAAW,KAAK,aAAa;AAE3B,YAAM,MACJ,QAAQ,MAAO,SAAS,QAAQ,OAAO,SAAS;AAElD,SAAG,GAAG,IAAI,YAAY,KAAK,YAAY,OAAO,KAAK,OAAO,GAAG;AAC7D,YAAM,KAAK,MAAM,KAAK;AAAA,IACxB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,eAAe;AACb,SAAK,cAAc,KAAK;AAAA,EAC1B;AACF;AAMO,MAAM,2BAA2B,MAAM;AAAA,EACnC,OAAO;AAClB;"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cvr-purger.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"cvr-purger.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAQ3C,KAAK,OAAO,GAAG;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,qBAAa,SAAU,YAAW,OAAO;;IACvC,QAAQ,CAAC,EAAE,YAAY;gBAYrB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAC,qBAAqB,EAAE,gBAAgB,EAAE,iBAAiB,EAAC,EAAE,OAAO,EACrE,uBAAuB,SAA4B;IAc/C,GAAG;IAgDT,iBAAiB,CACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,CAAC;IAkF/C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAItB"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
|
|
2
2
|
import { READ_COMMITTED } from "../../db/mode-enum.js";
|
|
3
|
-
import {
|
|
3
|
+
import { runTx } from "../../db/run-transaction.js";
|
|
4
|
+
import "../../types/pg.js";
|
|
4
5
|
import { cvrSchema } from "../../types/shards.js";
|
|
5
6
|
import { RunningState } from "../running-state.js";
|
|
6
7
|
const MINUTE = 60 * 1e3;
|
|
@@ -65,32 +66,33 @@ class CVRPurger {
|
|
|
65
66
|
}
|
|
66
67
|
// Exported for testing.
|
|
67
68
|
purgeInactiveCVRs(maxCVRs) {
|
|
68
|
-
return
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
69
|
+
return runTx(
|
|
70
|
+
this.#db,
|
|
71
|
+
async (sql) => {
|
|
72
|
+
const now = Date.now();
|
|
73
|
+
const threshold = now - this.#inactivityThresholdMs;
|
|
74
|
+
const tombstonePurgeThreshold = now - this.#tombstonePurgeThresholdMs;
|
|
75
|
+
const ids = (await sql`
|
|
74
76
|
SELECT "clientGroupID" FROM ${sql(this.#schema)}.instances
|
|
75
77
|
WHERE NOT "deleted" AND "lastActive" < ${threshold}
|
|
76
78
|
ORDER BY "lastActive" ASC
|
|
77
79
|
LIMIT ${maxCVRs}
|
|
78
80
|
FOR UPDATE SKIP LOCKED
|
|
79
81
|
`.values()).flat();
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
82
|
+
if (ids.length > 0) {
|
|
83
|
+
const stmts = [
|
|
84
|
+
"desires",
|
|
85
|
+
"queries",
|
|
86
|
+
"clients",
|
|
87
|
+
"rows",
|
|
88
|
+
"rowsVersion"
|
|
89
|
+
].map(
|
|
90
|
+
(table) => sql`
|
|
89
91
|
DELETE FROM ${sql(this.#schema)}.${sql(table)}
|
|
90
92
|
WHERE "clientGroupID" IN ${sql(ids)}`.execute()
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
93
|
+
);
|
|
94
|
+
stmts.push(
|
|
95
|
+
sql`
|
|
94
96
|
UPDATE ${sql(this.#schema)}.instances
|
|
95
97
|
SET "deleted" = TRUE,
|
|
96
98
|
"version" = '00',
|
|
@@ -100,21 +102,23 @@ class CVRPurger {
|
|
|
100
102
|
"grantedAt" = NULL,
|
|
101
103
|
"clientSchema" = NULL
|
|
102
104
|
WHERE "clientGroupID" IN ${sql(ids)}`.execute()
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
105
|
+
);
|
|
106
|
+
stmts.push(
|
|
107
|
+
sql`
|
|
106
108
|
DELETE FROM ${sql(this.#schema)}.instances
|
|
107
109
|
WHERE "deleted" AND "lastActive" < ${tombstonePurgeThreshold}
|
|
108
110
|
`.execute()
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
);
|
|
112
|
+
await Promise.all(stmts);
|
|
113
|
+
}
|
|
114
|
+
const [{ remaining }] = await sql`
|
|
113
115
|
SELECT COUNT(*) AS remaining FROM ${sql(this.#schema)}.instances
|
|
114
116
|
WHERE NOT "deleted" AND "lastActive" < ${threshold}
|
|
115
117
|
`;
|
|
116
|
-
|
|
117
|
-
|
|
118
|
+
return { purged: ids.length, remaining: Number(remaining) };
|
|
119
|
+
},
|
|
120
|
+
{ mode: READ_COMMITTED }
|
|
121
|
+
);
|
|
118
122
|
}
|
|
119
123
|
stop() {
|
|
120
124
|
this.#state.stop(this.#lc);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cvr-purger.js","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {READ_COMMITTED} from '../../db/mode-enum.ts';\nimport {
|
|
1
|
+
{"version":3,"file":"cvr-purger.js","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {READ_COMMITTED} from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../types/pg.ts';\nimport {cvrSchema, type ShardID} from '../../types/shards.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst MINUTE = 60 * 1000;\nconst MAX_PURGE_INTERVAL_MS = 16 * MINUTE;\n\n// Purge tombstones after 31 days to facilitate up to a 30-day actives metric.\nconst TOMBSTONE_PURGE_THRESHOLD = 31 * 24 * 60 * 60 * 1000;\n\ntype Options = {\n inactivityThresholdMs: number;\n initialBatchSize: number;\n initialIntervalMs: number;\n};\n\nexport class CVRPurger implements Service {\n readonly id = 'reaper';\n\n readonly #lc: LogContext;\n readonly #db: PostgresDB;\n readonly #schema: string;\n readonly #inactivityThresholdMs: number;\n readonly #tombstonePurgeThresholdMs: number;\n readonly #initialBatchSize: number;\n readonly #initialIntervalMs: number;\n readonly #state = new RunningState('reaper');\n\n constructor(\n lc: LogContext,\n db: PostgresDB,\n shard: ShardID,\n {inactivityThresholdMs, initialBatchSize, initialIntervalMs}: Options,\n tombstonePurgeThreshold = TOMBSTONE_PURGE_THRESHOLD,\n ) {\n this.#lc = lc;\n this.#db = db;\n this.#schema = cvrSchema(shard);\n this.#inactivityThresholdMs = inactivityThresholdMs;\n this.#tombstonePurgeThresholdMs = Math.max(\n tombstonePurgeThreshold,\n inactivityThresholdMs,\n );\n this.#initialBatchSize = initialBatchSize;\n this.#initialIntervalMs = initialIntervalMs;\n }\n\n async run() {\n let purgeable: number | undefined;\n let maxCVRsPerPurge = this.#initialBatchSize;\n let purgeInterval = this.#initialIntervalMs;\n\n if (this.#initialBatchSize === 0) {\n this.#lc.warn?.(\n `CVR garbage collection is disabled (initialBatchSize = 0)`,\n );\n // Do nothing and just wait to be stopped.\n await this.#state.stopped();\n } else {\n this.#lc.info?.(\n `running cvr-purger with`,\n await this.#db`SHOW statement_timeout`,\n );\n }\n\n while (this.#state.shouldRun()) {\n try {\n const start = performance.now();\n const {purged, remaining} =\n await this.purgeInactiveCVRs(maxCVRsPerPurge);\n\n if (purgeable !== undefined && remaining > purgeable) {\n // If the number of purgeable CVRs has grown even after the purge,\n // increase the number purged per round to achieve a steady state.\n maxCVRsPerPurge += this.#initialBatchSize;\n this.#lc.info?.(`increased CVRs per purge to ${maxCVRsPerPurge}`);\n }\n purgeable = remaining;\n\n purgeInterval =\n purgeable > 0\n ? this.#initialIntervalMs\n : Math.min(purgeInterval * 2, MAX_PURGE_INTERVAL_MS);\n const elapsed = performance.now() - start;\n this.#lc.info?.(\n `purged ${purged} inactive CVRs (${elapsed.toFixed(2)} ms). Next purge in ${purgeInterval} ms`,\n );\n await this.#state.sleep(purgeInterval);\n } catch (e) {\n this.#lc.warn?.(`error encountered while garbage collecting CVRs`, e);\n }\n }\n }\n\n // Exported for testing.\n purgeInactiveCVRs(\n maxCVRs: number,\n ): Promise<{purged: number; remaining: number}> {\n return runTx(\n this.#db,\n async sql => {\n const now = Date.now();\n const threshold = now - this.#inactivityThresholdMs;\n const tombstonePurgeThreshold = now - this.#tombstonePurgeThresholdMs;\n // Implementation note: `FOR UPDATE` will prevent a syncer from\n // concurrently updating the CVR, since the update also performs\n // a `SELECT ... FOR UPDATE`, instead causing that update to\n // fail, which will cause the client to create a new CVR.\n //\n // `SKIP LOCKED` will skip over CVRs that a syncer is already\n // in the process of updating. In this manner, an in-progress\n // update effectively excludes the CVR from the purge.\n const ids = (\n await sql<{clientGroupID: string}[]>`\n SELECT \"clientGroupID\" FROM ${sql(this.#schema)}.instances\n WHERE NOT \"deleted\" AND \"lastActive\" < ${threshold}\n ORDER BY \"lastActive\" ASC\n LIMIT ${maxCVRs}\n FOR UPDATE SKIP LOCKED\n `.values()\n ).flat();\n\n if (ids.length > 0) {\n // Explicitly delete rows from cvr tables from \"bottom\" up. Relying on\n // foreign key cascading deletes can be suboptimal when the foreign key\n // is not a prefix of the primary key (e.g. the \"desires\" foreign key\n // reference to the \"queries\" table is not a prefix of the \"desires\"\n // primary key).\n const stmts = [\n 'desires',\n 'queries',\n 'clients',\n 'rows',\n 'rowsVersion',\n ].map(table =>\n sql`\n DELETE FROM ${sql(this.#schema)}.${sql(table)} \n WHERE \"clientGroupID\" IN ${sql(ids)}`.execute(),\n );\n // Tombstones are written for the `instances` rows, preserving the\n // \"profileID\" and \"lastActive\" columns for computing usage stats.\n //\n // For backwards compatibility (i.e. older zero-caches that do not\n // check the \"deleted\" column) reset the \"version\" to '00' to trigger\n // the ClientNotFound error via\n // view-syncer.ts:checkClientAndCVRVersions()\n stmts.push(\n sql`\n UPDATE ${sql(this.#schema)}.instances\n SET \"deleted\" = TRUE, \n \"version\" = '00', \n \"ttlClock\" = 0,\n \"replicaVersion\" = NULL, \n \"owner\" = NULL,\n \"grantedAt\" = NULL,\n \"clientSchema\" = NULL\n WHERE \"clientGroupID\" IN ${sql(ids)}`.execute(),\n );\n // Tombstone rows are deleted after the tombstonePurgeThreshold.\n stmts.push(\n sql`\n DELETE FROM ${sql(this.#schema)}.instances\n WHERE \"deleted\" AND \"lastActive\" < ${tombstonePurgeThreshold}\n `.execute(),\n );\n await Promise.all(stmts);\n }\n\n const [{remaining}] = await sql<[{remaining: bigint}]>`\n SELECT COUNT(*) AS remaining FROM ${sql(this.#schema)}.instances\n WHERE NOT \"deleted\" AND \"lastActive\" < ${threshold}\n `;\n\n return {purged: ids.length, remaining: Number(remaining)};\n },\n {mode: READ_COMMITTED},\n );\n }\n\n stop(): Promise<void> {\n this.#state.stop(this.#lc);\n return promiseVoid;\n }\n}\n"],"names":[],"mappings":";;;;;;AASA,MAAM,SAAS,KAAK;AACpB,MAAM,wBAAwB,KAAK;AAGnC,MAAM,4BAA4B,KAAK,KAAK,KAAK,KAAK;AAQ/C,MAAM,UAA6B;AAAA,EAC/B,KAAK;AAAA,EAEL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,aAAa,QAAQ;AAAA,EAE3C,YACE,IACA,IACA,OACA,EAAC,uBAAuB,kBAAkB,kBAAA,GAC1C,0BAA0B,2BAC1B;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,UAAU,UAAU,KAAK;AAC9B,SAAK,yBAAyB;AAC9B,SAAK,6BAA6B,KAAK;AAAA,MACrC;AAAA,MACA;AAAA,IAAA;AAEF,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM;AACV,QAAI;AACJ,QAAI,kBAAkB,KAAK;AAC3B,QAAI,gBAAgB,KAAK;AAEzB,QAAI,KAAK,sBAAsB,GAAG;AAChC,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAGF,YAAM,KAAK,OAAO,QAAA;AAAA,IACpB,OAAO;AACL,WAAK,IAAI;AAAA,QACP;AAAA,QACA,MAAM,KAAK;AAAA,MAAA;AAAA,IAEf;AAEA,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACF,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,EAAC,QAAQ,UAAA,IACb,MAAM,KAAK,kBAAkB,eAAe;AAE9C,YAAI,cAAc,UAAa,YAAY,WAAW;AAGpD,6BAAmB,KAAK;AACxB,eAAK,IAAI,OAAO,+BAA+B,eAAe,EAAE;AAAA,QAClE;AACA,oBAAY;AAEZ,wBACE,YAAY,IACR,KAAK,qBACL,KAAK,IAAI,gBAAgB,GAAG,qBAAqB;AACvD,cAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,aAAK,IAAI;AAAA,UACP,UAAU,MAAM,mBAAmB,QAAQ,QAAQ,CAAC,CAAC,uBAAuB,aAAa;AAAA,QAAA;AAE3F,cAAM,KAAK,OAAO,MAAM,aAAa;AAAA,MACvC,SAAS,GAAG;AACV,aAAK,IAAI,OAAO,mDAAmD,CAAC;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,kBACE,SAC8C;AAC9C,WAAO;AAAA,MACL,KAAK;AAAA,MACL,OAAM,QAAO;AACX,cAAM,MAAM,KAAK,IAAA;AACjB,cAAM,YAAY,MAAM,KAAK;AAC7B,cAAM,0BAA0B,MAAM,KAAK;AAS3C,cAAM,OACJ,MAAM;AAAA,wCACwB,IAAI,KAAK,OAAO,CAAC;AAAA,qDACJ,SAAS;AAAA;AAAA,oBAE1C,OAAO;AAAA;AAAA,QAEnB,OAAA,GACE,KAAA;AAEF,YAAI,IAAI,SAAS,GAAG;AAMlB,gBAAM,QAAQ;AAAA,YACZ;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UAAA,EACA;AAAA,YAAI,CAAA,UACJ;AAAA,0BACc,IAAI,KAAK,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC;AAAA,yCAChB,IAAI,GAAG,CAAC,GAAG,QAAA;AAAA,UAAQ;AASlD,gBAAM;AAAA,YACJ;AAAA,qBACS,IAAI,KAAK,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yCAQG,IAAI,GAAG,CAAC,GAAG,QAAA;AAAA,UAAQ;AAGlD,gBAAM;AAAA,YACJ;AAAA,0BACc,IAAI,KAAK,OAAO,CAAC;AAAA,mDACQ,uBAAuB;AAAA,YAC9D,QAAA;AAAA,UAAQ;AAEV,gBAAM,QAAQ,IAAI,KAAK;AAAA,QACzB;AAEA,cAAM,CAAC,EAAC,WAAU,IAAI,MAAM;AAAA,4CACQ,IAAI,KAAK,OAAO,CAAC;AAAA,mDACV,SAAS;AAAA;AAGpD,eAAO,EAAC,QAAQ,IAAI,QAAQ,WAAW,OAAO,SAAS,EAAA;AAAA,MACzD;AAAA,MACA,EAAC,MAAM,eAAA;AAAA,IAAc;AAAA,EAEzB;AAAA,EAEA,OAAsB;AACpB,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,WAAO;AAAA,EACT;AACF;"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cvr-store.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAiBjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;
|
|
1
|
+
{"version":3,"file":"cvr-store.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAiBjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAMnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AAEvE,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,KAAK,EAAC,GAAG,EAAE,WAAW,EAAC,MAAM,UAAU,CAAC;AAE/C,OAAO,EAKL,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,KAAK,YAAY,EAGjB,KAAK,UAAU,EAGf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,WAAW,EAEhB,KAAK,KAAK,EACV,KAAK,SAAS,EAGf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AA4FF,qBAAa,QAAQ;;gBAmCjB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,EACjC,qBAAqB,SAA2B,EAChD,eAAe,SAAoB,EACnC,yBAAyB,SAAM,EAAE,qBAAqB;IACtD,YAAY,oBAAa;IA6B3B,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAyN3D,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAIvD,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI;IAIlC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;IAI7B;;;;OAIG;IACH,YAAY,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE;IAM5B;;;;OAIG;IACG,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3E;;;;;OAKG;IACG,WAAW,IAAI,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAelD,WAAW,CAAC,EACV,OAAO,EACP,cAAc,EACd,UAAU,EACV,YAAY,EACZ,SAAS,EACT,QAAQ,GACT,EAAE,IAAI,CACL,WAAW,EACT,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,cAAc,GACd,WAAW,GACX,UAAU,CACb,GAAG,IAAI;IAqBR,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IASrE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAelC,WAAW,CAAC,KAAK,EAAE,WAAW;IAc9B,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAYxC,YAAY,CAAC,QAAQ,EAAE,MAAM;IAU7B,eAAe,CACb,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACnB,MAAM,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACpB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,QAAQ,GAAG,SAAS,EACnC,GAAG,EAAE,MAAM,GACV,IAAI;IAkBP,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,EACnB,kBAAkB,GAAE,MAAM,EAAO,GAChC,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;IAUvC,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,cAAc,EAAE,CAAC;IA4d5B,IAAI,QAAQ,IAAI,MAAM,CAErB;IAEK,KAAK,CACT,EAAE,EAAE,UAAU,EACd,sBAAsB,EAAE,UAAU,EAClC,GAAG,EAAE,WAAW,EAChB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAkChC,iBAAiB,IAAI,OAAO;IAI5B,qDAAqD;IACrD,OAAO,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhC,cAAc,CAClB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,EAAE,CAAC;CAsC9B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,mBAAmB,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,sBAAsB,EAAE,UAAU,GACjC,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;gBACjD,OAAO,EAAE,MAAM;CAU5B;AAED,qBAAa,+BAAgC,SAAQ,sBAAsB;IACzE,QAAQ,CAAC,IAAI,qCAAqC;gBAEtC,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;CAU3D;AAED,qBAAa,cAAe,SAAQ,sBAAsB;IACxD,QAAQ,CAAC,IAAI,oBAAoB;gBAG/B,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,eAAe,EAAE,MAAM;CAe1B;AAED,qBAAa,wBAAyB,SAAQ,sBAAsB;IAClE,QAAQ,CAAC,IAAI,8BAA8B;gBAE/B,KAAK,EAAE,OAAO;CAW3B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,4BAA4B;IACzC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAExB,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;CAK3D"}
|