@rocicorp/zero 1.11.0-canary.35 → 1.11.0-canary.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +28 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +83 -0
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +2 -1
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +5 -1
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts +4 -0
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +18 -4
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +12 -6
- package/out/zero-cache/src/server/runner/run-worker.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +17 -1
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +27 -7
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +124 -62
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +9 -3
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +19 -6
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +8 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +131 -62
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +124 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js +33 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/drain-tracker.d.ts +24 -0
- package/out/zero-cache/src/services/change-streamer/drain-tracker.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/drain-tracker.js +67 -0
- package/out/zero-cache/src/services/change-streamer/drain-tracker.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/restore-admission.d.ts +214 -0
- package/out/zero-cache/src/services/change-streamer/restore-admission.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/restore-admission.js +294 -0
- package/out/zero-cache/src/services/change-streamer/restore-admission.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +13 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +7 -6
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +2 -2
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +2 -2
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscribe.d.ts +39 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscribe.js +19 -2
- package/out/zero-cache/src/services/change-streamer/subscribe.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts +22 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js +41 -15
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/http-service.d.ts +6 -0
- package/out/zero-cache/src/services/http-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/http-service.js +14 -2
- package/out/zero-cache/src/services/http-service.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts +33 -0
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +56 -10
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts +8 -0
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +22 -1
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/replicator/drain-handler.d.ts +74 -0
- package/out/zero-cache/src/services/replicator/drain-handler.d.ts.map +1 -0
- package/out/zero-cache/src/services/replicator/drain-handler.js +185 -0
- package/out/zero-cache/src/services/replicator/drain-handler.js.map +1 -0
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +2 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +66 -13
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +2 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +2 -2
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/batch-pacer.d.ts +79 -0
- package/out/zero-cache/src/services/view-syncer/batch-pacer.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/batch-pacer.js +249 -0
- package/out/zero-cache/src/services/view-syncer/batch-pacer.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +27 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +107 -17
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts +85 -3
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +301 -20
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/push-cost.d.ts +132 -0
- package/out/zero-cache/src/services/view-syncer/push-cost.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/push-cost.js +164 -0
- package/out/zero-cache/src/services/view-syncer/push-cost.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/push-cutoff.d.ts +55 -0
- package/out/zero-cache/src/services/view-syncer/push-cutoff.d.ts.map +1 -0
- package/out/zero-cache/src/services/view-syncer/push-cutoff.js +56 -0
- package/out/zero-cache/src/services/view-syncer/push-cutoff.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/query-stats.d.ts +7 -0
- package/out/zero-cache/src/services/view-syncer/query-stats.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/query-stats.js +20 -1
- package/out/zero-cache/src/services/view-syncer/query-stats.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +136 -5
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +246 -18
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +8 -2
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +381 -16
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/processes.d.ts +8 -0
- package/out/zero-cache/src/types/processes.d.ts.map +1 -1
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-react/src/use-query.js +1 -1
- package/out/zero-react/src/use-query.js.map +1 -1
- package/out/zero-server/src/queries/process-queries.d.ts.map +1 -1
- package/out/zero-server/src/queries/process-queries.js +1 -1
- package/out/zero-server/src/queries/process-queries.js.map +1 -1
- package/package.json +1 -1
|
@@ -126,10 +126,54 @@ export declare class Snapshotter {
|
|
|
126
126
|
* {@link SnapshotRowCache}. See {@link PrevWrites}.
|
|
127
127
|
*/
|
|
128
128
|
advance(syncableTables: Map<string, LiteAndZqlSpec>, allTableNames: Set<string>, observedTables?: TableFilter | undefined, prevWrites?: PrevWrites): SnapshotDiff;
|
|
129
|
+
/**
|
|
130
|
+
* The diff between the two snapshots of the last
|
|
131
|
+
* {@link advanceWithoutDiff()}, as {@link advance()} returns it.
|
|
132
|
+
*
|
|
133
|
+
* @param counts The counts of the change log entries between the two
|
|
134
|
+
* snapshots, if the caller has counted them already (see
|
|
135
|
+
* `Snapshot.countChangesSince()`). The diff then does not count
|
|
136
|
+
* them again.
|
|
137
|
+
* @param readTables If set, only the change log entries of these tables
|
|
138
|
+
* (and of the permissions table) are read, along with the
|
|
139
|
+
* table-wide ops of every table, where `observedTables` reads the
|
|
140
|
+
* entries of every table and skips those it does not have. For a
|
|
141
|
+
* caller that does not need the changes of the other tables, this
|
|
142
|
+
* saves reading and parsing their entries. This requires `counts`.
|
|
143
|
+
* If they show that there is no entry of these tables and no
|
|
144
|
+
* table-wide op, the change log is not read at all, and the
|
|
145
|
+
* diff's `changes` is 0.
|
|
146
|
+
*/
|
|
147
|
+
diff(syncableTables: Map<string, LiteAndZqlSpec>, allTableNames: Set<string>, observedTables?: TableFilter | undefined, prevWrites?: PrevWrites, counts?: ChangeLogCounts | undefined, readTables?: Iterable<string> | undefined): SnapshotDiff;
|
|
129
148
|
advanceWithoutDiff(): {
|
|
130
149
|
prev: Snapshot;
|
|
131
150
|
curr: Snapshot;
|
|
132
151
|
};
|
|
152
|
+
/**
|
|
153
|
+
* The reset of the pipelines that iterating over the changes from the
|
|
154
|
+
* previous snapshot to the current one ends in, if it does: the first of
|
|
155
|
+
* a schema change, a truncation, and a change of the permissions. It is
|
|
156
|
+
* found without iterating, so that a caller that would reset the pipelines
|
|
157
|
+
* anyway does not first process the changes that come before it.
|
|
158
|
+
*
|
|
159
|
+
* If there is none, iterating over a {@link diff()} of the two snapshots
|
|
160
|
+
* does not throw a {@link ResetPipelinesSignal}, and can be stopped at any
|
|
161
|
+
* point without missing one.
|
|
162
|
+
*
|
|
163
|
+
* @param counts The changes from the previous snapshot to the current
|
|
164
|
+
* one, from which it is known whether there is anything to look
|
|
165
|
+
* for.
|
|
166
|
+
*/
|
|
167
|
+
pendingReset(syncableTables: Map<string, LiteAndZqlSpec>, counts: ChangeLogCounts): ResetPipelinesSignal | undefined;
|
|
168
|
+
/**
|
|
169
|
+
* Releases the previous snapshot, i.e. the `prev` of the last
|
|
170
|
+
* {@link advance()} or {@link advanceWithoutDiff()}. It is only read
|
|
171
|
+
* while the diff that it is the start of is iterated over, but its read
|
|
172
|
+
* transaction otherwise stays open until the next advance, which keeps
|
|
173
|
+
* the replica from checkpointing its write-ahead log past it. A diff that
|
|
174
|
+
* starts at it can no longer be iterated over.
|
|
175
|
+
*/
|
|
176
|
+
releasePrevious(): void;
|
|
133
177
|
/**
|
|
134
178
|
* Call this to close the database connections when the Snapshotter is
|
|
135
179
|
* no longer needed.
|
|
@@ -148,6 +192,32 @@ export type Change = {
|
|
|
148
192
|
readonly nextValue: Readonly<Row> | null;
|
|
149
193
|
readonly rowKey: RowKey;
|
|
150
194
|
};
|
|
195
|
+
/** See `Snapshot.countChangesSince()`. */
|
|
196
|
+
/** Where an entry is in the order that the change log is read in. */
|
|
197
|
+
type ChangeLogPosition = {
|
|
198
|
+
readonly stateVersion: string;
|
|
199
|
+
readonly pos: number;
|
|
200
|
+
};
|
|
201
|
+
export type ChangeLogCounts = {
|
|
202
|
+
/** The number of entries. */
|
|
203
|
+
readonly total: number;
|
|
204
|
+
/** The number of them that are table-wide ops (a RESET or TRUNCATE). */
|
|
205
|
+
readonly tableWideOps: number;
|
|
206
|
+
/**
|
|
207
|
+
* The number of entries of each table. If {@link sampled}, these are
|
|
208
|
+
* estimates, and a table with few entries may be missing.
|
|
209
|
+
*/
|
|
210
|
+
readonly byTable: ReadonlyMap<string, number>;
|
|
211
|
+
/** Whether {@link byTable} was estimated from a sample of the entries. */
|
|
212
|
+
readonly sampled: boolean;
|
|
213
|
+
};
|
|
214
|
+
/**
|
|
215
|
+
* The most change log entries that are read to count a batch's changes by
|
|
216
|
+
* table. Reading an entry costs microseconds, and nothing else runs on the
|
|
217
|
+
* sync worker in the meantime, so the cost of counting must not grow with
|
|
218
|
+
* the number of changes that a client group has to catch up on.
|
|
219
|
+
*/
|
|
220
|
+
export declare const CHANGE_LOG_SAMPLE_SIZE = 1000;
|
|
151
221
|
export interface TableFilter {
|
|
152
222
|
has(table: string): boolean;
|
|
153
223
|
}
|
|
@@ -200,8 +270,20 @@ export interface SnapshotDiff extends Iterable<Change> {
|
|
|
200
270
|
readonly changes: number;
|
|
201
271
|
/** The number of raw ChangeLog entries read so far, including skipped entries. */
|
|
202
272
|
readonly changesScanned: number;
|
|
273
|
+
/**
|
|
274
|
+
* The raw ChangeLog entries read so far, by table. Like
|
|
275
|
+
* {@link changesScanned}, this includes the entries that are not yielded,
|
|
276
|
+
* so it counts the same entries as {@link ChangeLogCounts.byTable} does.
|
|
277
|
+
*/
|
|
278
|
+
readonly changesScannedByTable: ReadonlyMap<string, number>;
|
|
203
279
|
/** The scanned entries omitted because their table is unobserved/non-syncable or the change is a no-op. */
|
|
204
280
|
readonly changesSkipped: number;
|
|
281
|
+
/**
|
|
282
|
+
* The scanned entries whose rows were looked up in the snapshots. Unlike
|
|
283
|
+
* the entries that are yielded, this includes the no-op changes, which are
|
|
284
|
+
* only known to be that from their rows.
|
|
285
|
+
*/
|
|
286
|
+
readonly changesLookedUp: number;
|
|
205
287
|
/** Timing metrics recorded during diff iteration. */
|
|
206
288
|
readonly stats?: DiffTimingStats | undefined;
|
|
207
289
|
/**
|
|
@@ -230,8 +312,30 @@ declare class Snapshot {
|
|
|
230
312
|
static create(lc: LogContext, dbFile: string, appID: string, pageCacheSizeKib: number | undefined): Snapshot;
|
|
231
313
|
readonly db: StatementRunner;
|
|
232
314
|
readonly version: string;
|
|
233
|
-
|
|
315
|
+
/**
|
|
316
|
+
* @param begun Whether `db` is already in a `BEGIN CONCURRENT` transaction
|
|
317
|
+
* that has not read anything.
|
|
318
|
+
*/
|
|
319
|
+
constructor(db: StatementRunner, appID: string, begun?: boolean);
|
|
234
320
|
numChangesSince(prevVersion: string): any;
|
|
321
|
+
/**
|
|
322
|
+
* Counts the change log entries after `prevVersion`. The change log keeps
|
|
323
|
+
* the latest entry of a row, so these are counts of the rows that changed,
|
|
324
|
+
* which is what a diff from `prevVersion` to this snapshot reads.
|
|
325
|
+
*
|
|
326
|
+
* The total, and the number of table-wide ops (a RESET or TRUNCATE), are
|
|
327
|
+
* exact, and are counted from the index alone. The counts by table are
|
|
328
|
+
* exact if there are at most `sampleSize` entries. Otherwise they are
|
|
329
|
+
* estimated from the newest `sampleSize` entries, so that at most that
|
|
330
|
+
* many entries are read however many there are.
|
|
331
|
+
*/
|
|
332
|
+
countChangesSince(prevVersion: string, sampleSize?: number): ChangeLogCounts;
|
|
333
|
+
/**
|
|
334
|
+
* Whether an entry of `table` was logged after `prevVersion`. This reads
|
|
335
|
+
* the table's entries in the change log, however old, so it is for tables
|
|
336
|
+
* with few rows.
|
|
337
|
+
*/
|
|
338
|
+
tableChangedSince(table: string, prevVersion: string): boolean;
|
|
235
339
|
/**
|
|
236
340
|
* Whether a table-wide RESET op (i.e. a schema change) was logged after
|
|
237
341
|
* `prevVersion`. Table specs computed at or before `prevVersion` are
|
|
@@ -244,7 +348,27 @@ declare class Snapshot {
|
|
|
244
348
|
* abort with a {@link ResetPipelinesSignal} if so.
|
|
245
349
|
*/
|
|
246
350
|
hasTableWideOpSince(prevVersion: string): boolean;
|
|
247
|
-
|
|
351
|
+
/**
|
|
352
|
+
* The first table-wide op (a schema change or a truncation) logged after
|
|
353
|
+
* `prevVersion`, in the order that {@link changesSince} reads the entries.
|
|
354
|
+
*/
|
|
355
|
+
firstTableWideOpSince(prevVersion: string): ChangeLogPosition & {
|
|
356
|
+
table: string;
|
|
357
|
+
op: string;
|
|
358
|
+
};
|
|
359
|
+
/**
|
|
360
|
+
* The entries of `table` logged after `prevVersion`, in the order that
|
|
361
|
+
* {@link changesSince} reads them. For a table with few rows.
|
|
362
|
+
*/
|
|
363
|
+
entriesSince(table: string, prevVersion: string): (ChangeLogPosition & {
|
|
364
|
+
rowKey: string | null;
|
|
365
|
+
op: string;
|
|
366
|
+
})[];
|
|
367
|
+
/**
|
|
368
|
+
* @param tables If set, only the entries of these tables, and the
|
|
369
|
+
* table-wide ops of every table, are read.
|
|
370
|
+
*/
|
|
371
|
+
changesSince(prevVersion: string, tables?: string | undefined): {
|
|
248
372
|
changes: IterableIterator<unknown>;
|
|
249
373
|
cleanup: () => void;
|
|
250
374
|
};
|
|
@@ -267,11 +391,18 @@ declare class Snapshot {
|
|
|
267
391
|
*/
|
|
268
392
|
getRows(table: LiteTableSpecWithKeysAndVersion, keys: PrimaryKey[], row: RowValue, cache?: SnapshotRowCache, tag?: string): any[];
|
|
269
393
|
/**
|
|
270
|
-
* Whether the
|
|
271
|
-
* {@link resetToHead()}),
|
|
272
|
-
* {@link version}.
|
|
394
|
+
* Whether the snapshot has been released, or its connection reset to a
|
|
395
|
+
* newer snapshot (i.e. via {@link release()} or {@link resetToHead()}),
|
|
396
|
+
* after which reads no longer reflect {@link version}.
|
|
273
397
|
*/
|
|
274
398
|
get reset(): boolean;
|
|
399
|
+
/**
|
|
400
|
+
* Ends the snapshot's read transaction, after which reads no longer
|
|
401
|
+
* reflect {@link version}. An open read transaction keeps SQLite from
|
|
402
|
+
* checkpointing the write-ahead log past the snapshot, so a snapshot that
|
|
403
|
+
* is no longer read should not be left open.
|
|
404
|
+
*/
|
|
405
|
+
release(): void;
|
|
275
406
|
resetToHead(): Snapshot;
|
|
276
407
|
}
|
|
277
408
|
export declare class InvalidDiffError extends Error {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAIrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAYjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,yBAAyB,CAAC;AAE9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;IAStB;;;;;OAKG;IACH,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,EACzB,QAAQ,CAAC,EAAE,gBAAgB,EAO5B;IAED;;;;OAIG;IACH,IAAI,IAAI,IAAI,CAUX;IAED,WAAW,IAAI,OAAO,CAErB;IAED,+EAA+E;IAC/E,OAAO,IAAI,QAAQ,CAGlB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,cAAc,CAAC,EAAE,WAAW,GAAG,SAAS,EACxC,UAAU,GAAE,UAAsB,GACjC,YAAY,
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAY,KAAK,SAAS,EAAC,MAAM,uCAAuC,CAAC;AAGhF,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,uCAAuC,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sCAAsC,CAAC;AAIrE,OAAO,KAAK,EACV,cAAc,EACd,+BAA+B,EAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,uBAAuB,CAAC;AAYjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,yBAAyB,CAAC;AAE9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,qBAAa,WAAW;;IAStB;;;;;OAKG;IACH,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAC,KAAK,EAAC,EAAE,KAAK,EACd,gBAAgB,CAAC,EAAE,MAAM,EACzB,QAAQ,CAAC,EAAE,gBAAgB,EAO5B;IAED;;;;OAIG;IACH,IAAI,IAAI,IAAI,CAUX;IAED,WAAW,IAAI,OAAO,CAErB;IAED,+EAA+E;IAC/E,OAAO,IAAI,QAAQ,CAGlB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,OAAO,CACL,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,cAAc,CAAC,EAAE,WAAW,GAAG,SAAS,EACxC,UAAU,GAAE,UAAsB,GACjC,YAAY,CAGd;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,IAAI,CACF,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,cAAc,CAAC,EAAE,WAAW,GAAG,SAAS,EACxC,UAAU,GAAE,UAAsB,EAClC,MAAM,CAAC,EAAE,eAAe,GAAG,SAAS,EACpC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,GACxC,YAAY,CAiBd;IAED,kBAAkB;QAYR,IAAI;QAAc,IAAI;MAC/B;IAED;;;;;;;;;;;;;;OAcG;IACH,YAAY,CACV,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EAC3C,MAAM,EAAE,eAAe,GACtB,oBAAoB,GAAG,SAAS,CAwElC;IAED;;;;;;;OAOG;IACH,eAAe,IAAI,IAAI,CAEtB;IAED;;;OAGG;IACH,OAAO,SASN;CACF;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,0CAA0C;AAC1C,qEAAqE;AACrE,KAAK,iBAAiB,GAAG;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,MAAM,MAAM,eAAe,GAAG;IAC5B,6BAA6B;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,wEAAwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,OAAO,CAAC;AAE3C,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,WAAW,YAAa,SAAQ,QAAQ,CAAC,MAAM,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;QAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF;;;;;OAKG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,kFAAkF;IAClF,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC;;;;OAIG;IACH,QAAQ,CAAC,qBAAqB,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5D,2GAA2G;IAC3G,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC;;;;OAIG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;IAE7C;;;;;;OAMG;IACH,aAAa,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,CAAC;CAC7C;AAED;;;;GAIG;AACH,MAAM,MAAM,oBAAoB,GAC5B,qBAAqB,GACrB,iBAAiB,GACjB,eAAe,GACf,YAAY,GACZ,oBAAoB,CAAC;AAEzB,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IACtC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAExC,YAAY,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,MAAM,EAIxE;CACF;AAoED,cAAM,QAAQ;;IACZ,MAAM,CAAC,MAAM,CACX,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,GAAG,SAAS,YAqBrC;IAED,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC;IAE7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAGzB;;;OAGG;IACH,YAAY,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,UAAQ,EAY5D;IAED,eAAe,CAAC,WAAW,EAAE,MAAM,OAMlC;IAED;;;;;;;;;;OAUG;IACH,iBAAiB,CACf,WAAW,EAAE,MAAM,EACnB,UAAU,SAAyB,GAClC,eAAe,CA6BjB;IAED;;;;OAIG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAQ7D;IAED;;;;OAIG;IACH,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAQ/C;IAED;;;;OAIG;IACH,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAShD;IAED;;;OAGG;IACH,qBAAqB,CACnB,WAAW,EAAE,MAAM,GAClB,iBAAiB,GAAG;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,CAOjD;IAED;;;OAGG;IACH,YAAY,CACV,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,GAClB,CAAC,iBAAiB,GAAG;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,CAAC,EAAE,CAQ7D;IAED;;;OAGG;IACH,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS;QAYzD,OAAO;QASP,OAAO;MAEV;IAED;;;;;;;OAOG;IACH,MAAM,CACJ,KAAK,EAAE,+BAA+B,EACtC,MAAM,EAAE,SAAS,EACjB,KAAK,CAAC,EAAE,gBAAgB,EACxB,GAAG,CAAC,EAAE,MAAM,OA4Bb;IAED;;;;;;;OAOG;IACH,OAAO,CACL,KAAK,EAAE,+BAA+B,EACtC,IAAI,EAAE,UAAU,EAAE,EAClB,GAAG,EAAE,QAAQ,EACb,KAAK,CAAC,EAAE,gBAAgB,EACxB,GAAG,CAAC,EAAE,MAAM,SA0Eb;IAiBD;;;;OAIG;IACH,IAAI,KAAK,IAAI,OAAO,CAEnB;IAED;;;;;OAKG;IACH,OAAO,IAAI,IAAI,CAWd;IAED,WAAW,IAAI,QAAQ,CAGtB;CACF;AA2WD,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,YAAY,GAAG,EAAE,MAAM,EAEtB;CACF"}
|
|
@@ -153,8 +153,30 @@ var Snapshotter = class {
|
|
|
153
153
|
* {@link SnapshotRowCache}. See {@link PrevWrites}.
|
|
154
154
|
*/
|
|
155
155
|
advance(syncableTables, allTableNames, observedTables, prevWrites = "uniform") {
|
|
156
|
-
|
|
157
|
-
return
|
|
156
|
+
this.advanceWithoutDiff();
|
|
157
|
+
return this.diff(syncableTables, allTableNames, observedTables, prevWrites);
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* The diff between the two snapshots of the last
|
|
161
|
+
* {@link advanceWithoutDiff()}, as {@link advance()} returns it.
|
|
162
|
+
*
|
|
163
|
+
* @param counts The counts of the change log entries between the two
|
|
164
|
+
* snapshots, if the caller has counted them already (see
|
|
165
|
+
* `Snapshot.countChangesSince()`). The diff then does not count
|
|
166
|
+
* them again.
|
|
167
|
+
* @param readTables If set, only the change log entries of these tables
|
|
168
|
+
* (and of the permissions table) are read, along with the
|
|
169
|
+
* table-wide ops of every table, where `observedTables` reads the
|
|
170
|
+
* entries of every table and skips those it does not have. For a
|
|
171
|
+
* caller that does not need the changes of the other tables, this
|
|
172
|
+
* saves reading and parsing their entries. This requires `counts`.
|
|
173
|
+
* If they show that there is no entry of these tables and no
|
|
174
|
+
* table-wide op, the change log is not read at all, and the
|
|
175
|
+
* diff's `changes` is 0.
|
|
176
|
+
*/
|
|
177
|
+
diff(syncableTables, allTableNames, observedTables, prevWrites = "uniform", counts, readTables) {
|
|
178
|
+
assert(this.#prev !== void 0 && this.#curr !== void 0, "Snapshotter has not been advanced");
|
|
179
|
+
return new Diff(this.#appID, syncableTables, allTableNames, this.#prev, this.#curr, observedTables, this.#rowCache, prevWrites, counts, readTables);
|
|
158
180
|
}
|
|
159
181
|
advanceWithoutDiff() {
|
|
160
182
|
assert(this.#curr !== void 0, "Snapshotter has not been initialized");
|
|
@@ -167,6 +189,72 @@ var Snapshotter = class {
|
|
|
167
189
|
};
|
|
168
190
|
}
|
|
169
191
|
/**
|
|
192
|
+
* The reset of the pipelines that iterating over the changes from the
|
|
193
|
+
* previous snapshot to the current one ends in, if it does: the first of
|
|
194
|
+
* a schema change, a truncation, and a change of the permissions. It is
|
|
195
|
+
* found without iterating, so that a caller that would reset the pipelines
|
|
196
|
+
* anyway does not first process the changes that come before it.
|
|
197
|
+
*
|
|
198
|
+
* If there is none, iterating over a {@link diff()} of the two snapshots
|
|
199
|
+
* does not throw a {@link ResetPipelinesSignal}, and can be stopped at any
|
|
200
|
+
* point without missing one.
|
|
201
|
+
*
|
|
202
|
+
* @param counts The changes from the previous snapshot to the current
|
|
203
|
+
* one, from which it is known whether there is anything to look
|
|
204
|
+
* for.
|
|
205
|
+
*/
|
|
206
|
+
pendingReset(syncableTables, counts) {
|
|
207
|
+
assert(this.#prev !== void 0 && this.#curr !== void 0, "Snapshotter has not been advanced");
|
|
208
|
+
const prev = this.#prev;
|
|
209
|
+
const curr = this.#curr;
|
|
210
|
+
let first;
|
|
211
|
+
if (counts.tableWideOps > 0) {
|
|
212
|
+
const { stateVersion, pos, table, op } = curr.firstTableWideOpSince(prev.version);
|
|
213
|
+
first = {
|
|
214
|
+
stateVersion,
|
|
215
|
+
pos,
|
|
216
|
+
signal: op === "t" ? new ResetPipelinesSignal(`table ${table} has been truncated`, "truncation") : new ResetPipelinesSignal(`schema for table ${table} has changed`, "schema-change")
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
const permissionsTable = `${this.#appID}.permissions`;
|
|
220
|
+
const specs = syncableTables.get(permissionsTable);
|
|
221
|
+
if (specs && (counts.sampled || (counts.byTable.get(permissionsTable) ?? 0) > 0)) {
|
|
222
|
+
const { tableSpec } = specs;
|
|
223
|
+
for (const entry of curr.entriesSince(permissionsTable, prev.version)) {
|
|
224
|
+
if (first && !isBefore(entry, first)) break;
|
|
225
|
+
const rowKey = entry.rowKey && JSON.parse(entry.rowKey);
|
|
226
|
+
if (rowKey === null) continue;
|
|
227
|
+
const nextValue = entry.op === "s" ? curr.getRow(tableSpec, rowKey) : null;
|
|
228
|
+
let prevValues;
|
|
229
|
+
if (nextValue) prevValues = prev.getRows(tableSpec, tableSpec.uniqueKeys, nextValue);
|
|
230
|
+
else {
|
|
231
|
+
const prevValue = prev.getRow(tableSpec, rowKey);
|
|
232
|
+
prevValues = prevValue ? [prevValue] : [];
|
|
233
|
+
}
|
|
234
|
+
const changed = prevValues.find((prevValue) => prevValue.permissions !== nextValue?.permissions);
|
|
235
|
+
if (changed) {
|
|
236
|
+
first = {
|
|
237
|
+
...entry,
|
|
238
|
+
signal: new ResetPipelinesSignal(`Permissions have changed ${changed.hash} => ${nextValue?.hash}`, "permissions-change")
|
|
239
|
+
};
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return first?.signal;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Releases the previous snapshot, i.e. the `prev` of the last
|
|
248
|
+
* {@link advance()} or {@link advanceWithoutDiff()}. It is only read
|
|
249
|
+
* while the diff that it is the start of is iterated over, but its read
|
|
250
|
+
* transaction otherwise stays open until the next advance, which keeps
|
|
251
|
+
* the replica from checkpointing its write-ahead log past it. A diff that
|
|
252
|
+
* starts at it can no longer be iterated over.
|
|
253
|
+
*/
|
|
254
|
+
releasePrevious() {
|
|
255
|
+
this.#prev?.release();
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
170
258
|
* Call this to close the database connections when the Snapshotter is
|
|
171
259
|
* no longer needed.
|
|
172
260
|
*/
|
|
@@ -176,6 +264,16 @@ var Snapshotter = class {
|
|
|
176
264
|
this.#lc.debug?.("closed database connections");
|
|
177
265
|
}
|
|
178
266
|
};
|
|
267
|
+
function isBefore(a, b) {
|
|
268
|
+
return a.stateVersion === b.stateVersion ? a.pos < b.pos : a.stateVersion < b.stateVersion;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* The most change log entries that are read to count a batch's changes by
|
|
272
|
+
* table. Reading an entry costs microseconds, and nothing else runs on the
|
|
273
|
+
* sync worker in the meantime, so the cost of counting must not grow with
|
|
274
|
+
* the number of changes that a client group has to catch up on.
|
|
275
|
+
*/
|
|
276
|
+
var CHANGE_LOG_SAMPLE_SIZE = 1e3;
|
|
179
277
|
var ResetPipelinesSignal = class extends Error {
|
|
180
278
|
name = "ResetPipelinesSignal";
|
|
181
279
|
reason;
|
|
@@ -212,6 +310,19 @@ var MAX_MEMOIZED_UNIQUE_KEYS = 30;
|
|
|
212
310
|
* index is ever renamed.
|
|
213
311
|
*/
|
|
214
312
|
var CHANGE_LOG_STATE_VERSION_INDEX = "sqlite_autoindex__zero.changeLog2_1";
|
|
313
|
+
/**
|
|
314
|
+
* The index that SQLite creates for the `UNIQUE("table", "rowKey")` of
|
|
315
|
+
* `_zero.changeLog2` (the second constraint of its CREATE TABLE statement).
|
|
316
|
+
*/
|
|
317
|
+
var CHANGE_LOG_TABLE_INDEX = "sqlite_autoindex__zero.changeLog2_2";
|
|
318
|
+
/**
|
|
319
|
+
* Selects the change log entries of the tables in a JSON array of names,
|
|
320
|
+
* and the table-wide ops (RESET and TRUNCATE) of every table, which abort
|
|
321
|
+
* the diff whatever table they are for. Binds the two ops, then the array.
|
|
322
|
+
* The array is a single parameter so that the statement is the same, and is
|
|
323
|
+
* prepared once, for any set of tables.
|
|
324
|
+
*/
|
|
325
|
+
var CHANGE_LOG_TABLES_FILTER = `("op" IN (?, ?) OR "table" IN (SELECT "value" FROM json_each(?)))`;
|
|
215
326
|
var Snapshot = class Snapshot {
|
|
216
327
|
static create(lc, dbFile, appID, pageCacheSizeKib) {
|
|
217
328
|
const conn = new Database(lc, dbFile);
|
|
@@ -225,8 +336,12 @@ var Snapshot = class Snapshot {
|
|
|
225
336
|
#appID;
|
|
226
337
|
version;
|
|
227
338
|
#reset = false;
|
|
228
|
-
|
|
229
|
-
|
|
339
|
+
/**
|
|
340
|
+
* @param begun Whether `db` is already in a `BEGIN CONCURRENT` transaction
|
|
341
|
+
* that has not read anything.
|
|
342
|
+
*/
|
|
343
|
+
constructor(db, appID, begun = false) {
|
|
344
|
+
if (!begun) db.beginConcurrent();
|
|
230
345
|
const { stateVersion } = getReplicationState(db);
|
|
231
346
|
this.db = db;
|
|
232
347
|
this.#appID = appID;
|
|
@@ -237,6 +352,53 @@ var Snapshot = class Snapshot {
|
|
|
237
352
|
return count;
|
|
238
353
|
}
|
|
239
354
|
/**
|
|
355
|
+
* Counts the change log entries after `prevVersion`. The change log keeps
|
|
356
|
+
* the latest entry of a row, so these are counts of the rows that changed,
|
|
357
|
+
* which is what a diff from `prevVersion` to this snapshot reads.
|
|
358
|
+
*
|
|
359
|
+
* The total, and the number of table-wide ops (a RESET or TRUNCATE), are
|
|
360
|
+
* exact, and are counted from the index alone. The counts by table are
|
|
361
|
+
* exact if there are at most `sampleSize` entries. Otherwise they are
|
|
362
|
+
* estimated from the newest `sampleSize` entries, so that at most that
|
|
363
|
+
* many entries are read however many there are.
|
|
364
|
+
*/
|
|
365
|
+
countChangesSince(prevVersion, sampleSize = CHANGE_LOG_SAMPLE_SIZE) {
|
|
366
|
+
const { total, tableWideOps } = this.db.get(`SELECT COUNT(*) AS total, COALESCE(SUM(pos < 0), 0) AS tableWideOps
|
|
367
|
+
FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
|
|
368
|
+
WHERE stateVersion > ?`, prevVersion);
|
|
369
|
+
const byTable = /* @__PURE__ */ new Map();
|
|
370
|
+
if (total === 0) return {
|
|
371
|
+
total,
|
|
372
|
+
tableWideOps,
|
|
373
|
+
byTable,
|
|
374
|
+
sampled: false
|
|
375
|
+
};
|
|
376
|
+
const rows = this.db.all(`SELECT "table", COUNT(*) AS count FROM (
|
|
377
|
+
SELECT "table"
|
|
378
|
+
FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
|
|
379
|
+
WHERE stateVersion > ?
|
|
380
|
+
ORDER BY stateVersion DESC, pos DESC LIMIT ?
|
|
381
|
+
) GROUP BY "table"`, prevVersion, sampleSize);
|
|
382
|
+
const sampled = total > sampleSize;
|
|
383
|
+
const scale = sampled ? total / sampleSize : 1;
|
|
384
|
+
for (const { table, count } of rows) byTable.set(table, count * scale);
|
|
385
|
+
return {
|
|
386
|
+
total,
|
|
387
|
+
tableWideOps,
|
|
388
|
+
byTable,
|
|
389
|
+
sampled
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Whether an entry of `table` was logged after `prevVersion`. This reads
|
|
394
|
+
* the table's entries in the change log, however old, so it is for tables
|
|
395
|
+
* with few rows.
|
|
396
|
+
*/
|
|
397
|
+
tableChangedSince(table, prevVersion) {
|
|
398
|
+
return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_TABLE_INDEX}"
|
|
399
|
+
WHERE "table" = ? AND stateVersion > ? LIMIT 1`, table, prevVersion) !== void 0;
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
240
402
|
* Whether a table-wide RESET op (i.e. a schema change) was logged after
|
|
241
403
|
* `prevVersion`. Table specs computed at or before `prevVersion` are
|
|
242
404
|
* stale with respect to this snapshot if so.
|
|
@@ -254,11 +416,35 @@ var Snapshot = class Snapshot {
|
|
|
254
416
|
return this.db.get(`SELECT 1 FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
|
|
255
417
|
WHERE stateVersion > ? AND op IN (?, ?) LIMIT 1`, prevVersion, "r", "t") !== void 0;
|
|
256
418
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
419
|
+
/**
|
|
420
|
+
* The first table-wide op (a schema change or a truncation) logged after
|
|
421
|
+
* `prevVersion`, in the order that {@link changesSince} reads the entries.
|
|
422
|
+
*/
|
|
423
|
+
firstTableWideOpSince(prevVersion) {
|
|
424
|
+
return this.db.get(`SELECT "stateVersion", "pos", "table", "op" FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_STATE_VERSION_INDEX}"
|
|
425
|
+
WHERE "stateVersion" > ? AND "pos" < 0
|
|
426
|
+
ORDER BY "stateVersion" ASC, "pos" ASC LIMIT 1`, prevVersion);
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* The entries of `table` logged after `prevVersion`, in the order that
|
|
430
|
+
* {@link changesSince} reads them. For a table with few rows.
|
|
431
|
+
*/
|
|
432
|
+
entriesSince(table, prevVersion) {
|
|
433
|
+
return this.db.all(`SELECT "stateVersion", "pos", "rowKey", "op" FROM "_zero.changeLog2" INDEXED BY "${CHANGE_LOG_TABLE_INDEX}"
|
|
434
|
+
WHERE "table" = ? AND "stateVersion" > ?
|
|
435
|
+
ORDER BY "stateVersion" ASC, "pos" ASC`, table, prevVersion);
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* @param tables If set, only the entries of these tables, and the
|
|
439
|
+
* table-wide ops of every table, are read.
|
|
440
|
+
*/
|
|
441
|
+
changesSince(prevVersion, tables) {
|
|
442
|
+
const cached = this.db.statementCache.get(tables === void 0 ? `SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
|
|
443
|
+
WHERE "stateVersion" > ? ORDER BY "stateVersion" ASC, "pos" ASC` : `SELECT "stateVersion", "table", "rowKey", "op" FROM "_zero.changeLog2"
|
|
444
|
+
WHERE "stateVersion" > ? AND ${CHANGE_LOG_TABLES_FILTER}
|
|
445
|
+
ORDER BY "stateVersion" ASC, "pos" ASC`);
|
|
260
446
|
return {
|
|
261
|
-
changes: cached.statement.iterate(prevVersion),
|
|
447
|
+
changes: tables === void 0 ? cached.statement.iterate(prevVersion) : cached.statement.iterate(prevVersion, "r", "t", tables),
|
|
262
448
|
cleanup: () => this.db.statementCache.return(cached)
|
|
263
449
|
};
|
|
264
450
|
}
|
|
@@ -350,17 +536,29 @@ var Snapshot = class Snapshot {
|
|
|
350
536
|
if ((row["_0_version"] ?? "~") > this.version) throw new InvalidDiffError(`Diff is no longer valid. db has advanced past ${this.version}.`);
|
|
351
537
|
}
|
|
352
538
|
/**
|
|
353
|
-
* Whether the
|
|
354
|
-
* {@link resetToHead()}),
|
|
355
|
-
* {@link version}.
|
|
539
|
+
* Whether the snapshot has been released, or its connection reset to a
|
|
540
|
+
* newer snapshot (i.e. via {@link release()} or {@link resetToHead()}),
|
|
541
|
+
* after which reads no longer reflect {@link version}.
|
|
356
542
|
*/
|
|
357
543
|
get reset() {
|
|
358
544
|
return this.#reset;
|
|
359
545
|
}
|
|
546
|
+
/**
|
|
547
|
+
* Ends the snapshot's read transaction, after which reads no longer
|
|
548
|
+
* reflect {@link version}. An open read transaction keeps SQLite from
|
|
549
|
+
* checkpointing the write-ahead log past the snapshot, so a snapshot that
|
|
550
|
+
* is no longer read should not be left open.
|
|
551
|
+
*/
|
|
552
|
+
release() {
|
|
553
|
+
if (!this.#reset) {
|
|
554
|
+
this.#reset = true;
|
|
555
|
+
this.db.rollback();
|
|
556
|
+
this.db.beginConcurrent();
|
|
557
|
+
}
|
|
558
|
+
}
|
|
360
559
|
resetToHead() {
|
|
361
|
-
this
|
|
362
|
-
this.db
|
|
363
|
-
return new Snapshot(this.db, this.#appID);
|
|
560
|
+
this.release();
|
|
561
|
+
return new Snapshot(this.db, this.#appID, true);
|
|
364
562
|
}
|
|
365
563
|
};
|
|
366
564
|
var Diff = class {
|
|
@@ -369,6 +567,10 @@ var Diff = class {
|
|
|
369
567
|
#allTableNames;
|
|
370
568
|
#observedTables;
|
|
371
569
|
#rowCache;
|
|
570
|
+
/** The JSON array of the names of the tables whose entries are read. */
|
|
571
|
+
#readTables;
|
|
572
|
+
#permissionsChanged;
|
|
573
|
+
#nothingToRead = false;
|
|
372
574
|
#prevWrites;
|
|
373
575
|
#prevTagUniform;
|
|
374
576
|
#prevTagDivergent;
|
|
@@ -377,16 +579,20 @@ var Diff = class {
|
|
|
377
579
|
curr;
|
|
378
580
|
changes;
|
|
379
581
|
changesScanned = 0;
|
|
582
|
+
changesScannedByTable = /* @__PURE__ */ new Map();
|
|
380
583
|
changesSkipped = 0;
|
|
584
|
+
changesLookedUp = 0;
|
|
381
585
|
stats = {
|
|
382
586
|
changeLogReadMs: 0,
|
|
383
587
|
rowReadMs: 0,
|
|
384
588
|
parseMs: 0,
|
|
385
589
|
totalDiffMs: 0
|
|
386
590
|
};
|
|
387
|
-
constructor(appID, syncableTables, allTableNames, prev, curr, observedTables, rowCache, prevWrites = "uniform") {
|
|
591
|
+
constructor(appID, syncableTables, allTableNames, prev, curr, observedTables, rowCache, prevWrites = "uniform", counts, readTables) {
|
|
388
592
|
this.#prevWrites = prevWrites;
|
|
389
593
|
this.#permissionsTable = `${appID}.permissions`;
|
|
594
|
+
const read = readTables && new Set(readTables).add(this.#permissionsTable);
|
|
595
|
+
this.#readTables = read && JSON.stringify([...read]);
|
|
390
596
|
this.#syncableTables = syncableTables;
|
|
391
597
|
this.#allTableNames = allTableNames;
|
|
392
598
|
this.#observedTables = observedTables;
|
|
@@ -394,8 +600,27 @@ var Diff = class {
|
|
|
394
600
|
this.curr = curr;
|
|
395
601
|
this.#prevTagUniform = `p:${prev.version}`;
|
|
396
602
|
this.#prevTagDivergent = `p:${prev.version}:${curr.version}`;
|
|
397
|
-
|
|
398
|
-
|
|
603
|
+
if (counts === void 0) {
|
|
604
|
+
assert(read === void 0, "readTables requires counts");
|
|
605
|
+
this.changes = curr.numChangesSince(prev.version);
|
|
606
|
+
} else {
|
|
607
|
+
const hasEntries = (table) => this.#hasEntries(counts, table);
|
|
608
|
+
this.#nothingToRead = counts.total === 0 || read !== void 0 && counts.tableWideOps === 0 && ![...read].some(hasEntries);
|
|
609
|
+
this.changes = this.#nothingToRead ? 0 : counts.total;
|
|
610
|
+
}
|
|
611
|
+
const hasTableWideOp = counts ? counts.tableWideOps > 0 : curr.hasTableWideOpSince(prev.version);
|
|
612
|
+
this.#rowCache = rowCache && !hasTableWideOp ? rowCache : void 0;
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* Whether the diff has an entry of `table`. Sampled counts cannot show
|
|
616
|
+
* that a table has none, so the answer is then `true`, except for the
|
|
617
|
+
* permissions table: it has few rows, so whether it has an entry is
|
|
618
|
+
* looked up instead.
|
|
619
|
+
*/
|
|
620
|
+
#hasEntries(counts, table) {
|
|
621
|
+
if (!counts.sampled) return (counts.byTable.get(table) ?? 0) > 0;
|
|
622
|
+
if (table !== this.#permissionsTable) return true;
|
|
623
|
+
return this.#permissionsChanged ??= this.curr.tableChangedSince(table, this.prev.version);
|
|
399
624
|
}
|
|
400
625
|
setPrevWrites(prevWrites) {
|
|
401
626
|
assert(!this.#iterated || prevWrites === "divergent", "prevWrites must be set before iterating the diff");
|
|
@@ -403,7 +628,8 @@ var Diff = class {
|
|
|
403
628
|
}
|
|
404
629
|
[Symbol.iterator]() {
|
|
405
630
|
this.#iterated = true;
|
|
406
|
-
|
|
631
|
+
if (this.#nothingToRead) return [][Symbol.iterator]();
|
|
632
|
+
const { changes, cleanup: done } = this.curr.changesSince(this.prev.version, this.#readTables);
|
|
407
633
|
const cleanup = () => {
|
|
408
634
|
try {
|
|
409
635
|
changes.return?.(void 0);
|
|
@@ -430,6 +656,7 @@ var Diff = class {
|
|
|
430
656
|
const parseStart = performance.now();
|
|
431
657
|
const { table, rowKey, op, stateVersion } = parse(value, changeLogEntrySchema);
|
|
432
658
|
this.stats.parseMs += performance.now() - parseStart;
|
|
659
|
+
this.changesScannedByTable.set(table, (this.changesScannedByTable.get(table) ?? 0) + 1);
|
|
433
660
|
if (op === "r") throw new ResetPipelinesSignal(`schema for table ${table} has changed`, "schema-change");
|
|
434
661
|
if (op === "t") throw new ResetPipelinesSignal(`table ${table} has been truncated`, "truncation");
|
|
435
662
|
if (this.#observedTables && !this.#observedTables.has(table) && table !== this.#permissionsTable) {
|
|
@@ -455,6 +682,7 @@ var Diff = class {
|
|
|
455
682
|
const prevDependsOnWrites = tableSpec.uniqueKeys.length > 1 && this.#prevWrites !== "none";
|
|
456
683
|
const prevCache = prevDependsOnWrites && this.#prevWrites === "divergent" ? void 0 : cache;
|
|
457
684
|
const prevTag = prevDependsOnWrites ? this.#prevTagDivergent : this.#prevTagUniform;
|
|
685
|
+
this.changesLookedUp++;
|
|
458
686
|
const rowStart = performance.now();
|
|
459
687
|
const nextValue = op === "s" ? this.curr.getRow(tableSpec, rowKey, cache, `n:${stateVersion}`) : null;
|
|
460
688
|
let prevValues;
|