@syncular/server 0.85.0 → 0.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/frame-bytes.js +10 -0
- package/dist/handler.js +59 -9
- package/dist/postgres-storage.d.ts +1 -0
- package/dist/postgres-storage.js +30 -6
- package/dist/pull.d.ts +6 -1
- package/dist/pull.js +41 -8
- package/dist/relational-rows.d.ts +15 -0
- package/dist/relational-rows.js +27 -0
- package/dist/sqlite-dialect.d.ts +2 -0
- package/dist/sqlite-dialect.js +30 -6
- package/dist/storage-query.d.ts +11 -4
- package/dist/storage-query.js +16 -0
- package/dist/storage.d.ts +7 -0
- package/package.json +2 -2
- package/src/frame-bytes.ts +10 -0
- package/src/handler.ts +104 -32
- package/src/postgres-storage.ts +45 -5
- package/src/pull.ts +57 -8
- package/src/relational-rows.ts +28 -0
- package/src/sqlite-dialect.ts +42 -5
- package/src/storage-query.ts +21 -4
- package/src/storage.ts +11 -0
package/src/frame-bytes.ts
CHANGED
|
@@ -112,6 +112,16 @@ function wrapperFor(frame: ResponseFrame): {
|
|
|
112
112
|
frames: [stubHeader(), STUB_LIVE_START, frame, STUB_LIVE_END],
|
|
113
113
|
index: 2,
|
|
114
114
|
};
|
|
115
|
+
case 'UNIT_EXTENT':
|
|
116
|
+
return {
|
|
117
|
+
frames: [
|
|
118
|
+
stubHeader(),
|
|
119
|
+
{ ...STUB_SUB_START, bootstrap: true },
|
|
120
|
+
frame,
|
|
121
|
+
STUB_SUB_END,
|
|
122
|
+
],
|
|
123
|
+
index: 2,
|
|
124
|
+
};
|
|
115
125
|
case 'SEGMENT_REF':
|
|
116
126
|
case 'SEGMENT_INLINE':
|
|
117
127
|
return {
|
package/src/handler.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
decodeRequest,
|
|
15
15
|
echoDigest,
|
|
16
16
|
type ProgressFrame,
|
|
17
|
+
type RestrictFrame,
|
|
17
18
|
type PullHeaderFrame,
|
|
18
19
|
type PushCommitFrame,
|
|
19
20
|
type PushResultDetailsFrame,
|
|
@@ -82,6 +83,8 @@ interface PlannedUnit {
|
|
|
82
83
|
readonly plan: UnitPlan;
|
|
83
84
|
/** Absent for a live unit. */
|
|
84
85
|
readonly progress: ProgressFrame | undefined;
|
|
86
|
+
/** §4.3: the restriction that follows the unit's `PROGRESS`. */
|
|
87
|
+
readonly restrict: RestrictFrame | undefined;
|
|
85
88
|
}
|
|
86
89
|
|
|
87
90
|
interface PullPlan {
|
|
@@ -259,7 +262,11 @@ async function planRequest(
|
|
|
259
262
|
let pullHeader: PullHeaderFrame | undefined;
|
|
260
263
|
const unitFrames: UnitFrame[] = [];
|
|
261
264
|
const progressFrames: ProgressFrame[] = [];
|
|
265
|
+
// §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
|
|
266
|
+
const restricts = new Map<string, RestrictFrame>();
|
|
262
267
|
for (const frame of request.frames) {
|
|
268
|
+
if (frame.type === 'RESTRICT')
|
|
269
|
+
restricts.set(progressFrames.at(-1)!.id, frame);
|
|
263
270
|
if (frame.type === 'ROUND_FAILURE') roundFailure = frame;
|
|
264
271
|
else if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
|
|
265
272
|
else if (frame.type === 'PULL_HEADER') pullHeader = frame;
|
|
@@ -425,6 +432,22 @@ async function planRequest(
|
|
|
425
432
|
}
|
|
426
433
|
progressed.add(frame.id);
|
|
427
434
|
}
|
|
435
|
+
for (const { id, tables } of list) {
|
|
436
|
+
const restrict = restricts.get(id);
|
|
437
|
+
if (
|
|
438
|
+
restrict !== undefined &&
|
|
439
|
+
(restrict.tables.length >= tables.length ||
|
|
440
|
+
restrict.tables.some(
|
|
441
|
+
(name) => !tables.some((table) => table.name === name),
|
|
442
|
+
))
|
|
443
|
+
) {
|
|
444
|
+
throw syncError(
|
|
445
|
+
'sync.invalid_subscription',
|
|
446
|
+
'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)',
|
|
447
|
+
JSON.stringify({ reason: 'restrict_tables', id }),
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
428
451
|
}
|
|
429
452
|
}
|
|
430
453
|
|
|
@@ -450,7 +473,11 @@ async function planRequest(
|
|
|
450
473
|
registration,
|
|
451
474
|
units: list?.map(({ scopes, tables, id }) => {
|
|
452
475
|
const plan = planUnit(scopes, tables, resolved, id);
|
|
453
|
-
return {
|
|
476
|
+
return {
|
|
477
|
+
plan,
|
|
478
|
+
progress: progress.get(plan.id),
|
|
479
|
+
restrict: restricts.get(plan.id),
|
|
480
|
+
};
|
|
454
481
|
}),
|
|
455
482
|
},
|
|
456
483
|
record,
|
|
@@ -848,26 +875,47 @@ async function* streamResponse(
|
|
|
848
875
|
| { kind: 'post'; cursor: number }
|
|
849
876
|
| {
|
|
850
877
|
kind: 'bootstrap';
|
|
878
|
+
/** The join cursor (§4.7): the pin, or the kept cursor. */
|
|
851
879
|
pin: number;
|
|
880
|
+
restricted: boolean;
|
|
852
881
|
frames: ResponseFrame[];
|
|
853
882
|
token: string | undefined;
|
|
854
883
|
};
|
|
855
|
-
const states: UnitState[] = units.map(
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
884
|
+
const states: UnitState[] = units.map(
|
|
885
|
+
({ plan: unit, progress, restrict }): UnitState => {
|
|
886
|
+
if (unit.status === 'revoked') return { kind: 'revoked' };
|
|
887
|
+
if (progress === undefined) return { kind: 'live' };
|
|
888
|
+
// §4.6: a kept cursor outside [horizon, max] cannot join.
|
|
889
|
+
if (restrict !== undefined)
|
|
890
|
+
return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
|
|
891
|
+
? { kind: 'reset' }
|
|
892
|
+
: {
|
|
893
|
+
kind: 'bootstrap',
|
|
894
|
+
pin: -1,
|
|
895
|
+
restricted: true,
|
|
896
|
+
frames: [],
|
|
897
|
+
token: '',
|
|
898
|
+
};
|
|
899
|
+
// §4.7: a token, cursor -1 or a cursor from the future bootstraps.
|
|
900
|
+
if (
|
|
901
|
+
progress.bootstrapState === undefined &&
|
|
902
|
+
progress.cursor >= 0 &&
|
|
903
|
+
progress.cursor <= maxSeq
|
|
904
|
+
) {
|
|
905
|
+
return progress.cursor < horizonSeq
|
|
906
|
+
? { kind: 'reset' }
|
|
907
|
+
: { kind: 'post', cursor: progress.cursor };
|
|
908
|
+
}
|
|
909
|
+
// Replaced by the section the bootstrap below produces.
|
|
910
|
+
return {
|
|
911
|
+
kind: 'bootstrap',
|
|
912
|
+
pin: -1,
|
|
913
|
+
restricted: false,
|
|
914
|
+
frames: [],
|
|
915
|
+
token: '',
|
|
916
|
+
};
|
|
917
|
+
},
|
|
918
|
+
);
|
|
871
919
|
const traces = units.map((): PullSectionTrace | undefined =>
|
|
872
920
|
events !== undefined ? { segments: [] } : undefined,
|
|
873
921
|
);
|
|
@@ -899,19 +947,36 @@ async function* streamResponse(
|
|
|
899
947
|
pages += reads;
|
|
900
948
|
next += 1;
|
|
901
949
|
}
|
|
902
|
-
const batch = window.map((index) =>
|
|
903
|
-
index
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
950
|
+
const batch = window.map((index) => {
|
|
951
|
+
const { plan: unit, progress, restrict } = units[index]!;
|
|
952
|
+
// §4.7: a restricted bootstrap serves only the RESTRICT tables.
|
|
953
|
+
const only =
|
|
954
|
+
restrict === undefined ? undefined : new Set(restrict.tables);
|
|
955
|
+
return {
|
|
956
|
+
index,
|
|
957
|
+
plan: planBootstrap(
|
|
958
|
+
ctx,
|
|
959
|
+
schema,
|
|
960
|
+
limits,
|
|
961
|
+
only === undefined
|
|
962
|
+
? unit
|
|
963
|
+
: {
|
|
964
|
+
...unit,
|
|
965
|
+
tables: unit.tables.filter((table) =>
|
|
966
|
+
only.has(table.name),
|
|
967
|
+
),
|
|
968
|
+
pairs: new Map(
|
|
969
|
+
[...unit.pairs].filter(([name]) => only.has(name)),
|
|
970
|
+
),
|
|
971
|
+
},
|
|
972
|
+
progress?.bootstrapState,
|
|
973
|
+
maxSeq,
|
|
974
|
+
horizonSeq,
|
|
975
|
+
plan.logEpoch,
|
|
976
|
+
restrict?.cursor,
|
|
977
|
+
),
|
|
978
|
+
};
|
|
979
|
+
});
|
|
915
980
|
for (const { index, plan: bootstrap } of batch) {
|
|
916
981
|
const frames: ResponseFrame[] = [];
|
|
917
982
|
const segments = bootstrapSegments(
|
|
@@ -927,9 +992,11 @@ async function* streamResponse(
|
|
|
927
992
|
frames.push(step.value);
|
|
928
993
|
step = await segments.next();
|
|
929
994
|
}
|
|
995
|
+
const restrict = units[index]!.restrict;
|
|
930
996
|
states[index] = {
|
|
931
997
|
kind: 'bootstrap',
|
|
932
|
-
pin: bootstrap.pin,
|
|
998
|
+
pin: restrict?.cursor ?? bootstrap.pin,
|
|
999
|
+
restricted: restrict !== undefined,
|
|
933
1000
|
frames,
|
|
934
1001
|
token: step.value,
|
|
935
1002
|
};
|
|
@@ -975,7 +1042,12 @@ async function* streamResponse(
|
|
|
975
1042
|
const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
976
1043
|
let expired = false;
|
|
977
1044
|
for (const [index, state] of states.entries()) {
|
|
978
|
-
|
|
1045
|
+
// §4.6: a restricted bootstrap's kept cursor is rechecked too.
|
|
1046
|
+
const afterSeq =
|
|
1047
|
+
bound(state) ??
|
|
1048
|
+
(state.kind === 'bootstrap' && state.restricted
|
|
1049
|
+
? state.pin
|
|
1050
|
+
: undefined);
|
|
979
1051
|
if (afterSeq === undefined || afterSeq >= recheck) continue;
|
|
980
1052
|
expired = true;
|
|
981
1053
|
if (state.kind === 'live') liveReset = true;
|
package/src/postgres-storage.ts
CHANGED
|
@@ -100,6 +100,7 @@ import {
|
|
|
100
100
|
SYNC_SCOPES_COLUMN,
|
|
101
101
|
SYNC_VERSION_COLUMN,
|
|
102
102
|
type StoredColumnLayout,
|
|
103
|
+
countRowsSql,
|
|
103
104
|
scanRowBatchSql,
|
|
104
105
|
scanProbeSql,
|
|
105
106
|
scanRowPageSql,
|
|
@@ -3301,6 +3302,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3301
3302
|
created_at_ms: unknown;
|
|
3302
3303
|
key: unknown;
|
|
3303
3304
|
bytes: unknown;
|
|
3305
|
+
changes: unknown;
|
|
3304
3306
|
}>(
|
|
3305
3307
|
`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
3306
3308
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
|
|
@@ -3310,10 +3312,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3310
3312
|
ORDER BY commit_seq LIMIT $4) s
|
|
3311
3313
|
ORDER BY s.commit_seq LIMIT $4)
|
|
3312
3314
|
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
3313
|
-
z.key AS key, z.bytes AS bytes
|
|
3315
|
+
z.key AS key, z.bytes AS bytes, z.changes AS changes
|
|
3314
3316
|
FROM c
|
|
3315
3317
|
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
3316
|
-
LEFT JOIN LATERAL (SELECT k.i AS key,
|
|
3318
|
+
LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
|
|
3317
3319
|
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
3318
3320
|
FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
|
|
3319
3321
|
WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
@@ -3333,7 +3335,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3333
3335
|
}),
|
|
3334
3336
|
...(record.key === null
|
|
3335
3337
|
? {}
|
|
3336
|
-
: {
|
|
3338
|
+
: {
|
|
3339
|
+
key: asNumber(record.key),
|
|
3340
|
+
bytes: asNumber(record.bytes),
|
|
3341
|
+
changes: asNumber(record.changes),
|
|
3342
|
+
}),
|
|
3337
3343
|
}));
|
|
3338
3344
|
},
|
|
3339
3345
|
async (keys, commitSeqs) => {
|
|
@@ -3357,9 +3363,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3357
3363
|
);
|
|
3358
3364
|
}
|
|
3359
3365
|
|
|
3360
|
-
|
|
3366
|
+
#scanVariable(
|
|
3367
|
+
partition: string,
|
|
3368
|
+
query: RowScanQuery,
|
|
3369
|
+
): Promise<string | undefined> {
|
|
3361
3370
|
// The probes of concurrent scans leave as one statement, like their pages.
|
|
3362
|
-
|
|
3371
|
+
return chooseScanVariable(
|
|
3363
3372
|
partition,
|
|
3364
3373
|
query,
|
|
3365
3374
|
this.#scanVariables,
|
|
@@ -3376,6 +3385,37 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3376
3385
|
}),
|
|
3377
3386
|
),
|
|
3378
3387
|
);
|
|
3388
|
+
}
|
|
3389
|
+
|
|
3390
|
+
async countScopedRows(
|
|
3391
|
+
partition: string,
|
|
3392
|
+
query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
|
|
3393
|
+
): Promise<number> {
|
|
3394
|
+
const variable = await this.#scanVariable(partition, {
|
|
3395
|
+
...query,
|
|
3396
|
+
afterRowId: null,
|
|
3397
|
+
limit: 0,
|
|
3398
|
+
});
|
|
3399
|
+
if (variable === undefined) return 0;
|
|
3400
|
+
// The counts of concurrent bootstraps leave as one statement.
|
|
3401
|
+
const [row] = await this.#batchedPage<{ n: unknown }>(
|
|
3402
|
+
'countRows',
|
|
3403
|
+
countRowsSql('postgres'),
|
|
3404
|
+
partition,
|
|
3405
|
+
{
|
|
3406
|
+
tbl: query.table,
|
|
3407
|
+
var: variable,
|
|
3408
|
+
vals: query.scopeFilter[variable],
|
|
3409
|
+
rest: Object.entries(query.scopeFilter).flatMap(([name, vals]) =>
|
|
3410
|
+
name === variable ? [] : [{ var: name, vals }],
|
|
3411
|
+
),
|
|
3412
|
+
},
|
|
3413
|
+
);
|
|
3414
|
+
return row === undefined ? 0 : asNumber(row.n);
|
|
3415
|
+
}
|
|
3416
|
+
|
|
3417
|
+
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
|
3418
|
+
const variable = await this.#scanVariable(partition, query);
|
|
3379
3419
|
if (variable === undefined) return [];
|
|
3380
3420
|
// Candidates via the inverted index (ordered + LIMITed at the covering
|
|
3381
3421
|
// PK) LEFT JOINed to the row table — one round trip per page, never one
|
package/src/pull.ts
CHANGED
|
@@ -216,6 +216,9 @@ interface BootstrapToken {
|
|
|
216
216
|
rowCursor: string | null;
|
|
217
217
|
/** Present when the table at `tableIndex` continues on the image lane. */
|
|
218
218
|
image?: true;
|
|
219
|
+
/** The §4.7 extent: present together, or absent for the bootstrap. */
|
|
220
|
+
rowsTotal?: number;
|
|
221
|
+
rowsDone?: number;
|
|
219
222
|
}
|
|
220
223
|
|
|
221
224
|
/**
|
|
@@ -246,7 +249,12 @@ function parseBootstrapToken(
|
|
|
246
249
|
parsed.tableIndex >= tables.length ||
|
|
247
250
|
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
|
|
248
251
|
(parsed.image !== undefined &&
|
|
249
|
-
(parsed.image !== true || parsed.rowCursor === null))
|
|
252
|
+
(parsed.image !== true || parsed.rowCursor === null)) ||
|
|
253
|
+
(parsed.rowsTotal === undefined) !== (parsed.rowsDone === undefined) ||
|
|
254
|
+
[parsed.rowsTotal, parsed.rowsDone].some(
|
|
255
|
+
(count) =>
|
|
256
|
+
count !== undefined && (!Number.isSafeInteger(count) || count < 0),
|
|
257
|
+
)
|
|
250
258
|
) {
|
|
251
259
|
return undefined;
|
|
252
260
|
}
|
|
@@ -256,6 +264,9 @@ function parseBootstrapToken(
|
|
|
256
264
|
tableIndex: parsed.tableIndex,
|
|
257
265
|
rowCursor: parsed.rowCursor ?? null,
|
|
258
266
|
...(parsed.image === true ? { image: true as const } : {}),
|
|
267
|
+
...(parsed.rowsTotal !== undefined && parsed.rowsDone !== undefined
|
|
268
|
+
? { rowsTotal: parsed.rowsTotal, rowsDone: parsed.rowsDone }
|
|
269
|
+
: {}),
|
|
259
270
|
};
|
|
260
271
|
} catch {
|
|
261
272
|
return undefined;
|
|
@@ -625,7 +636,7 @@ async function* tableSegments(
|
|
|
625
636
|
logEpoch: string,
|
|
626
637
|
): AsyncGenerator<
|
|
627
638
|
ResponseFrame,
|
|
628
|
-
{ pages: number; rowCursor: string | null; image: boolean }
|
|
639
|
+
{ pages: number; rows: number; rowCursor: string | null; image: boolean }
|
|
629
640
|
> {
|
|
630
641
|
const { storage, segments, partition } = ctx;
|
|
631
642
|
const serverLimits = limitsOf(ctx);
|
|
@@ -634,6 +645,7 @@ async function* tableSegments(
|
|
|
634
645
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
635
646
|
let rowCursor = startRowCursor;
|
|
636
647
|
let image = start.image;
|
|
648
|
+
let rows = 0;
|
|
637
649
|
for (let page = 0; page < pageBudget; page++) {
|
|
638
650
|
const scanned =
|
|
639
651
|
page === 0
|
|
@@ -672,8 +684,9 @@ async function* tableSegments(
|
|
|
672
684
|
scanned,
|
|
673
685
|
);
|
|
674
686
|
if (record !== undefined) {
|
|
687
|
+
rows += record.rowCount;
|
|
675
688
|
if (record.nextRowCursor === null)
|
|
676
|
-
return { pages: page + 1, rowCursor: null, image: false };
|
|
689
|
+
return { pages: page + 1, rows, rowCursor: null, image: false };
|
|
677
690
|
rowCursor = record.nextRowCursor;
|
|
678
691
|
continue;
|
|
679
692
|
}
|
|
@@ -681,6 +694,7 @@ async function* tableSegments(
|
|
|
681
694
|
}
|
|
682
695
|
const pageRows = scanned.slice(0, limits.limitSnapshotRows);
|
|
683
696
|
const hasMore = scanned.length > limits.limitSnapshotRows;
|
|
697
|
+
rows += pageRows.length;
|
|
684
698
|
// §5.2: every row record carries the row's current server_version.
|
|
685
699
|
const decoded = pageRows.map((row: StoredRow) => ({
|
|
686
700
|
serverVersion: row.serverVersion,
|
|
@@ -740,10 +754,11 @@ async function* tableSegments(
|
|
|
740
754
|
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
741
755
|
);
|
|
742
756
|
}
|
|
743
|
-
if (!hasMore)
|
|
757
|
+
if (!hasMore)
|
|
758
|
+
return { pages: page + 1, rows, rowCursor: null, image: false };
|
|
744
759
|
rowCursor = nextRowCursor;
|
|
745
760
|
}
|
|
746
|
-
return { pages: pageBudget, rowCursor, image: image !== undefined };
|
|
761
|
+
return { pages: pageBudget, rows, rowCursor, image: image !== undefined };
|
|
747
762
|
}
|
|
748
763
|
|
|
749
764
|
/** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
|
|
@@ -757,6 +772,10 @@ export interface BootstrapPlan {
|
|
|
757
772
|
readonly image: boolean;
|
|
758
773
|
/** Table index → its start reads. */
|
|
759
774
|
readonly starts: ReadonlyMap<number, Promise<TableStart>>;
|
|
775
|
+
/** The §4.7 extent the section states, absent for an uncounted bootstrap. */
|
|
776
|
+
readonly extent:
|
|
777
|
+
| Promise<{ readonly rowsTotal: number; readonly rowsDone: number }>
|
|
778
|
+
| undefined;
|
|
760
779
|
}
|
|
761
780
|
|
|
762
781
|
/**
|
|
@@ -775,12 +794,15 @@ export function planBootstrap(
|
|
|
775
794
|
maxSeq: number,
|
|
776
795
|
horizonSeq: number,
|
|
777
796
|
logEpoch: string,
|
|
797
|
+
keptCursor = 0,
|
|
778
798
|
): BootstrapPlan {
|
|
779
|
-
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
780
|
-
// fell behind the horizon
|
|
799
|
+
// §4.7: resume at the token's pin unless the token is unusable, or the pin
|
|
800
|
+
// fell behind the horizon or a restricted bootstrap's kept cursor; then
|
|
801
|
+
// restart from a fresh pin.
|
|
781
802
|
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
782
803
|
const resume =
|
|
783
|
-
token !== undefined &&
|
|
804
|
+
token !== undefined &&
|
|
805
|
+
token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
|
|
784
806
|
? token
|
|
785
807
|
: undefined;
|
|
786
808
|
const tableIndex = resume?.tableIndex ?? 0;
|
|
@@ -811,6 +833,25 @@ export function planBootstrap(
|
|
|
811
833
|
start.catch(() => undefined);
|
|
812
834
|
starts.set(index, start);
|
|
813
835
|
}
|
|
836
|
+
// §4.7: a fresh pin counts every active pair once; a resume carries the
|
|
837
|
+
// token's counts, and a token minted without them stays without.
|
|
838
|
+
const extent =
|
|
839
|
+
resume === undefined
|
|
840
|
+
? Promise.all(
|
|
841
|
+
[...unit.pairs].map(([table, scopeFilter]) =>
|
|
842
|
+
ctx.storage.countScopedRows(ctx.partition, { table, scopeFilter }),
|
|
843
|
+
),
|
|
844
|
+
).then((counts) => ({
|
|
845
|
+
rowsTotal: counts.reduce((sum, count) => sum + count, 0),
|
|
846
|
+
rowsDone: 0,
|
|
847
|
+
}))
|
|
848
|
+
: resume.rowsTotal !== undefined && resume.rowsDone !== undefined
|
|
849
|
+
? Promise.resolve({
|
|
850
|
+
rowsTotal: resume.rowsTotal,
|
|
851
|
+
rowsDone: resume.rowsDone,
|
|
852
|
+
})
|
|
853
|
+
: undefined;
|
|
854
|
+
extent?.catch(() => undefined);
|
|
814
855
|
return {
|
|
815
856
|
unit,
|
|
816
857
|
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
@@ -818,6 +859,7 @@ export function planBootstrap(
|
|
|
818
859
|
rowCursor,
|
|
819
860
|
image,
|
|
820
861
|
starts,
|
|
862
|
+
extent,
|
|
821
863
|
};
|
|
822
864
|
}
|
|
823
865
|
|
|
@@ -838,6 +880,9 @@ export async function* bootstrapSegments(
|
|
|
838
880
|
const { unit, pin } = plan;
|
|
839
881
|
let { tableIndex, rowCursor, image } = plan;
|
|
840
882
|
let pages = 0;
|
|
883
|
+
const extent = await plan.extent;
|
|
884
|
+
if (extent !== undefined) yield { type: 'UNIT_EXTENT', ...extent };
|
|
885
|
+
let rowsDone = extent?.rowsDone ?? 0;
|
|
841
886
|
for (;;) {
|
|
842
887
|
while (
|
|
843
888
|
tableIndex < unit.tables.length &&
|
|
@@ -855,6 +900,9 @@ export async function* bootstrapSegments(
|
|
|
855
900
|
tableIndex,
|
|
856
901
|
rowCursor,
|
|
857
902
|
...(image ? { image: true as const } : {}),
|
|
903
|
+
...(extent !== undefined
|
|
904
|
+
? { rowsTotal: extent.rowsTotal, rowsDone }
|
|
905
|
+
: {}),
|
|
858
906
|
};
|
|
859
907
|
return JSON.stringify(token);
|
|
860
908
|
}
|
|
@@ -872,6 +920,7 @@ export async function* bootstrapSegments(
|
|
|
872
920
|
logEpoch,
|
|
873
921
|
);
|
|
874
922
|
pages += outcome.pages;
|
|
923
|
+
rowsDone += outcome.rows;
|
|
875
924
|
rowCursor = outcome.rowCursor;
|
|
876
925
|
image = outcome.image;
|
|
877
926
|
if (rowCursor === null) tableIndex += 1;
|
package/src/relational-rows.ts
CHANGED
|
@@ -719,6 +719,34 @@ export function scanRowPageSql(
|
|
|
719
719
|
ORDER BY c.row_id`;
|
|
720
720
|
}
|
|
721
721
|
|
|
722
|
+
/**
|
|
723
|
+
* The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
|
|
724
|
+
* entries of the seek variable whose row also holds an entry of every other
|
|
725
|
+
* variable of the filter. A row holds one entry per variable, so the count is
|
|
726
|
+
* the matching rows; the index changes in the transaction of each row write.
|
|
727
|
+
*
|
|
728
|
+
* Bind order:
|
|
729
|
+
* - postgres: [partition, records as JSON text]; one `(sub, n)` row per
|
|
730
|
+
* `{i, tbl, var, vals, rest}` record, so the counts of concurrent
|
|
731
|
+
* bootstraps share one statement;
|
|
732
|
+
* - sqlite: [partition, tbl, var, values as JSON array text, rest as
|
|
733
|
+
* JSON text]; one `n` row.
|
|
734
|
+
* `rest` is the JSON array of `{var, vals}` for the other variables.
|
|
735
|
+
*/
|
|
736
|
+
export function countRowsSql(dialect: RelationalDialect): string {
|
|
737
|
+
return dialect === 'sqlite'
|
|
738
|
+
? `SELECT count(*) AS n FROM sync_row_scopes a
|
|
739
|
+
WHERE a.partition=?1 AND a.tbl=?2 AND a.var=?3 AND a.value IN (SELECT value FROM json_each(?4))
|
|
740
|
+
AND NOT EXISTS (SELECT 1 FROM json_each(?5) f WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
|
|
741
|
+
WHERE b.partition=?1 AND b.tbl=?2 AND b.var=json_extract(f.value, '$.var')
|
|
742
|
+
AND b.value IN (SELECT value FROM json_each(f.value, '$.vals')) AND b.row_id=a.row_id))`
|
|
743
|
+
: `SELECT s.i AS sub, (SELECT count(*) FROM sync_row_scopes a
|
|
744
|
+
WHERE a.partition=$1 AND a.tbl=s.tbl AND a.var=s.var AND a.value=ANY(s.vals)
|
|
745
|
+
AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset(s.rest) AS f(var text, vals text[]) WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
|
|
746
|
+
WHERE b.partition=$1 AND b.tbl=s.tbl AND b.var=f.var AND b.value=ANY(f.vals) AND b.row_id=a.row_id))) AS n
|
|
747
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], rest jsonb)`;
|
|
748
|
+
}
|
|
749
|
+
|
|
722
750
|
/**
|
|
723
751
|
* Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
|
|
724
752
|
* record of the JSON array, the count of its `sync_row_scopes` entries up
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
deleteSqliteRowScopesSql,
|
|
29
29
|
indexRowPageStatement,
|
|
30
30
|
quoteIdent,
|
|
31
|
+
countRowsSql,
|
|
31
32
|
scanProbeSql,
|
|
32
33
|
scanRowPageSql,
|
|
33
34
|
rowsReferencingBlobSql,
|
|
@@ -982,6 +983,7 @@ export abstract class SqliteFamilyStorage {
|
|
|
982
983
|
created_at_ms: number | null;
|
|
983
984
|
key: number | null;
|
|
984
985
|
bytes: number | null;
|
|
986
|
+
changes: number | null;
|
|
985
987
|
}>(
|
|
986
988
|
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
987
989
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
@@ -990,10 +992,10 @@ export abstract class SqliteFamilyStorage {
|
|
|
990
992
|
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
991
993
|
ORDER BY s.commit_seq LIMIT ?4)
|
|
992
994
|
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
993
|
-
z.i AS key, z.bytes AS bytes
|
|
995
|
+
z.i AS key, z.bytes AS bytes, z.changes AS changes
|
|
994
996
|
FROM c
|
|
995
997
|
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
996
|
-
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
|
|
998
|
+
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
|
|
997
999
|
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
998
1000
|
FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
999
1001
|
CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
|
|
@@ -1012,7 +1014,11 @@ export abstract class SqliteFamilyStorage {
|
|
|
1012
1014
|
}
|
|
1013
1015
|
: {}),
|
|
1014
1016
|
...(record.key !== null
|
|
1015
|
-
? {
|
|
1017
|
+
? {
|
|
1018
|
+
key: record.key,
|
|
1019
|
+
bytes: record.bytes ?? 0,
|
|
1020
|
+
changes: record.changes ?? 0,
|
|
1021
|
+
}
|
|
1016
1022
|
: {}),
|
|
1017
1023
|
})),
|
|
1018
1024
|
async (keys, commitSeqs) =>
|
|
@@ -1033,8 +1039,11 @@ export abstract class SqliteFamilyStorage {
|
|
|
1033
1039
|
);
|
|
1034
1040
|
}
|
|
1035
1041
|
|
|
1036
|
-
|
|
1037
|
-
|
|
1042
|
+
private scanVariable(
|
|
1043
|
+
partition: string,
|
|
1044
|
+
query: RowScanQuery,
|
|
1045
|
+
): Promise<string | undefined> {
|
|
1046
|
+
return chooseScanVariable(
|
|
1038
1047
|
partition,
|
|
1039
1048
|
query,
|
|
1040
1049
|
this.scanVariables,
|
|
@@ -1045,6 +1054,34 @@ export abstract class SqliteFamilyStorage {
|
|
|
1045
1054
|
cap,
|
|
1046
1055
|
]),
|
|
1047
1056
|
);
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
async countScopedRows(
|
|
1060
|
+
partition: string,
|
|
1061
|
+
query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
|
|
1062
|
+
): Promise<number> {
|
|
1063
|
+
const variable = await this.scanVariable(partition, {
|
|
1064
|
+
...query,
|
|
1065
|
+
afterRowId: null,
|
|
1066
|
+
limit: 0,
|
|
1067
|
+
});
|
|
1068
|
+
if (variable === undefined) return 0;
|
|
1069
|
+
const [row] = await this.guardedAll<{ n: number }>(countRowsSql('sqlite'), [
|
|
1070
|
+
partition,
|
|
1071
|
+
query.table,
|
|
1072
|
+
variable,
|
|
1073
|
+
JSON.stringify(query.scopeFilter[variable]),
|
|
1074
|
+
JSON.stringify(
|
|
1075
|
+
Object.entries(query.scopeFilter).flatMap(([name, vals]) =>
|
|
1076
|
+
name === variable ? [] : [{ var: name, vals }],
|
|
1077
|
+
),
|
|
1078
|
+
),
|
|
1079
|
+
]);
|
|
1080
|
+
return Number(row?.n ?? 0);
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
|
1084
|
+
const variable = await this.scanVariable(partition, query);
|
|
1048
1085
|
if (variable === undefined) return [];
|
|
1049
1086
|
const values = query.scopeFilter[variable]!;
|
|
1050
1087
|
// Candidates via the inverted index LEFT JOINed to the row table — one
|