@syncular/server 0.66.1 → 0.67.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handler.js +3 -3
- package/dist/postgres-storage.js +20 -10
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +4 -4
- package/dist/sqlite-dialect.js +21 -5
- package/dist/storage.d.ts +7 -6
- package/package.json +2 -2
- package/src/handler.ts +6 -4
- package/src/postgres-storage.ts +22 -10
- package/src/pull.ts +4 -3
- package/src/sqlite-dialect.ts +25 -7
- package/src/storage.ts +7 -6
package/dist/handler.js
CHANGED
|
@@ -217,7 +217,7 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
217
217
|
throw syncError('sync.invalid_subscription', 'the registration list names a scope map twice (§4.1)', JSON.stringify({ reason: 'duplicate_unit', scopes }));
|
|
218
218
|
}
|
|
219
219
|
ids.add(id);
|
|
220
|
-
return { scopes, tables };
|
|
220
|
+
return { scopes, tables, id };
|
|
221
221
|
});
|
|
222
222
|
const progressed = new Set();
|
|
223
223
|
for (const frame of progressFrames) {
|
|
@@ -246,8 +246,8 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
246
246
|
header: pullHeader,
|
|
247
247
|
limits: clampPullLimits(pullHeader),
|
|
248
248
|
registration,
|
|
249
|
-
units: list?.map(({ scopes, tables }) => {
|
|
250
|
-
const plan = planUnit(scopes, tables, resolved);
|
|
249
|
+
units: list?.map(({ scopes, tables, id }) => {
|
|
250
|
+
const plan = planUnit(scopes, tables, resolved, id);
|
|
251
251
|
return { plan, progress: progress.get(plan.id) };
|
|
252
252
|
}),
|
|
253
253
|
},
|
package/dist/postgres-storage.js
CHANGED
|
@@ -2229,17 +2229,27 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2229
2229
|
};
|
|
2230
2230
|
}
|
|
2231
2231
|
async latestScopedChangeSeq(partition, query) {
|
|
2232
|
-
const
|
|
2233
|
-
|
|
2234
|
-
if (variable === undefined || value === undefined)
|
|
2232
|
+
const keys = Object.entries(query.scopes).sort(([a], [b]) => a < b ? -1 : 1);
|
|
2233
|
+
if (keys.length === 0)
|
|
2235
2234
|
return 0;
|
|
2236
|
-
// Concurrent reads of a pull share one statement across tables
|
|
2237
|
-
//
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2235
|
+
// Concurrent reads of a pull share one statement across tables. Per spec
|
|
2236
|
+
// a leapfrog over the unit's key ranges: each step seeks the next key's
|
|
2237
|
+
// newest commit at or below the current bound.
|
|
2238
|
+
const [row] = await this.#batchedPage('latestScopedChange', `SELECT s.i AS sub, (
|
|
2239
|
+
WITH RECURSIVE step(seq, i, same) AS (
|
|
2240
|
+
SELECT 9007199254740991::bigint, 0, 0
|
|
2241
|
+
UNION ALL
|
|
2242
|
+
SELECT x.seq, step.i+1,
|
|
2243
|
+
CASE WHEN x.seq=step.seq THEN step.same+1 ELSE 1 END
|
|
2244
|
+
FROM step CROSS JOIN LATERAL (
|
|
2245
|
+
SELECT max(c.commit_seq) AS seq FROM sync_change_scopes AS c
|
|
2246
|
+
WHERE c.partition=$1 AND c.tbl=s.tbl
|
|
2247
|
+
AND c.var=s.keys->(step.i%s.n)->>0 AND c.value=s.keys->(step.i%s.n)->>1
|
|
2248
|
+
AND c.commit_seq<=step.seq) AS x
|
|
2249
|
+
WHERE step.seq IS NOT NULL AND step.same<s.n)
|
|
2250
|
+
SELECT seq FROM step WHERE same>=s.n) AS seq
|
|
2251
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, keys jsonb, n int)
|
|
2252
|
+
ORDER BY s.i`, partition, { tbl: query.table, keys, n: keys.length });
|
|
2243
2253
|
return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
|
|
2244
2254
|
}
|
|
2245
2255
|
readLiveWindow(partition, query) {
|
package/dist/pull.d.ts
CHANGED
|
@@ -40,7 +40,7 @@ export interface UnitPlan {
|
|
|
40
40
|
*/
|
|
41
41
|
export declare function coveredTables(schema: CompiledSchema, scopes: UnitScopes): CompiledTable[];
|
|
42
42
|
/** §3.2 per covered table: the unit's active pairs, status and echo. */
|
|
43
|
-
export declare function planUnit(scopes: UnitScopes, tables: readonly CompiledTable[], resolved: ResolvedScopes): UnitPlan;
|
|
43
|
+
export declare function planUnit(scopes: UnitScopes, tables: readonly CompiledTable[], resolved: ResolvedScopes, id?: string): UnitPlan;
|
|
44
44
|
/**
|
|
45
45
|
* The `COMMIT` frame of one live-section commit (§4.5): its frame-local
|
|
46
46
|
* table dictionary and every change, projected onto the client's codec.
|
package/dist/pull.js
CHANGED
|
@@ -52,7 +52,7 @@ export function coveredTables(schema, scopes) {
|
|
|
52
52
|
return tables;
|
|
53
53
|
}
|
|
54
54
|
/** §3.2 per covered table: the unit's active pairs, status and echo. */
|
|
55
|
-
export function planUnit(scopes, tables, resolved) {
|
|
55
|
+
export function planUnit(scopes, tables, resolved, id = unitId(scopes)) {
|
|
56
56
|
const requested = {};
|
|
57
57
|
for (const [key, value] of Object.entries(scopes))
|
|
58
58
|
requested[key] = [value];
|
|
@@ -76,7 +76,7 @@ export function planUnit(scopes, tables, resolved) {
|
|
|
76
76
|
}
|
|
77
77
|
const active = pairs.size > 0;
|
|
78
78
|
return {
|
|
79
|
-
id
|
|
79
|
+
id,
|
|
80
80
|
scopes,
|
|
81
81
|
tables,
|
|
82
82
|
pairs,
|
|
@@ -225,8 +225,8 @@ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, m
|
|
|
225
225
|
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
226
226
|
return { firstPage: await scan };
|
|
227
227
|
}
|
|
228
|
-
// §4.7: a sqlite-lane table pins at the newest
|
|
229
|
-
//
|
|
228
|
+
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
229
|
+
// unit's values (never below the horizon, never above maxCommitSeq), so a
|
|
230
230
|
// stored image stays the reuse key's match until this scope changes
|
|
231
231
|
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
232
232
|
const [firstPage, latest] = await Promise.all([
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -763,12 +763,28 @@ export class SqliteFamilyStorage {
|
|
|
763
763
|
};
|
|
764
764
|
}
|
|
765
765
|
async latestScopedChangeSeq(partition, query) {
|
|
766
|
-
const
|
|
767
|
-
|
|
768
|
-
if (variable === undefined || value === undefined)
|
|
766
|
+
const keys = Object.entries(query.scopes).sort(([a], [b]) => a < b ? -1 : 1);
|
|
767
|
+
if (keys.length === 0)
|
|
769
768
|
return 0;
|
|
770
|
-
//
|
|
771
|
-
|
|
769
|
+
// Leapfrog over the unit's key ranges: each step seeks the next key's
|
|
770
|
+
// newest commit at or below the current bound, so the pin costs one
|
|
771
|
+
// seek per key per range it jumps, whatever the history above it.
|
|
772
|
+
const [row] = await this.guardedAll(`WITH RECURSIVE step(seq, i, same) AS (
|
|
773
|
+
SELECT 9007199254740991, 0, 0
|
|
774
|
+
UNION ALL
|
|
775
|
+
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
776
|
+
WHERE partition=?1 AND tbl=?2
|
|
777
|
+
AND var=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][0]')
|
|
778
|
+
AND value=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][1]')
|
|
779
|
+
AND commit_seq<=step.seq),
|
|
780
|
+
step.i+1,
|
|
781
|
+
CASE WHEN (SELECT max(commit_seq) FROM sync_change_scopes
|
|
782
|
+
WHERE partition=?1 AND tbl=?2
|
|
783
|
+
AND var=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][0]')
|
|
784
|
+
AND value=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][1]')
|
|
785
|
+
AND commit_seq<=step.seq)=step.seq THEN step.same+1 ELSE 1 END
|
|
786
|
+
FROM step WHERE step.seq IS NOT NULL AND step.same<?4)
|
|
787
|
+
SELECT seq FROM step WHERE same>=?4`, [partition, query.table, JSON.stringify(keys), keys.length]);
|
|
772
788
|
return row?.seq ?? 0;
|
|
773
789
|
}
|
|
774
790
|
readLiveWindow(partition, query) {
|
package/dist/storage.d.ts
CHANGED
|
@@ -812,12 +812,13 @@ export interface ServerStorage {
|
|
|
812
812
|
*/
|
|
813
813
|
readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
|
|
814
814
|
/**
|
|
815
|
-
* Newest commitSeq whose
|
|
816
|
-
*
|
|
817
|
-
*
|
|
818
|
-
*
|
|
819
|
-
*
|
|
820
|
-
*
|
|
815
|
+
* Newest commitSeq whose changes to `query.table` carry every value of
|
|
816
|
+
* the unit's scope map in the change-scope index. Every change the unit
|
|
817
|
+
* would receive for the table, and each half of a scope move into or out
|
|
818
|
+
* of it, carries all of those values, so none sits above it. 0 when the
|
|
819
|
+
* index holds none (never written, or pruned). The §4.7 sqlite-lane scope
|
|
820
|
+
* pin reads it; it MUST seek the keys' index ranges (one seek per key per
|
|
821
|
+
* jump between them) and never scan the commits above the result.
|
|
821
822
|
*/
|
|
822
823
|
latestScopedChangeSeq(partition: string, query: {
|
|
823
824
|
readonly table: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.67.1",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.67.1"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/handler.ts
CHANGED
|
@@ -352,7 +352,9 @@ async function planRequest(
|
|
|
352
352
|
|
|
353
353
|
// §4.1: the effective list is the traveling list, or the stored list the
|
|
354
354
|
// digest names; a digest naming neither answers `unregistered`.
|
|
355
|
-
let list:
|
|
355
|
+
let list:
|
|
356
|
+
| { scopes: UnitScopes; tables: CompiledTable[]; id: string }[]
|
|
357
|
+
| undefined;
|
|
356
358
|
let registration: ClientRegistration | undefined;
|
|
357
359
|
if (pullHeader !== undefined) {
|
|
358
360
|
// §4.2: rows support is mandatory (bits 0 and 1).
|
|
@@ -399,7 +401,7 @@ async function planRequest(
|
|
|
399
401
|
);
|
|
400
402
|
}
|
|
401
403
|
ids.add(id);
|
|
402
|
-
return { scopes, tables };
|
|
404
|
+
return { scopes, tables, id };
|
|
403
405
|
});
|
|
404
406
|
const progressed = new Set<string>();
|
|
405
407
|
for (const frame of progressFrames) {
|
|
@@ -438,8 +440,8 @@ async function planRequest(
|
|
|
438
440
|
header: pullHeader,
|
|
439
441
|
limits: clampPullLimits(pullHeader),
|
|
440
442
|
registration,
|
|
441
|
-
units: list?.map(({ scopes, tables }) => {
|
|
442
|
-
const plan = planUnit(scopes, tables, resolved);
|
|
443
|
+
units: list?.map(({ scopes, tables, id }) => {
|
|
444
|
+
const plan = planUnit(scopes, tables, resolved, id);
|
|
443
445
|
return { plan, progress: progress.get(plan.id) };
|
|
444
446
|
}),
|
|
445
447
|
},
|
package/src/postgres-storage.ts
CHANGED
|
@@ -3280,20 +3280,32 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3280
3280
|
partition: string,
|
|
3281
3281
|
query: { readonly table: string; readonly scopes: UnitScopes },
|
|
3282
3282
|
): Promise<number> {
|
|
3283
|
-
const
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
//
|
|
3283
|
+
const keys = Object.entries(query.scopes).sort(([a], [b]) =>
|
|
3284
|
+
a < b ? -1 : 1,
|
|
3285
|
+
);
|
|
3286
|
+
if (keys.length === 0) return 0;
|
|
3287
|
+
// Concurrent reads of a pull share one statement across tables. Per spec
|
|
3288
|
+
// a leapfrog over the unit's key ranges: each step seeks the next key's
|
|
3289
|
+
// newest commit at or below the current bound.
|
|
3288
3290
|
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
3289
3291
|
'latestScopedChange',
|
|
3290
|
-
`SELECT s.i AS sub,
|
|
3291
|
-
(
|
|
3292
|
-
|
|
3293
|
-
|
|
3292
|
+
`SELECT s.i AS sub, (
|
|
3293
|
+
WITH RECURSIVE step(seq, i, same) AS (
|
|
3294
|
+
SELECT 9007199254740991::bigint, 0, 0
|
|
3295
|
+
UNION ALL
|
|
3296
|
+
SELECT x.seq, step.i+1,
|
|
3297
|
+
CASE WHEN x.seq=step.seq THEN step.same+1 ELSE 1 END
|
|
3298
|
+
FROM step CROSS JOIN LATERAL (
|
|
3299
|
+
SELECT max(c.commit_seq) AS seq FROM sync_change_scopes AS c
|
|
3300
|
+
WHERE c.partition=$1 AND c.tbl=s.tbl
|
|
3301
|
+
AND c.var=s.keys->(step.i%s.n)->>0 AND c.value=s.keys->(step.i%s.n)->>1
|
|
3302
|
+
AND c.commit_seq<=step.seq) AS x
|
|
3303
|
+
WHERE step.seq IS NOT NULL AND step.same<s.n)
|
|
3304
|
+
SELECT seq FROM step WHERE same>=s.n) AS seq
|
|
3305
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, keys jsonb, n int)
|
|
3294
3306
|
ORDER BY s.i`,
|
|
3295
3307
|
partition,
|
|
3296
|
-
{ tbl: query.table,
|
|
3308
|
+
{ tbl: query.table, keys, n: keys.length },
|
|
3297
3309
|
);
|
|
3298
3310
|
return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
|
|
3299
3311
|
}
|
package/src/pull.ts
CHANGED
|
@@ -115,6 +115,7 @@ export function planUnit(
|
|
|
115
115
|
scopes: UnitScopes,
|
|
116
116
|
tables: readonly CompiledTable[],
|
|
117
117
|
resolved: ResolvedScopes,
|
|
118
|
+
id = unitId(scopes),
|
|
118
119
|
): UnitPlan {
|
|
119
120
|
const requested: ScopeMap = {};
|
|
120
121
|
for (const [key, value] of Object.entries(scopes)) requested[key] = [value];
|
|
@@ -144,7 +145,7 @@ export function planUnit(
|
|
|
144
145
|
}
|
|
145
146
|
const active = pairs.size > 0;
|
|
146
147
|
return {
|
|
147
|
-
id
|
|
148
|
+
id,
|
|
148
149
|
scopes,
|
|
149
150
|
tables,
|
|
150
151
|
pairs,
|
|
@@ -371,8 +372,8 @@ async function readTableStart(
|
|
|
371
372
|
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
372
373
|
return { firstPage: await scan };
|
|
373
374
|
}
|
|
374
|
-
// §4.7: a sqlite-lane table pins at the newest
|
|
375
|
-
//
|
|
375
|
+
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
376
|
+
// unit's values (never below the horizon, never above maxCommitSeq), so a
|
|
376
377
|
// stored image stays the reuse key's match until this scope changes
|
|
377
378
|
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
378
379
|
const [firstPage, latest] = await Promise.all([
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -1166,13 +1166,31 @@ export abstract class SqliteFamilyStorage {
|
|
|
1166
1166
|
partition: string,
|
|
1167
1167
|
query: { readonly table: string; readonly scopes: UnitScopes },
|
|
1168
1168
|
): Promise<number> {
|
|
1169
|
-
const
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1169
|
+
const keys = Object.entries(query.scopes).sort(([a], [b]) =>
|
|
1170
|
+
a < b ? -1 : 1,
|
|
1171
|
+
);
|
|
1172
|
+
if (keys.length === 0) return 0;
|
|
1173
|
+
// Leapfrog over the unit's key ranges: each step seeks the next key's
|
|
1174
|
+
// newest commit at or below the current bound, so the pin costs one
|
|
1175
|
+
// seek per key per range it jumps, whatever the history above it.
|
|
1176
|
+
const [row] = await this.guardedAll<{ seq: number }>(
|
|
1177
|
+
`WITH RECURSIVE step(seq, i, same) AS (
|
|
1178
|
+
SELECT 9007199254740991, 0, 0
|
|
1179
|
+
UNION ALL
|
|
1180
|
+
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
1181
|
+
WHERE partition=?1 AND tbl=?2
|
|
1182
|
+
AND var=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][0]')
|
|
1183
|
+
AND value=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][1]')
|
|
1184
|
+
AND commit_seq<=step.seq),
|
|
1185
|
+
step.i+1,
|
|
1186
|
+
CASE WHEN (SELECT max(commit_seq) FROM sync_change_scopes
|
|
1187
|
+
WHERE partition=?1 AND tbl=?2
|
|
1188
|
+
AND var=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][0]')
|
|
1189
|
+
AND value=json_extract(?3, '$[' || CAST(step.i%?4 AS INTEGER) || '][1]')
|
|
1190
|
+
AND commit_seq<=step.seq)=step.seq THEN step.same+1 ELSE 1 END
|
|
1191
|
+
FROM step WHERE step.seq IS NOT NULL AND step.same<?4)
|
|
1192
|
+
SELECT seq FROM step WHERE same>=?4`,
|
|
1193
|
+
[partition, query.table, JSON.stringify(keys), keys.length],
|
|
1176
1194
|
);
|
|
1177
1195
|
return row?.seq ?? 0;
|
|
1178
1196
|
}
|
package/src/storage.ts
CHANGED
|
@@ -1121,12 +1121,13 @@ export interface ServerStorage {
|
|
|
1121
1121
|
): Promise<LiveWindow>;
|
|
1122
1122
|
|
|
1123
1123
|
/**
|
|
1124
|
-
* Newest commitSeq whose
|
|
1125
|
-
*
|
|
1126
|
-
*
|
|
1127
|
-
*
|
|
1128
|
-
*
|
|
1129
|
-
*
|
|
1124
|
+
* Newest commitSeq whose changes to `query.table` carry every value of
|
|
1125
|
+
* the unit's scope map in the change-scope index. Every change the unit
|
|
1126
|
+
* would receive for the table, and each half of a scope move into or out
|
|
1127
|
+
* of it, carries all of those values, so none sits above it. 0 when the
|
|
1128
|
+
* index holds none (never written, or pruned). The §4.7 sqlite-lane scope
|
|
1129
|
+
* pin reads it; it MUST seek the keys' index ranges (one seek per key per
|
|
1130
|
+
* jump between them) and never scan the commits above the result.
|
|
1130
1131
|
*/
|
|
1131
1132
|
latestScopedChangeSeq(
|
|
1132
1133
|
partition: string,
|