@syncular/server 0.95.0 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/context.d.ts +4 -16
- package/dist/context.js +0 -3
- package/dist/errors.js +8 -0
- package/dist/frame-bytes.js +1 -0
- package/dist/handler.d.ts +20 -2
- package/dist/handler.js +195 -58
- package/dist/postgres-storage.d.ts +3 -2
- package/dist/postgres-storage.js +18 -2
- package/dist/pull.d.ts +2 -2
- package/dist/pull.js +12 -11
- package/dist/push.js +41 -77
- package/dist/realtime.d.ts +5 -11
- package/dist/realtime.js +10 -23
- package/dist/relational-rows.d.ts +4 -2
- package/dist/relational-rows.js +8 -3
- package/dist/schema-window.js +2 -2
- package/dist/schema.d.ts +2 -2
- package/dist/schema.js +2 -2
- package/dist/sqlite-dialect.d.ts +3 -1
- package/dist/sqlite-dialect.js +37 -5
- package/dist/storage.d.ts +12 -0
- package/dist/validate.d.ts +21 -39
- package/dist/validate.js +4 -3
- package/package.json +2 -2
- package/src/context.ts +4 -23
- package/src/errors.ts +8 -0
- package/src/frame-bytes.ts +1 -0
- package/src/handler.ts +251 -72
- package/src/postgres-storage.ts +32 -6
- package/src/pull.ts +22 -12
- package/src/push.ts +51 -129
- package/src/realtime.ts +10 -22
- package/src/relational-rows.ts +10 -3
- package/src/schema-window.ts +2 -2
- package/src/schema.ts +4 -4
- package/src/sqlite-dialect.ts +47 -6
- package/src/storage.ts +13 -0
- package/src/validate.ts +21 -48
package/src/handler.ts
CHANGED
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
RESOLVER_OUTAGE,
|
|
38
38
|
touchAuthenticatedPartition,
|
|
39
39
|
} from './context';
|
|
40
|
-
import { reportError, SyncError, syncError } from './errors';
|
|
40
|
+
import { adapterSyncError, reportError, SyncError, syncError } from './errors';
|
|
41
41
|
import {
|
|
42
42
|
emitEvent,
|
|
43
43
|
type PullUnitSummary,
|
|
@@ -855,7 +855,8 @@ async function* streamResponse(
|
|
|
855
855
|
);
|
|
856
856
|
}
|
|
857
857
|
} else {
|
|
858
|
-
const
|
|
858
|
+
const horizons = await ctx.storage.getHorizons(ctx.partition);
|
|
859
|
+
const { seq: horizonSeq, tables: tableHorizons } = horizons;
|
|
859
860
|
const R = pull.header.cursor;
|
|
860
861
|
const echo = echoDigest(
|
|
861
862
|
units.map(({ plan: unit }) => ({
|
|
@@ -875,6 +876,20 @@ async function* streamResponse(
|
|
|
875
876
|
);
|
|
876
877
|
// §4.6 replica reset: a live unit with R outside [horizon, max].
|
|
877
878
|
let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
|
|
879
|
+
// §4.6 unit reset: `cursor` lies below the table horizon of one of
|
|
880
|
+
// the unit's active pairs outside its RESTRICT tables.
|
|
881
|
+
const belowTable = (
|
|
882
|
+
index: number,
|
|
883
|
+
cursor: number,
|
|
884
|
+
horizons: ReadonlyMap<string, number>,
|
|
885
|
+
): boolean => {
|
|
886
|
+
const { plan: unit, restrict } = units[index]!;
|
|
887
|
+
return [...unit.pairs.keys()].some(
|
|
888
|
+
(table) =>
|
|
889
|
+
cursor < (horizons.get(table) ?? -1) &&
|
|
890
|
+
restrict?.tables.includes(table) !== true,
|
|
891
|
+
);
|
|
892
|
+
};
|
|
878
893
|
// Per unit: its section kind and its scan-set bound (§4.5).
|
|
879
894
|
type UnitState =
|
|
880
895
|
| { kind: 'revoked' }
|
|
@@ -890,12 +905,17 @@ async function* streamResponse(
|
|
|
890
905
|
token: string | undefined;
|
|
891
906
|
};
|
|
892
907
|
const states: UnitState[] = units.map(
|
|
893
|
-
({ plan: unit, progress, restrict }): UnitState => {
|
|
908
|
+
({ plan: unit, progress, restrict }, index): UnitState => {
|
|
894
909
|
if (unit.status === 'revoked') return { kind: 'revoked' };
|
|
895
|
-
if (progress === undefined)
|
|
910
|
+
if (progress === undefined)
|
|
911
|
+
return !liveReset && belowTable(index, R, tableHorizons)
|
|
912
|
+
? { kind: 'reset' }
|
|
913
|
+
: { kind: 'live' };
|
|
896
914
|
// §4.6: a kept cursor outside [horizon, max] cannot join.
|
|
897
915
|
if (restrict !== undefined)
|
|
898
|
-
return restrict.cursor < horizonSeq ||
|
|
916
|
+
return restrict.cursor < horizonSeq ||
|
|
917
|
+
restrict.cursor > maxSeq ||
|
|
918
|
+
belowTable(index, restrict.cursor, tableHorizons)
|
|
899
919
|
? { kind: 'reset' }
|
|
900
920
|
: {
|
|
901
921
|
kind: 'bootstrap',
|
|
@@ -910,7 +930,8 @@ async function* streamResponse(
|
|
|
910
930
|
progress.cursor >= 0 &&
|
|
911
931
|
progress.cursor <= maxSeq
|
|
912
932
|
) {
|
|
913
|
-
return progress.cursor < horizonSeq
|
|
933
|
+
return progress.cursor < horizonSeq ||
|
|
934
|
+
belowTable(index, progress.cursor, tableHorizons)
|
|
914
935
|
? { kind: 'reset' }
|
|
915
936
|
: { kind: 'post', cursor: progress.cursor };
|
|
916
937
|
}
|
|
@@ -945,7 +966,6 @@ async function* streamResponse(
|
|
|
945
966
|
// pull holds the encoded bytes of the puts in flight.
|
|
946
967
|
const puts: Promise<ResponseFrame>[] = [];
|
|
947
968
|
const inFlight = new Set<Promise<void>>();
|
|
948
|
-
const maxPuts = limitsOf(ctx).maxConcurrentSegmentPuts;
|
|
949
969
|
let failed = false;
|
|
950
970
|
for (let next = 0; next < bootstrapping.length;) {
|
|
951
971
|
const window: number[] = [];
|
|
@@ -985,7 +1005,7 @@ async function* streamResponse(
|
|
|
985
1005
|
},
|
|
986
1006
|
progress?.bootstrapState,
|
|
987
1007
|
maxSeq,
|
|
988
|
-
|
|
1008
|
+
horizons,
|
|
989
1009
|
plan.logEpoch,
|
|
990
1010
|
restrict?.cursor,
|
|
991
1011
|
),
|
|
@@ -1006,7 +1026,9 @@ async function* streamResponse(
|
|
|
1006
1026
|
const frame = step.value;
|
|
1007
1027
|
if (frame.type === 'PENDING_REF') {
|
|
1008
1028
|
// A rows segment's put runs while later pages are scanned;
|
|
1009
|
-
// the next page waits while
|
|
1029
|
+
// the next page waits while six puts are in flight: a
|
|
1030
|
+
// Workers invocation holds six connections open at once and
|
|
1031
|
+
// an R2 put uses one, so past six the runtime queues the put.
|
|
1010
1032
|
const settled: Promise<void> = frame.ref.then(
|
|
1011
1033
|
() => void inFlight.delete(settled),
|
|
1012
1034
|
() => {
|
|
@@ -1017,7 +1039,7 @@ async function* streamResponse(
|
|
|
1017
1039
|
inFlight.add(settled);
|
|
1018
1040
|
puts.push(frame.ref);
|
|
1019
1041
|
frames.push(frame.ref);
|
|
1020
|
-
while (inFlight.size >=
|
|
1042
|
+
while (inFlight.size >= 6) {
|
|
1021
1043
|
await Promise.race(inFlight);
|
|
1022
1044
|
if (failed) break;
|
|
1023
1045
|
}
|
|
@@ -1058,10 +1080,27 @@ async function* streamResponse(
|
|
|
1058
1080
|
let live: LiveWindow | undefined;
|
|
1059
1081
|
for (;;) {
|
|
1060
1082
|
const scanned = units.flatMap(({ plan: unit }, index) => {
|
|
1061
|
-
const
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1083
|
+
const bounded = bound(states[index]!);
|
|
1084
|
+
if (bounded === undefined) return [];
|
|
1085
|
+
// §4.5: a table reaches the unit only above its table horizon,
|
|
1086
|
+
// which lies above the bound only for a RESTRICT table.
|
|
1087
|
+
const byBound = new Map<number, Map<string, ScopeMap>>();
|
|
1088
|
+
for (const [table, scopes] of unit.pairs) {
|
|
1089
|
+
const afterSeq = Math.max(
|
|
1090
|
+
bounded,
|
|
1091
|
+
tableHorizons.get(table) ?? -1,
|
|
1092
|
+
);
|
|
1093
|
+
byBound.set(
|
|
1094
|
+
afterSeq,
|
|
1095
|
+
(byBound.get(afterSeq) ?? new Map()).set(table, scopes),
|
|
1096
|
+
);
|
|
1097
|
+
}
|
|
1098
|
+
return [...byBound].map(([afterSeq, tables]) => ({
|
|
1099
|
+
index,
|
|
1100
|
+
scopes: unit.scopes,
|
|
1101
|
+
afterSeq,
|
|
1102
|
+
tables,
|
|
1103
|
+
}));
|
|
1065
1104
|
});
|
|
1066
1105
|
if (liveReset || scanned.length === 0) break;
|
|
1067
1106
|
live = await ctx.storage.readLiveWindow(ctx.partition, {
|
|
@@ -1081,18 +1120,33 @@ async function* streamResponse(
|
|
|
1081
1120
|
live = undefined;
|
|
1082
1121
|
continue;
|
|
1083
1122
|
}
|
|
1084
|
-
const recheck
|
|
1123
|
+
const { seq: recheck, tables: recheckTables } =
|
|
1124
|
+
await ctx.storage.getHorizons(ctx.partition);
|
|
1085
1125
|
let expired = false;
|
|
1086
1126
|
for (const [index, state] of states.entries()) {
|
|
1087
|
-
// §4.6: a restricted bootstrap's kept cursor is rechecked too
|
|
1127
|
+
// §4.6: a restricted bootstrap's kept cursor is rechecked too,
|
|
1128
|
+
// and a table horizon that rose during the round resets it: its
|
|
1129
|
+
// segments were read before the re-key.
|
|
1130
|
+
const restricted = state.kind === 'bootstrap' && state.restricted;
|
|
1088
1131
|
const afterSeq =
|
|
1089
|
-
bound(state) ??
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1132
|
+
bound(state) ?? (restricted ? state.pin : undefined);
|
|
1133
|
+
if (
|
|
1134
|
+
afterSeq === undefined ||
|
|
1135
|
+
(afterSeq >= recheck &&
|
|
1136
|
+
!belowTable(index, afterSeq, recheckTables) &&
|
|
1137
|
+
!(
|
|
1138
|
+
restricted &&
|
|
1139
|
+
[...units[index]!.plan.pairs.keys()].some(
|
|
1140
|
+
(table) =>
|
|
1141
|
+
(recheckTables.get(table) ?? -1) >
|
|
1142
|
+
(tableHorizons.get(table) ?? -1),
|
|
1143
|
+
)
|
|
1144
|
+
))
|
|
1145
|
+
)
|
|
1146
|
+
continue;
|
|
1094
1147
|
expired = true;
|
|
1095
|
-
|
|
1148
|
+
// A live unit below a table horizon alone takes a unit reset.
|
|
1149
|
+
if (state.kind === 'live' && afterSeq < recheck) liveReset = true;
|
|
1096
1150
|
else states[index] = { kind: 'reset' };
|
|
1097
1151
|
}
|
|
1098
1152
|
if (!expired) break;
|
|
@@ -1308,18 +1362,7 @@ async function* streamResponse(
|
|
|
1308
1362
|
report.errorCode = error.code;
|
|
1309
1363
|
await recordRoundFailure(ctx, clientId, error);
|
|
1310
1364
|
// §1.6: in-band ERROR, then END, nothing else.
|
|
1311
|
-
yield
|
|
1312
|
-
{
|
|
1313
|
-
type: 'ERROR',
|
|
1314
|
-
code: error.code,
|
|
1315
|
-
message: error.message,
|
|
1316
|
-
category: error.category,
|
|
1317
|
-
retryable: error.retryable,
|
|
1318
|
-
recommendedAction: error.recommendedAction,
|
|
1319
|
-
...(error.details !== undefined ? { details: error.details } : {}),
|
|
1320
|
-
},
|
|
1321
|
-
plan.wireVersion,
|
|
1322
|
-
);
|
|
1365
|
+
yield errorFrameBytes(error, plan.wireVersion);
|
|
1323
1366
|
yield END_FRAME_BYTES;
|
|
1324
1367
|
return;
|
|
1325
1368
|
}
|
|
@@ -1328,15 +1371,43 @@ async function* streamResponse(
|
|
|
1328
1371
|
yield END_FRAME_BYTES;
|
|
1329
1372
|
}
|
|
1330
1373
|
|
|
1374
|
+
/** The in-band `ERROR` frame of a failure (§1.6). */
|
|
1375
|
+
function errorFrameBytes(error: SyncError, wireVersion: number): Uint8Array {
|
|
1376
|
+
return encodeResponseFrame(
|
|
1377
|
+
{
|
|
1378
|
+
type: 'ERROR',
|
|
1379
|
+
code: error.code,
|
|
1380
|
+
message: error.message,
|
|
1381
|
+
category: error.category,
|
|
1382
|
+
retryable: error.retryable,
|
|
1383
|
+
recommendedAction: error.recommendedAction,
|
|
1384
|
+
...(error.details !== undefined ? { details: error.details } : {}),
|
|
1385
|
+
},
|
|
1386
|
+
wireVersion,
|
|
1387
|
+
);
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
/**
|
|
1391
|
+
* A response's chunks, one per encoded frame (§1.4). `live` turns true when
|
|
1392
|
+
* the response goes out before it is built: its header frames and
|
|
1393
|
+
* `KEEPALIVE` frames (§1.6) precede the rest, and a later failure is an
|
|
1394
|
+
* in-band `ERROR`.
|
|
1395
|
+
*/
|
|
1396
|
+
export interface SyncResponseStream extends AsyncIterable<Uint8Array> {
|
|
1397
|
+
readonly live: boolean;
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1331
1400
|
/**
|
|
1332
1401
|
* Decode and validate the request, then return the streaming response.
|
|
1333
1402
|
* Request-validation failures (§1.7) throw `SyncError` here, before any
|
|
1334
|
-
* bytes are produced; adapters map them to HTTP JSON errors (§1.1).
|
|
1403
|
+
* bytes are produced; adapters map them to HTTP JSON errors (§1.1). The
|
|
1404
|
+
* first chunk arrives once the response is built or its first `KEEPALIVE`
|
|
1405
|
+
* is due (§1.6); a failure before then throws from it, before any byte.
|
|
1335
1406
|
*/
|
|
1336
1407
|
export async function createSyncResponseStream(
|
|
1337
1408
|
bytes: Uint8Array,
|
|
1338
1409
|
ctx: SyncRequestContext,
|
|
1339
|
-
): Promise<
|
|
1410
|
+
): Promise<SyncResponseStream> {
|
|
1340
1411
|
const sink = ctx.events;
|
|
1341
1412
|
if (sink === undefined) {
|
|
1342
1413
|
try {
|
|
@@ -1448,7 +1519,7 @@ async function createStreamCore(
|
|
|
1448
1519
|
ctx: SyncRequestContext,
|
|
1449
1520
|
events: SyncularServerEvents | undefined,
|
|
1450
1521
|
startedAtMs = 0,
|
|
1451
|
-
): Promise<
|
|
1522
|
+
): Promise<SyncResponseStream> {
|
|
1452
1523
|
let request: RequestMessage;
|
|
1453
1524
|
try {
|
|
1454
1525
|
request = decodeRequest(bytes);
|
|
@@ -1507,19 +1578,23 @@ async function createStreamCore(
|
|
|
1507
1578
|
);
|
|
1508
1579
|
}
|
|
1509
1580
|
const report: RequestReport = { outcome: 'ok' };
|
|
1510
|
-
// RFC 0007 read-verify-refuse for the streamed read path: the
|
|
1511
|
-
//
|
|
1512
|
-
//
|
|
1513
|
-
//
|
|
1514
|
-
//
|
|
1581
|
+
// RFC 0007 read-verify-refuse for the streamed read path: the build
|
|
1582
|
+
// buffers the whole response (its data reads), then compares the gate token
|
|
1583
|
+
// across those reads. No push result or section escapes before the
|
|
1584
|
+
// verification, and the buffered size is bounded by the request's pull
|
|
1585
|
+
// limits and the server's inline segment cap. Every request verifies,
|
|
1515
1586
|
// including a mixed push+pull: a write that already applied when the token
|
|
1516
1587
|
// changed is durable and replayable under the same commit id / idempotency
|
|
1517
1588
|
// key, so refusing the request is safe (proved in the serve-gate lane).
|
|
1518
|
-
const
|
|
1519
|
-
|
|
1589
|
+
const buffered: Uint8Array[] = [];
|
|
1590
|
+
// Set at the build deadline: a build that finishes later stops before the
|
|
1591
|
+
// serve-gate read and persists nothing.
|
|
1592
|
+
let abandoned = false;
|
|
1593
|
+
const built = (async () => {
|
|
1520
1594
|
for await (const chunk of streamResponse(plan, ctx, served, report)) {
|
|
1521
1595
|
buffered.push(chunk);
|
|
1522
1596
|
}
|
|
1597
|
+
if (abandoned) return false;
|
|
1523
1598
|
const after = await ctx.storage.readServeGate(
|
|
1524
1599
|
ctx.partition,
|
|
1525
1600
|
schema.version,
|
|
@@ -1533,24 +1608,138 @@ async function createStreamCore(
|
|
|
1533
1608
|
if (afterRefusal !== undefined) throw serveNotReadyError(afterRefusal);
|
|
1534
1609
|
if (changed !== undefined) throw serveNotReadyError(changed);
|
|
1535
1610
|
await report.persist?.();
|
|
1536
|
-
|
|
1611
|
+
return true;
|
|
1612
|
+
})();
|
|
1613
|
+
// The races below observe the failure; this keeps an abandoned stream's
|
|
1614
|
+
// failure from surfacing as unhandled.
|
|
1615
|
+
built.catch(() => {});
|
|
1616
|
+
// §1.6 header frames: the envelope header, RESP_HEADER and its LEASE.
|
|
1617
|
+
const headerChunks = plan.leaseToEmit !== undefined ? 3 : 2;
|
|
1618
|
+
let live = false;
|
|
1619
|
+
const verified = async function* (): AsyncGenerator<Uint8Array> {
|
|
1620
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1621
|
+
let elapsedMs = 0;
|
|
1622
|
+
try {
|
|
1623
|
+
for (;;) {
|
|
1624
|
+
let done: boolean;
|
|
1625
|
+
try {
|
|
1626
|
+
done = await Promise.race([
|
|
1627
|
+
built,
|
|
1628
|
+
// §1.6: every 10 s, a third of the native client's 30 s socket
|
|
1629
|
+
// round silence bound (§1.8).
|
|
1630
|
+
new Promise<false>((resolve) => {
|
|
1631
|
+
timer = setTimeout(() => resolve(false), 10_000);
|
|
1632
|
+
}),
|
|
1633
|
+
]);
|
|
1634
|
+
if (!done) elapsedMs += 10_000;
|
|
1635
|
+
// §1.6 build deadline: 300 s, 2.5 times the slowest cold build
|
|
1636
|
+
// measured in production (119.7 s for 67 segment puts). A build
|
|
1637
|
+
// still running then is abandoned like a failed put.
|
|
1638
|
+
if (elapsedMs === 300_000) {
|
|
1639
|
+
abandoned = true;
|
|
1640
|
+
throw syncError(
|
|
1641
|
+
'sync.segment_build_timeout',
|
|
1642
|
+
'the response was not built within the server build deadline',
|
|
1643
|
+
);
|
|
1644
|
+
}
|
|
1645
|
+
} catch (thrown) {
|
|
1646
|
+
if (!live) throw thrown;
|
|
1647
|
+
// The header frames went out: the failure is in-band (§1.6), and
|
|
1648
|
+
// one outside the catalog answers `sync.internal_error`.
|
|
1649
|
+
const error = adapterSyncError(
|
|
1650
|
+
thrown,
|
|
1651
|
+
ctx.onError,
|
|
1652
|
+
'sync',
|
|
1653
|
+
ctx.mapError,
|
|
1654
|
+
);
|
|
1655
|
+
report.outcome = 'error';
|
|
1656
|
+
report.errorCode = error.code;
|
|
1657
|
+
await recordRoundFailure(ctx, plan.header.clientId, error);
|
|
1658
|
+
yield errorFrameBytes(error, plan.wireVersion);
|
|
1659
|
+
yield END_FRAME_BYTES;
|
|
1660
|
+
return;
|
|
1661
|
+
}
|
|
1662
|
+
clearTimeout(timer);
|
|
1663
|
+
if (done) break;
|
|
1664
|
+
if (!live && buffered.length < headerChunks) continue;
|
|
1665
|
+
if (!live) {
|
|
1666
|
+
live = true;
|
|
1667
|
+
yield* buffered.slice(0, headerChunks);
|
|
1668
|
+
}
|
|
1669
|
+
yield encodeResponseFrame({ type: 'KEEPALIVE' }, plan.wireVersion);
|
|
1670
|
+
}
|
|
1671
|
+
} finally {
|
|
1672
|
+
clearTimeout(timer);
|
|
1673
|
+
}
|
|
1674
|
+
yield* buffered.slice(live ? headerChunks : 0);
|
|
1537
1675
|
};
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
),
|
|
1676
|
+
const stream = recordedStream(
|
|
1677
|
+
events === undefined
|
|
1678
|
+
? verified()
|
|
1679
|
+
: instrumentedStream(
|
|
1680
|
+
verified(),
|
|
1681
|
+
ctx,
|
|
1682
|
+
events,
|
|
1683
|
+
plan,
|
|
1684
|
+
report,
|
|
1685
|
+
bytes.length,
|
|
1686
|
+
startedAtMs,
|
|
1687
|
+
),
|
|
1551
1688
|
ctx,
|
|
1552
1689
|
plan.header.clientId,
|
|
1553
1690
|
);
|
|
1691
|
+
return {
|
|
1692
|
+
get live() {
|
|
1693
|
+
return live;
|
|
1694
|
+
},
|
|
1695
|
+
[Symbol.asyncIterator]: () => stream,
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
/**
|
|
1700
|
+
* The `POST /sync` body (§1.1): the whole response when it was built before
|
|
1701
|
+
* its first `KEEPALIVE` was due, otherwise a stream that carries the header
|
|
1702
|
+
* frames and `KEEPALIVE` frames first (§1.6). Adapters compress only the
|
|
1703
|
+
* whole response (§1.3).
|
|
1704
|
+
*/
|
|
1705
|
+
export async function createSyncResponseBody(
|
|
1706
|
+
bytes: Uint8Array,
|
|
1707
|
+
ctx: SyncRequestContext,
|
|
1708
|
+
): Promise<Uint8Array | ReadableStream<Uint8Array>> {
|
|
1709
|
+
const stream = await createSyncResponseStream(bytes, ctx);
|
|
1710
|
+
const iterator = stream[Symbol.asyncIterator]();
|
|
1711
|
+
const first = await iterator.next();
|
|
1712
|
+
if (stream.live) {
|
|
1713
|
+
return new ReadableStream<Uint8Array>({
|
|
1714
|
+
start(controller) {
|
|
1715
|
+
if (!first.done) controller.enqueue(first.value);
|
|
1716
|
+
},
|
|
1717
|
+
async pull(controller) {
|
|
1718
|
+
const step = await iterator.next();
|
|
1719
|
+
if (step.done) controller.close();
|
|
1720
|
+
else controller.enqueue(step.value);
|
|
1721
|
+
},
|
|
1722
|
+
async cancel() {
|
|
1723
|
+
await iterator.return?.();
|
|
1724
|
+
},
|
|
1725
|
+
});
|
|
1726
|
+
}
|
|
1727
|
+
const chunks: Uint8Array[] = first.done ? [] : [first.value];
|
|
1728
|
+
for (
|
|
1729
|
+
let step = await iterator.next();
|
|
1730
|
+
!step.done;
|
|
1731
|
+
step = await iterator.next()
|
|
1732
|
+
)
|
|
1733
|
+
chunks.push(step.value);
|
|
1734
|
+
const out = new Uint8Array(
|
|
1735
|
+
chunks.reduce((sum, chunk) => sum + chunk.length, 0),
|
|
1736
|
+
);
|
|
1737
|
+
let offset = 0;
|
|
1738
|
+
for (const chunk of chunks) {
|
|
1739
|
+
out.set(chunk, offset);
|
|
1740
|
+
offset += chunk.length;
|
|
1741
|
+
}
|
|
1742
|
+
return out;
|
|
1554
1743
|
}
|
|
1555
1744
|
|
|
1556
1745
|
/** Counts response bytes and emits `request.handled` when the stream
|
|
@@ -1605,18 +1794,8 @@ export async function handleSyncRequest(
|
|
|
1605
1794
|
bytes: Uint8Array,
|
|
1606
1795
|
ctx: SyncRequestContext,
|
|
1607
1796
|
): Promise<Uint8Array> {
|
|
1608
|
-
const
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
chunks.push(chunk);
|
|
1613
|
-
total += chunk.length;
|
|
1614
|
-
}
|
|
1615
|
-
const out = new Uint8Array(total);
|
|
1616
|
-
let offset = 0;
|
|
1617
|
-
for (const chunk of chunks) {
|
|
1618
|
-
out.set(chunk, offset);
|
|
1619
|
-
offset += chunk.length;
|
|
1620
|
-
}
|
|
1621
|
-
return out;
|
|
1797
|
+
const body = await createSyncResponseBody(bytes, ctx);
|
|
1798
|
+
return body instanceof Uint8Array
|
|
1799
|
+
? body
|
|
1800
|
+
: new Uint8Array(await new Response(body).arrayBuffer());
|
|
1622
1801
|
}
|
package/src/postgres-storage.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune';
|
|
2
2
|
import { StorageQueryError } from './storage-errors';
|
|
3
|
-
import type {
|
|
3
|
+
import type {
|
|
4
|
+
CommitPruneQuery,
|
|
5
|
+
CommitPruneResult,
|
|
6
|
+
PartitionHorizons,
|
|
7
|
+
} from './storage';
|
|
4
8
|
import {
|
|
5
9
|
asJson,
|
|
6
10
|
CLIENT_RECORD_COLUMNS,
|
|
@@ -222,6 +226,11 @@ CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
|
222
226
|
epoch_required BOOLEAN NOT NULL DEFAULT FALSE,
|
|
223
227
|
last_authenticated_at_ms BIGINT NOT NULL
|
|
224
228
|
);
|
|
229
|
+
CREATE TABLE IF NOT EXISTS sync_table_horizons(
|
|
230
|
+
partition TEXT NOT NULL, tbl TEXT NOT NULL,
|
|
231
|
+
commit_seq BIGINT NOT NULL,
|
|
232
|
+
PRIMARY KEY(partition, tbl)
|
|
233
|
+
);
|
|
225
234
|
CREATE TABLE IF NOT EXISTS sync_row_scopes(
|
|
226
235
|
partition TEXT NOT NULL, tbl TEXT NOT NULL,
|
|
227
236
|
var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
@@ -2794,6 +2803,20 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2794
2803
|
return rows[0] === undefined ? 0 : asNumber(rows[0].horizon_seq);
|
|
2795
2804
|
}
|
|
2796
2805
|
|
|
2806
|
+
async getHorizons(partition: string): Promise<PartitionHorizons> {
|
|
2807
|
+
const { rows } = await this.#exec.query<{ seq: unknown; tables: string }>(
|
|
2808
|
+
`SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=$1),0) AS seq,
|
|
2809
|
+
coalesce((SELECT jsonb_object_agg(tbl, commit_seq) FROM sync_table_horizons WHERE partition=$1),'{}')::text AS tables`,
|
|
2810
|
+
[partition],
|
|
2811
|
+
);
|
|
2812
|
+
return {
|
|
2813
|
+
seq: asNumber(rows[0]?.seq),
|
|
2814
|
+
tables: new Map(
|
|
2815
|
+
Object.entries(asJson<Record<string, number>>(rows[0]?.tables ?? '{}')),
|
|
2816
|
+
),
|
|
2817
|
+
};
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2797
2820
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
2798
2821
|
await this.#exec.query(
|
|
2799
2822
|
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
|
|
@@ -2845,11 +2868,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2845
2868
|
'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
|
|
2846
2869
|
[partition, horizonSeq],
|
|
2847
2870
|
);
|
|
2848
|
-
// §4.6: tombstones prune in the same pass as the
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2871
|
+
// §4.6: tombstones and table horizons prune in the same pass as the
|
|
2872
|
+
// commit log.
|
|
2873
|
+
for (const table of ['sync_tombstones', 'sync_table_horizons']) {
|
|
2874
|
+
await client.query(
|
|
2875
|
+
`DELETE FROM ${table} WHERE partition=$1 AND commit_seq<=$2`,
|
|
2876
|
+
[partition, horizonSeq],
|
|
2877
|
+
);
|
|
2878
|
+
}
|
|
2853
2879
|
return {
|
|
2854
2880
|
previousHorizonSeq,
|
|
2855
2881
|
horizonSeq,
|
package/src/pull.ts
CHANGED
|
@@ -26,7 +26,12 @@ import {
|
|
|
26
26
|
import { computeEffective, type ResolvedScopes, scopeDigest } from './scopes';
|
|
27
27
|
import type { SegmentRecord, SegmentStore } from './segment-store';
|
|
28
28
|
import { issueSegmentUrl } from './signed-url';
|
|
29
|
-
import type {
|
|
29
|
+
import type {
|
|
30
|
+
PartitionHorizons,
|
|
31
|
+
ServerStorage,
|
|
32
|
+
StoredCommit,
|
|
33
|
+
StoredRow,
|
|
34
|
+
} from './storage';
|
|
30
35
|
import { projectRowPayload, projectRowValues } from './schema-window';
|
|
31
36
|
|
|
32
37
|
// One artifact build per owning storage pair and complete immutable identity.
|
|
@@ -433,9 +438,9 @@ async function readTableStart(
|
|
|
433
438
|
return { firstPage: await scan };
|
|
434
439
|
}
|
|
435
440
|
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
436
|
-
// unit's values (never below the horizon, never above
|
|
437
|
-
// stored image stays the reuse key's match until this
|
|
438
|
-
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
441
|
+
// unit's values (never below the table's horizon, never above
|
|
442
|
+
// maxCommitSeq), so a stored image stays the reuse key's match until this
|
|
443
|
+
// scope changes (§5.3). The server computes the pin; a client token never supplies it.
|
|
439
444
|
const [firstPage, latest] = await Promise.all([
|
|
440
445
|
scan,
|
|
441
446
|
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
@@ -734,7 +739,7 @@ async function* tableSegments(
|
|
|
734
739
|
yield { type: 'SEGMENT_INLINE', payload: bytes };
|
|
735
740
|
} else {
|
|
736
741
|
// The page's put runs while the next page is scanned; the caller keeps
|
|
737
|
-
//
|
|
742
|
+
// six puts in flight and emits the frames in order.
|
|
738
743
|
trace?.segments.push({
|
|
739
744
|
mediaType: 'rows',
|
|
740
745
|
delivery: 'ref',
|
|
@@ -806,17 +811,22 @@ export function planBootstrap(
|
|
|
806
811
|
unit: UnitPlan,
|
|
807
812
|
bootstrapState: string | undefined,
|
|
808
813
|
maxSeq: number,
|
|
809
|
-
|
|
814
|
+
horizon: PartitionHorizons,
|
|
810
815
|
logEpoch: string,
|
|
811
816
|
keptCursor = 0,
|
|
812
817
|
): BootstrapPlan {
|
|
813
818
|
// §4.7: resume at the token's pin unless the token is unusable, or the pin
|
|
814
|
-
// fell behind the horizon
|
|
815
|
-
// restart from a fresh pin.
|
|
819
|
+
// fell behind the horizon, the table horizon of a bootstrap table, or a
|
|
820
|
+
// restricted bootstrap's kept cursor; then restart from a fresh pin.
|
|
816
821
|
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
817
822
|
const resume =
|
|
818
823
|
token !== undefined &&
|
|
819
|
-
token.asOfCommitSeq >=
|
|
824
|
+
token.asOfCommitSeq >=
|
|
825
|
+
Math.max(
|
|
826
|
+
horizon.seq,
|
|
827
|
+
keptCursor,
|
|
828
|
+
...unit.tables.map((table) => horizon.tables.get(table.name) ?? -1),
|
|
829
|
+
)
|
|
820
830
|
? token
|
|
821
831
|
: undefined;
|
|
822
832
|
const tableIndex = resume?.tableIndex ?? 0;
|
|
@@ -840,7 +850,7 @@ export function planBootstrap(
|
|
|
840
850
|
index === tableIndex ? rowCursor : null,
|
|
841
851
|
index === tableIndex && image,
|
|
842
852
|
maxSeq,
|
|
843
|
-
|
|
853
|
+
Math.max(horizon.seq, horizon.tables.get(table.name) ?? -1),
|
|
844
854
|
logEpoch,
|
|
845
855
|
);
|
|
846
856
|
// A section that stops early never awaits the later starts.
|
|
@@ -1021,7 +1031,7 @@ export async function publishSqliteImage(
|
|
|
1021
1031
|
accept: ACCEPT_SQLITE,
|
|
1022
1032
|
};
|
|
1023
1033
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
1024
|
-
const
|
|
1034
|
+
const horizons = await config.storage.getHorizons(partition);
|
|
1025
1035
|
const trace: PullSectionTrace = { segments: [] };
|
|
1026
1036
|
const plan = planBootstrap(
|
|
1027
1037
|
ctx,
|
|
@@ -1030,7 +1040,7 @@ export async function publishSqliteImage(
|
|
|
1030
1040
|
unit,
|
|
1031
1041
|
undefined,
|
|
1032
1042
|
maxSeq,
|
|
1033
|
-
|
|
1043
|
+
horizons,
|
|
1034
1044
|
logEpoch,
|
|
1035
1045
|
);
|
|
1036
1046
|
const refs: SegmentRefFrame[] = [];
|