@syncular/server 0.15.47 → 0.15.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/admin.d.ts +1 -5
- package/dist/admin.js +2 -7
- package/dist/blob-handlers.js +4 -1
- package/dist/context.d.ts +3 -1
- package/dist/context.js +4 -0
- package/dist/d1-storage.d.ts +4 -1
- package/dist/d1-storage.js +75 -11
- package/dist/errors.js +6 -0
- package/dist/events.d.ts +2 -1
- package/dist/frame-bytes.d.ts +2 -2
- package/dist/frame-bytes.js +33 -14
- package/dist/handler.js +58 -14
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/operations.js +2 -1
- package/dist/postgres-storage.d.ts +5 -2
- package/dist/postgres-storage.js +71 -10
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +11 -5
- package/dist/realtime.d.ts +4 -1
- package/dist/realtime.js +33 -9
- package/dist/restore.d.ts +13 -0
- package/dist/restore.js +13 -0
- package/dist/s3-segment-store.js +10 -1
- package/dist/seed.js +36 -4
- package/dist/segment-download.js +5 -2
- package/dist/segment-store.d.ts +3 -0
- package/dist/segment-store.js +1 -0
- package/dist/sqlite-bun-driver.d.ts +2 -1
- package/dist/sqlite-bun-driver.js +5 -2
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +7 -0
- package/dist/sqlite-segment-store.js +14 -5
- package/dist/sqlite-storage.d.ts +4 -1
- package/dist/sqlite-storage.js +81 -11
- package/dist/storage.d.ts +18 -5
- package/package.json +2 -2
- package/src/admin.ts +3 -9
- package/src/blob-handlers.ts +8 -1
- package/src/context.ts +16 -1
- package/src/d1-storage.ts +107 -12
- package/src/errors.ts +6 -0
- package/src/events.ts +2 -1
- package/src/frame-bytes.ts +40 -14
- package/src/handler.ts +102 -29
- package/src/index.ts +1 -0
- package/src/operations.ts +6 -1
- package/src/postgres-storage.ts +102 -13
- package/src/pull.ts +11 -0
- package/src/realtime.ts +46 -7
- package/src/restore.ts +28 -0
- package/src/s3-segment-store.ts +10 -1
- package/src/seed.ts +46 -4
- package/src/segment-download.ts +11 -2
- package/src/segment-store.ts +4 -0
- package/src/sqlite-bun-driver.ts +6 -2
- package/src/sqlite-dialect.ts +7 -0
- package/src/sqlite-segment-store.ts +18 -4
- package/src/sqlite-storage.ts +118 -12
- package/src/storage.ts +28 -5
package/src/frame-bytes.ts
CHANGED
|
@@ -18,7 +18,14 @@ import {
|
|
|
18
18
|
type SubStartFrame,
|
|
19
19
|
} from '@syncular/core';
|
|
20
20
|
|
|
21
|
-
const
|
|
21
|
+
const stubHeader = (wireVersion: number): RespHeaderFrame =>
|
|
22
|
+
wireVersion >= 2
|
|
23
|
+
? {
|
|
24
|
+
type: 'RESP_HEADER',
|
|
25
|
+
logEpoch: 'frame-probe',
|
|
26
|
+
resetRequired: false,
|
|
27
|
+
}
|
|
28
|
+
: { type: 'RESP_HEADER' };
|
|
22
29
|
const STUB_SUB_START: SubStartFrame = {
|
|
23
30
|
type: 'SUB_START',
|
|
24
31
|
id: '',
|
|
@@ -30,18 +37,28 @@ const STUB_SUB_START: SubStartFrame = {
|
|
|
30
37
|
const STUB_SUB_END: SubEndFrame = { type: 'SUB_END', nextCursor: 0 };
|
|
31
38
|
|
|
32
39
|
const probe = encodeMessage({
|
|
33
|
-
wireVersion:
|
|
40
|
+
wireVersion: 1,
|
|
34
41
|
msgKind: 'response',
|
|
35
|
-
frames: [
|
|
42
|
+
frames: [stubHeader(1)],
|
|
36
43
|
});
|
|
37
44
|
|
|
38
45
|
/** The 8-byte SSP2 response envelope header (§1.2). */
|
|
39
|
-
export
|
|
46
|
+
export function responseEnvelopeHeader(wireVersion: number): Uint8Array {
|
|
47
|
+
const encoded = encodeMessage({
|
|
48
|
+
wireVersion,
|
|
49
|
+
msgKind: 'response',
|
|
50
|
+
frames: [stubHeader(wireVersion)],
|
|
51
|
+
});
|
|
52
|
+
return encoded.slice(0, 8);
|
|
53
|
+
}
|
|
40
54
|
|
|
41
55
|
/** The terminating END frame (§1.2 rule 1). */
|
|
42
56
|
export const END_FRAME_BYTES: Uint8Array = probe.slice(probe.length - 5);
|
|
43
57
|
|
|
44
|
-
function wrapperFor(
|
|
58
|
+
function wrapperFor(
|
|
59
|
+
frame: ResponseFrame,
|
|
60
|
+
wireVersion: number,
|
|
61
|
+
): {
|
|
45
62
|
frames: ResponseFrame[];
|
|
46
63
|
index: number;
|
|
47
64
|
} {
|
|
@@ -50,11 +67,11 @@ function wrapperFor(frame: ResponseFrame): {
|
|
|
50
67
|
return { frames: [frame], index: 0 };
|
|
51
68
|
case 'LEASE':
|
|
52
69
|
// §7.3.2: LEASE rides immediately after RESP_HEADER.
|
|
53
|
-
return { frames: [
|
|
70
|
+
return { frames: [stubHeader(wireVersion), frame], index: 1 };
|
|
54
71
|
case 'PUSH_RESULT':
|
|
55
72
|
case 'ERROR':
|
|
56
73
|
case 'UNKNOWN':
|
|
57
|
-
return { frames: [
|
|
74
|
+
return { frames: [stubHeader(wireVersion), frame], index: 1 };
|
|
58
75
|
case 'PUSH_RESULT_DETAILS': {
|
|
59
76
|
const result: PushResultFrame = {
|
|
60
77
|
type: 'PUSH_RESULT',
|
|
@@ -68,17 +85,23 @@ function wrapperFor(frame: ResponseFrame): {
|
|
|
68
85
|
retryable: false,
|
|
69
86
|
})),
|
|
70
87
|
};
|
|
71
|
-
return { frames: [
|
|
88
|
+
return { frames: [stubHeader(wireVersion), result, frame], index: 2 };
|
|
72
89
|
}
|
|
73
90
|
case 'SUB_START':
|
|
74
|
-
return {
|
|
91
|
+
return {
|
|
92
|
+
frames: [stubHeader(wireVersion), frame, STUB_SUB_END],
|
|
93
|
+
index: 1,
|
|
94
|
+
};
|
|
75
95
|
case 'SUB_END':
|
|
76
|
-
return {
|
|
96
|
+
return {
|
|
97
|
+
frames: [stubHeader(wireVersion), STUB_SUB_START, frame],
|
|
98
|
+
index: 2,
|
|
99
|
+
};
|
|
77
100
|
case 'COMMIT':
|
|
78
101
|
case 'SEGMENT_REF':
|
|
79
102
|
case 'SEGMENT_INLINE':
|
|
80
103
|
return {
|
|
81
|
-
frames: [
|
|
104
|
+
frames: [stubHeader(wireVersion), STUB_SUB_START, frame, STUB_SUB_END],
|
|
82
105
|
index: 2,
|
|
83
106
|
};
|
|
84
107
|
}
|
|
@@ -88,10 +111,13 @@ function wrapperFor(frame: ResponseFrame): {
|
|
|
88
111
|
* Encode one response frame (5-byte frame header + payload) using the
|
|
89
112
|
* reference codec.
|
|
90
113
|
*/
|
|
91
|
-
export function encodeResponseFrame(
|
|
92
|
-
|
|
114
|
+
export function encodeResponseFrame(
|
|
115
|
+
frame: ResponseFrame,
|
|
116
|
+
wireVersion = PROTOCOL_WIRE_VERSION,
|
|
117
|
+
): Uint8Array {
|
|
118
|
+
const { frames, index } = wrapperFor(frame, wireVersion);
|
|
93
119
|
const encoded = encodeMessage({
|
|
94
|
-
wireVersion
|
|
120
|
+
wireVersion,
|
|
95
121
|
msgKind: 'response',
|
|
96
122
|
frames,
|
|
97
123
|
});
|
package/src/handler.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
limitsOf,
|
|
29
29
|
REMOTE_COMMAND_CLIENT_ID_PREFIX,
|
|
30
30
|
RESOLVER_OUTAGE,
|
|
31
|
+
touchAuthenticatedPartition,
|
|
31
32
|
} from './context';
|
|
32
33
|
import { SyncError, syncError } from './errors';
|
|
33
34
|
import {
|
|
@@ -38,7 +39,7 @@ import {
|
|
|
38
39
|
import {
|
|
39
40
|
END_FRAME_BYTES,
|
|
40
41
|
encodeResponseFrame,
|
|
41
|
-
|
|
42
|
+
responseEnvelopeHeader,
|
|
42
43
|
} from './frame-bytes';
|
|
43
44
|
import type { LeaseRecord } from './lease-store';
|
|
44
45
|
import {
|
|
@@ -53,9 +54,12 @@ import { type ProcessedPushCommit, processPushCommitWithTrace } from './push';
|
|
|
53
54
|
import type { CompiledSchema } from './schema';
|
|
54
55
|
import { compileSchema } from './schema';
|
|
55
56
|
import { computeEffective, type ResolvedScopes } from './scopes';
|
|
56
|
-
import type { ClientSubscription } from './storage';
|
|
57
|
+
import type { ClientSubscription, PartitionRegistryEntry } from './storage';
|
|
57
58
|
|
|
58
59
|
interface RequestPlan {
|
|
60
|
+
readonly wireVersion: number;
|
|
61
|
+
readonly logEpoch: string;
|
|
62
|
+
readonly epochReset: boolean;
|
|
59
63
|
readonly header: ReqHeaderFrame;
|
|
60
64
|
readonly pushes: readonly PushCommitFrame[];
|
|
61
65
|
readonly pull: PullHeaderFrame | undefined;
|
|
@@ -192,6 +196,7 @@ async function planRequest(
|
|
|
192
196
|
request: RequestMessage,
|
|
193
197
|
ctx: SyncRequestContext,
|
|
194
198
|
schema: CompiledSchema,
|
|
199
|
+
registry: PartitionRegistryEntry,
|
|
195
200
|
): Promise<RequestPlan> {
|
|
196
201
|
const header = request.frames[0];
|
|
197
202
|
if (header === undefined || header.type !== 'REQ_HEADER') {
|
|
@@ -206,10 +211,32 @@ async function planRequest(
|
|
|
206
211
|
else if (frame.type === 'SUBSCRIPTION') subFrames.push(frame);
|
|
207
212
|
}
|
|
208
213
|
|
|
214
|
+
if (request.wireVersion === 1 && registry.epochRequired) {
|
|
215
|
+
throw syncError(
|
|
216
|
+
'sync.client_wire_unsupported',
|
|
217
|
+
'this partition requires a client that validates log epochs (§2.1)',
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
if (
|
|
221
|
+
request.wireVersion >= 2 &&
|
|
222
|
+
header.logEpoch === undefined &&
|
|
223
|
+
pushes.length > 0
|
|
224
|
+
) {
|
|
225
|
+
throw syncError(
|
|
226
|
+
'sync.invalid_request',
|
|
227
|
+
'epoch acquisition requests must not carry push commits (§2.1)',
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
const epochReset =
|
|
231
|
+
request.wireVersion >= 2 && header.logEpoch !== registry.logEpoch;
|
|
232
|
+
|
|
209
233
|
if (header.schemaVersion !== schema.version) {
|
|
210
234
|
// §2.4: no degraded encoding — answer with the schema floor (§1.6).
|
|
211
235
|
// No lease is issued on a floor round (§7.3.3).
|
|
212
236
|
return {
|
|
237
|
+
wireVersion: request.wireVersion,
|
|
238
|
+
logEpoch: registry.logEpoch,
|
|
239
|
+
epochReset,
|
|
213
240
|
header,
|
|
214
241
|
pushes,
|
|
215
242
|
pull,
|
|
@@ -220,6 +247,21 @@ async function planRequest(
|
|
|
220
247
|
};
|
|
221
248
|
}
|
|
222
249
|
|
|
250
|
+
if (epochReset) {
|
|
251
|
+
return {
|
|
252
|
+
wireVersion: request.wireVersion,
|
|
253
|
+
logEpoch: registry.logEpoch,
|
|
254
|
+
epochReset: true,
|
|
255
|
+
header,
|
|
256
|
+
pushes: [],
|
|
257
|
+
pull: undefined,
|
|
258
|
+
subscriptions: [],
|
|
259
|
+
resolved: { ok: false },
|
|
260
|
+
schemaFloor: false,
|
|
261
|
+
leaseToEmit: undefined,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
223
265
|
if (header.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
224
266
|
throw syncError(
|
|
225
267
|
'sync.invalid_client_id',
|
|
@@ -313,6 +355,9 @@ async function planRequest(
|
|
|
313
355
|
});
|
|
314
356
|
|
|
315
357
|
return {
|
|
358
|
+
wireVersion: request.wireVersion,
|
|
359
|
+
logEpoch: registry.logEpoch,
|
|
360
|
+
epochReset: false,
|
|
316
361
|
header,
|
|
317
362
|
pushes,
|
|
318
363
|
pull,
|
|
@@ -325,7 +370,7 @@ async function planRequest(
|
|
|
325
370
|
|
|
326
371
|
/** Mutable outcome box shared with the instrumented stream wrapper. */
|
|
327
372
|
interface RequestReport {
|
|
328
|
-
outcome: 'ok' | 'schema_floor' | 'error';
|
|
373
|
+
outcome: 'ok' | 'schema_floor' | 'reset' | 'error';
|
|
329
374
|
errorCode?: string;
|
|
330
375
|
}
|
|
331
376
|
|
|
@@ -413,28 +458,48 @@ async function* streamResponse(
|
|
|
413
458
|
report?: RequestReport,
|
|
414
459
|
): AsyncGenerator<Uint8Array> {
|
|
415
460
|
const events = ctx.events;
|
|
416
|
-
yield
|
|
461
|
+
yield responseEnvelopeHeader(plan.wireVersion);
|
|
417
462
|
if (plan.schemaFloor) {
|
|
418
463
|
if (report !== undefined) report.outcome = 'schema_floor';
|
|
419
|
-
yield encodeResponseFrame(
|
|
464
|
+
yield encodeResponseFrame(
|
|
465
|
+
{
|
|
466
|
+
type: 'RESP_HEADER',
|
|
467
|
+
requiredSchemaVersion: schema.version,
|
|
468
|
+
latestSchemaVersion: schema.version,
|
|
469
|
+
...(plan.wireVersion >= 2
|
|
470
|
+
? { logEpoch: plan.logEpoch, resetRequired: plan.epochReset }
|
|
471
|
+
: {}),
|
|
472
|
+
},
|
|
473
|
+
plan.wireVersion,
|
|
474
|
+
);
|
|
475
|
+
yield END_FRAME_BYTES;
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
yield encodeResponseFrame(
|
|
479
|
+
{
|
|
420
480
|
type: 'RESP_HEADER',
|
|
421
|
-
requiredSchemaVersion: schema.version,
|
|
422
481
|
latestSchemaVersion: schema.version,
|
|
423
|
-
|
|
482
|
+
...(plan.wireVersion >= 2
|
|
483
|
+
? { logEpoch: plan.logEpoch, resetRequired: plan.epochReset }
|
|
484
|
+
: {}),
|
|
485
|
+
},
|
|
486
|
+
plan.wireVersion,
|
|
487
|
+
);
|
|
488
|
+
if (plan.epochReset) {
|
|
489
|
+
if (report !== undefined) report.outcome = 'reset';
|
|
424
490
|
yield END_FRAME_BYTES;
|
|
425
491
|
return;
|
|
426
492
|
}
|
|
427
|
-
yield encodeResponseFrame({
|
|
428
|
-
type: 'RESP_HEADER',
|
|
429
|
-
latestSchemaVersion: schema.version,
|
|
430
|
-
});
|
|
431
493
|
// §7.3.2: the LEASE frame rides immediately after RESP_HEADER.
|
|
432
494
|
if (plan.leaseToEmit !== undefined) {
|
|
433
|
-
yield encodeResponseFrame(
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
495
|
+
yield encodeResponseFrame(
|
|
496
|
+
{
|
|
497
|
+
type: 'LEASE',
|
|
498
|
+
leaseId: plan.leaseToEmit.leaseId,
|
|
499
|
+
expiresAtMs: plan.leaseToEmit.expiresAtMs,
|
|
500
|
+
},
|
|
501
|
+
plan.wireVersion,
|
|
502
|
+
);
|
|
438
503
|
if (events !== undefined) {
|
|
439
504
|
emitEvent(events, {
|
|
440
505
|
type: 'lease.issued',
|
|
@@ -461,9 +526,11 @@ async function* streamResponse(
|
|
|
461
526
|
if (events !== undefined) {
|
|
462
527
|
emitPushEvent(events, ctx, plan.header.clientId, push, processed);
|
|
463
528
|
}
|
|
464
|
-
yield encodeResponseFrame(frame);
|
|
529
|
+
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
465
530
|
const details = pushResultDetailsFrame(frame);
|
|
466
|
-
if (details !== undefined)
|
|
531
|
+
if (details !== undefined) {
|
|
532
|
+
yield encodeResponseFrame(details, plan.wireVersion);
|
|
533
|
+
}
|
|
467
534
|
}
|
|
468
535
|
|
|
469
536
|
// Pull half (§4): subscriptions echoed in request order.
|
|
@@ -485,6 +552,7 @@ async function* streamResponse(
|
|
|
485
552
|
maxSeq,
|
|
486
553
|
horizonSeq,
|
|
487
554
|
trace,
|
|
555
|
+
plan.logEpoch,
|
|
488
556
|
);
|
|
489
557
|
let status: 'active' | 'revoked' | 'reset' = 'active';
|
|
490
558
|
let bootstrap = false;
|
|
@@ -502,7 +570,7 @@ async function* streamResponse(
|
|
|
502
570
|
changes += frame.changes.length;
|
|
503
571
|
}
|
|
504
572
|
}
|
|
505
|
-
yield encodeResponseFrame(frame);
|
|
573
|
+
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
506
574
|
step = await section.next();
|
|
507
575
|
}
|
|
508
576
|
if (step.value.active) cursors.push(step.value.nextCursor);
|
|
@@ -556,6 +624,7 @@ async function* streamResponse(
|
|
|
556
624
|
await ctx.storage.putClientRecord(ctx.partition, {
|
|
557
625
|
clientId: plan.header.clientId,
|
|
558
626
|
actorId: ctx.actorId,
|
|
627
|
+
wireVersion: plan.wireVersion,
|
|
559
628
|
cursor,
|
|
560
629
|
updatedAtMs: clockOf(ctx)(),
|
|
561
630
|
subscriptions,
|
|
@@ -567,15 +636,18 @@ async function* streamResponse(
|
|
|
567
636
|
report.errorCode = error.code;
|
|
568
637
|
}
|
|
569
638
|
// §1.6: in-band ERROR, then END, nothing else.
|
|
570
|
-
yield encodeResponseFrame(
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
639
|
+
yield encodeResponseFrame(
|
|
640
|
+
{
|
|
641
|
+
type: 'ERROR',
|
|
642
|
+
code: error.code,
|
|
643
|
+
message: error.message,
|
|
644
|
+
category: error.category,
|
|
645
|
+
retryable: error.retryable,
|
|
646
|
+
recommendedAction: error.recommendedAction,
|
|
647
|
+
...(error.details !== undefined ? { details: error.details } : {}),
|
|
648
|
+
},
|
|
649
|
+
plan.wireVersion,
|
|
650
|
+
);
|
|
579
651
|
yield END_FRAME_BYTES;
|
|
580
652
|
return;
|
|
581
653
|
}
|
|
@@ -644,7 +716,8 @@ async function createStreamCore(
|
|
|
644
716
|
// Relational row tables: create/
|
|
645
717
|
// migrate on first contact; memoized per storage instance thereafter.
|
|
646
718
|
await ctx.storage.ensureSchema(schema);
|
|
647
|
-
const
|
|
719
|
+
const registry = await touchAuthenticatedPartition(ctx);
|
|
720
|
+
const plan = await planRequest(request, ctx, schema, registry);
|
|
648
721
|
if (events === undefined) return streamResponse(plan, ctx, schema);
|
|
649
722
|
const report: RequestReport = { outcome: 'ok' };
|
|
650
723
|
return instrumentedStream(
|
package/src/index.ts
CHANGED
package/src/operations.ts
CHANGED
|
@@ -9,7 +9,11 @@ import {
|
|
|
9
9
|
type ScopeMap,
|
|
10
10
|
} from '@syncular/core';
|
|
11
11
|
import type { SyncRequestContext } from './context';
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
REMOTE_COMMAND_CLIENT_ID_PREFIX,
|
|
14
|
+
RESOLVER_OUTAGE,
|
|
15
|
+
touchAuthenticatedPartition,
|
|
16
|
+
} from './context';
|
|
13
17
|
import { SyncError, syncError } from './errors';
|
|
14
18
|
import { processPushOperationsWithTrace } from './push';
|
|
15
19
|
import { compileSchema } from './schema';
|
|
@@ -673,6 +677,7 @@ export async function handleRemoteOperation(
|
|
|
673
677
|
return encodeRemoteOperationError(error, 'operation.invalid_request');
|
|
674
678
|
}
|
|
675
679
|
try {
|
|
680
|
+
await touchAuthenticatedPartition(ctx);
|
|
676
681
|
if (request.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
677
682
|
throw syncError(
|
|
678
683
|
'sync.invalid_client_id',
|
package/src/postgres-storage.ts
CHANGED
|
@@ -85,6 +85,7 @@ import type {
|
|
|
85
85
|
IndexRowScanQuery,
|
|
86
86
|
NewCommit,
|
|
87
87
|
NewReaction,
|
|
88
|
+
PartitionRegistryEntry,
|
|
88
89
|
PrunedReactionCounts,
|
|
89
90
|
ReactionClaimQuery,
|
|
90
91
|
ReactionFailure,
|
|
@@ -136,6 +137,12 @@ CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
|
136
137
|
max_commit_seq BIGINT NOT NULL DEFAULT 0,
|
|
137
138
|
horizon_seq BIGINT NOT NULL DEFAULT 0
|
|
138
139
|
);
|
|
140
|
+
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
141
|
+
partition TEXT PRIMARY KEY,
|
|
142
|
+
log_epoch TEXT NOT NULL,
|
|
143
|
+
epoch_required BOOLEAN NOT NULL DEFAULT FALSE,
|
|
144
|
+
last_authenticated_at_ms BIGINT NOT NULL
|
|
145
|
+
);
|
|
139
146
|
CREATE TABLE IF NOT EXISTS sync_row_scopes(
|
|
140
147
|
partition TEXT NOT NULL, tbl TEXT NOT NULL,
|
|
141
148
|
var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
@@ -189,10 +196,13 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
189
196
|
ON sync_reactions(partition, status, available_at_ms, idempotency_key);
|
|
190
197
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
191
198
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
199
|
+
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
192
200
|
cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
|
|
193
201
|
updated_at_ms BIGINT NOT NULL,
|
|
194
202
|
PRIMARY KEY(partition, client_id)
|
|
195
203
|
);
|
|
204
|
+
ALTER TABLE sync_clients
|
|
205
|
+
ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;
|
|
196
206
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
197
207
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
198
208
|
blob_id TEXT NOT NULL,
|
|
@@ -998,6 +1008,86 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
998
1008
|
this.#schemaVersion = schema.version;
|
|
999
1009
|
}
|
|
1000
1010
|
|
|
1011
|
+
async touchPartition(
|
|
1012
|
+
partition: string,
|
|
1013
|
+
authenticatedAtMs: number,
|
|
1014
|
+
initialLogEpoch: string,
|
|
1015
|
+
): Promise<PartitionRegistryEntry> {
|
|
1016
|
+
if (initialLogEpoch.length === 0) {
|
|
1017
|
+
throw new Error('initial log epoch must be non-empty');
|
|
1018
|
+
}
|
|
1019
|
+
const { rows } = await this.#exec.query<{
|
|
1020
|
+
log_epoch: string;
|
|
1021
|
+
epoch_required: boolean;
|
|
1022
|
+
last_authenticated_at_ms: unknown;
|
|
1023
|
+
}>(
|
|
1024
|
+
`INSERT INTO sync_partition_registry(
|
|
1025
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
1026
|
+
) VALUES ($1,$2,$3)
|
|
1027
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1028
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
|
|
1029
|
+
RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
|
|
1030
|
+
[partition, initialLogEpoch, authenticatedAtMs],
|
|
1031
|
+
);
|
|
1032
|
+
const row = rows[0];
|
|
1033
|
+
if (row === undefined)
|
|
1034
|
+
throw new Error('partition registry write did not persist');
|
|
1035
|
+
return {
|
|
1036
|
+
partition,
|
|
1037
|
+
logEpoch: row.log_epoch,
|
|
1038
|
+
epochRequired: row.epoch_required,
|
|
1039
|
+
lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
async rotatePartitionLogEpoch(
|
|
1044
|
+
partition: string,
|
|
1045
|
+
logEpoch: string,
|
|
1046
|
+
authenticatedAtMs: number,
|
|
1047
|
+
): Promise<PartitionRegistryEntry> {
|
|
1048
|
+
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
1049
|
+
await this.#exec.transaction(async (client) => {
|
|
1050
|
+
await client.query(
|
|
1051
|
+
`INSERT INTO sync_partition_registry(
|
|
1052
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1053
|
+
) VALUES ($1,$2,TRUE,$3)
|
|
1054
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1055
|
+
log_epoch=EXCLUDED.log_epoch,
|
|
1056
|
+
epoch_required=TRUE,
|
|
1057
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms`,
|
|
1058
|
+
[partition, logEpoch, authenticatedAtMs],
|
|
1059
|
+
);
|
|
1060
|
+
await client.query('DELETE FROM sync_clients WHERE partition=$1', [
|
|
1061
|
+
partition,
|
|
1062
|
+
]);
|
|
1063
|
+
});
|
|
1064
|
+
return {
|
|
1065
|
+
partition,
|
|
1066
|
+
logEpoch,
|
|
1067
|
+
epochRequired: true,
|
|
1068
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
|
|
1073
|
+
const { rows } = await this.#exec.query<{
|
|
1074
|
+
partition: string;
|
|
1075
|
+
log_epoch: string;
|
|
1076
|
+
epoch_required: boolean;
|
|
1077
|
+
last_authenticated_at_ms: unknown;
|
|
1078
|
+
}>(
|
|
1079
|
+
`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1080
|
+
FROM sync_partition_registry ORDER BY partition`,
|
|
1081
|
+
[],
|
|
1082
|
+
);
|
|
1083
|
+
return rows.map((row) => ({
|
|
1084
|
+
partition: row.partition,
|
|
1085
|
+
logEpoch: row.log_epoch,
|
|
1086
|
+
epochRequired: row.epoch_required,
|
|
1087
|
+
lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
|
|
1088
|
+
}));
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1001
1091
|
/**
|
|
1002
1092
|
* Open a real Postgres transaction. The push handler drives the returned
|
|
1003
1093
|
* `StorageTransaction` imperatively (getRow/upsert/…/commit), but the
|
|
@@ -1496,11 +1586,12 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1496
1586
|
const { rows } = await this.#exec.query<{
|
|
1497
1587
|
client_id: string;
|
|
1498
1588
|
actor_id: string;
|
|
1589
|
+
wire_version: unknown;
|
|
1499
1590
|
cursor: unknown;
|
|
1500
1591
|
subscriptions: unknown;
|
|
1501
1592
|
updated_at_ms: unknown;
|
|
1502
1593
|
}>(
|
|
1503
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
1594
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
1504
1595
|
[partition, clientId],
|
|
1505
1596
|
);
|
|
1506
1597
|
const record = rows[0];
|
|
@@ -1508,6 +1599,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1508
1599
|
return {
|
|
1509
1600
|
clientId: record.client_id,
|
|
1510
1601
|
actorId: record.actor_id,
|
|
1602
|
+
wireVersion: asNumber(record.wire_version),
|
|
1511
1603
|
cursor: asNumber(record.cursor),
|
|
1512
1604
|
updatedAtMs: asNumber(record.updated_at_ms),
|
|
1513
1605
|
subscriptions: asJson<ClientSubscription[]>(record.subscriptions),
|
|
@@ -1519,16 +1611,18 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1519
1611
|
record: ClientRecord,
|
|
1520
1612
|
): Promise<void> {
|
|
1521
1613
|
await this.#exec.query(
|
|
1522
|
-
`INSERT INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms)
|
|
1523
|
-
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1614
|
+
`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms)
|
|
1615
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
1524
1616
|
ON CONFLICT (partition, client_id) DO UPDATE
|
|
1525
|
-
SET actor_id=EXCLUDED.actor_id,
|
|
1617
|
+
SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
|
|
1618
|
+
cursor=EXCLUDED.cursor,
|
|
1526
1619
|
subscriptions=EXCLUDED.subscriptions,
|
|
1527
1620
|
updated_at_ms=EXCLUDED.updated_at_ms`,
|
|
1528
1621
|
[
|
|
1529
1622
|
partition,
|
|
1530
1623
|
record.clientId,
|
|
1531
1624
|
record.actorId,
|
|
1625
|
+
record.wireVersion,
|
|
1532
1626
|
record.cursor,
|
|
1533
1627
|
JSON.stringify(record.subscriptions),
|
|
1534
1628
|
record.updatedAtMs,
|
|
@@ -1608,16 +1702,18 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1608
1702
|
const { rows } = await this.#exec.query<{
|
|
1609
1703
|
client_id: string;
|
|
1610
1704
|
actor_id: string;
|
|
1705
|
+
wire_version: unknown;
|
|
1611
1706
|
cursor: unknown;
|
|
1612
1707
|
subscriptions: unknown;
|
|
1613
1708
|
updated_at_ms: unknown;
|
|
1614
1709
|
}>(
|
|
1615
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
1710
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
1616
1711
|
[partition],
|
|
1617
1712
|
);
|
|
1618
1713
|
return rows.map((r) => ({
|
|
1619
1714
|
clientId: r.client_id,
|
|
1620
1715
|
actorId: r.actor_id,
|
|
1716
|
+
wireVersion: asNumber(r.wire_version),
|
|
1621
1717
|
cursor: asNumber(r.cursor),
|
|
1622
1718
|
updatedAtMs: asNumber(r.updated_at_ms),
|
|
1623
1719
|
subscriptions: (typeof r.subscriptions === 'string'
|
|
@@ -1742,13 +1838,6 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1742
1838
|
}
|
|
1743
1839
|
|
|
1744
1840
|
async listPartitions(): Promise<string[]> {
|
|
1745
|
-
|
|
1746
|
-
// first pull — a partition with only one of the two still shows up.
|
|
1747
|
-
const { rows } = await this.#exec.query<{ partition: string }>(
|
|
1748
|
-
`SELECT partition FROM sync_partitions
|
|
1749
|
-
UNION SELECT partition FROM sync_clients ORDER BY partition`,
|
|
1750
|
-
[],
|
|
1751
|
-
);
|
|
1752
|
-
return rows.map((r) => r.partition);
|
|
1841
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1753
1842
|
}
|
|
1754
1843
|
}
|
package/src/pull.ts
CHANGED
|
@@ -212,12 +212,14 @@ async function* sqliteImageSegment(
|
|
|
212
212
|
asOf: number,
|
|
213
213
|
digest: string,
|
|
214
214
|
trace: PullSectionTrace | undefined,
|
|
215
|
+
logEpoch: string,
|
|
215
216
|
): AsyncGenerator<ResponseFrame, boolean> {
|
|
216
217
|
const { storage, segments, partition } = ctx;
|
|
217
218
|
const now = clockOf(ctx)();
|
|
218
219
|
const existing = await segments.find(
|
|
219
220
|
{
|
|
220
221
|
partition,
|
|
222
|
+
logEpoch,
|
|
221
223
|
table: plan.table.name,
|
|
222
224
|
schemaVersion: schema.version,
|
|
223
225
|
mediaType: 'sqlite',
|
|
@@ -286,6 +288,7 @@ async function* sqliteImageSegment(
|
|
|
286
288
|
const record = await segments.put(
|
|
287
289
|
{
|
|
288
290
|
partition,
|
|
291
|
+
logEpoch,
|
|
289
292
|
table: plan.table.name,
|
|
290
293
|
schemaVersion: schema.version,
|
|
291
294
|
mediaType: 'sqlite',
|
|
@@ -320,6 +323,7 @@ async function* bootstrapSegments(
|
|
|
320
323
|
asOf: number,
|
|
321
324
|
startRowCursor: string | null,
|
|
322
325
|
trace: PullSectionTrace | undefined,
|
|
326
|
+
logEpoch: string,
|
|
323
327
|
): AsyncGenerator<
|
|
324
328
|
ResponseFrame,
|
|
325
329
|
{ complete: boolean; rowCursor: string | null }
|
|
@@ -347,6 +351,7 @@ async function* bootstrapSegments(
|
|
|
347
351
|
asOf,
|
|
348
352
|
digest,
|
|
349
353
|
trace,
|
|
354
|
+
logEpoch,
|
|
350
355
|
);
|
|
351
356
|
if (imaged) return { complete: true, rowCursor: null };
|
|
352
357
|
}
|
|
@@ -393,6 +398,7 @@ async function* bootstrapSegments(
|
|
|
393
398
|
const record = await segments.put(
|
|
394
399
|
{
|
|
395
400
|
partition,
|
|
401
|
+
logEpoch,
|
|
396
402
|
table: plan.table.name,
|
|
397
403
|
schemaVersion: schema.version,
|
|
398
404
|
mediaType: 'rows',
|
|
@@ -435,8 +441,12 @@ export async function* subscriptionSection(
|
|
|
435
441
|
maxSeq: number,
|
|
436
442
|
horizonSeq: number,
|
|
437
443
|
trace?: PullSectionTrace,
|
|
444
|
+
logEpoch?: string,
|
|
438
445
|
): AsyncGenerator<ResponseFrame, SubscriptionResult> {
|
|
439
446
|
const sub = plan.frame;
|
|
447
|
+
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
448
|
+
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
449
|
+
}
|
|
440
450
|
|
|
441
451
|
if (plan.status === 'revoked') {
|
|
442
452
|
yield {
|
|
@@ -496,6 +506,7 @@ export async function* subscriptionSection(
|
|
|
496
506
|
asOf,
|
|
497
507
|
startCursor,
|
|
498
508
|
trace,
|
|
509
|
+
logEpoch,
|
|
499
510
|
);
|
|
500
511
|
if (outcome.complete) {
|
|
501
512
|
yield { type: 'SUB_END', nextCursor: asOf };
|