@syncular/server 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/dist/errors.js
CHANGED
|
@@ -204,6 +204,14 @@ export const ERROR_CATALOG = {
|
|
|
204
204
|
recommendedAction: 'splitBatch',
|
|
205
205
|
httpStatus: 400,
|
|
206
206
|
},
|
|
207
|
+
// §1.7 / D3: a request or watch session past a configured `ServerLimits`
|
|
208
|
+
// cap; `details.limit` names the cap.
|
|
209
|
+
'sync.limit_exceeded': {
|
|
210
|
+
category: 'invalid-request',
|
|
211
|
+
retryable: false,
|
|
212
|
+
recommendedAction: 'fixRequest',
|
|
213
|
+
httpStatus: 413,
|
|
214
|
+
},
|
|
207
215
|
'sync.not_found': {
|
|
208
216
|
category: 'not-found',
|
|
209
217
|
retryable: false,
|
package/dist/events.d.ts
CHANGED
|
@@ -213,7 +213,7 @@ export interface BlobSweptEvent {
|
|
|
213
213
|
readonly partition: string;
|
|
214
214
|
/** Blobs deleted this pass (unreferenced and older than the grace period). */
|
|
215
215
|
readonly swept: number;
|
|
216
|
-
/**
|
|
216
|
+
/** Old candidates a live row still references (§5.9.4 reference index). */
|
|
217
217
|
readonly referenced: number;
|
|
218
218
|
/** The grace period applied (ms) — the upload→push race protection. */
|
|
219
219
|
readonly graceMs: number;
|
package/dist/handler.js
CHANGED
|
@@ -119,12 +119,10 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
119
119
|
else if (frame.type === 'SUBSCRIPTION')
|
|
120
120
|
subFrames.push(frame);
|
|
121
121
|
}
|
|
122
|
-
if (
|
|
123
|
-
header.logEpoch === undefined &&
|
|
124
|
-
pushes.length > 0) {
|
|
122
|
+
if (header.logEpoch === undefined && pushes.length > 0) {
|
|
125
123
|
throw syncError('sync.invalid_request', 'epoch acquisition requests must not carry push commits (§2.1)');
|
|
126
124
|
}
|
|
127
|
-
const epochReset =
|
|
125
|
+
const epochReset = header.logEpoch !== registry.logEpoch;
|
|
128
126
|
if (header.schemaVersion !== schema.version) {
|
|
129
127
|
// §2.4: no degraded encoding — answer with the schema floor (§1.6).
|
|
130
128
|
// No lease is issued on a floor round (§7.3.3).
|
|
@@ -171,6 +169,27 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
171
169
|
if (totalOperations > limits.maxOperationsPerRequest) {
|
|
172
170
|
throw syncError('sync.too_many_operations', `request carries ${totalOperations} operations (cap ${limits.maxOperationsPerRequest}, §6.1)`);
|
|
173
171
|
}
|
|
172
|
+
// §1.7: subscription caps, checked before any subscription is planned.
|
|
173
|
+
if (subFrames.length > limits.maxSubscriptionsPerRequest) {
|
|
174
|
+
throw syncError('sync.limit_exceeded', 'request carries more subscriptions than the server accepts (§1.7)', JSON.stringify({
|
|
175
|
+
limit: 'maxSubscriptionsPerRequest',
|
|
176
|
+
max: limits.maxSubscriptionsPerRequest,
|
|
177
|
+
actual: subFrames.length,
|
|
178
|
+
}));
|
|
179
|
+
}
|
|
180
|
+
for (const frame of subFrames) {
|
|
181
|
+
let values = 0;
|
|
182
|
+
for (const list of Object.values(frame.scopes))
|
|
183
|
+
values += list.length;
|
|
184
|
+
if (values > limits.maxScopeValuesPerSubscription) {
|
|
185
|
+
throw syncError('sync.limit_exceeded', 'subscription requests more scope values than the server accepts (§1.7)', JSON.stringify({
|
|
186
|
+
limit: 'maxScopeValuesPerSubscription',
|
|
187
|
+
max: limits.maxScopeValuesPerSubscription,
|
|
188
|
+
actual: values,
|
|
189
|
+
subscriptionId: frame.id,
|
|
190
|
+
}));
|
|
191
|
+
}
|
|
192
|
+
}
|
|
174
193
|
if (pull !== undefined) {
|
|
175
194
|
// §4.2: rows support is mandatory (bits 0 and 1) — reject its absence
|
|
176
195
|
// as request validation (the MAY of §1.7).
|
|
@@ -206,7 +225,7 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
206
225
|
const table = schema.tables.get(frame.table);
|
|
207
226
|
if (table === undefined)
|
|
208
227
|
throw new Error('unreachable: validated table');
|
|
209
|
-
const outcome = computeEffective(frame.scopes, resolved);
|
|
228
|
+
const outcome = computeEffective(table.declaredVariables, frame.scopes, resolved);
|
|
210
229
|
if (outcome.status === 'active') {
|
|
211
230
|
return { frame, table, status: 'active', effective: outcome.effective };
|
|
212
231
|
}
|
|
@@ -298,15 +317,13 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
298
317
|
const events = ctx.events;
|
|
299
318
|
yield responseEnvelopeHeader(plan.wireVersion);
|
|
300
319
|
if (plan.schemaFloor) {
|
|
301
|
-
|
|
302
|
-
report.outcome = 'schema_floor';
|
|
320
|
+
report.outcome = 'schema_floor';
|
|
303
321
|
yield encodeResponseFrame({
|
|
304
322
|
type: 'RESP_HEADER',
|
|
305
323
|
requiredSchemaVersion: schemaWindowOf(ctx.schema, ctx.schemaWindow).at(-1).version,
|
|
306
324
|
latestSchemaVersion: ctx.schema.version,
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
: {}),
|
|
325
|
+
logEpoch: plan.logEpoch,
|
|
326
|
+
resetRequired: plan.epochReset,
|
|
310
327
|
}, plan.wireVersion);
|
|
311
328
|
yield END_FRAME_BYTES;
|
|
312
329
|
return;
|
|
@@ -314,15 +331,12 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
314
331
|
yield encodeResponseFrame({
|
|
315
332
|
type: 'RESP_HEADER',
|
|
316
333
|
latestSchemaVersion: ctx.schema.version,
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
: {}),
|
|
334
|
+
logEpoch: plan.logEpoch,
|
|
335
|
+
resetRequired: plan.epochReset,
|
|
320
336
|
}, plan.wireVersion);
|
|
321
337
|
if (plan.epochReset) {
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
plan.header.logEpoch === undefined ? 'epoch_acquired' : 'reset';
|
|
325
|
-
}
|
|
338
|
+
report.outcome =
|
|
339
|
+
plan.header.logEpoch === undefined ? 'epoch_acquired' : 'reset';
|
|
326
340
|
yield END_FRAME_BYTES;
|
|
327
341
|
return;
|
|
328
342
|
}
|
|
@@ -399,62 +413,73 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
399
413
|
if (maxSeq < acceptedThrough)
|
|
400
414
|
throw syncError('sync.storage_stale_read', 'storage returned a commit sequence older than the accepted push; disable query caching for sync storage', JSON.stringify({ acceptedThrough, storageMaxCommitSeq: maxSeq }));
|
|
401
415
|
const horizonSeq = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
const
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
416
|
+
// A window of sections runs its pre-frame reads together, so a batching
|
|
417
|
+
// storage answers them in one statement per table. The window holds at
|
|
418
|
+
// most 100,000 first-page or window rows (two maximal §4.2 pages): a
|
|
419
|
+
// request's read memory follows the configured limits, never its
|
|
420
|
+
// subscription count.
|
|
421
|
+
const window = Math.max(1, Math.floor(100_000 /
|
|
422
|
+
(Math.max(limits.limitSnapshotRows, limits.limitCommits) + 1)));
|
|
423
|
+
for (let start = 0; start < plan.subscriptions.length; start += window) {
|
|
424
|
+
const batch = plan.subscriptions.slice(start, start + window);
|
|
425
|
+
const prepared = await prepareSections(ctx, schema, limits, batch, maxSeq, horizonSeq, plan.logEpoch);
|
|
426
|
+
for (const [index, subscription] of batch.entries()) {
|
|
427
|
+
const trace = summaries !== undefined ? { segments: [] } : undefined;
|
|
428
|
+
const reads = prepared[index];
|
|
429
|
+
if (reads === undefined) {
|
|
430
|
+
throw new Error('prepareSections lost a plan');
|
|
431
|
+
}
|
|
432
|
+
const section = preparedSection(ctx, schema, limits, subscription, maxSeq, reads, trace, plan.logEpoch);
|
|
433
|
+
let status = 'active';
|
|
434
|
+
let bootstrap = false;
|
|
435
|
+
let commits = 0;
|
|
436
|
+
let changes = 0;
|
|
437
|
+
let effectiveScopes = {};
|
|
438
|
+
let firstCommitSeq;
|
|
439
|
+
let lastCommitSeq;
|
|
440
|
+
let step = await section.next();
|
|
441
|
+
while (!step.done) {
|
|
442
|
+
const frame = step.value;
|
|
443
|
+
if (summaries !== undefined) {
|
|
444
|
+
if (frame.type === 'SUB_START') {
|
|
445
|
+
status = frame.status;
|
|
446
|
+
bootstrap = frame.bootstrap;
|
|
447
|
+
effectiveScopes = frame.effectiveScopes;
|
|
448
|
+
}
|
|
449
|
+
else if (frame.type === 'COMMIT') {
|
|
450
|
+
commits += 1;
|
|
451
|
+
changes += frame.changes.length;
|
|
452
|
+
firstCommitSeq ??= frame.commitSeq;
|
|
453
|
+
lastCommitSeq = frame.commitSeq;
|
|
454
|
+
}
|
|
430
455
|
}
|
|
456
|
+
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
457
|
+
step = await section.next();
|
|
458
|
+
}
|
|
459
|
+
if (step.value.active)
|
|
460
|
+
cursors.push(step.value.nextCursor);
|
|
461
|
+
if (summaries !== undefined && trace !== undefined) {
|
|
462
|
+
summaries.push({
|
|
463
|
+
id: subscription.frame.id,
|
|
464
|
+
table: subscription.frame.table,
|
|
465
|
+
status,
|
|
466
|
+
mode: status !== 'active'
|
|
467
|
+
? 'none'
|
|
468
|
+
: bootstrap
|
|
469
|
+
? 'bootstrap'
|
|
470
|
+
: 'incremental',
|
|
471
|
+
requestedScopes: subscription.frame.scopes,
|
|
472
|
+
effectiveScopes,
|
|
473
|
+
...(firstCommitSeq !== undefined && lastCommitSeq !== undefined
|
|
474
|
+
? { firstCommitSeq, lastCommitSeq }
|
|
475
|
+
: {}),
|
|
476
|
+
fromCursor: subscription.frame.cursor,
|
|
477
|
+
nextCursor: step.value.nextCursor,
|
|
478
|
+
commits,
|
|
479
|
+
changes,
|
|
480
|
+
segments: trace.segments,
|
|
481
|
+
});
|
|
431
482
|
}
|
|
432
|
-
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
433
|
-
step = await section.next();
|
|
434
|
-
}
|
|
435
|
-
if (step.value.active)
|
|
436
|
-
cursors.push(step.value.nextCursor);
|
|
437
|
-
if (summaries !== undefined && trace !== undefined) {
|
|
438
|
-
summaries.push({
|
|
439
|
-
id: subscription.frame.id,
|
|
440
|
-
table: subscription.frame.table,
|
|
441
|
-
status,
|
|
442
|
-
mode: status !== 'active'
|
|
443
|
-
? 'none'
|
|
444
|
-
: bootstrap
|
|
445
|
-
? 'bootstrap'
|
|
446
|
-
: 'incremental',
|
|
447
|
-
requestedScopes: subscription.frame.scopes,
|
|
448
|
-
effectiveScopes,
|
|
449
|
-
...(firstCommitSeq !== undefined && lastCommitSeq !== undefined
|
|
450
|
-
? { firstCommitSeq, lastCommitSeq }
|
|
451
|
-
: {}),
|
|
452
|
-
fromCursor: subscription.frame.cursor,
|
|
453
|
-
nextCursor: step.value.nextCursor,
|
|
454
|
-
commits,
|
|
455
|
-
changes,
|
|
456
|
-
segments: trace.segments,
|
|
457
|
-
});
|
|
458
483
|
}
|
|
459
484
|
}
|
|
460
485
|
if (events !== undefined && summaries !== undefined) {
|
|
@@ -470,41 +495,47 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
470
495
|
});
|
|
471
496
|
}
|
|
472
497
|
}
|
|
473
|
-
// §4.5/§8.1:
|
|
474
|
-
//
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
498
|
+
// §4.5/§8.1: the cursor watermark and the request's subscription list
|
|
499
|
+
// per (partition, clientId), persisted only after the serve gate
|
|
500
|
+
// verified the round (§4.5: a failed round leaves the cursor alone).
|
|
501
|
+
report.persist = async () => {
|
|
502
|
+
const previous = await ctx.storage.getClientRecord(ctx.partition, plan.header.clientId);
|
|
503
|
+
const subscriptions = plan.pull !== undefined
|
|
504
|
+
? plan.subscriptions.map((s) => ({
|
|
505
|
+
id: s.frame.id,
|
|
506
|
+
table: s.frame.table,
|
|
507
|
+
scopes: s.frame.scopes,
|
|
508
|
+
schemaVersion: schema.version,
|
|
509
|
+
}))
|
|
510
|
+
: [...(previous?.subscriptions ?? [])];
|
|
511
|
+
let cursor = cursors[0] ?? previous?.cursor ?? -1;
|
|
512
|
+
for (const next of cursors)
|
|
513
|
+
cursor = Math.min(cursor, next);
|
|
514
|
+
await ctx.storage.putClientRecord(ctx.partition, {
|
|
515
|
+
clientId: plan.header.clientId,
|
|
516
|
+
actorId: ctx.actorId,
|
|
517
|
+
wireVersion: plan.wireVersion,
|
|
518
|
+
cursor,
|
|
519
|
+
updatedAtMs: clockOf(ctx)(),
|
|
520
|
+
subscriptions,
|
|
521
|
+
},
|
|
522
|
+
// §4.5: a round that carried a `ROUND_FAILURE` report completes
|
|
523
|
+
// without clearing the reported code and streak; the next request
|
|
524
|
+
// without the frame clears them as a plain completed round does.
|
|
525
|
+
plan.roundFailure !== undefined
|
|
526
|
+
? { keepRoundOutcome: true }
|
|
527
|
+
: undefined);
|
|
528
|
+
if (ctx.realtime?.notifySubscriptions !== undefined &&
|
|
529
|
+
JSON.stringify((previous?.subscriptions ?? []).map((s) => [s.id, s.table, s.scopes])) !==
|
|
530
|
+
JSON.stringify(subscriptions.map((s) => [s.id, s.table, s.scopes]))) {
|
|
531
|
+
await ctx.realtime.notifySubscriptions(ctx.partition, plan.header.clientId);
|
|
532
|
+
}
|
|
533
|
+
};
|
|
501
534
|
}
|
|
502
535
|
catch (error) {
|
|
503
536
|
if (error instanceof SyncError) {
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
report.errorCode = error.code;
|
|
507
|
-
}
|
|
537
|
+
report.outcome = 'error';
|
|
538
|
+
report.errorCode = error.code;
|
|
508
539
|
await recordRoundFailure(ctx, plan.header.clientId, error);
|
|
509
540
|
// §1.6: in-band ERROR, then END, nothing else.
|
|
510
541
|
yield encodeResponseFrame({
|
|
@@ -690,6 +721,7 @@ async function createStreamCore(bytes, ctx, events, startedAtMs = 0) {
|
|
|
690
721
|
throw serveNotReadyError(afterRefusal);
|
|
691
722
|
if (changed !== undefined)
|
|
692
723
|
throw serveNotReadyError(changed);
|
|
724
|
+
await report.persist?.();
|
|
693
725
|
for (const chunk of buffered)
|
|
694
726
|
yield chunk;
|
|
695
727
|
};
|
package/dist/local-authority.js
CHANGED
|
@@ -268,7 +268,13 @@ export async function openLocalAuthority(options) {
|
|
|
268
268
|
uploading = false;
|
|
269
269
|
}
|
|
270
270
|
}
|
|
271
|
-
writeMeta({
|
|
271
|
+
writeMeta({
|
|
272
|
+
state: 'local',
|
|
273
|
+
attach_id: null,
|
|
274
|
+
sent: null,
|
|
275
|
+
applied: null,
|
|
276
|
+
undo: null,
|
|
277
|
+
});
|
|
272
278
|
state = 'local';
|
|
273
279
|
},
|
|
274
280
|
};
|
|
@@ -287,6 +293,11 @@ export async function openLocalAuthority(options) {
|
|
|
287
293
|
throw error;
|
|
288
294
|
}
|
|
289
295
|
}
|
|
296
|
+
else if (readMeta('undo') !== undefined) {
|
|
297
|
+
// §12.3: resuming after a started rollback would count `cached`
|
|
298
|
+
// uploads whose rows the rollback deleted.
|
|
299
|
+
throw new LocalAuthorityAttachError({ reason: 'undo_pending' });
|
|
300
|
+
}
|
|
290
301
|
const attachId = attachIdOf();
|
|
291
302
|
const commits = planAttach(database, compiled, attachId);
|
|
292
303
|
const send = await connect(transport, `authority-${attachId}`);
|
|
@@ -312,7 +323,8 @@ export async function openLocalAuthority(options) {
|
|
|
312
323
|
}
|
|
313
324
|
/**
|
|
314
325
|
* §12.3 abandon: settle the one commit whose outcome may be unknown, then
|
|
315
|
-
* delete the rows of every applied commit, children before parents
|
|
326
|
+
* delete the rows of every applied commit, children before parents, under
|
|
327
|
+
* commit ids fresh to this attempt (the persisted `undo` marker).
|
|
316
328
|
*/
|
|
317
329
|
async function rollback(transport) {
|
|
318
330
|
const attachId = attachIdOf();
|
|
@@ -335,10 +347,12 @@ export async function openLocalAuthority(options) {
|
|
|
335
347
|
rowId: operation.rowId,
|
|
336
348
|
op: 'delete',
|
|
337
349
|
}));
|
|
350
|
+
const attempt = ordinal('undo') + 1;
|
|
351
|
+
writeMeta({ undo: String(attempt) });
|
|
338
352
|
for (let start = 0; start < deletes.length; start += ATTACH_OPERATION_BUDGET) {
|
|
339
353
|
const commit = {
|
|
340
354
|
type: 'PUSH_COMMIT',
|
|
341
|
-
clientCommitId: `${attachId}:undo:${start / ATTACH_OPERATION_BUDGET}`,
|
|
355
|
+
clientCommitId: `${attachId}:undo:${attempt}:${start / ATTACH_OPERATION_BUDGET}`,
|
|
342
356
|
operations: deletes.slice(start, start + ATTACH_OPERATION_BUDGET),
|
|
343
357
|
};
|
|
344
358
|
const result = await send(commit);
|
package/dist/object-bucket.d.ts
CHANGED
|
@@ -32,6 +32,8 @@ export interface BucketObject {
|
|
|
32
32
|
export interface BucketObjectHead {
|
|
33
33
|
readonly etag: string | undefined;
|
|
34
34
|
readonly size: number | undefined;
|
|
35
|
+
/** Backend last-modified time (epoch ms): S3 `Last-Modified`, R2 `uploaded`. */
|
|
36
|
+
readonly lastModifiedMs: number | undefined;
|
|
35
37
|
readonly metadata: Readonly<Record<string, string>>;
|
|
36
38
|
}
|
|
37
39
|
export interface BucketWriteOptions {
|
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
import { decodeRemoteOperationRealtimeMessage, encodeRemoteOperationRealtimeMessage, } from '@syncular/core';
|
|
2
|
-
import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context.js';
|
|
2
|
+
import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context.js';
|
|
3
3
|
import { reportError, SyncError, syncError } from './errors.js';
|
|
4
|
+
/**
|
|
5
|
+
* One socket's registered-query watches. A session runs one refresh at a
|
|
6
|
+
* time: `notify` queues each affected watch once, and the drain reruns the
|
|
7
|
+
* queue in order, so a commit costs at most one query per watch and never
|
|
8
|
+
* more than one query at a time.
|
|
9
|
+
*/
|
|
4
10
|
class WatchSession {
|
|
5
11
|
#ctx;
|
|
6
12
|
#registry;
|
|
7
13
|
#send;
|
|
8
14
|
#closed;
|
|
15
|
+
#maxWatches;
|
|
9
16
|
#watches = new Map();
|
|
17
|
+
/** Watches waiting for a refresh, in arrival order, each at most once. */
|
|
18
|
+
#pending = new Set();
|
|
19
|
+
#draining;
|
|
10
20
|
#isClosed = false;
|
|
11
21
|
get partition() {
|
|
12
22
|
return this.#ctx.partition;
|
|
@@ -16,6 +26,7 @@ class WatchSession {
|
|
|
16
26
|
this.#registry = registry;
|
|
17
27
|
this.#send = send;
|
|
18
28
|
this.#closed = closed;
|
|
29
|
+
this.#maxWatches = limitsOf(ctx).maxWatchesPerSession;
|
|
19
30
|
}
|
|
20
31
|
async receive(bytes) {
|
|
21
32
|
if (this.#isClosed)
|
|
@@ -46,6 +57,9 @@ class WatchSession {
|
|
|
46
57
|
throw syncError('operation.invalid_request');
|
|
47
58
|
}
|
|
48
59
|
if (message.kind === 'unwatch') {
|
|
60
|
+
const state = this.#watches.get(message.watchId);
|
|
61
|
+
if (state !== undefined)
|
|
62
|
+
this.#pending.delete(state);
|
|
49
63
|
this.#watches.delete(message.watchId);
|
|
50
64
|
return;
|
|
51
65
|
}
|
|
@@ -68,73 +82,77 @@ class WatchSession {
|
|
|
68
82
|
this.#sendError(message.watchId, syncError('operation.unknown'));
|
|
69
83
|
return;
|
|
70
84
|
}
|
|
85
|
+
if (this.#watches.size >= this.#maxWatches) {
|
|
86
|
+
this.#sendError(message.watchId, syncError('sync.limit_exceeded', 'watch session holds the maximum number of watches', JSON.stringify({
|
|
87
|
+
limit: 'maxWatchesPerSession',
|
|
88
|
+
max: this.#maxWatches,
|
|
89
|
+
})));
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
71
92
|
const state = {
|
|
72
93
|
watchId: message.watchId,
|
|
73
94
|
clientId: message.clientId,
|
|
74
95
|
operation,
|
|
75
96
|
params: message.params,
|
|
76
|
-
running: false,
|
|
77
|
-
dirty: false,
|
|
78
97
|
};
|
|
79
98
|
this.#watches.set(message.watchId, state);
|
|
80
99
|
await this.#refresh(state);
|
|
81
100
|
}
|
|
82
101
|
notify(tables) {
|
|
83
102
|
for (const state of this.#watches.values()) {
|
|
84
|
-
if (
|
|
85
|
-
continue;
|
|
86
|
-
if (state.running)
|
|
87
|
-
state.dirty = true;
|
|
88
|
-
else
|
|
103
|
+
if (state.operation.tables.some((table) => tables.has(table))) {
|
|
89
104
|
void this.#refresh(state);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
/** Queue `state` and resolve when the session's queue has drained. */
|
|
109
|
+
#refresh(state) {
|
|
110
|
+
this.#pending.add(state);
|
|
111
|
+
this.#draining ??= this.#drain();
|
|
112
|
+
return this.#draining;
|
|
113
|
+
}
|
|
114
|
+
async #drain() {
|
|
115
|
+
try {
|
|
116
|
+
// A watch queued again while it runs is appended and runs once more.
|
|
117
|
+
for (const state of this.#pending) {
|
|
118
|
+
this.#pending.delete(state);
|
|
119
|
+
await this.#run(state);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
this.#draining = undefined;
|
|
90
124
|
}
|
|
91
125
|
}
|
|
92
|
-
async #
|
|
126
|
+
async #run(state) {
|
|
93
127
|
if (this.#isClosed || this.#watches.get(state.watchId) !== state)
|
|
94
128
|
return;
|
|
95
|
-
if (state.running) {
|
|
96
|
-
state.dirty = true;
|
|
97
|
-
return;
|
|
98
|
-
}
|
|
99
|
-
state.running = true;
|
|
100
129
|
try {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
rows: response.rows,
|
|
117
|
-
maxCommitSeq: response.maxCommitSeq,
|
|
118
|
-
})))
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
121
|
-
catch (error) {
|
|
122
|
-
if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
|
|
123
|
-
return;
|
|
124
|
-
}
|
|
125
|
-
if (!(error instanceof SyncError)) {
|
|
126
|
-
reportError(error, this.#ctx.onError, 'operations');
|
|
127
|
-
}
|
|
128
|
-
this.#sendError(state.watchId, error instanceof SyncError
|
|
129
|
-
? error
|
|
130
|
-
: syncError('operation.query_failed'));
|
|
131
|
-
}
|
|
132
|
-
} while (state.dirty &&
|
|
133
|
-
!this.#isClosed &&
|
|
134
|
-
this.#watches.get(state.watchId) === state);
|
|
130
|
+
const response = await state.operation.run(this.#ctx, state.clientId, state.params);
|
|
131
|
+
if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
if (response.kind !== 'query') {
|
|
135
|
+
throw syncError('operation.query_failed');
|
|
136
|
+
}
|
|
137
|
+
this.#emit(encodeRemoteOperationRealtimeMessage({
|
|
138
|
+
revision: 1,
|
|
139
|
+
kind: 'snapshot',
|
|
140
|
+
watchId: state.watchId,
|
|
141
|
+
operationId: response.operationId,
|
|
142
|
+
rows: response.rows,
|
|
143
|
+
maxCommitSeq: response.maxCommitSeq,
|
|
144
|
+
}));
|
|
135
145
|
}
|
|
136
|
-
|
|
137
|
-
state.
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (!(error instanceof SyncError)) {
|
|
151
|
+
reportError(error, this.#ctx.onError, 'operations');
|
|
152
|
+
}
|
|
153
|
+
this.#sendError(state.watchId, error instanceof SyncError
|
|
154
|
+
? error
|
|
155
|
+
: syncError('operation.query_failed'));
|
|
138
156
|
}
|
|
139
157
|
}
|
|
140
158
|
#sendError(watchId, error) {
|
|
@@ -164,6 +182,7 @@ class WatchSession {
|
|
|
164
182
|
return;
|
|
165
183
|
this.#isClosed = true;
|
|
166
184
|
this.#watches.clear();
|
|
185
|
+
this.#pending.clear();
|
|
167
186
|
this.#closed();
|
|
168
187
|
}
|
|
169
188
|
}
|
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
* `PgExecutor` interface (see the server README).
|
|
9
9
|
*
|
|
10
10
|
* pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
|
|
11
|
-
* `ROLLBACK` on the one connection.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* `ROLLBACK` on the one connection. Transactions and pool-level `query` calls
|
|
12
|
+
* share one promise chain (mirroring `SqliteServerStorage`): a nested BEGIN
|
|
13
|
+
* on Postgres is a warning-level no-op, so interleaved scopes would silently
|
|
14
|
+
* collapse into one SQL transaction, and a pool-level statement issued while
|
|
15
|
+
* a transaction is open would read its uncommitted rows or join it (§6.4).
|
|
16
16
|
*/
|
|
17
17
|
import type { PGlite } from '@electric-sql/pglite';
|
|
18
18
|
import type { PgExecutor } from './pg-executor.js';
|
|
@@ -20,7 +20,6 @@ import type { PgExecutor } from './pg-executor.js';
|
|
|
20
20
|
interface PgliteLike {
|
|
21
21
|
query<Row>(query: string, params?: unknown[]): Promise<{
|
|
22
22
|
rows: Row[];
|
|
23
|
-
affectedRows?: number;
|
|
24
23
|
}>;
|
|
25
24
|
exec(query: string): Promise<unknown>;
|
|
26
25
|
close(): Promise<void>;
|
|
@@ -3,28 +3,36 @@ function queryable(db) {
|
|
|
3
3
|
return {
|
|
4
4
|
async query(text, params) {
|
|
5
5
|
const result = await db.query(text, params ? [...params] : undefined);
|
|
6
|
-
return {
|
|
7
|
-
rows: result.rows,
|
|
8
|
-
rowCount: result.affectedRows ?? result.rows.length,
|
|
9
|
-
};
|
|
6
|
+
return { rows: result.rows };
|
|
10
7
|
},
|
|
11
8
|
};
|
|
12
9
|
}
|
|
13
10
|
export function pgliteExecutor(db) {
|
|
14
11
|
const like = db;
|
|
15
12
|
const q = queryable(like);
|
|
16
|
-
// One
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
// One caller at a time on the single connection (see the file header).
|
|
14
|
+
let tail = Promise.resolve();
|
|
15
|
+
const acquire = async () => {
|
|
16
|
+
const previous = tail;
|
|
17
|
+
let release;
|
|
18
|
+
tail = new Promise((resolve) => {
|
|
19
|
+
release = resolve;
|
|
20
|
+
});
|
|
21
|
+
await previous;
|
|
22
|
+
return release;
|
|
23
|
+
};
|
|
19
24
|
return {
|
|
20
|
-
|
|
25
|
+
async query(text, params) {
|
|
26
|
+
const release = await acquire();
|
|
27
|
+
try {
|
|
28
|
+
return await q.query(text, params);
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
release();
|
|
32
|
+
}
|
|
33
|
+
},
|
|
21
34
|
async transaction(fn) {
|
|
22
|
-
const
|
|
23
|
-
let release;
|
|
24
|
-
transactionTail = new Promise((resolve) => {
|
|
25
|
-
release = resolve;
|
|
26
|
-
});
|
|
27
|
-
await previous;
|
|
35
|
+
const release = await acquire();
|
|
28
36
|
try {
|
|
29
37
|
await like.exec('BEGIN');
|
|
30
38
|
try {
|