@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/src/admin.ts
CHANGED
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
type PushOperationResult,
|
|
20
20
|
renderRowValues,
|
|
21
21
|
type ScopeMap,
|
|
22
|
+
unitId,
|
|
23
|
+
type UnitScopes,
|
|
22
24
|
} from '@syncular/core';
|
|
23
25
|
import type { BlobStore, BlobStoreStats } from './blob-store';
|
|
24
26
|
import {
|
|
@@ -52,6 +54,7 @@ import type {
|
|
|
52
54
|
CheckpointDeclaration,
|
|
53
55
|
ClientRecord,
|
|
54
56
|
CommitMetadata,
|
|
57
|
+
PushWatermark,
|
|
55
58
|
ReactionStatus,
|
|
56
59
|
StoredPushResult,
|
|
57
60
|
StoredRow,
|
|
@@ -61,13 +64,22 @@ import type {
|
|
|
61
64
|
StoredReaction,
|
|
62
65
|
} from './storage';
|
|
63
66
|
|
|
67
|
+
/** One registered unit of a client record. */
|
|
68
|
+
export interface AdminClientUnit {
|
|
69
|
+
/** The unit id (§4.1). */
|
|
70
|
+
readonly id: string;
|
|
71
|
+
readonly scopes: UnitScopes;
|
|
72
|
+
/** The unit's cursor while it catches up; absent for a live unit. */
|
|
73
|
+
readonly catchingUpCursor?: number;
|
|
74
|
+
}
|
|
75
|
+
|
|
64
76
|
/** A connected/known client as the console sees it (§4.5, §8.1). */
|
|
65
77
|
export interface AdminClient {
|
|
66
78
|
readonly clientId: string;
|
|
67
79
|
readonly actorId: string;
|
|
68
80
|
/** SSP2 wire version last accepted from this client. */
|
|
69
81
|
readonly wireVersion: number;
|
|
70
|
-
/**
|
|
82
|
+
/** Schema version of its latest completed round, when recorded. */
|
|
71
83
|
readonly schemaVersion?: number;
|
|
72
84
|
readonly cursor: number;
|
|
73
85
|
/**
|
|
@@ -76,11 +88,8 @@ export interface AdminClient {
|
|
|
76
88
|
*/
|
|
77
89
|
readonly lag: number;
|
|
78
90
|
readonly updatedAtMs: number;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
readonly table: string;
|
|
82
|
-
readonly scopes: Record<string, readonly string[]>;
|
|
83
|
-
}[];
|
|
91
|
+
/** The stored registration list (§4.1); empty when none is stored. */
|
|
92
|
+
readonly units: readonly AdminClientUnit[];
|
|
84
93
|
/** True when the cursor record was touched within the active window. */
|
|
85
94
|
readonly active: boolean;
|
|
86
95
|
/**
|
|
@@ -265,11 +274,11 @@ export interface AdminCommitDetail extends CommitMetadata {
|
|
|
265
274
|
* The trace verdict, evaluated in this order:
|
|
266
275
|
* `client-unknown`: no client record for the id.
|
|
267
276
|
* `row-missing`: no row and no retained change to it.
|
|
268
|
-
* `no-
|
|
277
|
+
* `no-unit`: no registered unit of the client covers the table.
|
|
269
278
|
* `row-deleted`: the row's newest retained change is a delete the client
|
|
270
279
|
* has pulled.
|
|
271
|
-
* `scope-mismatch`: no
|
|
272
|
-
* `scope-denied`: a
|
|
280
|
+
* `scope-mismatch`: no covering unit requests the row's stored scopes.
|
|
281
|
+
* `scope-denied`: a covering unit requests them, but the actor's allowed
|
|
273
282
|
* scopes (the host `resolveScopes`) exclude the row.
|
|
274
283
|
* `pending-pull`: the client's cursor is below the row's newest change.
|
|
275
284
|
* `delivered`: the client has pulled the row's newest change.
|
|
@@ -277,24 +286,24 @@ export interface AdminCommitDetail extends CommitMetadata {
|
|
|
277
286
|
export type AdminTraceVerdict =
|
|
278
287
|
| 'client-unknown'
|
|
279
288
|
| 'row-missing'
|
|
280
|
-
| 'no-
|
|
289
|
+
| 'no-unit'
|
|
281
290
|
| 'row-deleted'
|
|
282
291
|
| 'scope-mismatch'
|
|
283
292
|
| 'scope-denied'
|
|
284
293
|
| 'pending-pull'
|
|
285
294
|
| 'delivered';
|
|
286
295
|
|
|
287
|
-
/** One of the client's
|
|
288
|
-
export interface
|
|
296
|
+
/** One of the client's units covering the traced table. */
|
|
297
|
+
export interface AdminTraceUnit {
|
|
289
298
|
readonly id: string;
|
|
290
|
-
readonly scopes:
|
|
291
|
-
/** The row's stored scopes
|
|
299
|
+
readonly scopes: UnitScopes;
|
|
300
|
+
/** The row's stored scopes carry the unit's values. */
|
|
292
301
|
readonly requestedMatch: boolean;
|
|
293
|
-
/** Each
|
|
302
|
+
/** Each unit variable the row's stored value fails. */
|
|
294
303
|
readonly mismatches: readonly {
|
|
295
304
|
readonly variable: string;
|
|
296
305
|
readonly rowValue: string | null;
|
|
297
|
-
readonly requested:
|
|
306
|
+
readonly requested: string;
|
|
298
307
|
}[];
|
|
299
308
|
/**
|
|
300
309
|
* The row also satisfies requested ∩ allowed (§3.2). Absent when the
|
|
@@ -320,7 +329,7 @@ export interface AdminRowTrace {
|
|
|
320
329
|
* when it returned the §7.3.3 outage sentinel. Absent without a resolver.
|
|
321
330
|
*/
|
|
322
331
|
readonly allowed?: ScopeMap | 'outage';
|
|
323
|
-
readonly
|
|
332
|
+
readonly units: readonly AdminTraceUnit[];
|
|
324
333
|
/** The client's retained push rejections and conflicts, newest first. */
|
|
325
334
|
readonly rejections: readonly SyncularServerEvent[];
|
|
326
335
|
}
|
|
@@ -392,6 +401,13 @@ export interface AdminWriteLookup {
|
|
|
392
401
|
readonly clientCommitId: string;
|
|
393
402
|
/** One outcome per client that sent this clientCommitId. */
|
|
394
403
|
readonly outcomes: readonly AdminWriteOutcome[];
|
|
404
|
+
/**
|
|
405
|
+
* With no stored outcome: the §6.3.2 watermarks of the clients whose
|
|
406
|
+
* retained commit log carries this clientCommitId. A commit below an
|
|
407
|
+
* outbox's `throughSeq` lost its result to the watermark ("expired below
|
|
408
|
+
* sequence N").
|
|
409
|
+
*/
|
|
410
|
+
readonly watermarks: readonly PushWatermark[];
|
|
395
411
|
/** Retained ring events carrying this clientCommitId, newest first. */
|
|
396
412
|
readonly events: readonly SyncularServerEvent[];
|
|
397
413
|
}
|
|
@@ -525,7 +541,7 @@ function required<T>(value: T | undefined, what: string): T {
|
|
|
525
541
|
const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
|
|
526
542
|
'client-unknown',
|
|
527
543
|
'row-missing',
|
|
528
|
-
'no-
|
|
544
|
+
'no-unit',
|
|
529
545
|
'row-deleted',
|
|
530
546
|
'scope-mismatch',
|
|
531
547
|
'scope-denied',
|
|
@@ -533,7 +549,7 @@ const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
|
|
|
533
549
|
'delivered',
|
|
534
550
|
];
|
|
535
551
|
|
|
536
|
-
/** One known client's
|
|
552
|
+
/** One known client's units covering the row's table and its verdict. */
|
|
537
553
|
function traceVerdict(
|
|
538
554
|
declared: ReadonlySet<string>,
|
|
539
555
|
record: ClientRecord,
|
|
@@ -542,34 +558,40 @@ function traceVerdict(
|
|
|
542
558
|
horizonSeq: number,
|
|
543
559
|
allowed: ScopeMap | 'outage' | undefined,
|
|
544
560
|
): {
|
|
545
|
-
readonly
|
|
561
|
+
readonly units: AdminTraceUnit[];
|
|
546
562
|
readonly verdict: AdminTraceVerdict;
|
|
547
563
|
} {
|
|
548
564
|
const rowScopes = row.scopes ?? {};
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
.
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
.map(([variable,
|
|
565
|
+
// §4.1: a unit covers every table that declares all of its variables.
|
|
566
|
+
const units = (record.registration?.units ?? [])
|
|
567
|
+
.filter((scopes) =>
|
|
568
|
+
Object.keys(scopes).every((variable) => declared.has(variable)),
|
|
569
|
+
)
|
|
570
|
+
.map((scopes): AdminTraceUnit => {
|
|
571
|
+
const mismatches = Object.entries(scopes)
|
|
572
|
+
.filter(([variable, value]) => rowScopes[variable] !== value)
|
|
573
|
+
.map(([variable, value]) => ({
|
|
558
574
|
variable,
|
|
559
575
|
rowValue: rowScopes[variable] ?? null,
|
|
560
|
-
requested:
|
|
576
|
+
requested: value,
|
|
561
577
|
}));
|
|
562
578
|
const traced = {
|
|
563
|
-
id:
|
|
564
|
-
scopes
|
|
579
|
+
id: unitId(scopes),
|
|
580
|
+
scopes,
|
|
565
581
|
requestedMatch: mismatches.length === 0,
|
|
566
582
|
mismatches,
|
|
567
583
|
};
|
|
568
584
|
if (allowed === undefined || allowed === 'outage') return traced;
|
|
569
|
-
const effective = computeEffective(
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
585
|
+
const effective = computeEffective(
|
|
586
|
+
declared,
|
|
587
|
+
Object.fromEntries(
|
|
588
|
+
Object.entries(scopes).map(([variable, value]) => [
|
|
589
|
+
variable,
|
|
590
|
+
[value],
|
|
591
|
+
]),
|
|
592
|
+
),
|
|
593
|
+
{ ok: true, allowed },
|
|
594
|
+
);
|
|
573
595
|
return {
|
|
574
596
|
...traced,
|
|
575
597
|
effectiveMatch:
|
|
@@ -583,20 +605,20 @@ function traceVerdict(
|
|
|
583
605
|
const verdict: AdminTraceVerdict =
|
|
584
606
|
!row.exists && lastChange === undefined
|
|
585
607
|
? 'row-missing'
|
|
586
|
-
:
|
|
587
|
-
? 'no-
|
|
608
|
+
: units.length === 0
|
|
609
|
+
? 'no-unit'
|
|
588
610
|
: !row.exists
|
|
589
611
|
? pulled
|
|
590
612
|
? 'row-deleted'
|
|
591
613
|
: 'pending-pull'
|
|
592
|
-
: !
|
|
614
|
+
: !units.some((unit) => unit.requestedMatch)
|
|
593
615
|
? 'scope-mismatch'
|
|
594
|
-
:
|
|
616
|
+
: units.every((unit) => unit.effectiveMatch === false)
|
|
595
617
|
? 'scope-denied'
|
|
596
618
|
: pulled
|
|
597
619
|
? 'delivered'
|
|
598
620
|
: 'pending-pull';
|
|
599
|
-
return {
|
|
621
|
+
return { units, verdict };
|
|
600
622
|
}
|
|
601
623
|
|
|
602
624
|
/** The console query surface plus its operator actions. One per process. */
|
|
@@ -870,7 +892,7 @@ export class SyncularAdmin {
|
|
|
870
892
|
|
|
871
893
|
/**
|
|
872
894
|
* Trace whether one client receives one row, combining the row's stored
|
|
873
|
-
* scopes and newest change with the client's
|
|
895
|
+
* scopes and newest change with the client's units, cursor, the
|
|
874
896
|
* actor's allowed scopes, and its retained push failures. See
|
|
875
897
|
* `AdminTraceVerdict` for the verdict order.
|
|
876
898
|
*/
|
|
@@ -903,7 +925,7 @@ export class SyncularAdmin {
|
|
|
903
925
|
...(lastChange !== undefined ? { lastChange } : {}),
|
|
904
926
|
};
|
|
905
927
|
if (record === undefined) {
|
|
906
|
-
return { ...base, verdict: 'client-unknown',
|
|
928
|
+
return { ...base, verdict: 'client-unknown', units: [] };
|
|
907
929
|
}
|
|
908
930
|
const allowed = await this.#allowedScopes(partition, record);
|
|
909
931
|
return {
|
|
@@ -1134,7 +1156,14 @@ export class SyncularAdmin {
|
|
|
1134
1156
|
(event) =>
|
|
1135
1157
|
'clientCommitId' in event && event.clientCommitId === clientCommitId,
|
|
1136
1158
|
);
|
|
1137
|
-
|
|
1159
|
+
const watermarks =
|
|
1160
|
+
outcomes.length > 0
|
|
1161
|
+
? []
|
|
1162
|
+
: await required(
|
|
1163
|
+
this.#storage.listPushWatermarks?.bind(this.#storage),
|
|
1164
|
+
'storage',
|
|
1165
|
+
)(partition, clientCommitId);
|
|
1166
|
+
return { clientCommitId, outcomes, watermarks, events };
|
|
1138
1167
|
}
|
|
1139
1168
|
|
|
1140
1169
|
/**
|
|
@@ -1624,30 +1653,29 @@ export class SyncularAdmin {
|
|
|
1624
1653
|
}
|
|
1625
1654
|
}
|
|
1626
1655
|
return (record) => {
|
|
1627
|
-
let schemaVersion: number | undefined;
|
|
1628
|
-
for (const sub of record.subscriptions) {
|
|
1629
|
-
if (
|
|
1630
|
-
sub.schemaVersion !== undefined &&
|
|
1631
|
-
(schemaVersion === undefined || sub.schemaVersion > schemaVersion)
|
|
1632
|
-
) {
|
|
1633
|
-
schemaVersion = sub.schemaVersion;
|
|
1634
|
-
}
|
|
1635
|
-
}
|
|
1636
1656
|
const connectedSinceMs = connectedSince.get(record.clientId);
|
|
1637
1657
|
return {
|
|
1638
1658
|
clientId: record.clientId,
|
|
1639
1659
|
actorId: record.actorId,
|
|
1640
1660
|
wireVersion: record.wireVersion,
|
|
1641
|
-
...(schemaVersion !== undefined
|
|
1661
|
+
...(record.schemaVersion !== undefined
|
|
1662
|
+
? { schemaVersion: record.schemaVersion }
|
|
1663
|
+
: {}),
|
|
1642
1664
|
cursor: record.cursor,
|
|
1643
1665
|
// A never-pulled cursor (-1) has seen nothing: it lags the whole log.
|
|
1644
1666
|
lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
|
|
1645
1667
|
updatedAtMs: record.updatedAtMs,
|
|
1646
|
-
|
|
1647
|
-
id
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1668
|
+
units: (record.registration?.units ?? []).map((scopes) => {
|
|
1669
|
+
const id = unitId(scopes);
|
|
1670
|
+
const catchingUp = record.catchingUp.find((unit) => unit.id === id);
|
|
1671
|
+
return {
|
|
1672
|
+
id,
|
|
1673
|
+
scopes,
|
|
1674
|
+
...(catchingUp !== undefined
|
|
1675
|
+
? { catchingUpCursor: catchingUp.cursor }
|
|
1676
|
+
: {}),
|
|
1677
|
+
};
|
|
1678
|
+
}),
|
|
1651
1679
|
active: record.updatedAtMs >= activeFloorMs,
|
|
1652
1680
|
...(sockets !== undefined
|
|
1653
1681
|
? { connected: connectedSinceMs !== undefined }
|
package/src/context.ts
CHANGED
|
@@ -58,7 +58,7 @@ export type ResolverOutage = typeof RESOLVER_OUTAGE;
|
|
|
58
58
|
/**
|
|
59
59
|
* Host callback: actor → allowed scopes (§3.2 step 3). Resolved at most
|
|
60
60
|
* once per request and memoized (§3.4 step 1). `'*'` means "any value for
|
|
61
|
-
* this variable". A throwing resolver revokes
|
|
61
|
+
* this variable". A throwing resolver revokes units and rejects
|
|
62
62
|
* writes with `sync.forbidden` — fail loud, never leak. Returning
|
|
63
63
|
* `RESOLVER_OUTAGE` instead opts the request into lease authorization
|
|
64
64
|
* (§7.3.3) — a deliberate outage signal, not a failure.
|
|
@@ -87,10 +87,8 @@ export interface ServerLimits {
|
|
|
87
87
|
* decoding, and a socket round's assembled request (§8.7).
|
|
88
88
|
*/
|
|
89
89
|
readonly maxRequestBytes: number;
|
|
90
|
-
/** §1.7 cap on `
|
|
90
|
+
/** §1.7 cap on `UNIT` frames per request: the units of a traveling list. */
|
|
91
91
|
readonly maxSubscriptionsPerRequest: number;
|
|
92
|
-
/** §1.7 cap on requested scope values, summed over a subscription's keys. */
|
|
93
|
-
readonly maxScopeValuesPerSubscription: number;
|
|
94
92
|
/** Cap on concurrent registered-query watches per watch session. */
|
|
95
93
|
readonly maxWatchesPerSession: number;
|
|
96
94
|
}
|
|
@@ -101,10 +99,8 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
101
99
|
maxCascadeOperationsPerCommit: 1_000,
|
|
102
100
|
// The size above which adapters already refuse to buffer a segment body.
|
|
103
101
|
maxRequestBytes: 16 * 1024 * 1024,
|
|
104
|
-
// One
|
|
102
|
+
// One unit per pushable operation: the existing per-request cap.
|
|
105
103
|
maxSubscriptionsPerRequest: 500,
|
|
106
|
-
// One default snapshot page (§4.2): a longer list wants a coarser scope.
|
|
107
|
-
maxScopeValuesPerSubscription: 1_000,
|
|
108
104
|
// Every commit reruns a session's affected watches one after another.
|
|
109
105
|
maxWatchesPerSession: 100,
|
|
110
106
|
};
|
|
@@ -113,10 +109,10 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
113
109
|
export interface RealtimeNotifier {
|
|
114
110
|
notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
|
|
115
111
|
/**
|
|
116
|
-
* A pull changed the client's
|
|
117
|
-
* re-registers that client's live sockets, so a client that runs its
|
|
118
|
-
* rounds over HTTP while a socket is open receives deltas for the
|
|
119
|
-
*
|
|
112
|
+
* A pull changed the client's registration list or live set (§8.1). A
|
|
113
|
+
* hub re-registers that client's live sockets, so a client that runs its
|
|
114
|
+
* rounds over HTTP while a socket is open receives deltas for the units
|
|
115
|
+
* that joined.
|
|
120
116
|
*/
|
|
121
117
|
notifySubscriptions?(
|
|
122
118
|
partition: string,
|
package/src/d1-storage.ts
CHANGED
|
@@ -78,9 +78,10 @@ import {
|
|
|
78
78
|
type SqliteStatementSpec,
|
|
79
79
|
SqliteFamilyStorage,
|
|
80
80
|
SqliteFamilyTransaction,
|
|
81
|
-
SQLITE_ADDED_COLUMNS,
|
|
82
81
|
type SqliteTombstoneRecord,
|
|
83
82
|
sqliteDdlStatements,
|
|
83
|
+
SQLITE_UPGRADE_PROBE,
|
|
84
|
+
sqliteUpgradeDdl,
|
|
84
85
|
toStoredRow,
|
|
85
86
|
toTombstone,
|
|
86
87
|
} from './sqlite-dialect';
|
|
@@ -93,6 +94,8 @@ import type {
|
|
|
93
94
|
CommitPruneResult,
|
|
94
95
|
IndexRowScanQuery,
|
|
95
96
|
NewCommit,
|
|
97
|
+
OutboxSequence,
|
|
98
|
+
OutboxSettlement,
|
|
96
99
|
PreparedPushApply,
|
|
97
100
|
RowScanQuery,
|
|
98
101
|
ScopeValueHistoryQuery,
|
|
@@ -104,6 +107,7 @@ import type {
|
|
|
104
107
|
StoredRow,
|
|
105
108
|
Tombstone,
|
|
106
109
|
} from './storage';
|
|
110
|
+
import { commitDeleteFloor } from './storage';
|
|
107
111
|
import {
|
|
108
112
|
isD1ConstraintError,
|
|
109
113
|
StorageConstraintError,
|
|
@@ -528,6 +532,7 @@ class D1Transaction
|
|
|
528
532
|
readonly table: string;
|
|
529
533
|
readonly rowIds: readonly string[];
|
|
530
534
|
}[] = [],
|
|
535
|
+
outbox?: OutboxSettlement,
|
|
531
536
|
): Promise<PreparedPushApply> {
|
|
532
537
|
this.assertOpen();
|
|
533
538
|
if (!this.#pushApplySerialized) {
|
|
@@ -537,11 +542,33 @@ class D1Transaction
|
|
|
537
542
|
}
|
|
538
543
|
// D1 has no interactive lock. The caller explicitly asserted that every
|
|
539
544
|
// write for this partition is already serialized (normally by its DO).
|
|
545
|
+
// The settlement's writes sit below the checkpoint, so a rejection keeps
|
|
546
|
+
// them.
|
|
547
|
+
const settled =
|
|
548
|
+
outbox === undefined
|
|
549
|
+
? undefined
|
|
550
|
+
: await this.settleOutbox(clientId, clientCommitId, outbox);
|
|
540
551
|
this.#pushApplyCheckpoint = this.#buffer.length;
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
552
|
+
// The serve gate and the delete floor (§2.2) in one round trip.
|
|
553
|
+
const read = await this.#db
|
|
554
|
+
.prepare(
|
|
555
|
+
'SELECT (SELECT schema_version FROM sync_schema_meta WHERE id=1) AS schema_version, (SELECT delete_floor FROM sync_partitions WHERE partition=?) AS delete_floor',
|
|
556
|
+
)
|
|
557
|
+
.bind(this.partition)
|
|
558
|
+
.first<{ schema_version: number | null; delete_floor: number | null }>();
|
|
559
|
+
const gate: ServeGate = {
|
|
560
|
+
storedSchemaVersion: read?.schema_version ?? 0,
|
|
561
|
+
logEpoch: undefined,
|
|
562
|
+
pending: [],
|
|
544
563
|
};
|
|
564
|
+
const deleteFloor = read?.delete_floor ?? 0;
|
|
565
|
+
return settled === undefined
|
|
566
|
+
? {
|
|
567
|
+
gate,
|
|
568
|
+
deleteFloor,
|
|
569
|
+
persisted: await this.getPushResult(clientId, clientCommitId),
|
|
570
|
+
}
|
|
571
|
+
: { gate, deleteFloor, ...settled };
|
|
545
572
|
}
|
|
546
573
|
|
|
547
574
|
advanceCheckpoint(
|
|
@@ -573,6 +600,7 @@ class D1Transaction
|
|
|
573
600
|
clientId: string,
|
|
574
601
|
clientCommitId: string,
|
|
575
602
|
result: StoredPushResult,
|
|
603
|
+
sequence?: OutboxSequence,
|
|
576
604
|
): Promise<StoredPushResult> {
|
|
577
605
|
this.assertOpen();
|
|
578
606
|
const checkpoint = this.#pushApplyCheckpoint;
|
|
@@ -584,7 +612,7 @@ class D1Transaction
|
|
|
584
612
|
this.#clearedTombstones.clear();
|
|
585
613
|
this.#maxCommitSeq = undefined;
|
|
586
614
|
this.#applicationOpIndexes.clear();
|
|
587
|
-
await this.putPushResult(clientId, clientCommitId, result);
|
|
615
|
+
await this.putPushResult(clientId, clientCommitId, result, sequence);
|
|
588
616
|
await this.commit();
|
|
589
617
|
// The idempotency re-check found no record under the serialized apply, so
|
|
590
618
|
// the insert above is the canonical outcome.
|
|
@@ -710,8 +738,8 @@ class D1Transaction
|
|
|
710
738
|
const commitSeq = this.#maxCommitSeq + 1;
|
|
711
739
|
this.#maxCommitSeq = commitSeq;
|
|
712
740
|
this.run(
|
|
713
|
-
'INSERT INTO sync_partitions(partition, max_commit_seq) VALUES (
|
|
714
|
-
[p, commitSeq],
|
|
741
|
+
'INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES (?,?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq, delete_floor=max(delete_floor, excluded.delete_floor)',
|
|
742
|
+
[p, commitSeq, commitDeleteFloor(commit.changes)],
|
|
715
743
|
);
|
|
716
744
|
this.run(
|
|
717
745
|
'INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)',
|
|
@@ -794,6 +822,7 @@ const D1_CORE_TABLES = [
|
|
|
794
822
|
'sync_changes',
|
|
795
823
|
'sync_change_scopes',
|
|
796
824
|
'sync_push_results',
|
|
825
|
+
'sync_push_watermarks',
|
|
797
826
|
'sync_reactions',
|
|
798
827
|
'sync_clients',
|
|
799
828
|
'sync_blob_refs',
|
|
@@ -840,9 +869,9 @@ export class D1ServerStorage
|
|
|
840
869
|
protected readonly maxBindParams = 100;
|
|
841
870
|
/**
|
|
842
871
|
* Page reads waiting for the end of the current microtask turn. A pull
|
|
843
|
-
* starts the
|
|
872
|
+
* starts the first snapshot page and scope pin of every bootstrapping unit
|
|
844
873
|
* before awaiting any of them, so they leave as ONE `db.batch` round trip
|
|
845
|
-
* instead of one per
|
|
874
|
+
* instead of one per unit (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
846
875
|
*/
|
|
847
876
|
#pendingReads: PendingReads | undefined;
|
|
848
877
|
|
|
@@ -924,33 +953,18 @@ export class D1ServerStorage
|
|
|
924
953
|
for (const statement of sqliteDdlStatements()) {
|
|
925
954
|
await this.#db.exec(`${statement.replace(/\s+/g, ' ')};`);
|
|
926
955
|
}
|
|
927
|
-
for (const sql of await this.#
|
|
956
|
+
for (const sql of await this.#upgradeDdl()) {
|
|
928
957
|
await this.#db.exec(`${sql};`);
|
|
929
958
|
}
|
|
930
959
|
}
|
|
931
960
|
|
|
932
|
-
/**
|
|
933
|
-
|
|
934
|
-
* lacks, read in one statement.
|
|
935
|
-
*/
|
|
936
|
-
async #addedColumnDdl(): Promise<string[]> {
|
|
961
|
+
/** `sqliteUpgradeDdl` for this database, read in one statement. */
|
|
962
|
+
async #upgradeDdl(): Promise<string[]> {
|
|
937
963
|
const { results } = await this.#db
|
|
938
|
-
.prepare(
|
|
939
|
-
|
|
940
|
-
)
|
|
941
|
-
.bind(
|
|
942
|
-
JSON.stringify([
|
|
943
|
-
...new Set(SQLITE_ADDED_COLUMNS.map(([table]) => table)),
|
|
944
|
-
]),
|
|
945
|
-
)
|
|
964
|
+
.prepare(SQLITE_UPGRADE_PROBE.sql)
|
|
965
|
+
.bind(...SQLITE_UPGRADE_PROBE.params)
|
|
946
966
|
.all<{ tbl: string; name: string }>();
|
|
947
|
-
return
|
|
948
|
-
([table, name]) =>
|
|
949
|
-
!results.some((column) => column.tbl === table && column.name === name),
|
|
950
|
-
).map(
|
|
951
|
-
([table, name, definition]) =>
|
|
952
|
-
`ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
|
|
953
|
-
);
|
|
967
|
+
return sqliteUpgradeDdl(results);
|
|
954
968
|
}
|
|
955
969
|
|
|
956
970
|
async ensureSchema(
|
|
@@ -1070,9 +1084,11 @@ export class D1ServerStorage
|
|
|
1070
1084
|
'sqlite',
|
|
1071
1085
|
);
|
|
1072
1086
|
}
|
|
1073
|
-
// A database from
|
|
1087
|
+
// A database from an earlier release gains the core-table columns and
|
|
1088
|
+
// indexes it lacks in place, and loses the columns wire version 4
|
|
1089
|
+
// deleted.
|
|
1074
1090
|
statementsExecuted++;
|
|
1075
|
-
const added = await this.#
|
|
1091
|
+
const added = await this.#upgradeDdl();
|
|
1076
1092
|
if (added.length > 0) {
|
|
1077
1093
|
await batch(added.map((sql) => this.#db.prepare(sql)));
|
|
1078
1094
|
}
|
|
@@ -1211,7 +1227,7 @@ export class D1ServerStorage
|
|
|
1211
1227
|
// The column read, the checkpoint pair, and every missing column.
|
|
1212
1228
|
if (remaining < 4) break;
|
|
1213
1229
|
statementsExecuted++;
|
|
1214
|
-
const added = await this.#
|
|
1230
|
+
const added = await this.#upgradeDdl();
|
|
1215
1231
|
if (remaining < 3 + added.length) break;
|
|
1216
1232
|
if (
|
|
1217
1233
|
!(await save(
|
package/src/errors.ts
CHANGED
|
@@ -176,6 +176,14 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
|
|
|
176
176
|
recommendedAction: 'retryLater',
|
|
177
177
|
httpStatus: 500,
|
|
178
178
|
},
|
|
179
|
+
// §6.3.2: below the outbox watermark with no retained result. A push
|
|
180
|
+
// operation-result error record only, never persisted.
|
|
181
|
+
'sync.commit_expired': {
|
|
182
|
+
category: 'invalid-request',
|
|
183
|
+
retryable: false,
|
|
184
|
+
recommendedAction: 'resetClientId',
|
|
185
|
+
httpStatus: 400,
|
|
186
|
+
},
|
|
179
187
|
// §2.4 on a non-D1 backend: the server refuses while a declared backfill
|
|
180
188
|
// checkpoint is incomplete or the database is newer than the running build.
|
|
181
189
|
// Formerly host-only; promoted to the §10.2 catalogue so an in-flight
|
package/src/events.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* - Timestamps and durations come from the ctx clock where one exists, so
|
|
14
14
|
* the conformance virtual-clock discipline stays intact.
|
|
15
15
|
*/
|
|
16
|
-
import type {
|
|
16
|
+
import type { FencedRecord, UnitScopes, WakeReason } from '@syncular/core';
|
|
17
17
|
|
|
18
18
|
/** One `POST /sync` request, emitted once per request after the response
|
|
19
19
|
* bytes are fully produced (or the request was rejected up front). */
|
|
@@ -48,7 +48,8 @@ export interface RequestHandledEvent {
|
|
|
48
48
|
readonly errorCode?: string;
|
|
49
49
|
readonly pushCommits: number;
|
|
50
50
|
readonly pulled: boolean;
|
|
51
|
-
|
|
51
|
+
/** Units of the request's effective list (§4.1); 0 without one. */
|
|
52
|
+
readonly units: number;
|
|
52
53
|
}
|
|
53
54
|
|
|
54
55
|
interface PushEventBase {
|
|
@@ -154,7 +155,7 @@ export interface ReactionPruneCompletedEvent {
|
|
|
154
155
|
readonly mayHaveMore: boolean;
|
|
155
156
|
}
|
|
156
157
|
|
|
157
|
-
/** One emitted segment within a
|
|
158
|
+
/** One emitted segment within a unit section. */
|
|
158
159
|
export interface PullSegmentSummary {
|
|
159
160
|
readonly mediaType: 'rows' | 'sqlite';
|
|
160
161
|
readonly delivery: 'inline' | 'ref';
|
|
@@ -164,25 +165,40 @@ export interface PullSegmentSummary {
|
|
|
164
165
|
readonly rows: number;
|
|
165
166
|
}
|
|
166
167
|
|
|
167
|
-
/** One
|
|
168
|
-
export interface
|
|
168
|
+
/** One unit of a served pull's effective list (§4.1). */
|
|
169
|
+
export interface PullUnitSummary {
|
|
169
170
|
readonly id: string;
|
|
170
|
-
readonly
|
|
171
|
+
readonly scopes: UnitScopes;
|
|
171
172
|
readonly status: 'active' | 'revoked' | 'reset';
|
|
172
|
-
/**
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
readonly
|
|
173
|
+
/**
|
|
174
|
+
* `live` and `catching-up` (a post-bootstrap unit) units take part in the
|
|
175
|
+
* live section; `bootstrap` sections carry segments; `none` for revoked and
|
|
176
|
+
* reset units.
|
|
177
|
+
*/
|
|
178
|
+
readonly mode: 'live' | 'catching-up' | 'bootstrap' | 'none';
|
|
179
|
+
/** Whether the response carried a unit section for it (§4.4). */
|
|
180
|
+
readonly section: boolean;
|
|
181
|
+
/** The unit's echo (§4.4). */
|
|
182
|
+
readonly fenced: readonly FencedRecord[];
|
|
183
|
+
/** The cursor the request held for the unit: its `PROGRESS` cursor or R. */
|
|
178
184
|
readonly fromCursor: number;
|
|
179
|
-
readonly
|
|
185
|
+
readonly segments: readonly PullSegmentSummary[];
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** The live section of a served pull (§4.5). */
|
|
189
|
+
export interface PullLiveSummary {
|
|
190
|
+
readonly status: 'active' | 'reset' | 'unregistered';
|
|
191
|
+
/** The request's replica cursor R. */
|
|
192
|
+
readonly fromCursor: number;
|
|
193
|
+
/** `LIVE_END.throughSeq`. */
|
|
194
|
+
readonly throughSeq: number;
|
|
180
195
|
readonly commits: number;
|
|
181
196
|
readonly changes: number;
|
|
182
|
-
readonly
|
|
197
|
+
readonly firstCommitSeq?: number;
|
|
198
|
+
readonly lastCommitSeq?: number;
|
|
183
199
|
}
|
|
184
200
|
|
|
185
|
-
/** The pull half of a request, emitted once after
|
|
201
|
+
/** The pull half of a request, emitted once after every section streamed. */
|
|
186
202
|
export interface PullServedEvent {
|
|
187
203
|
readonly type: 'pull.served';
|
|
188
204
|
readonly atMs: number;
|
|
@@ -191,7 +207,10 @@ export interface PullServedEvent {
|
|
|
191
207
|
readonly clientId: string;
|
|
192
208
|
readonly acceptedThrough: number;
|
|
193
209
|
readonly storageMaxCommitSeq: number;
|
|
194
|
-
|
|
210
|
+
/** The request carried its registration list (§4.1). */
|
|
211
|
+
readonly registrationTraveled: boolean;
|
|
212
|
+
readonly units: readonly PullUnitSummary[];
|
|
213
|
+
readonly live: PullLiveSummary;
|
|
195
214
|
}
|
|
196
215
|
|
|
197
216
|
/** A direct segment download (§5.5), success or failure. */
|
|
@@ -292,7 +311,7 @@ export interface PruneCompletedEvent {
|
|
|
292
311
|
}
|
|
293
312
|
|
|
294
313
|
/** The host `resolveScopes` callback threw — the §3.2/§3.4 fail-loud
|
|
295
|
-
* path (
|
|
314
|
+
* path (units revoke, writes reject, downloads deny). */
|
|
296
315
|
export interface ScopesResolveFailedEvent {
|
|
297
316
|
readonly type: 'scopes.resolve_failed';
|
|
298
317
|
readonly atMs: number;
|