@syncular/server 0.59.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +0 -97
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +0 -105
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/src/push.ts
CHANGED
|
@@ -48,6 +48,8 @@ import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes';
|
|
|
48
48
|
import { decodeColumnVersions, encodeColumnVersions } from './relational-rows';
|
|
49
49
|
import type {
|
|
50
50
|
NewChange,
|
|
51
|
+
OutboxSequence,
|
|
52
|
+
OutboxSettlement,
|
|
51
53
|
PreparedPushApply,
|
|
52
54
|
StorageTransaction,
|
|
53
55
|
StoredCommit,
|
|
@@ -344,6 +346,8 @@ async function applyOperation(
|
|
|
344
346
|
validators: ValidatorRegistry | undefined,
|
|
345
347
|
partition: string,
|
|
346
348
|
actorId: string,
|
|
349
|
+
/** §2.2: the partition's delete floor, raised by this commit's deletes. */
|
|
350
|
+
deleteFloor: number,
|
|
347
351
|
): Promise<OperationOutcome> {
|
|
348
352
|
const table = schema.tables.get(op.table);
|
|
349
353
|
const wireTable = wireSchema.tables.get(op.table);
|
|
@@ -369,8 +373,9 @@ async function applyOperation(
|
|
|
369
373
|
|
|
370
374
|
if (op.op === 'rescope') {
|
|
371
375
|
// §2.2 explicit scope move. One stored row changes its scope values;
|
|
372
|
-
// apply rewrites it in place,
|
|
373
|
-
// delete under the old scopes plus the upsert under the new ones
|
|
376
|
+
// apply rewrites it in place, advances `server_version` by 1, and emits
|
|
377
|
+
// the delete under the old scopes plus the upsert under the new ones,
|
|
378
|
+
// both at the new version.
|
|
374
379
|
if (stored === undefined) {
|
|
375
380
|
return errorRecord(
|
|
376
381
|
opIndex,
|
|
@@ -502,6 +507,22 @@ async function applyOperation(
|
|
|
502
507
|
);
|
|
503
508
|
if (blobCheck !== undefined) return blobCheck;
|
|
504
509
|
const payload = encodeRow(table.columns, values);
|
|
510
|
+
// §2.2: the moved scope columns take the new version as their column
|
|
511
|
+
// version; every other column keeps its own (a row stored before column
|
|
512
|
+
// versions holds every column at its current version).
|
|
513
|
+
const movedVersion = stored.serverVersion + 1;
|
|
514
|
+
const storedVersions = decodeColumnVersions(stored.columnVersions);
|
|
515
|
+
const movedVersions = new Map<number, number>();
|
|
516
|
+
for (let i = 0; i < table.columns.length; i++) {
|
|
517
|
+
const version =
|
|
518
|
+
scopeIndexes.has(i) &&
|
|
519
|
+
!sameValue(values[i] ?? null, storedValues[i] ?? null)
|
|
520
|
+
? movedVersion
|
|
521
|
+
: (storedVersions.get(i) ??
|
|
522
|
+
(stored.columnVersions === undefined ? stored.serverVersion : 1));
|
|
523
|
+
if (version > 1) movedVersions.set(i, version);
|
|
524
|
+
}
|
|
525
|
+
const columnVersions = encodeColumnVersions(movedVersions);
|
|
505
526
|
// The move clears any tombstone from an earlier incarnation: the row is
|
|
506
527
|
// live at the end of this commit, so the next move must not hit §5.2.
|
|
507
528
|
await tx.clearTombstone(op.table, op.rowId);
|
|
@@ -509,12 +530,10 @@ async function applyOperation(
|
|
|
509
530
|
op.table,
|
|
510
531
|
{
|
|
511
532
|
rowId: op.rowId,
|
|
512
|
-
serverVersion:
|
|
533
|
+
serverVersion: movedVersion,
|
|
513
534
|
scopes: extracted.scopes,
|
|
514
535
|
payload,
|
|
515
|
-
...(
|
|
516
|
-
? { columnVersions: stored.columnVersions }
|
|
517
|
-
: {}),
|
|
536
|
+
...(columnVersions !== undefined ? { columnVersions } : {}),
|
|
518
537
|
},
|
|
519
538
|
{ opIndex },
|
|
520
539
|
);
|
|
@@ -526,13 +545,14 @@ async function applyOperation(
|
|
|
526
545
|
table: op.table,
|
|
527
546
|
rowId: op.rowId,
|
|
528
547
|
op: 'delete',
|
|
548
|
+
rowVersion: movedVersion,
|
|
529
549
|
scopes: stored.scopes,
|
|
530
550
|
},
|
|
531
551
|
{
|
|
532
552
|
table: op.table,
|
|
533
553
|
rowId: op.rowId,
|
|
534
554
|
op: 'upsert',
|
|
535
|
-
rowVersion:
|
|
555
|
+
rowVersion: movedVersion,
|
|
536
556
|
scopes: extracted.scopes,
|
|
537
557
|
payload,
|
|
538
558
|
},
|
|
@@ -555,7 +575,7 @@ async function applyOperation(
|
|
|
555
575
|
row: toValidateRow(table.columns, values),
|
|
556
576
|
stored: storedRow,
|
|
557
577
|
storedServerVersion: stored.serverVersion,
|
|
558
|
-
nextServerVersion:
|
|
578
|
+
nextServerVersion: movedVersion,
|
|
559
579
|
},
|
|
560
580
|
],
|
|
561
581
|
staged: {
|
|
@@ -635,7 +655,9 @@ async function applyOperation(
|
|
|
635
655
|
table: op.table,
|
|
636
656
|
rowId: op.rowId,
|
|
637
657
|
op: 'delete',
|
|
638
|
-
|
|
658
|
+
// §2.2: a delete advances the version by 1; its tombstone records
|
|
659
|
+
// it and the delete floor rises to it.
|
|
660
|
+
rowVersion: stored.serverVersion + 1,
|
|
639
661
|
scopes: stored.scopes,
|
|
640
662
|
},
|
|
641
663
|
],
|
|
@@ -985,11 +1007,11 @@ async function applyOperation(
|
|
|
985
1007
|
actorId,
|
|
986
1008
|
);
|
|
987
1009
|
if (insertReject !== undefined) return insertReject;
|
|
988
|
-
// §6.2:
|
|
989
|
-
//
|
|
990
|
-
//
|
|
1010
|
+
// §2.2, §6.2: every insert starts one above the delete floor, over a
|
|
1011
|
+
// tombstone or not; without a column-versions blob every column sits at
|
|
1012
|
+
// that version.
|
|
991
1013
|
if (tombstone !== undefined) await tx.clearTombstone(op.table, op.rowId);
|
|
992
|
-
const serverVersion =
|
|
1014
|
+
const serverVersion = deleteFloor + 1;
|
|
993
1015
|
const newRow = {
|
|
994
1016
|
rowId: op.rowId,
|
|
995
1017
|
serverVersion,
|
|
@@ -1579,7 +1601,7 @@ async function enforceReferences(
|
|
|
1579
1601
|
table: child.name,
|
|
1580
1602
|
rowId: childRow.rowId,
|
|
1581
1603
|
op: 'delete',
|
|
1582
|
-
|
|
1604
|
+
rowVersion: childRow.serverVersion + 1,
|
|
1583
1605
|
scopes: childRow.scopes,
|
|
1584
1606
|
});
|
|
1585
1607
|
operations.push({
|
|
@@ -1694,16 +1716,28 @@ export interface ProcessedPushCommit {
|
|
|
1694
1716
|
readonly replayed: boolean;
|
|
1695
1717
|
readonly recordedAtMs?: number;
|
|
1696
1718
|
readonly cacheIdentity?: string;
|
|
1719
|
+
/** The outbox watermark `W` the commit's settlement read (§6.3.2). */
|
|
1720
|
+
readonly watermark?: number;
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
/** A pushed commit's outbox position and the request's watermark (§6.3.2). */
|
|
1724
|
+
export interface PushOutbox extends OutboxSequence {
|
|
1725
|
+
/** `W` after the request's first commit. */
|
|
1726
|
+
readonly watermark: number;
|
|
1727
|
+
/** Present with the request's first commit: steps 1 and 2 run with it. */
|
|
1728
|
+
readonly settlement?: OutboxSettlement;
|
|
1697
1729
|
}
|
|
1698
1730
|
|
|
1699
1731
|
function processedPushCommit(
|
|
1700
1732
|
clientCommitId: string,
|
|
1701
1733
|
stored: StoredPushResult,
|
|
1702
1734
|
replayed: boolean,
|
|
1735
|
+
watermark?: number,
|
|
1703
1736
|
): ProcessedPushCommit {
|
|
1704
1737
|
return {
|
|
1705
1738
|
frame: resultFrame(clientCommitId, stored, replayed),
|
|
1706
1739
|
replayed,
|
|
1740
|
+
...(watermark !== undefined ? { watermark } : {}),
|
|
1707
1741
|
...(stored.recordedAtMs !== undefined
|
|
1708
1742
|
? { recordedAtMs: stored.recordedAtMs }
|
|
1709
1743
|
: {}),
|
|
@@ -1744,6 +1778,24 @@ function idempotencyCacheMissFrame(
|
|
|
1744
1778
|
};
|
|
1745
1779
|
}
|
|
1746
1780
|
|
|
1781
|
+
/** §6.3: the unpersisted refusal of a commit below its outbox's watermark. */
|
|
1782
|
+
function commitExpiredFrame(clientCommitId: string): PushResultFrame {
|
|
1783
|
+
return {
|
|
1784
|
+
type: 'PUSH_RESULT',
|
|
1785
|
+
clientCommitId,
|
|
1786
|
+
status: 'rejected',
|
|
1787
|
+
results: [
|
|
1788
|
+
{
|
|
1789
|
+
opIndex: 0,
|
|
1790
|
+
status: 'error',
|
|
1791
|
+
code: 'sync.commit_expired',
|
|
1792
|
+
message: 'the commit lies below its outbox watermark (§6.3.2)',
|
|
1793
|
+
retryable: false,
|
|
1794
|
+
},
|
|
1795
|
+
],
|
|
1796
|
+
};
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1747
1799
|
export interface AppliedCommitEvent {
|
|
1748
1800
|
readonly commit: StoredCommit;
|
|
1749
1801
|
}
|
|
@@ -1776,6 +1828,7 @@ export async function processPushCommitWithTrace(
|
|
|
1776
1828
|
clientId: string,
|
|
1777
1829
|
frame: PushCommitFrame,
|
|
1778
1830
|
wireSchema: CompiledSchema = schema,
|
|
1831
|
+
outbox?: PushOutbox,
|
|
1779
1832
|
): Promise<ProcessedPushCommit> {
|
|
1780
1833
|
// The frame names its rows before the transaction opens, so the admission
|
|
1781
1834
|
// read carries the commit's read-ahead and the apply issues no separate
|
|
@@ -1800,6 +1853,7 @@ export async function processPushCommitWithTrace(
|
|
|
1800
1853
|
targets.size === 0
|
|
1801
1854
|
? undefined
|
|
1802
1855
|
: [...targets].map(([table, rowIds]) => ({ table, rowIds })),
|
|
1856
|
+
outbox,
|
|
1803
1857
|
);
|
|
1804
1858
|
}
|
|
1805
1859
|
|
|
@@ -1822,6 +1876,7 @@ export async function processPushOperationsWithTrace(
|
|
|
1822
1876
|
readonly table: string;
|
|
1823
1877
|
readonly rowIds: readonly string[];
|
|
1824
1878
|
}[],
|
|
1879
|
+
outbox?: PushOutbox,
|
|
1825
1880
|
): Promise<ProcessedPushCommit> {
|
|
1826
1881
|
const { partition } = ctx;
|
|
1827
1882
|
const createdAtMs = clockOf(ctx)();
|
|
@@ -1835,6 +1890,13 @@ export async function processPushOperationsWithTrace(
|
|
|
1835
1890
|
const commitValidator = ctx.commitValidator;
|
|
1836
1891
|
const reactionPlanner = ctx.reactionPlanner;
|
|
1837
1892
|
const tx = await ctx.storage.begin(partition);
|
|
1893
|
+
// §6.3.2: the settlement's writes commit whatever the commit's outcome.
|
|
1894
|
+
const release = (): Promise<void> =>
|
|
1895
|
+
outbox?.settlement !== undefined ? tx.commit() : tx.rollback();
|
|
1896
|
+
const sequence =
|
|
1897
|
+
outbox === undefined
|
|
1898
|
+
? undefined
|
|
1899
|
+
: { outboxId: outbox.outboxId, clientSeq: outbox.clientSeq };
|
|
1838
1900
|
try {
|
|
1839
1901
|
if (reactionPlanner !== undefined && tx.enqueueReactions === undefined) {
|
|
1840
1902
|
throw new Error(
|
|
@@ -1854,6 +1916,7 @@ export async function processPushOperationsWithTrace(
|
|
|
1854
1916
|
clientCommitId,
|
|
1855
1917
|
schema.version,
|
|
1856
1918
|
prefetchTargets,
|
|
1919
|
+
outbox?.settlement,
|
|
1857
1920
|
);
|
|
1858
1921
|
} catch (error) {
|
|
1859
1922
|
if (
|
|
@@ -1862,7 +1925,7 @@ export async function processPushOperationsWithTrace(
|
|
|
1862
1925
|
) {
|
|
1863
1926
|
// §6.3: answer the retryable cache-miss for this commit rather than
|
|
1864
1927
|
// re-applying. Not persisted — a retry may find a readable record.
|
|
1865
|
-
await
|
|
1928
|
+
await release();
|
|
1866
1929
|
return {
|
|
1867
1930
|
frame: idempotencyCacheMissFrame(clientCommitId, error),
|
|
1868
1931
|
replayed: false,
|
|
@@ -1879,9 +1942,10 @@ export async function processPushOperationsWithTrace(
|
|
|
1879
1942
|
ctx.checkpoints,
|
|
1880
1943
|
);
|
|
1881
1944
|
if (refusal !== undefined) {
|
|
1882
|
-
await
|
|
1945
|
+
await release();
|
|
1883
1946
|
throw serveNotReadyError(refusal);
|
|
1884
1947
|
}
|
|
1948
|
+
const watermark = prepared.watermark ?? outbox?.watermark;
|
|
1885
1949
|
if (prepared.persisted !== undefined) {
|
|
1886
1950
|
const conflict = prepared.persisted.results.find(
|
|
1887
1951
|
(result) => result.status === 'conflict',
|
|
@@ -1895,7 +1959,7 @@ export async function processPushOperationsWithTrace(
|
|
|
1895
1959
|
table === undefined ||
|
|
1896
1960
|
!authorizeWrite(table, authorization.scopes, resolved))
|
|
1897
1961
|
) {
|
|
1898
|
-
await
|
|
1962
|
+
await release();
|
|
1899
1963
|
return processedPushCommit(
|
|
1900
1964
|
clientCommitId,
|
|
1901
1965
|
{
|
|
@@ -1911,10 +1975,29 @@ export async function processPushOperationsWithTrace(
|
|
|
1911
1975
|
],
|
|
1912
1976
|
},
|
|
1913
1977
|
true,
|
|
1978
|
+
watermark,
|
|
1914
1979
|
);
|
|
1915
1980
|
}
|
|
1916
|
-
await
|
|
1917
|
-
return processedPushCommit(
|
|
1981
|
+
await release();
|
|
1982
|
+
return processedPushCommit(
|
|
1983
|
+
clientCommitId,
|
|
1984
|
+
prepared.persisted,
|
|
1985
|
+
true,
|
|
1986
|
+
watermark,
|
|
1987
|
+
);
|
|
1988
|
+
}
|
|
1989
|
+
// §6.3.2 step 3: below the watermark without a retained result.
|
|
1990
|
+
if (
|
|
1991
|
+
outbox !== undefined &&
|
|
1992
|
+
watermark !== undefined &&
|
|
1993
|
+
outbox.clientSeq < watermark
|
|
1994
|
+
) {
|
|
1995
|
+
await release();
|
|
1996
|
+
return {
|
|
1997
|
+
frame: commitExpiredFrame(clientCommitId),
|
|
1998
|
+
replayed: false,
|
|
1999
|
+
watermark,
|
|
2000
|
+
};
|
|
1918
2001
|
}
|
|
1919
2002
|
const results: PushOperationResult[] = [];
|
|
1920
2003
|
const changes: NewChange[] = [];
|
|
@@ -1973,6 +2056,9 @@ export async function processPushOperationsWithTrace(
|
|
|
1973
2056
|
// §6.4: the row each touched (table, rowId) held before this commit, so a
|
|
1974
2057
|
// conflict reports the row as stored once the commit rolls back.
|
|
1975
2058
|
const storedBeforeCommit = new Map<string, StoredRow | undefined>();
|
|
2059
|
+
// §2.2: inserts start above the floor, which this commit's deletes raise
|
|
2060
|
+
// as they stage.
|
|
2061
|
+
let deleteFloor = prepared.deleteFloor;
|
|
1976
2062
|
for (let opIndex = 0; opIndex < operations.length; opIndex++) {
|
|
1977
2063
|
const op = operations[opIndex];
|
|
1978
2064
|
if (op === undefined) continue;
|
|
@@ -1988,11 +2074,17 @@ export async function processPushOperationsWithTrace(
|
|
|
1988
2074
|
validators,
|
|
1989
2075
|
partition,
|
|
1990
2076
|
ctx.actorId,
|
|
2077
|
+
deleteFloor,
|
|
1991
2078
|
);
|
|
1992
2079
|
if (outcome.kind === 'terminate') {
|
|
1993
2080
|
terminated = outcome.record;
|
|
1994
2081
|
break;
|
|
1995
2082
|
}
|
|
2083
|
+
for (const change of outcome.changes) {
|
|
2084
|
+
if (change.op === 'delete') {
|
|
2085
|
+
deleteFloor = Math.max(deleteFloor, change.rowVersion);
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
1996
2088
|
const rowKey = `${op.table}\u0000${op.rowId}`;
|
|
1997
2089
|
if (!storedBeforeCommit.has(rowKey)) {
|
|
1998
2090
|
storedBeforeCommit.set(rowKey, outcome.staged.stored);
|
|
@@ -2122,11 +2214,13 @@ export async function processPushOperationsWithTrace(
|
|
|
2122
2214
|
clientId,
|
|
2123
2215
|
clientCommitId,
|
|
2124
2216
|
stored,
|
|
2217
|
+
sequence,
|
|
2125
2218
|
);
|
|
2126
2219
|
return processedPushCommit(
|
|
2127
2220
|
clientCommitId,
|
|
2128
2221
|
canonical,
|
|
2129
2222
|
canonical.cacheIdentity !== stored.cacheIdentity,
|
|
2223
|
+
watermark,
|
|
2130
2224
|
);
|
|
2131
2225
|
}
|
|
2132
2226
|
|
|
@@ -2155,7 +2249,7 @@ export async function processPushOperationsWithTrace(
|
|
|
2155
2249
|
commitSeq,
|
|
2156
2250
|
results,
|
|
2157
2251
|
});
|
|
2158
|
-
await tx.putPushResult(clientId, clientCommitId, stored);
|
|
2252
|
+
await tx.putPushResult(clientId, clientCommitId, stored, sequence);
|
|
2159
2253
|
await tx.commit();
|
|
2160
2254
|
if (ctx.events !== undefined) {
|
|
2161
2255
|
const atMs = clockOf(ctx)();
|
|
@@ -2189,7 +2283,7 @@ export async function processPushOperationsWithTrace(
|
|
|
2189
2283
|
changes,
|
|
2190
2284
|
});
|
|
2191
2285
|
}
|
|
2192
|
-
return processedPushCommit(clientCommitId, stored, false);
|
|
2286
|
+
return processedPushCommit(clientCommitId, stored, false, watermark);
|
|
2193
2287
|
} catch (error) {
|
|
2194
2288
|
if (error instanceof StorageConstraintError) {
|
|
2195
2289
|
const stored = newStoredPushResult(createdAtMs, {
|
|
@@ -2209,6 +2303,7 @@ export async function processPushOperationsWithTrace(
|
|
|
2209
2303
|
clientId,
|
|
2210
2304
|
clientCommitId,
|
|
2211
2305
|
stored,
|
|
2306
|
+
sequence,
|
|
2212
2307
|
);
|
|
2213
2308
|
return processedPushCommit(
|
|
2214
2309
|
clientCommitId,
|