@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/dist/sqlite-storage.js
CHANGED
|
@@ -10,10 +10,10 @@
|
|
|
10
10
|
* match against the stored scope map — never a log scan.
|
|
11
11
|
*/
|
|
12
12
|
import { isSqliteConstraintError, retainRollbackFailure, rollbackAfterError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
|
|
13
|
-
import { serveGateRefusal } from './storage.js';
|
|
13
|
+
import { commitDeleteFloor, serveGateRefusal } from './storage.js';
|
|
14
14
|
import { serveNotReadyError } from './readiness.js';
|
|
15
15
|
import { assertPhysicalColumns, assertStoredLayouts, dropTableDdl, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
|
|
16
|
-
import {
|
|
16
|
+
import { SQLITE_DDL, SQLITE_UPGRADE_PROBE, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteUpgradeDdl, } from './sqlite-dialect.js';
|
|
17
17
|
import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
|
|
18
18
|
function toStoredCheckpoint(record) {
|
|
19
19
|
return {
|
|
@@ -67,15 +67,25 @@ class SqliteTransaction extends SqliteFamilyTransaction {
|
|
|
67
67
|
// BEGIN IMMEDIATE and observes every row staged so far.
|
|
68
68
|
return this.#storage.runAuthoritativeQuery(this.partition, query);
|
|
69
69
|
}
|
|
70
|
-
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = []) {
|
|
70
|
+
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = [], outbox) {
|
|
71
71
|
this.assertOpen();
|
|
72
72
|
// BEGIN IMMEDIATE in the constructor already owns SQLite's writer lock.
|
|
73
|
+
// The settlement precedes the savepoint, so a rejection keeps it.
|
|
74
|
+
const settled = outbox === undefined
|
|
75
|
+
? undefined
|
|
76
|
+
: await this.settleOutbox(clientId, clientCommitId, outbox);
|
|
73
77
|
this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
|
|
74
78
|
this.#pushApplySavepoint = true;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
+
const gate = await this.#storage.readServeGate(this.partition, runningSchemaVersion);
|
|
80
|
+
const [floor] = await this.all('SELECT delete_floor FROM sync_partitions WHERE partition=?', [this.partition]);
|
|
81
|
+
const deleteFloor = floor?.delete_floor ?? 0;
|
|
82
|
+
return settled === undefined
|
|
83
|
+
? {
|
|
84
|
+
gate,
|
|
85
|
+
deleteFloor,
|
|
86
|
+
persisted: await this.#storage.getPushResult(this.partition, clientId, clientCommitId),
|
|
87
|
+
}
|
|
88
|
+
: { gate, deleteFloor, ...settled };
|
|
79
89
|
}
|
|
80
90
|
async lockPartition() {
|
|
81
91
|
this.assertOpen();
|
|
@@ -96,7 +106,7 @@ class SqliteTransaction extends SqliteFamilyTransaction {
|
|
|
96
106
|
// BEGIN IMMEDIATE, so the gate and the write share one transaction.
|
|
97
107
|
return this.#storage.readServeGate(this.partition, runningSchemaVersion);
|
|
98
108
|
}
|
|
99
|
-
async commitRejectedPushResult(clientId, clientCommitId, result) {
|
|
109
|
+
async commitRejectedPushResult(clientId, clientCommitId, result, sequence) {
|
|
100
110
|
this.assertOpen();
|
|
101
111
|
if (!this.#pushApplySavepoint) {
|
|
102
112
|
throw new Error('push rejection requires its apply savepoint');
|
|
@@ -104,7 +114,7 @@ class SqliteTransaction extends SqliteFamilyTransaction {
|
|
|
104
114
|
this.#storage.db.exec('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
105
115
|
this.#storage.db.exec('RELEASE SAVEPOINT syncular_push_candidate');
|
|
106
116
|
this.#pushApplySavepoint = false;
|
|
107
|
-
await this.putPushResult(clientId, clientCommitId, result);
|
|
117
|
+
await this.putPushResult(clientId, clientCommitId, result, sequence);
|
|
108
118
|
await this.commit();
|
|
109
119
|
// The idempotency re-check found no record under the writer lock, so the
|
|
110
120
|
// insert above is the canonical outcome.
|
|
@@ -129,10 +139,7 @@ class SqliteTransaction extends SqliteFamilyTransaction {
|
|
|
129
139
|
]);
|
|
130
140
|
const [row] = await this.all('SELECT max_commit_seq FROM sync_partitions WHERE partition=?', [p]);
|
|
131
141
|
const commitSeq = (row?.max_commit_seq ?? 0) + 1;
|
|
132
|
-
this.run('UPDATE sync_partitions SET max_commit_seq
|
|
133
|
-
commitSeq,
|
|
134
|
-
p,
|
|
135
|
-
]);
|
|
142
|
+
this.run('UPDATE sync_partitions SET max_commit_seq=?, delete_floor=max(delete_floor, ?) WHERE partition=?', [commitSeq, commitDeleteFloor(commit.changes), p]);
|
|
136
143
|
this.run('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version) VALUES (?,?,?,?,?,?,?)', [
|
|
137
144
|
p,
|
|
138
145
|
commitSeq,
|
|
@@ -216,13 +223,10 @@ export class SqliteServerStorage extends SqliteFamilyStorage {
|
|
|
216
223
|
super();
|
|
217
224
|
this.db = db;
|
|
218
225
|
this.db.exec(SQLITE_DDL);
|
|
219
|
-
for (const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
if (!columns.some((column) => column.name === name)) {
|
|
224
|
-
this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`);
|
|
225
|
-
}
|
|
226
|
+
for (const statement of sqliteUpgradeDdl(this.db
|
|
227
|
+
.query(SQLITE_UPGRADE_PROBE.sql)
|
|
228
|
+
.all(...SQLITE_UPGRADE_PROBE.params))) {
|
|
229
|
+
this.db.exec(statement);
|
|
226
230
|
}
|
|
227
231
|
const commitColumns = this.db
|
|
228
232
|
.query('PRAGMA table_info("sync_commits")')
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CompiledTable, IndexSchema } from './schema.js';
|
|
2
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, IndexRowScanQuery, RowScanQuery, StorageTransaction } from './storage.js';
|
|
2
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, IndexRowScanQuery, LiveWindow, LiveWindowQuery, RowScanQuery, StorageTransaction, StoredChange } from './storage.js';
|
|
3
3
|
/**
|
|
4
4
|
* Fail loudly instead of making an unsupported unscoped scan look empty.
|
|
5
5
|
* Returns the first scope variable in sorted order — the one every adapter
|
|
@@ -15,3 +15,32 @@ export declare function resolveIndexRowScan(table: CompiledTable, query: IndexRo
|
|
|
15
15
|
* which reads outside the transaction.
|
|
16
16
|
*/
|
|
17
17
|
export declare function transactionQuery(tx: StorageTransaction, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
18
|
+
/** One index range of the live scan: a unit key and its exclusive lower bound. */
|
|
19
|
+
export interface LiveKey {
|
|
20
|
+
readonly var: string;
|
|
21
|
+
readonly value: string;
|
|
22
|
+
readonly after: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* One row of a live-window page: a candidate commit joined to its metadata
|
|
26
|
+
* (absent when the commit vanished) and one of its changes (absent when it
|
|
27
|
+
* has none), in (commitSeq, change index) order.
|
|
28
|
+
*/
|
|
29
|
+
export interface LiveWindowRow {
|
|
30
|
+
readonly commitSeq: number;
|
|
31
|
+
readonly meta?: {
|
|
32
|
+
readonly actorId: string;
|
|
33
|
+
readonly createdAtMs: number;
|
|
34
|
+
};
|
|
35
|
+
readonly change?: StoredChange;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The §4.5 live scan over a backend's page read. Each distinct unit key
|
|
39
|
+
* (the first variable, keys sorted) becomes one index range starting at the
|
|
40
|
+
* lowest bound of the units that share it; `readPage` returns up to `limit`
|
|
41
|
+
* candidate commits above every range's bound. A change is kept when a unit
|
|
42
|
+
* covers its table, the pair's effective scopes match it, and its commit
|
|
43
|
+
* lies above that unit's own bound. Every candidate advances the scan, so a
|
|
44
|
+
* page of non-matching candidates never repeats.
|
|
45
|
+
*/
|
|
46
|
+
export declare function readLiveWindowPages(query: LiveWindowQuery, readPage: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveWindowRow[]>): Promise<LiveWindow>;
|
package/dist/storage-query.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { matchesEffective } from './scopes.js';
|
|
1
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
2
3
|
/**
|
|
3
4
|
* Fail loudly instead of making an unsupported unscoped scan look empty.
|
|
@@ -46,3 +47,70 @@ export async function transactionQuery(tx, query) {
|
|
|
46
47
|
}
|
|
47
48
|
return tx.queryAuthoritative(query);
|
|
48
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* The §4.5 live scan over a backend's page read. Each distinct unit key
|
|
52
|
+
* (the first variable, keys sorted) becomes one index range starting at the
|
|
53
|
+
* lowest bound of the units that share it; `readPage` returns up to `limit`
|
|
54
|
+
* candidate commits above every range's bound. A change is kept when a unit
|
|
55
|
+
* covers its table, the pair's effective scopes match it, and its commit
|
|
56
|
+
* lies above that unit's own bound. Every candidate advances the scan, so a
|
|
57
|
+
* page of non-matching candidates never repeats.
|
|
58
|
+
*/
|
|
59
|
+
export async function readLiveWindowPages(query, readPage) {
|
|
60
|
+
const bounds = new Map();
|
|
61
|
+
for (const unit of query.units) {
|
|
62
|
+
if (unit.tables.size === 0)
|
|
63
|
+
continue;
|
|
64
|
+
const variable = Object.keys(unit.scopes).sort()[0];
|
|
65
|
+
const value = variable === undefined ? undefined : unit.scopes[variable];
|
|
66
|
+
if (variable === undefined || value === undefined) {
|
|
67
|
+
throw new StorageQueryError('sync.storage.scan_requires_scope');
|
|
68
|
+
}
|
|
69
|
+
const id = JSON.stringify([variable, value]);
|
|
70
|
+
const prior = bounds.get(id);
|
|
71
|
+
if (prior === undefined || unit.afterSeq < prior.after) {
|
|
72
|
+
bounds.set(id, { var: variable, value, after: unit.afterSeq });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const commits = [];
|
|
76
|
+
const limit = Math.max(64, query.limitChanges);
|
|
77
|
+
let delivered = 0;
|
|
78
|
+
let scanned = -1;
|
|
79
|
+
while (bounds.size > 0) {
|
|
80
|
+
const rows = await readPage([...bounds.values()].map((key) => ({
|
|
81
|
+
...key,
|
|
82
|
+
after: Math.max(key.after, scanned),
|
|
83
|
+
})), limit);
|
|
84
|
+
let candidates = 0;
|
|
85
|
+
for (let i = 0; i < rows.length;) {
|
|
86
|
+
const head = rows[i];
|
|
87
|
+
if (head === undefined)
|
|
88
|
+
break;
|
|
89
|
+
candidates += 1;
|
|
90
|
+
scanned = head.commitSeq;
|
|
91
|
+
const changes = [];
|
|
92
|
+
for (; rows[i]?.commitSeq === head.commitSeq; i++) {
|
|
93
|
+
const change = rows[i]?.change;
|
|
94
|
+
if (change !== undefined &&
|
|
95
|
+
query.units.some((unit) => {
|
|
96
|
+
const effective = unit.tables.get(change.table);
|
|
97
|
+
return (head.commitSeq > unit.afterSeq &&
|
|
98
|
+
effective !== undefined &&
|
|
99
|
+
matchesEffective(change.scopes, effective));
|
|
100
|
+
})) {
|
|
101
|
+
changes.push(change);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (head.meta === undefined || changes.length === 0)
|
|
105
|
+
continue;
|
|
106
|
+
commits.push({ commitSeq: head.commitSeq, ...head.meta, changes });
|
|
107
|
+
delivered += changes.length;
|
|
108
|
+
if (delivered >= query.limitChanges) {
|
|
109
|
+
return { commits, throughSeq: head.commitSeq };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (candidates < limit)
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
return { commits, throughSeq: query.throughSeq };
|
|
116
|
+
}
|
package/dist/storage.d.ts
CHANGED
|
@@ -11,13 +11,14 @@ import type { AuthoritativeRelationPlan } from './authoritative-query.js';
|
|
|
11
11
|
* - the pruning horizon per §4.6;
|
|
12
12
|
* - current-row reads for write authorization (§3.4) and conflict handling
|
|
13
13
|
* (§6.2);
|
|
14
|
-
* - per-(partition, clientId)
|
|
15
|
-
* (§4.5, §8.1)
|
|
14
|
+
* - the per-(partition, clientId) client record: registration list,
|
|
15
|
+
* catching-up units and record cursor (§4.5, §8.1);
|
|
16
|
+
* - the per-outbox acknowledged watermark (§6.3.2).
|
|
16
17
|
*
|
|
17
18
|
* The interface is async throughout so a Postgres implementation slots in
|
|
18
19
|
* without touching the core. All methods are partition-local (§2.1).
|
|
19
20
|
*/
|
|
20
|
-
import type { PushOperationResult, RowValue, ScopeMap } from '@syncular/core';
|
|
21
|
+
import type { PushOperationResult, RowValue, ScopeMap, UnitScopes } from '@syncular/core';
|
|
21
22
|
import type { CompiledSchema } from './schema.js';
|
|
22
23
|
export interface CommitPruneQuery {
|
|
23
24
|
readonly logEpoch: string;
|
|
@@ -77,6 +78,35 @@ export interface PreparedPushApply {
|
|
|
77
78
|
readonly gate: ServeGate;
|
|
78
79
|
/** The already-persisted outcome for this key, or `undefined` on a miss. */
|
|
79
80
|
readonly persisted: StoredPushResult | undefined;
|
|
81
|
+
/**
|
|
82
|
+
* The partition's delete floor (§2.2): the highest version any applied
|
|
83
|
+
* delete recorded, `0` before the first. An insert starts one above it.
|
|
84
|
+
*/
|
|
85
|
+
readonly deleteFloor: number;
|
|
86
|
+
/**
|
|
87
|
+
* The outbox watermark `W` after §6.3.2 step 1; present iff the call
|
|
88
|
+
* carried an {@link OutboxSettlement}. A commit of the request whose
|
|
89
|
+
* `persisted` is `undefined` and whose `clientSeq` is below `W` is
|
|
90
|
+
* expired (`sync.commit_expired`); at or above `W` it was never seen.
|
|
91
|
+
*/
|
|
92
|
+
readonly watermark?: number;
|
|
93
|
+
}
|
|
94
|
+
/** The outbox position of one pushed commit (§6.3.2). */
|
|
95
|
+
export interface OutboxSequence {
|
|
96
|
+
readonly outboxId: string;
|
|
97
|
+
readonly clientSeq: number;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* One push request's outbox, for the §9 settlement of results stored before
|
|
101
|
+
* wire version 4 and §6.3.2 steps 1 and 2.
|
|
102
|
+
*/
|
|
103
|
+
export interface OutboxSettlement {
|
|
104
|
+
readonly outboxId: string;
|
|
105
|
+
/** Every `PUSH_COMMIT` of the request in frame order. */
|
|
106
|
+
readonly commits: readonly {
|
|
107
|
+
readonly clientCommitId: string;
|
|
108
|
+
readonly clientSeq: number;
|
|
109
|
+
}[];
|
|
80
110
|
}
|
|
81
111
|
/** Why `serveGate` refuses, or `undefined` when this request may serve. */
|
|
82
112
|
export type ServeGateRefusal = {
|
|
@@ -126,13 +156,12 @@ export interface NewChange {
|
|
|
126
156
|
readonly table: string;
|
|
127
157
|
readonly rowId: string;
|
|
128
158
|
readonly op: 'upsert' | 'delete';
|
|
129
|
-
/** Present for upsert (§2.2). */
|
|
130
|
-
readonly rowVersion?: number;
|
|
131
159
|
/**
|
|
132
|
-
*
|
|
133
|
-
*
|
|
160
|
+
* The row's `server_version` after the write (§2.2): an upsert's written
|
|
161
|
+
* version, a delete's advanced version, which its tombstone records and
|
|
162
|
+
* the delete floor rises to.
|
|
134
163
|
*/
|
|
135
|
-
readonly
|
|
164
|
+
readonly rowVersion: number;
|
|
136
165
|
readonly scopes: Record<string, string>;
|
|
137
166
|
/** Present for upsert. */
|
|
138
167
|
readonly payload?: Uint8Array;
|
|
@@ -161,10 +190,16 @@ export interface NewCommit {
|
|
|
161
190
|
* its tombstone.
|
|
162
191
|
*/
|
|
163
192
|
export declare function rowsLiveAfterDelete(changes: readonly NewChange[]): Set<string>;
|
|
193
|
+
/**
|
|
194
|
+
* The delete floor a commit raises the partition's to (§2.2): the highest
|
|
195
|
+
* version of a delete that leaves a tombstone, `0` without one.
|
|
196
|
+
*/
|
|
197
|
+
export declare function commitDeleteFloor(changes: readonly NewChange[]): number;
|
|
164
198
|
export interface StoredChange {
|
|
165
199
|
readonly table: string;
|
|
166
200
|
readonly rowId: string;
|
|
167
201
|
readonly op: 'upsert' | 'delete';
|
|
202
|
+
/** Absent only on a delete logged before wire version 4. */
|
|
168
203
|
readonly rowVersion?: number;
|
|
169
204
|
readonly scopes: Record<string, string>;
|
|
170
205
|
readonly payload?: Uint8Array;
|
|
@@ -257,23 +292,43 @@ export interface StoredPushResult {
|
|
|
257
292
|
};
|
|
258
293
|
readonly results: readonly PushOperationResult[];
|
|
259
294
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
295
|
+
/** The registration list a client last sent, with its digest (§4.1). */
|
|
296
|
+
export interface ClientRegistration {
|
|
297
|
+
/** Scope maps in registration order. */
|
|
298
|
+
readonly units: readonly UnitScopes[];
|
|
299
|
+
/** The 32-byte registration digest. */
|
|
300
|
+
readonly digest: Uint8Array;
|
|
301
|
+
}
|
|
302
|
+
/** A unit still catching up after the client's last completed round (§4.1). */
|
|
303
|
+
export interface CatchingUpUnit {
|
|
304
|
+
/** The unit id (§4.1). */
|
|
263
305
|
readonly id: string;
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
readonly scopes: ScopeMap;
|
|
306
|
+
/** `-1`, the bootstrap pin, or the post-bootstrap cursor `P`. */
|
|
307
|
+
readonly cursor: number;
|
|
267
308
|
}
|
|
309
|
+
/** The §4.5 client record. */
|
|
268
310
|
export interface ClientRecord {
|
|
269
311
|
readonly clientId: string;
|
|
270
312
|
readonly actorId: string;
|
|
271
|
-
/** SSP2 version last accepted from this client
|
|
313
|
+
/** SSP2 version last accepted from this client. */
|
|
272
314
|
readonly wireVersion: number;
|
|
273
|
-
/**
|
|
315
|
+
/** The record cursor (§4.5); feeds the §4.6 retention watermark. */
|
|
274
316
|
readonly cursor: number;
|
|
275
317
|
readonly updatedAtMs: number;
|
|
276
|
-
|
|
318
|
+
/**
|
|
319
|
+
* Schema version of the latest completed round; realtime projects deltas
|
|
320
|
+
* through its codec. Absent on a record written before wire version 4.
|
|
321
|
+
*/
|
|
322
|
+
readonly schemaVersion?: number;
|
|
323
|
+
/**
|
|
324
|
+
* The stored registration list; absent when none is stored (§9: a record
|
|
325
|
+
* written before wire version 4 holds none). On `putClientRecord`, absent
|
|
326
|
+
* keeps the stored list and digest: a round writes them only when its
|
|
327
|
+
* list traveled.
|
|
328
|
+
*/
|
|
329
|
+
readonly registration?: ClientRegistration;
|
|
330
|
+
/** Catching-up units in registration order; the rest of the list is live. */
|
|
331
|
+
readonly catchingUp: readonly CatchingUpUnit[];
|
|
277
332
|
/** When this client's last sync round finished, successfully or not. */
|
|
278
333
|
readonly lastRoundAtMs?: number;
|
|
279
334
|
/** Code of the last round when it failed; absent after a successful round. */
|
|
@@ -292,23 +347,42 @@ export interface ClientRecordWriteOptions {
|
|
|
292
347
|
}
|
|
293
348
|
/** The round-state fields of a stored `sync_clients` row. */
|
|
294
349
|
export declare function clientRoundState(lastRoundAtMs: number | null, lastErrorCode: string | null, consecutiveFailures: number): Pick<ClientRecord, 'lastRoundAtMs' | 'lastErrorCode' | 'consecutiveFailures'>;
|
|
295
|
-
|
|
296
|
-
|
|
350
|
+
/** One unit of the §4.5 scan set. */
|
|
351
|
+
export interface LiveScanUnit {
|
|
352
|
+
/** The unit's scope map; its first variable (keys sorted) is its index key. */
|
|
353
|
+
readonly scopes: UnitScopes;
|
|
354
|
+
/** Exclusive lower bound: `R`, the unit's cursor `P`, or its pin. */
|
|
355
|
+
readonly afterSeq: number;
|
|
297
356
|
/**
|
|
298
|
-
*
|
|
299
|
-
*
|
|
300
|
-
*
|
|
357
|
+
* The unit's active (unit, table) pairs: covered table → effective scopes
|
|
358
|
+
* (§3.2). A change of table `t` matches iff its stored scopes satisfy the
|
|
359
|
+
* pair for `t` and its commit lies above `afterSeq`.
|
|
301
360
|
*/
|
|
302
|
-
readonly
|
|
303
|
-
|
|
304
|
-
|
|
361
|
+
readonly tables: ReadonlyMap<string, ScopeMap>;
|
|
362
|
+
}
|
|
363
|
+
export interface LiveWindowQuery {
|
|
364
|
+
readonly units: readonly LiveScanUnit[];
|
|
365
|
+
/** `maxCommitSeq` read at the start of the pull. */
|
|
305
366
|
readonly throughSeq: number;
|
|
306
|
-
/**
|
|
367
|
+
/** The `limitCommits` budget: stop after the commit that reaches it. */
|
|
307
368
|
readonly limitChanges: number;
|
|
308
369
|
}
|
|
370
|
+
/** The commits of one live section (§4.5). */
|
|
371
|
+
export interface LiveWindow {
|
|
372
|
+
/** Oldest first, each with its matching changes in change order. */
|
|
373
|
+
readonly commits: readonly StoredCommit[];
|
|
374
|
+
/**
|
|
375
|
+
* `T`: `query.throughSeq`, or the last delivered `commitSeq` when the
|
|
376
|
+
* budget cut the section.
|
|
377
|
+
*/
|
|
378
|
+
readonly throughSeq: number;
|
|
379
|
+
}
|
|
309
380
|
export interface RowScanQuery {
|
|
310
381
|
readonly table: string;
|
|
311
|
-
/**
|
|
382
|
+
/**
|
|
383
|
+
* Effective scopes (§3.2): a row matches iff, for every key, its stored
|
|
384
|
+
* value is in the list. Candidates come from the inverted row-scope index.
|
|
385
|
+
*/
|
|
312
386
|
readonly scopeFilter: ScopeMap;
|
|
313
387
|
/** Resume after this rowId (exclusive); `null` = start of table. */
|
|
314
388
|
readonly afterRowId: string | null;
|
|
@@ -388,6 +462,13 @@ export interface RowListQuery {
|
|
|
388
462
|
readonly afterRowId: string | null;
|
|
389
463
|
readonly limit: number;
|
|
390
464
|
}
|
|
465
|
+
/** One §6.3.2 watermark row. */
|
|
466
|
+
export interface PushWatermark {
|
|
467
|
+
readonly clientId: string;
|
|
468
|
+
readonly outboxId: string;
|
|
469
|
+
/** `W`: results of this outbox below it are deleted. */
|
|
470
|
+
readonly throughSeq: number;
|
|
471
|
+
}
|
|
391
472
|
/** A stored push outcome with the client that sent it. */
|
|
392
473
|
export interface ClientPushResult {
|
|
393
474
|
readonly clientId: string;
|
|
@@ -523,11 +604,18 @@ export interface StorageTransaction {
|
|
|
523
604
|
* are built; a storage with per-statement network cost folds that read-ahead
|
|
524
605
|
* into the admission read and answers the following `getRow` calls from it.
|
|
525
606
|
* Missing support fails closed before an app-row mutation.
|
|
607
|
+
*
|
|
608
|
+
* `outbox`, passed with the first commit of a request that carries an
|
|
609
|
+
* `outboxId`, settles the client's results stored before wire version 4
|
|
610
|
+
* (§9) and runs §6.3.2 steps 1 and 2 under the same lock, before the read
|
|
611
|
+
* of `persisted`. Those writes survive `commitRejectedPushResult`; the
|
|
612
|
+
* caller commits this transaction whatever the commit's outcome, so the
|
|
613
|
+
* settlement runs exactly once per request.
|
|
526
614
|
*/
|
|
527
615
|
preparePushApply(clientId: string, clientCommitId: string, runningSchemaVersion: number, targets?: readonly {
|
|
528
616
|
readonly table: string;
|
|
529
617
|
readonly rowIds: readonly string[];
|
|
530
|
-
}[]): Promise<PreparedPushApply>;
|
|
618
|
+
}[], outbox?: OutboxSettlement): Promise<PreparedPushApply>;
|
|
531
619
|
/**
|
|
532
620
|
* Rejection finalization while the push-apply serialization lock is still
|
|
533
621
|
* held: discard every candidate write, persist the rejected idempotency
|
|
@@ -536,7 +624,7 @@ export interface StorageTransaction {
|
|
|
536
624
|
* push so a concurrent duplicate cannot rerun operations, validators, or
|
|
537
625
|
* merges.
|
|
538
626
|
*/
|
|
539
|
-
commitRejectedPushResult(clientId: string, clientCommitId: string, result: StoredPushResult): Promise<StoredPushResult>;
|
|
627
|
+
commitRejectedPushResult(clientId: string, clientCommitId: string, result: StoredPushResult, sequence?: OutboxSequence): Promise<StoredPushResult>;
|
|
540
628
|
upsertRow(table: string, row: StoredRow, context?: {
|
|
541
629
|
readonly opIndex: number;
|
|
542
630
|
}): Promise<void>;
|
|
@@ -551,9 +639,11 @@ export interface StorageTransaction {
|
|
|
551
639
|
enqueueReactions?(reactions: readonly NewReaction[]): Promise<void>;
|
|
552
640
|
/**
|
|
553
641
|
* Persist an idempotency outcome only when the key is still absent. The
|
|
554
|
-
* first writer wins; callers read the canonical value after commit.
|
|
642
|
+
* first writer wins; callers read the canonical value after commit. A
|
|
643
|
+
* result without `sequence` (no `outboxId`, §6.10, §12.3) is never deleted
|
|
644
|
+
* by a watermark.
|
|
555
645
|
*/
|
|
556
|
-
putPushResult(clientId: string, clientCommitId: string, result: StoredPushResult): Promise<void>;
|
|
646
|
+
putPushResult(clientId: string, clientCommitId: string, result: StoredPushResult, sequence?: OutboxSequence): Promise<void>;
|
|
557
647
|
/**
|
|
558
648
|
* Advance this partition's checkpoint only while `owner_epoch` still
|
|
559
649
|
* matches, so the owner-epoch CAS rides inside the caller's transaction.
|
|
@@ -610,7 +700,10 @@ export interface ServerStorage {
|
|
|
610
700
|
* back without a write, so a request that only reads takes no write lock.
|
|
611
701
|
*/
|
|
612
702
|
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
613
|
-
/**
|
|
703
|
+
/**
|
|
704
|
+
* Rotate log continuity after restore and delete the partition's client
|
|
705
|
+
* records. Watermark rows stay (§6.3.2).
|
|
706
|
+
*/
|
|
614
707
|
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
615
708
|
/** Registry entries ordered by partition for maintenance loops. */
|
|
616
709
|
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
@@ -710,22 +803,25 @@ export interface ServerStorage {
|
|
|
710
803
|
/** Delete a bounded set of aged terminal rows. Never deletes active work. */
|
|
711
804
|
pruneReactions?(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
712
805
|
/**
|
|
713
|
-
*
|
|
714
|
-
*
|
|
715
|
-
*
|
|
806
|
+
* The §4.5 live section: every commit in `(min afterSeq, throughSeq]`
|
|
807
|
+
* carrying a change that matches a unit of the scan set, oldest first,
|
|
808
|
+
* each with its matching changes once, in change order, across tables.
|
|
809
|
+
* Stops after the commit whose matching changes reach `limitChanges`
|
|
810
|
+
* (never splitting a commit). Candidates come from the change-scope key
|
|
811
|
+
* index, one index range per distinct unit key, never from a log scan.
|
|
716
812
|
*/
|
|
717
|
-
|
|
813
|
+
readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
|
|
718
814
|
/**
|
|
719
|
-
* Newest commitSeq whose change to `query.table` carries
|
|
720
|
-
*
|
|
721
|
-
* the
|
|
722
|
-
*
|
|
723
|
-
*
|
|
724
|
-
*
|
|
815
|
+
* Newest commitSeq whose change to `query.table` carries the unit's value
|
|
816
|
+
* of its first variable (keys sorted): the `readLiveWindow` candidate rule,
|
|
817
|
+
* so no change the unit would receive for the table sits above it. 0 when
|
|
818
|
+
* the index holds none (never written, or pruned). The §4.7 sqlite-lane
|
|
819
|
+
* scope pin reads it; it MUST be one index lookup, independent of the
|
|
820
|
+
* table's history.
|
|
725
821
|
*/
|
|
726
822
|
latestScopedChangeSeq(partition: string, query: {
|
|
727
823
|
readonly table: string;
|
|
728
|
-
readonly
|
|
824
|
+
readonly scopes: UnitScopes;
|
|
729
825
|
}): Promise<number>;
|
|
730
826
|
/** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
|
|
731
827
|
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
@@ -756,10 +852,13 @@ export interface ServerStorage {
|
|
|
756
852
|
scanRowsByIndex?(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
757
853
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
758
854
|
/**
|
|
759
|
-
* Write the record for a completed round.
|
|
760
|
-
*
|
|
761
|
-
*
|
|
762
|
-
*
|
|
855
|
+
* Write the record for a completed round (§4.5). A record without
|
|
856
|
+
* `registration` keeps the stored list and digest. The round-state fields
|
|
857
|
+
* reset: the stored last round is a success at `lastRoundAtMs` (default
|
|
858
|
+
* `updatedAtMs`) with no failures, unless `keepRoundOutcome` is set, which
|
|
859
|
+
* leaves the reported code and streak in place (the round carried a
|
|
860
|
+
* `ROUND_FAILURE` report, §1.5/§4.5). An `unregistered` answer rewrites the
|
|
861
|
+
* stored record with only `lastRoundAtMs` moved.
|
|
763
862
|
*/
|
|
764
863
|
putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
|
|
765
864
|
/**
|
|
@@ -770,14 +869,13 @@ export interface ServerStorage {
|
|
|
770
869
|
*/
|
|
771
870
|
recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
|
|
772
871
|
/**
|
|
773
|
-
*
|
|
774
|
-
*
|
|
775
|
-
*
|
|
872
|
+
* §8.2 ack: atomically raise an existing record's cursor to `cursor`,
|
|
873
|
+
* capped at the lowest non-negative cursor of its catching-up units, and
|
|
874
|
+
* its activity timestamp, both monotonically. Every other field stays;
|
|
875
|
+
* requires the matching actor and the current log epoch. A missing record
|
|
876
|
+
* or a mismatched identity is unchanged.
|
|
776
877
|
*/
|
|
777
878
|
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
778
|
-
/** Advance an existing client's ACK cursor and timestamp atomically,
|
|
779
|
-
* preserving actor, wire version, and subscriptions. Missing records stay absent. */
|
|
780
|
-
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
781
879
|
/**
|
|
782
880
|
* Minimum non-negative cursor with updatedAtMs >= cutoff (§4.6); null when
|
|
783
881
|
* none are active. A bootstrapping client (cursor < 0) holds no cursor.
|
|
@@ -824,7 +922,7 @@ export interface ServerStorage {
|
|
|
824
922
|
* calls them.
|
|
825
923
|
*
|
|
826
924
|
* `listClientRecords`: every stored client record for the partition
|
|
827
|
-
* (cursor, last-seen,
|
|
925
|
+
* (cursor, last-seen, registration list): the "who's connected" view.
|
|
828
926
|
* `listCommitMetadata`: the commit log without payloads (metadata +
|
|
829
927
|
* touched tables), the newest-oriented window an operator inspects.
|
|
830
928
|
* `scopeActivity`: the recent commits touching one scope key, via the
|
|
@@ -839,6 +937,9 @@ export interface ServerStorage {
|
|
|
839
937
|
* `listPushResults`: every stored idempotency outcome for one
|
|
840
938
|
* clientCommitId across the partition's clients, scanning the retained
|
|
841
939
|
* outcomes without a dedicated index.
|
|
940
|
+
* `listPushWatermarks`: the §6.3.2 watermark rows of every client whose
|
|
941
|
+
* retained commit log carries the clientCommitId; a commit whose result
|
|
942
|
+
* the watermark deleted reads as "expired below sequence `throughSeq`".
|
|
842
943
|
* `listPartitions`: the partition-only compatibility view of
|
|
843
944
|
* `listPartitionRegistry`, sorted.
|
|
844
945
|
*/
|
|
@@ -850,6 +951,7 @@ export interface ServerStorage {
|
|
|
850
951
|
listRows?(partition: string, query: RowListQuery): Promise<StoredRow[]>;
|
|
851
952
|
countRows?(partition: string, table: string): Promise<number>;
|
|
852
953
|
listPushResults?(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
954
|
+
listPushWatermarks?(partition: string, clientCommitId: string): Promise<PushWatermark[]>;
|
|
853
955
|
listPartitions(): Promise<string[]>;
|
|
854
956
|
}
|
|
855
957
|
/** One scope binding of one table, budgeted (§4.6.1). */
|
package/dist/storage.js
CHANGED
|
@@ -64,6 +64,21 @@ export function rowsLiveAfterDelete(changes) {
|
|
|
64
64
|
}
|
|
65
65
|
return live;
|
|
66
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* The delete floor a commit raises the partition's to (§2.2): the highest
|
|
69
|
+
* version of a delete that leaves a tombstone, `0` without one.
|
|
70
|
+
*/
|
|
71
|
+
export function commitDeleteFloor(changes) {
|
|
72
|
+
const live = rowsLiveAfterDelete(changes);
|
|
73
|
+
let floor = 0;
|
|
74
|
+
for (const change of changes) {
|
|
75
|
+
if (change.op === 'delete' &&
|
|
76
|
+
!live.has(`${change.table}\u0000${change.rowId}`)) {
|
|
77
|
+
floor = Math.max(floor, change.rowVersion);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return floor;
|
|
81
|
+
}
|
|
67
82
|
/** The round-state fields of a stored `sync_clients` row. */
|
|
68
83
|
export function clientRoundState(lastRoundAtMs, lastErrorCode, consecutiveFailures) {
|
|
69
84
|
return {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.60.1",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.60.1"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|