@syncular/server 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +49 -36
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +46 -40
  4. package/dist/blob-handlers.d.ts +25 -31
  5. package/dist/blob-handlers.js +63 -41
  6. package/dist/blob-store.d.ts +42 -24
  7. package/dist/blob-store.js +0 -0
  8. package/dist/bucket-blob-store.d.ts +12 -8
  9. package/dist/bucket-blob-store.js +54 -45
  10. package/dist/bucket-segment-store.js +18 -16
  11. package/dist/content-encoding.d.ts +5 -2
  12. package/dist/content-encoding.js +42 -13
  13. package/dist/context.d.ts +19 -5
  14. package/dist/context.js +19 -1
  15. package/dist/d1-storage.d.ts +14 -98
  16. package/dist/d1-storage.js +249 -1119
  17. package/dist/errors.js +16 -0
  18. package/dist/events.d.ts +36 -18
  19. package/dist/frame-bytes.js +24 -6
  20. package/dist/handler.js +503 -208
  21. package/dist/local-authority.d.ts +2 -0
  22. package/dist/local-authority.js +24 -12
  23. package/dist/object-bucket.d.ts +2 -0
  24. package/dist/operations-realtime.js +69 -50
  25. package/dist/operations.d.ts +5 -2
  26. package/dist/operations.js +20 -29
  27. package/dist/pg-executor-pglite.d.ts +5 -6
  28. package/dist/pg-executor-pglite.js +22 -14
  29. package/dist/pg-executor.d.ts +4 -1
  30. package/dist/postgres-storage.d.ts +24 -19
  31. package/dist/postgres-storage.js +420 -262
  32. package/dist/pull.d.ts +70 -47
  33. package/dist/pull.js +270 -301
  34. package/dist/push.d.ts +12 -3
  35. package/dist/push.js +361 -173
  36. package/dist/realtime.d.ts +85 -39
  37. package/dist/realtime.js +355 -219
  38. package/dist/relational-rows.d.ts +21 -66
  39. package/dist/relational-rows.js +28 -109
  40. package/dist/s3-blob-store.d.ts +11 -12
  41. package/dist/s3-blob-store.js +20 -13
  42. package/dist/s3-object-bucket.d.ts +8 -1
  43. package/dist/s3-object-bucket.js +16 -3
  44. package/dist/schema.d.ts +14 -0
  45. package/dist/schema.js +23 -0
  46. package/dist/scopes.d.ts +6 -2
  47. package/dist/scopes.js +22 -4
  48. package/dist/seed.js +10 -16
  49. package/dist/segment-download.d.ts +4 -1
  50. package/dist/segment-download.js +22 -6
  51. package/dist/signed-url.d.ts +2 -2
  52. package/dist/signed-url.js +9 -2
  53. package/dist/sigv4.d.ts +8 -2
  54. package/dist/sigv4.js +8 -4
  55. package/dist/sqlite-blob-store.d.ts +1 -1
  56. package/dist/sqlite-blob-store.js +23 -22
  57. package/dist/sqlite-dialect.d.ts +164 -84
  58. package/dist/sqlite-dialect.js +1145 -150
  59. package/dist/sqlite-lease-store.js +13 -18
  60. package/dist/sqlite-storage.d.ts +9 -62
  61. package/dist/sqlite-storage.js +303 -1123
  62. package/dist/storage-query.d.ts +30 -1
  63. package/dist/storage-query.js +68 -0
  64. package/dist/storage.d.ts +185 -63
  65. package/dist/storage.js +15 -0
  66. package/package.json +2 -2
  67. package/src/admin.ts +99 -57
  68. package/src/blob-handlers.ts +91 -72
  69. package/src/blob-store.ts +0 -0
  70. package/src/bucket-blob-store.ts +75 -48
  71. package/src/bucket-segment-store.ts +20 -15
  72. package/src/content-encoding.ts +48 -16
  73. package/src/context.ts +33 -6
  74. package/src/d1-storage.ts +350 -1700
  75. package/src/errors.ts +16 -0
  76. package/src/events.ts +37 -18
  77. package/src/frame-bytes.ts +26 -6
  78. package/src/handler.ts +654 -261
  79. package/src/local-authority.ts +27 -14
  80. package/src/object-bucket.ts +2 -0
  81. package/src/operations-realtime.ts +85 -61
  82. package/src/operations.ts +44 -42
  83. package/src/pg-executor-pglite.ts +27 -23
  84. package/src/pg-executor.ts +5 -1
  85. package/src/postgres-storage.ts +557 -354
  86. package/src/pull.ts +406 -440
  87. package/src/push.ts +466 -220
  88. package/src/realtime.ts +447 -290
  89. package/src/relational-rows.ts +43 -121
  90. package/src/s3-blob-store.ts +23 -13
  91. package/src/s3-object-bucket.ts +20 -2
  92. package/src/schema.ts +31 -0
  93. package/src/scopes.ts +23 -3
  94. package/src/seed.ts +13 -20
  95. package/src/segment-download.ts +33 -7
  96. package/src/signed-url.ts +10 -4
  97. package/src/sigv4.ts +17 -4
  98. package/src/sqlite-blob-store.ts +27 -26
  99. package/src/sqlite-dialect.ts +1851 -199
  100. package/src/sqlite-lease-store.ts +12 -23
  101. package/src/sqlite-storage.ts +390 -1821
  102. package/src/storage-query.ts +102 -0
  103. package/src/storage.ts +215 -67
@@ -51,6 +51,8 @@ export type LocalAuthorityAttachDetails = {
51
51
  readonly retryable: boolean;
52
52
  } | {
53
53
  readonly reason: 'log_epoch_changed';
54
+ } | {
55
+ readonly reason: 'undo_pending';
54
56
  } | {
55
57
  readonly reason: 'schema_floor';
56
58
  readonly requiredSchemaVersion: number;
@@ -4,7 +4,7 @@
4
4
  * with no remote server, and later moves the authority's rows to a remote
5
5
  * server with `attach` (§12.3).
6
6
  */
7
- import { decodeMessage, decodeRow, encodeMessage, encodeSparseRow, PROTOCOL_WIRE_VERSION, } from '@syncular/core';
7
+ import { decodeResponse, decodeRow, encodeRequest, encodeSparseRow, PROTOCOL_WIRE_VERSION, registrationDigest, } from '@syncular/core';
8
8
  import { SyncError } from './errors.js';
9
9
  import { handleSyncRequest } from './handler.js';
10
10
  import { pruneCommitLog } from './prune.js';
@@ -138,18 +138,11 @@ export async function openLocalAuthority(options) {
138
138
  return attachId;
139
139
  };
140
140
  const exchange = async (transport, frames) => {
141
- const message = decodeMessage(await transport(encodeMessage({
141
+ const message = decodeResponse(await transport(encodeRequest({
142
142
  wireVersion: PROTOCOL_WIRE_VERSION,
143
143
  msgKind: 'request',
144
144
  frames,
145
145
  })));
146
- if (message.msgKind !== 'response') {
147
- throw new LocalAuthorityAttachError({
148
- reason: 'request_failed',
149
- code: 'sync.invalid_request',
150
- retryable: false,
151
- });
152
- }
153
146
  const header = message.frames[0];
154
147
  if (header?.type === 'RESP_HEADER' && header.requiredSchemaVersion) {
155
148
  throw new LocalAuthorityAttachError({
@@ -180,6 +173,9 @@ export async function openLocalAuthority(options) {
180
173
  limitSnapshotRows: 0,
181
174
  maxSnapshotPages: 0,
182
175
  accept: 0b0011,
176
+ cursor: -1,
177
+ registrationDigest: registrationDigest([]),
178
+ echoDigest: new Uint8Array(),
183
179
  },
184
180
  ]))[0];
185
181
  const logEpoch = acquired?.type === 'RESP_HEADER' ? acquired.logEpoch : undefined;
@@ -268,7 +264,13 @@ export async function openLocalAuthority(options) {
268
264
  uploading = false;
269
265
  }
270
266
  }
271
- writeMeta({ state: 'local', attach_id: null, sent: null, applied: null });
267
+ writeMeta({
268
+ state: 'local',
269
+ attach_id: null,
270
+ sent: null,
271
+ applied: null,
272
+ undo: null,
273
+ });
272
274
  state = 'local';
273
275
  },
274
276
  };
@@ -287,6 +289,11 @@ export async function openLocalAuthority(options) {
287
289
  throw error;
288
290
  }
289
291
  }
292
+ else if (readMeta('undo') !== undefined) {
293
+ // §12.3: resuming after a started rollback would count `cached`
294
+ // uploads whose rows the rollback deleted.
295
+ throw new LocalAuthorityAttachError({ reason: 'undo_pending' });
296
+ }
290
297
  const attachId = attachIdOf();
291
298
  const commits = planAttach(database, compiled, attachId);
292
299
  const send = await connect(transport, `authority-${attachId}`);
@@ -312,7 +319,8 @@ export async function openLocalAuthority(options) {
312
319
  }
313
320
  /**
314
321
  * §12.3 abandon: settle the one commit whose outcome may be unknown, then
315
- * delete the rows of every applied commit, children before parents.
322
+ * delete the rows of every applied commit, children before parents, under
323
+ * commit ids fresh to this attempt (the persisted `undo` marker).
316
324
  */
317
325
  async function rollback(transport) {
318
326
  const attachId = attachIdOf();
@@ -335,10 +343,13 @@ export async function openLocalAuthority(options) {
335
343
  rowId: operation.rowId,
336
344
  op: 'delete',
337
345
  }));
346
+ const attempt = ordinal('undo') + 1;
347
+ writeMeta({ undo: String(attempt) });
338
348
  for (let start = 0; start < deletes.length; start += ATTACH_OPERATION_BUDGET) {
339
349
  const commit = {
340
350
  type: 'PUSH_COMMIT',
341
- clientCommitId: `${attachId}:undo:${start / ATTACH_OPERATION_BUDGET}`,
351
+ clientCommitId: `${attachId}:undo:${attempt}:${start / ATTACH_OPERATION_BUDGET}`,
352
+ clientSeq: 0,
342
353
  operations: deletes.slice(start, start + ATTACH_OPERATION_BUDGET),
343
354
  };
344
355
  const result = await send(commit);
@@ -439,6 +450,7 @@ function planAttach(database, schema, attachId) {
439
450
  commits.push({
440
451
  type: 'PUSH_COMMIT',
441
452
  clientCommitId: `${attachId}:${commits.length}`,
453
+ clientSeq: 0,
442
454
  operations,
443
455
  });
444
456
  operations = [];
@@ -32,6 +32,8 @@ export interface BucketObject {
32
32
  export interface BucketObjectHead {
33
33
  readonly etag: string | undefined;
34
34
  readonly size: number | undefined;
35
+ /** Backend last-modified time (epoch ms): S3 `Last-Modified`, R2 `uploaded`. */
36
+ readonly lastModifiedMs: number | undefined;
35
37
  readonly metadata: Readonly<Record<string, string>>;
36
38
  }
37
39
  export interface BucketWriteOptions {
@@ -1,12 +1,22 @@
1
1
  import { decodeRemoteOperationRealtimeMessage, encodeRemoteOperationRealtimeMessage, } from '@syncular/core';
2
- import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context.js';
2
+ import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context.js';
3
3
  import { reportError, SyncError, syncError } from './errors.js';
4
+ /**
5
+ * One socket's registered-query watches. A session runs one refresh at a
6
+ * time: `notify` queues each affected watch once, and the drain reruns the
7
+ * queue in order, so a commit costs at most one query per watch and never
8
+ * more than one query at a time.
9
+ */
4
10
  class WatchSession {
5
11
  #ctx;
6
12
  #registry;
7
13
  #send;
8
14
  #closed;
15
+ #maxWatches;
9
16
  #watches = new Map();
17
+ /** Watches waiting for a refresh, in arrival order, each at most once. */
18
+ #pending = new Set();
19
+ #draining;
10
20
  #isClosed = false;
11
21
  get partition() {
12
22
  return this.#ctx.partition;
@@ -16,6 +26,7 @@ class WatchSession {
16
26
  this.#registry = registry;
17
27
  this.#send = send;
18
28
  this.#closed = closed;
29
+ this.#maxWatches = limitsOf(ctx).maxWatchesPerSession;
19
30
  }
20
31
  async receive(bytes) {
21
32
  if (this.#isClosed)
@@ -46,6 +57,9 @@ class WatchSession {
46
57
  throw syncError('operation.invalid_request');
47
58
  }
48
59
  if (message.kind === 'unwatch') {
60
+ const state = this.#watches.get(message.watchId);
61
+ if (state !== undefined)
62
+ this.#pending.delete(state);
49
63
  this.#watches.delete(message.watchId);
50
64
  return;
51
65
  }
@@ -68,73 +82,77 @@ class WatchSession {
68
82
  this.#sendError(message.watchId, syncError('operation.unknown'));
69
83
  return;
70
84
  }
85
+ if (this.#watches.size >= this.#maxWatches) {
86
+ this.#sendError(message.watchId, syncError('sync.limit_exceeded', 'watch session holds the maximum number of watches', JSON.stringify({
87
+ limit: 'maxWatchesPerSession',
88
+ max: this.#maxWatches,
89
+ })));
90
+ return;
91
+ }
71
92
  const state = {
72
93
  watchId: message.watchId,
73
94
  clientId: message.clientId,
74
95
  operation,
75
96
  params: message.params,
76
- running: false,
77
- dirty: false,
78
97
  };
79
98
  this.#watches.set(message.watchId, state);
80
99
  await this.#refresh(state);
81
100
  }
82
101
  notify(tables) {
83
102
  for (const state of this.#watches.values()) {
84
- if (!state.operation.tables.some((table) => tables.has(table)))
85
- continue;
86
- if (state.running)
87
- state.dirty = true;
88
- else
103
+ if (state.operation.tables.some((table) => tables.has(table))) {
89
104
  void this.#refresh(state);
105
+ }
106
+ }
107
+ }
108
+ /** Queue `state` and resolve when the session's queue has drained. */
109
+ #refresh(state) {
110
+ this.#pending.add(state);
111
+ this.#draining ??= this.#drain();
112
+ return this.#draining;
113
+ }
114
+ async #drain() {
115
+ try {
116
+ // A watch queued again while it runs is appended and runs once more.
117
+ for (const state of this.#pending) {
118
+ this.#pending.delete(state);
119
+ await this.#run(state);
120
+ }
121
+ }
122
+ finally {
123
+ this.#draining = undefined;
90
124
  }
91
125
  }
92
- async #refresh(state) {
126
+ async #run(state) {
93
127
  if (this.#isClosed || this.#watches.get(state.watchId) !== state)
94
128
  return;
95
- if (state.running) {
96
- state.dirty = true;
97
- return;
98
- }
99
- state.running = true;
100
129
  try {
101
- do {
102
- state.dirty = false;
103
- try {
104
- const response = await state.operation.run(this.#ctx, state.clientId, state.params);
105
- if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
106
- return;
107
- }
108
- if (response.kind !== 'query') {
109
- throw syncError('operation.query_failed');
110
- }
111
- if (!this.#emit(encodeRemoteOperationRealtimeMessage({
112
- revision: 1,
113
- kind: 'snapshot',
114
- watchId: state.watchId,
115
- operationId: response.operationId,
116
- rows: response.rows,
117
- maxCommitSeq: response.maxCommitSeq,
118
- })))
119
- return;
120
- }
121
- catch (error) {
122
- if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
123
- return;
124
- }
125
- if (!(error instanceof SyncError)) {
126
- reportError(error, this.#ctx.onError, 'operations');
127
- }
128
- this.#sendError(state.watchId, error instanceof SyncError
129
- ? error
130
- : syncError('operation.query_failed'));
131
- }
132
- } while (state.dirty &&
133
- !this.#isClosed &&
134
- this.#watches.get(state.watchId) === state);
130
+ const response = await state.operation.run(this.#ctx, state.clientId, state.params);
131
+ if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
132
+ return;
133
+ }
134
+ if (response.kind !== 'query') {
135
+ throw syncError('operation.query_failed');
136
+ }
137
+ this.#emit(encodeRemoteOperationRealtimeMessage({
138
+ revision: 1,
139
+ kind: 'snapshot',
140
+ watchId: state.watchId,
141
+ operationId: response.operationId,
142
+ rows: response.rows,
143
+ maxCommitSeq: response.maxCommitSeq,
144
+ }));
135
145
  }
136
- finally {
137
- state.running = false;
146
+ catch (error) {
147
+ if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
148
+ return;
149
+ }
150
+ if (!(error instanceof SyncError)) {
151
+ reportError(error, this.#ctx.onError, 'operations');
152
+ }
153
+ this.#sendError(state.watchId, error instanceof SyncError
154
+ ? error
155
+ : syncError('operation.query_failed'));
138
156
  }
139
157
  }
140
158
  #sendError(watchId, error) {
@@ -164,6 +182,7 @@ class WatchSession {
164
182
  return;
165
183
  this.#isClosed = true;
166
184
  this.#watches.clear();
185
+ this.#pending.clear();
167
186
  this.#closed();
168
187
  }
169
188
  }
@@ -7,11 +7,14 @@ export interface RemoteQueryDependency {
7
7
  readonly table: string;
8
8
  readonly scopeKeys?: readonly string[];
9
9
  }
10
+ /**
11
+ * The window units a scoped query reads (§4.8): the client's `WindowBase`
12
+ * (a variable and one value per other variable) and the variable's values.
13
+ */
10
14
  export interface RemoteQueryCoverage {
11
15
  readonly base: {
12
- readonly table: string;
13
16
  readonly variable: string;
14
- readonly fixedScopes?: Readonly<Record<string, readonly string[]>>;
17
+ readonly fixedScopes?: Readonly<Record<string, string>>;
15
18
  };
16
19
  readonly units: readonly string[];
17
20
  }
@@ -136,43 +136,34 @@ export function registerRemoteQuery(descriptor, options) {
136
136
  throw syncError('operation.forbidden', 'live scope authorization is unavailable for this query');
137
137
  }
138
138
  const coverage = descriptor.coverage(params);
139
- const coverageByTable = new Map();
140
- for (const entry of coverage) {
141
- if (coverageByTable.has(entry.base.table) ||
142
- !descriptor.tables.includes(entry.base.table)) {
143
- throw syncError('operation.invalid_request', 'scoped remote query has invalid generated scope coverage');
144
- }
145
- coverageByTable.set(entry.base.table, entry);
146
- }
147
- if (descriptor.tables.some((table) => !coverageByTable.has(table))) {
139
+ // A unit names every variable of a table it lets a query read in
140
+ // full: a variable left open would read rows outside the grant, and
141
+ // the wildcard fence (§3.2) has no counterpart in a SQL read.
142
+ const exact = (entry, declared) => {
143
+ const variables = new Set([
144
+ entry.base.variable,
145
+ ...Object.keys(entry.base.fixedScopes ?? {}),
146
+ ]);
147
+ return (variables.size === declared.size &&
148
+ [...declared].every((variable) => variables.has(variable)));
149
+ };
150
+ const declared = descriptor.tables.map((name) => schema.tables.get(name)?.declaredVariables);
151
+ if (declared.some((variables) => variables === undefined ||
152
+ !coverage.some((entry) => exact(entry, variables)))) {
148
153
  throw syncError('operation.invalid_request', 'scoped remote query lacks complete generated scope coverage');
149
154
  }
150
155
  for (const entry of coverage) {
151
- const table = schema.tables.get(entry.base.table);
152
156
  const fixedScopes = entry.base.fixedScopes ?? {};
153
- const coveredVariables = new Set([
154
- entry.base.variable,
155
- ...Object.keys(fixedScopes),
156
- ]);
157
- if (table === undefined ||
158
- Object.prototype.hasOwnProperty.call(fixedScopes, entry.base.variable) ||
159
- coveredVariables.size !== table.declaredVariables.size ||
160
- [...table.declaredVariables].some((variable) => !coveredVariables.has(variable))) {
161
- throw syncError('operation.invalid_request', 'scoped remote query lacks complete generated scope coverage');
157
+ if (Object.hasOwn(fixedScopes, entry.base.variable) ||
158
+ !declared.some((variables) => variables !== undefined && exact(entry, variables))) {
159
+ throw syncError('operation.invalid_request', 'scoped remote query has invalid generated scope coverage');
162
160
  }
163
161
  if (entry.units.length === 0) {
164
162
  throw syncError('operation.invalid_request', 'scoped remote query has an empty scope unit');
165
163
  }
166
- for (const value of entry.units) {
167
- if (!scopeAllowed(allowed, entry.base.variable, value)) {
168
- throw syncError('operation.forbidden');
169
- }
170
- }
171
- for (const [variable, values] of Object.entries(fixedScopes)) {
172
- if (values.length === 0 ||
173
- values.some((value) => !scopeAllowed(allowed, variable, value))) {
174
- throw syncError('operation.forbidden');
175
- }
164
+ if (entry.units.some((value) => !scopeAllowed(allowed, entry.base.variable, value)) ||
165
+ Object.entries(fixedScopes).some(([variable, value]) => !scopeAllowed(allowed, variable, value))) {
166
+ throw syncError('operation.forbidden');
176
167
  }
177
168
  }
178
169
  }
@@ -8,11 +8,11 @@
8
8
  * `PgExecutor` interface (see the server README).
9
9
  *
10
10
  * pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
11
- * `ROLLBACK` on the one connection. Overlapping `transaction` calls are
12
- * serialized through a promise chain (mirroring `SqliteServerStorage`'s
13
- * begin() FIFO): a nested BEGIN on Postgres is a warning-level no-op, so
14
- * interleaved scopes would silently collapse into one SQL transaction and
15
- * break the push layer's serialization guarantee on this dev driver.
11
+ * `ROLLBACK` on the one connection. Transactions and pool-level `query` calls
12
+ * share one promise chain (mirroring `SqliteServerStorage`): a nested BEGIN
13
+ * on Postgres is a warning-level no-op, so interleaved scopes would silently
14
+ * collapse into one SQL transaction, and a pool-level statement issued while
15
+ * a transaction is open would read its uncommitted rows or join it (§6.4).
16
16
  */
17
17
  import type { PGlite } from '@electric-sql/pglite';
18
18
  import type { PgExecutor } from './pg-executor.js';
@@ -20,7 +20,6 @@ import type { PgExecutor } from './pg-executor.js';
20
20
  interface PgliteLike {
21
21
  query<Row>(query: string, params?: unknown[]): Promise<{
22
22
  rows: Row[];
23
- affectedRows?: number;
24
23
  }>;
25
24
  exec(query: string): Promise<unknown>;
26
25
  close(): Promise<void>;
@@ -3,28 +3,36 @@ function queryable(db) {
3
3
  return {
4
4
  async query(text, params) {
5
5
  const result = await db.query(text, params ? [...params] : undefined);
6
- return {
7
- rows: result.rows,
8
- rowCount: result.affectedRows ?? result.rows.length,
9
- };
6
+ return { rows: result.rows };
10
7
  },
11
8
  };
12
9
  }
13
10
  export function pgliteExecutor(db) {
14
11
  const like = db;
15
12
  const q = queryable(like);
16
- // One BEGIN…COMMIT/ROLLBACK scope at a time on the single connection (see
17
- // the file header).
18
- let transactionTail = Promise.resolve();
13
+ // One caller at a time on the single connection (see the file header).
14
+ let tail = Promise.resolve();
15
+ const acquire = async () => {
16
+ const previous = tail;
17
+ let release;
18
+ tail = new Promise((resolve) => {
19
+ release = resolve;
20
+ });
21
+ await previous;
22
+ return release;
23
+ };
19
24
  return {
20
- query: q.query,
25
+ async query(text, params) {
26
+ const release = await acquire();
27
+ try {
28
+ return await q.query(text, params);
29
+ }
30
+ finally {
31
+ release();
32
+ }
33
+ },
21
34
  async transaction(fn) {
22
- const previous = transactionTail;
23
- let release;
24
- transactionTail = new Promise((resolve) => {
25
- release = resolve;
26
- });
27
- await previous;
35
+ const release = await acquire();
28
36
  try {
29
37
  await like.exec('BEGIN');
30
38
  try {
@@ -47,11 +47,14 @@ export type PgRow = Record<string, unknown>;
47
47
  * a time. An adapter does not need to queue calls itself, and must not
48
48
  * accept two in flight: node-postgres warns about that today and rejects it
49
49
  * in node-postgres 9.
50
+ *
51
+ * The result carries returned rows only. Drivers disagree on the affected-row
52
+ * count of a statement without result rows (Bun.sql reports none), so every
53
+ * write whose outcome matters ends in `RETURNING` and counts `rows`.
50
54
  */
51
55
  export interface PgQueryable {
52
56
  query<Row = PgRow>(text: string, params?: readonly unknown[]): Promise<{
53
57
  rows: Row[];
54
- rowCount: number;
55
58
  }>;
56
59
  }
57
60
  /**
@@ -9,13 +9,13 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
9
9
  *
10
10
  * - both the commit log and the current-row table carry a (table, var,
11
11
  * value) inverted scope index;
12
- * - `readCommitWindow`/`scanRows` select candidates through that index
13
- * (`sync_change_scopes` / `sync_row_scopes`), ordered + LIMITed at the
14
- * index, then verify the full multi-variable match against the stored
15
- * scope map — never a log or table scan;
12
+ * - `readLiveWindow`/`scanRows` select candidates through an index
13
+ * (`sync_change_scopes_by_key` / `sync_row_scopes`), ordered + LIMITed
14
+ * at the index, then verify the full multi-variable match against the
15
+ * stored scope map — never a log or table scan;
16
16
  * - the candidate indexes are *covering* for their selection: the ordered
17
- * column (`commit_seq` / `row_id`) sits in the index key after
18
- * (tbl, var, value), so the planner does an index range scan and returns
17
+ * column (`commit_seq` / `row_id`) sits in the index key after the scope
18
+ * key, so the planner does an index range scan and returns
19
19
  * already-ordered candidates without a heap sort. `test/postgres-explain
20
20
  * .test.ts` asserts an `Index` node (never `Seq Scan`) so the regression
21
21
  * cannot silently return.
@@ -36,21 +36,26 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
36
36
  * would leave gaps on rollback, which the pull-window arithmetic in §4.5
37
37
  * does not tolerate). Cross-partition pushes never contend.
38
38
  */
39
- import type { ScopeMap } from '@syncular/core';
39
+ import type { UnitScopes } from '@syncular/core';
40
40
  import { type PgExecutor } from './pg-executor.js';
41
41
  import type { CompiledSchema, CompiledTable } from './schema.js';
42
- import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
42
+ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, IndexRowScanQuery, LiveWindow, LiveWindowQuery, PartitionRegistryEntry, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
43
43
  /**
44
44
  * Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
45
45
  *
46
46
  * Covering index design (the reason this file exists):
47
- * - `sync_change_scopes_pk (partition, tbl, var, value, commit_seq)` — the
48
- * candidate scan for `readCommitWindow` ranges on the (partition, tbl,
49
- * var, value IN …) prefix and returns `commit_seq` already ascending;
50
- * - `sync_row_scopes_pk (partition, tbl, var, value, row_id)` — same shape
51
- * for `scanRows`, ordered by `row_id`.
52
- * Both are the PRIMARY KEY, so they are the clustering/covering index for
53
- * their table.
47
+ * - `sync_change_scopes_by_key (partition, var, value, commit_seq)`: the
48
+ * §4.5 live scan ranges once per unit key on the (partition, var, value)
49
+ * prefix across tables and returns `commit_seq` already ascending;
50
+ * - `sync_change_scopes_pk (partition, tbl, var, value, commit_seq)`:
51
+ * the §4.7 scope pin, erasure and pruning name the table;
52
+ * - `sync_row_scopes_pk (partition, tbl, var, value, row_id)`: the same
53
+ * shape for `scanRows`, ordered by `row_id`.
54
+ *
55
+ * The §6.3.2 watermark delete and the §9 settlement of results stored
56
+ * before wire version 4 (NULL `outbox_id`) range over the `(partition,
57
+ * client_id)` prefix of `sync_push_results`' PRIMARY KEY, which the
58
+ * watermark keeps to one request's results per outbox.
54
59
  *
55
60
  * `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
56
61
  * scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
@@ -69,7 +74,7 @@ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDec
69
74
  * set, §5.9.5) as an index range, never a scan. `postgres-explain
70
75
  * .test.ts` asserts an `Index` node here too.
71
76
  */
72
- export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n last_round_at_ms BIGINT, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
77
+ export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0,\n delete_floor BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_change_scopes_by_key\n ON sync_change_scopes(partition, var, value, commit_seq);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n outbox_id TEXT, client_seq BIGINT,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS outbox_id TEXT;\nALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS client_seq BIGINT;\nCREATE TABLE IF NOT EXISTS sync_push_watermarks(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,\n through_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, client_id, outbox_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n last_round_at_ms BIGINT, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n schema_version INTEGER, units JSONB, registration_digest BYTEA,\n catching_up JSONB NOT NULL DEFAULT '[]',\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS schema_version INTEGER;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS units JSONB;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS registration_digest BYTEA;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS catching_up JSONB NOT NULL DEFAULT '[]';\n-- \u00A79: a database from before wire version 4 (its client records still hold\n-- subscriptions) starts the horizon at its last commit.\nUPDATE sync_partitions SET horizon_seq = max_commit_seq\n WHERE EXISTS (SELECT 1 FROM information_schema.columns\n WHERE table_schema = current_schema()\n AND table_name = 'sync_clients'\n AND column_name = 'subscriptions');\nALTER TABLE sync_clients DROP COLUMN IF EXISTS subscriptions;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,\n PRIMARY KEY(partition, tbl, row_id)\n);\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;\nALTER TABLE sync_partitions\n ADD COLUMN IF NOT EXISTS delete_floor BIGINT NOT NULL DEFAULT 0;\nUPDATE sync_partitions SET delete_floor = t.version\n FROM (SELECT partition, max(server_version) AS version\n FROM sync_tombstones GROUP BY partition) t\n WHERE t.partition = sync_partitions.partition\n AND t.version > sync_partitions.delete_floor;\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
73
78
  export declare class PostgresServerStorage implements ServerStorage {
74
79
  #private;
75
80
  constructor(exec: PgExecutor);
@@ -123,16 +128,15 @@ export declare class PostgresServerStorage implements ServerStorage {
123
128
  pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
124
129
  latestScopedChangeSeq(partition: string, query: {
125
130
  readonly table: string;
126
- readonly scopeFilter: ScopeMap;
131
+ readonly scopes: UnitScopes;
127
132
  }): Promise<number>;
128
- readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
133
+ readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
129
134
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
130
135
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
131
136
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
132
137
  putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
133
138
  recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
134
139
  advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
135
- updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
136
140
  getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
137
141
  listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
138
142
  listRowsReferencingBlob(partition: string, blobId: string): Promise<{
@@ -150,6 +154,7 @@ export declare class PostgresServerStorage implements ServerStorage {
150
154
  listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
151
155
  countRows(partition: string, table: string): Promise<number>;
152
156
  listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
157
+ listPushWatermarks(partition: string, clientCommitId: string): Promise<PushWatermark[]>;
153
158
  scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
154
159
  listPartitions(): Promise<string[]>;
155
160
  }