@syncular/server 0.58.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
@@ -4,7 +4,7 @@ import {
4
4
  type RemoteOperationRealtimeMessage,
5
5
  } from '@syncular/core';
6
6
  import type { RealtimeNotifier, SyncRequestContext } from './context';
7
- import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context';
7
+ import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context';
8
8
  import { reportError, SyncError, syncError } from './errors';
9
9
  import type {
10
10
  RegisteredRemoteQuery,
@@ -22,16 +22,24 @@ interface WatchState {
22
22
  readonly clientId: string;
23
23
  readonly operation: RegisteredRemoteQuery;
24
24
  readonly params: unknown;
25
- running: boolean;
26
- dirty: boolean;
27
25
  }
28
26
 
27
+ /**
28
+ * One socket's registered-query watches. A session runs one refresh at a
29
+ * time: `notify` queues each affected watch once, and the drain reruns the
30
+ * queue in order, so a commit costs at most one query per watch and never
31
+ * more than one query at a time.
32
+ */
29
33
  class WatchSession implements RemoteOperationWatchSession {
30
34
  readonly #ctx: SyncRequestContext;
31
35
  readonly #registry: RemoteOperationRegistry;
32
36
  readonly #send: (bytes: Uint8Array) => void;
33
37
  readonly #closed: () => void;
38
+ readonly #maxWatches: number;
34
39
  readonly #watches = new Map<string, WatchState>();
40
+ /** Watches waiting for a refresh, in arrival order, each at most once. */
41
+ readonly #pending = new Set<WatchState>();
42
+ #draining: Promise<void> | undefined;
35
43
  #isClosed = false;
36
44
 
37
45
  get partition(): string {
@@ -48,6 +56,7 @@ class WatchSession implements RemoteOperationWatchSession {
48
56
  this.#registry = registry;
49
57
  this.#send = send;
50
58
  this.#closed = closed;
59
+ this.#maxWatches = limitsOf(ctx).maxWatchesPerSession;
51
60
  }
52
61
 
53
62
  async receive(bytes: Uint8Array): Promise<void> {
@@ -80,6 +89,8 @@ class WatchSession implements RemoteOperationWatchSession {
80
89
  throw syncError('operation.invalid_request');
81
90
  }
82
91
  if (message.kind === 'unwatch') {
92
+ const state = this.#watches.get(message.watchId);
93
+ if (state !== undefined) this.#pending.delete(state);
83
94
  this.#watches.delete(message.watchId);
84
95
  return;
85
96
  }
@@ -122,13 +133,25 @@ class WatchSession implements RemoteOperationWatchSession {
122
133
  this.#sendError(message.watchId, syncError('operation.unknown'));
123
134
  return;
124
135
  }
136
+ if (this.#watches.size >= this.#maxWatches) {
137
+ this.#sendError(
138
+ message.watchId,
139
+ syncError(
140
+ 'sync.limit_exceeded',
141
+ 'watch session holds the maximum number of watches',
142
+ JSON.stringify({
143
+ limit: 'maxWatchesPerSession',
144
+ max: this.#maxWatches,
145
+ }),
146
+ ),
147
+ );
148
+ return;
149
+ }
125
150
  const state: WatchState = {
126
151
  watchId: message.watchId,
127
152
  clientId: message.clientId,
128
153
  operation,
129
154
  params: message.params,
130
- running: false,
131
- dirty: false,
132
155
  };
133
156
  this.#watches.set(message.watchId, state);
134
157
  await this.#refresh(state);
@@ -136,68 +159,68 @@ class WatchSession implements RemoteOperationWatchSession {
136
159
 
137
160
  notify(tables: ReadonlySet<string>): void {
138
161
  for (const state of this.#watches.values()) {
139
- if (!state.operation.tables.some((table) => tables.has(table))) continue;
140
- if (state.running) state.dirty = true;
141
- else void this.#refresh(state);
162
+ if (state.operation.tables.some((table) => tables.has(table))) {
163
+ void this.#refresh(state);
164
+ }
142
165
  }
143
166
  }
144
167
 
145
- async #refresh(state: WatchState): Promise<void> {
146
- if (this.#isClosed || this.#watches.get(state.watchId) !== state) return;
147
- if (state.running) {
148
- state.dirty = true;
149
- return;
168
+ /** Queue `state` and resolve when the session's queue has drained. */
169
+ #refresh(state: WatchState): Promise<void> {
170
+ this.#pending.add(state);
171
+ this.#draining ??= this.#drain();
172
+ return this.#draining;
173
+ }
174
+
175
+ async #drain(): Promise<void> {
176
+ try {
177
+ // A watch queued again while it runs is appended and runs once more.
178
+ for (const state of this.#pending) {
179
+ this.#pending.delete(state);
180
+ await this.#run(state);
181
+ }
182
+ } finally {
183
+ this.#draining = undefined;
150
184
  }
151
- state.running = true;
185
+ }
186
+
187
+ async #run(state: WatchState): Promise<void> {
188
+ if (this.#isClosed || this.#watches.get(state.watchId) !== state) return;
152
189
  try {
153
- do {
154
- state.dirty = false;
155
- try {
156
- const response = await state.operation.run(
157
- this.#ctx,
158
- state.clientId,
159
- state.params,
160
- );
161
- if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
162
- return;
163
- }
164
- if (response.kind !== 'query') {
165
- throw syncError('operation.query_failed');
166
- }
167
- if (
168
- !this.#emit(
169
- encodeRemoteOperationRealtimeMessage({
170
- revision: 1,
171
- kind: 'snapshot',
172
- watchId: state.watchId,
173
- operationId: response.operationId,
174
- rows: response.rows,
175
- maxCommitSeq: response.maxCommitSeq,
176
- }),
177
- )
178
- )
179
- return;
180
- } catch (error) {
181
- if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
182
- return;
183
- }
184
- if (!(error instanceof SyncError)) {
185
- reportError(error, this.#ctx.onError, 'operations');
186
- }
187
- this.#sendError(
188
- state.watchId,
189
- error instanceof SyncError
190
- ? error
191
- : syncError('operation.query_failed'),
192
- );
193
- }
194
- } while (
195
- state.dirty &&
196
- !this.#isClosed &&
197
- this.#watches.get(state.watchId) === state
190
+ const response = await state.operation.run(
191
+ this.#ctx,
192
+ state.clientId,
193
+ state.params,
194
+ );
195
+ if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
196
+ return;
197
+ }
198
+ if (response.kind !== 'query') {
199
+ throw syncError('operation.query_failed');
200
+ }
201
+ this.#emit(
202
+ encodeRemoteOperationRealtimeMessage({
203
+ revision: 1,
204
+ kind: 'snapshot',
205
+ watchId: state.watchId,
206
+ operationId: response.operationId,
207
+ rows: response.rows,
208
+ maxCommitSeq: response.maxCommitSeq,
209
+ }),
210
+ );
211
+ } catch (error) {
212
+ if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
213
+ return;
214
+ }
215
+ if (!(error instanceof SyncError)) {
216
+ reportError(error, this.#ctx.onError, 'operations');
217
+ }
218
+ this.#sendError(
219
+ state.watchId,
220
+ error instanceof SyncError
221
+ ? error
222
+ : syncError('operation.query_failed'),
198
223
  );
199
- } finally {
200
- state.running = false;
201
224
  }
202
225
  }
203
226
 
@@ -229,6 +252,7 @@ class WatchSession implements RemoteOperationWatchSession {
229
252
  if (this.#isClosed) return;
230
253
  this.#isClosed = true;
231
254
  this.#watches.clear();
255
+ this.#pending.clear();
232
256
  this.#closed();
233
257
  }
234
258
  }
@@ -9,21 +9,18 @@ import { retainRollbackFailure } from './storage-errors';
9
9
  * `PgExecutor` interface (see the server README).
10
10
  *
11
11
  * pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
12
- * `ROLLBACK` on the one connection. Overlapping `transaction` calls are
13
- * serialized through a promise chain (mirroring `SqliteServerStorage`'s
14
- * begin() FIFO): a nested BEGIN on Postgres is a warning-level no-op, so
15
- * interleaved scopes would silently collapse into one SQL transaction and
16
- * break the push layer's serialization guarantee on this dev driver.
12
+ * `ROLLBACK` on the one connection. Transactions and pool-level `query` calls
13
+ * share one promise chain (mirroring `SqliteServerStorage`): a nested BEGIN
14
+ * on Postgres is a warning-level no-op, so interleaved scopes would silently
15
+ * collapse into one SQL transaction, and a pool-level statement issued while
16
+ * a transaction is open would read its uncommitted rows or join it (§6.4).
17
17
  */
18
18
  import type { PGlite } from '@electric-sql/pglite';
19
19
  import type { PgExecutor, PgQueryable, PgRow } from './pg-executor';
20
20
 
21
21
  /** Minimal structural type for a pglite instance (avoids a hard type dep). */
22
22
  interface PgliteLike {
23
- query<Row>(
24
- query: string,
25
- params?: unknown[],
26
- ): Promise<{ rows: Row[]; affectedRows?: number }>;
23
+ query<Row>(query: string, params?: unknown[]): Promise<{ rows: Row[] }>;
27
24
  exec(query: string): Promise<unknown>;
28
25
  close(): Promise<void>;
29
26
  }
@@ -35,10 +32,7 @@ function queryable(db: PgliteLike): PgQueryable {
35
32
  text,
36
33
  params ? [...params] : undefined,
37
34
  );
38
- return {
39
- rows: result.rows,
40
- rowCount: result.affectedRows ?? result.rows.length,
41
- };
35
+ return { rows: result.rows };
42
36
  },
43
37
  };
44
38
  }
@@ -46,18 +40,28 @@ function queryable(db: PgliteLike): PgQueryable {
46
40
  export function pgliteExecutor(db: PGlite | PgliteLike): PgExecutor {
47
41
  const like = db as PgliteLike;
48
42
  const q = queryable(like);
49
- // One BEGIN…COMMIT/ROLLBACK scope at a time on the single connection (see
50
- // the file header).
51
- let transactionTail: Promise<void> = Promise.resolve();
43
+ // One caller at a time on the single connection (see the file header).
44
+ let tail: Promise<void> = Promise.resolve();
45
+ const acquire = async (): Promise<() => void> => {
46
+ const previous = tail;
47
+ let release!: () => void;
48
+ tail = new Promise<void>((resolve) => {
49
+ release = resolve;
50
+ });
51
+ await previous;
52
+ return release;
53
+ };
52
54
  return {
53
- query: q.query,
55
+ async query<Row = PgRow>(text: string, params?: readonly unknown[]) {
56
+ const release = await acquire();
57
+ try {
58
+ return await q.query<Row>(text, params);
59
+ } finally {
60
+ release();
61
+ }
62
+ },
54
63
  async transaction<T>(fn: (client: PgQueryable) => Promise<T>): Promise<T> {
55
- const previous = transactionTail;
56
- let release!: () => void;
57
- transactionTail = new Promise<void>((resolve) => {
58
- release = resolve;
59
- });
60
- await previous;
64
+ const release = await acquire();
61
65
  try {
62
66
  await like.exec('BEGIN');
63
67
  try {
@@ -49,12 +49,16 @@ export type PgRow = Record<string, unknown>;
49
49
  * a time. An adapter does not need to queue calls itself, and must not
50
50
  * accept two in flight: node-postgres warns about that today and rejects it
51
51
  * in node-postgres 9.
52
+ *
53
+ * The result carries returned rows only. Drivers disagree on the affected-row
54
+ * count of a statement without result rows (Bun.sql reports none), so every
55
+ * write whose outcome matters ends in `RETURNING` and counts `rows`.
52
56
  */
53
57
  export interface PgQueryable {
54
58
  query<Row = PgRow>(
55
59
  text: string,
56
60
  params?: readonly unknown[],
57
- ): Promise<{ rows: Row[]; rowCount: number }>;
61
+ ): Promise<{ rows: Row[] }>;
58
62
  }
59
63
 
60
64
  /**