@syncular/server 0.58.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
@@ -138,38 +138,26 @@ export class BucketBlobStore {
138
138
  };
139
139
  const objectKey = this.objectKeyFor(partition, blobId);
140
140
  // Idempotent re-put (same content-address ⇒ same key): count each distinct
141
- // blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
142
- // is not reset by a re-upload (§5.9.2 upload age).
141
+ // blob once. A re-upload rewrites the object so its metadata carries the
142
+ // re-upload time (the sweep's grace clock, §5.9.2) and keeps a recorded
143
+ // media type.
143
144
  const preexisting = await this.#bucket.head(objectKey);
144
145
  const storedMeta = preexisting?.metadata[BLOB_META_KEY];
145
146
  const existing = storedMeta === undefined
146
147
  ? undefined
147
148
  : parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
148
- if (existing !== undefined &&
149
- (existing.mediaType !== undefined || mediaType === undefined)) {
150
- return {
151
- blobId,
152
- partition,
153
- byteLength: existing.byteLength,
154
- ...(existing.mediaType !== undefined
155
- ? { mediaType: existing.mediaType }
156
- : {}),
157
- createdAtMs: existing.createdAtMs,
158
- };
159
- }
160
- // A re-upload that names the media type an earlier upload left out
161
- // rewrites the object under the same precondition, keeping its age.
149
+ const storedType = existing?.mediaType ?? mediaType;
162
150
  const meta = {
163
151
  byteLength: bytes.length,
164
- ...(mediaType !== undefined ? { mediaType } : {}),
165
- createdAtMs: existing?.createdAtMs ?? nowMs,
152
+ ...(storedType !== undefined ? { mediaType: storedType } : {}),
153
+ createdAtMs: Math.max(existing?.createdAtMs ?? nowMs, nowMs),
166
154
  };
167
155
  // Conditional write: of two concurrent uploads of the same blob exactly
168
- // one lands, so the first createdAtMs survives and the blob counts once.
169
- // The loser re-reads and returns the winner's record.
156
+ // one lands per pre-state and the blob counts once. The loser re-reads
157
+ // and writes over the winner.
170
158
  const written = await this.#bucket.put(objectKey, {
171
159
  body: bytes,
172
- contentType: mediaType ?? 'application/octet-stream',
160
+ contentType: storedType ?? 'application/octet-stream',
173
161
  metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
174
162
  ...(preexisting === undefined
175
163
  ? { createOnly: true }
@@ -179,7 +167,11 @@ export class BucketBlobStore {
179
167
  return this.put(partition, blobId, bytes, nowMs, mediaType);
180
168
  if (preexisting === undefined)
181
169
  await this.#bumpStats({ count: 1, bytes: bytes.length });
182
- return { ...record, createdAtMs: meta.createdAtMs };
170
+ return {
171
+ ...record,
172
+ ...(storedType !== undefined ? { mediaType: storedType } : {}),
173
+ createdAtMs: meta.createdAtMs,
174
+ };
183
175
  }
184
176
  async delete(partition, blobId) {
185
177
  await this.#bucket.delete(this.objectKeyFor(partition, blobId));
@@ -191,35 +183,51 @@ export class BucketBlobStore {
191
183
  return head !== undefined;
192
184
  }
193
185
  async get(partition, blobId) {
194
- if (!BLOB_ID_PATTERN.test(blobId))
195
- return undefined;
196
- const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
186
+ const object = await this.#object(partition, blobId);
197
187
  if (object === undefined)
198
188
  return undefined;
199
189
  const bytes = new Uint8Array(await object.arrayBuffer());
190
+ return {
191
+ record: this.#record(partition, blobId, object, bytes.length),
192
+ bytes,
193
+ };
194
+ }
195
+ /** The record plus the object GET's body, never buffered (see `get`). */
196
+ async open(partition, blobId) {
197
+ const object = await this.#object(partition, blobId);
198
+ if (object === undefined || object.body === null)
199
+ return undefined;
200
+ const record = this.#record(partition, blobId, object, object.size ?? 0);
201
+ return { record, body: object.body, byteLength: record.byteLength };
202
+ }
203
+ async #object(partition, blobId) {
204
+ if (!BLOB_ID_PATTERN.test(blobId))
205
+ return undefined;
206
+ return this.#bucket.get(this.objectKeyFor(partition, blobId));
207
+ }
208
+ /** An object without Syncular metadata was written by a presigned PUT. */
209
+ #record(partition, blobId, object, byteLength) {
200
210
  const rawMeta = object.metadata[BLOB_META_KEY];
201
211
  const meta = rawMeta !== undefined
202
212
  ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
203
- : { byteLength: bytes.length, createdAtMs: 0 };
213
+ : { byteLength, createdAtMs: 0 };
204
214
  return {
205
- record: {
206
- blobId,
207
- partition,
208
- byteLength: meta.byteLength,
209
- ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
210
- createdAtMs: meta.createdAtMs,
211
- },
212
- bytes,
215
+ blobId,
216
+ partition,
217
+ byteLength: meta.byteLength,
218
+ ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
219
+ createdAtMs: meta.createdAtMs,
213
220
  };
214
221
  }
215
222
  /**
216
223
  * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
217
- * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
218
- * the ones both older than `olderThanMs` AND absent from
219
- * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
220
- * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
224
+ * {partition}/` prefix, HEAD each object for its upload time (the metadata
225
+ * `createdAtMs`, else the backend's last-modified time; an object with
226
+ * neither is skipped), and DELETE the ones older than `olderThanMs` that
227
+ * `isReferenced` denies right before the delete. Returns the deleted
228
+ * blobIds. Off the hot path — a host-scheduled GC job.
221
229
  */
222
- async sweepOrphans(partition, olderThanMs, referencedBlobIds) {
230
+ async sweepOrphans(partition, olderThanMs, isReferenced) {
223
231
  const listPrefix = `${this.#prefix}blob/${partition}/`;
224
232
  const swept = [];
225
233
  let deletedBytes = 0;
@@ -235,20 +243,21 @@ export class BucketBlobStore {
235
243
  const blobId = `sha256:${rest.slice('sha256/'.length)}`;
236
244
  if (!BLOB_ID_PATTERN.test(blobId))
237
245
  continue;
238
- if (referencedBlobIds.has(blobId))
239
- continue;
240
246
  const head = await this.#bucket.head(key);
241
247
  if (head === undefined)
242
248
  continue;
243
249
  const rawMeta = head.metadata[BLOB_META_KEY];
244
- const meta = rawMeta !== undefined
250
+ const uploadedAtMs = rawMeta !== undefined
245
251
  ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
246
- : { byteLength: head.size ?? 0, createdAtMs: 0 };
247
- if (meta.createdAtMs >= olderThanMs)
252
+ .createdAtMs
253
+ : head.lastModifiedMs;
254
+ if (uploadedAtMs === undefined || uploadedAtMs >= olderThanMs)
255
+ continue;
256
+ if (await isReferenced(blobId))
248
257
  continue;
249
258
  await this.#bucket.delete(key);
250
259
  swept.push(blobId);
251
- deletedBytes += meta.byteLength;
260
+ deletedBytes += head.size ?? 0;
252
261
  }
253
262
  } while (cursor !== undefined);
254
263
  if (swept.length > 0) {
@@ -1,4 +1,4 @@
1
- import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentBytesEqual, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
1
+ import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
2
2
  import { sha256Hex } from './sigv4.js';
3
3
  const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
4
4
  const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
@@ -311,25 +311,27 @@ export class BucketSegmentStore {
311
311
  throw new Error(`${this.#label}: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`);
312
312
  }
313
313
  /**
314
- * Write the immutable bytes object. Create-only, so a second publisher of
315
- * identical bytes is a no-op and a second publisher of DIFFERENT bytes under
316
- * the same content address is the hash collision §5.1 requires to fail
317
- * loudly instead of overwriting.
314
+ * Write the immutable bytes object unless it is stored already. A HEAD
315
+ * comes first, so republishing a segment (every bootstrap pull of an
316
+ * unchanged window) moves no bytes. The write is create-only, so two first
317
+ * publishers cannot overwrite each other. A stored object under the same
318
+ * content address with a different size is the hash collision §5.1
319
+ * requires to fail loudly; equal size under one SHA-256 address is the same
320
+ * content.
318
321
  */
319
322
  async #putBytes(segmentId, bytes) {
320
323
  const key = this.objectKeyFor(segmentId);
321
- const written = await this.#bucket.put(key, {
322
- body: bytes,
323
- contentType: 'application/octet-stream',
324
- createOnly: true,
325
- });
326
- if (written)
327
- return;
328
- const current = await this.#bucket.get(key);
329
- if (current === undefined)
324
+ const head = await this.#bucket.head(key);
325
+ if (head === undefined &&
326
+ (await this.#bucket.put(key, {
327
+ body: bytes,
328
+ contentType: 'application/octet-stream',
329
+ createOnly: true,
330
+ }))) {
330
331
  return;
331
- const stored = new Uint8Array(await current.arrayBuffer());
332
- if (!segmentBytesEqual(stored, bytes)) {
332
+ }
333
+ const stored = head ?? (await this.#bucket.head(key));
334
+ if (stored !== undefined && stored.size !== bytes.length) {
333
335
  throw new Error(`${this.#label}: content-address collision at ${segmentId} (§5.1)`);
334
336
  }
335
337
  }
@@ -40,6 +40,9 @@ export declare function encodeStream(body: ReadableStream<Uint8Array<ArrayBuffer
40
40
  * `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
41
41
  * servers MUST accept it). Returns `undefined` for any other declared
42
42
  * encoding — the adapter answers 415 rather than parse bytes it cannot read.
43
- * A gzip body that does not decompress is a `sync.invalid_request`.
43
+ * A gzip body that does not decompress is a `sync.invalid_request`. One read
44
+ * loop counts the decoded bytes of identity and gzip bodies alike and stops
45
+ * with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
46
+ * small compressed body never inflates past the cap.
44
47
  */
45
- export declare function decodeRequestBody(request: Request): Promise<Uint8Array | undefined>;
48
+ export declare function decodeRequestBody(request: Request, maxBytes: number): Promise<Uint8Array | undefined>;
@@ -1,7 +1,7 @@
1
1
  /**
2
- * §1.3 transport compression for every buffered HTTP body the adapters
3
- * serve — `<mount>/sync`, `<mount>/operations`, blob GETs and segment
4
- * downloads: negotiate `Content-Encoding` from the request's
2
+ * §1.3 transport compression for the HTTP bodies the adapters serve —
3
+ * `<mount>/sync`, `<mount>/operations` and segment downloads (blob downloads
4
+ * stream identity, §5.9.5): negotiate `Content-Encoding` from the request's
5
5
  * `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
6
6
  * offers neither), feature-detecting the runtime's codecs, zero
7
7
  * dependencies. Bun supplies zstd + gzip natively; every other runtime
@@ -121,26 +121,55 @@ export function encodeStream(body, acceptEncoding) {
121
121
  * `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
122
122
  * servers MUST accept it). Returns `undefined` for any other declared
123
123
  * encoding — the adapter answers 415 rather than parse bytes it cannot read.
124
- * A gzip body that does not decompress is a `sync.invalid_request`.
124
+ * A gzip body that does not decompress is a `sync.invalid_request`. One read
125
+ * loop counts the decoded bytes of identity and gzip bodies alike and stops
126
+ * with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
127
+ * small compressed body never inflates past the cap.
125
128
  */
126
- export async function decodeRequestBody(request) {
129
+ export async function decodeRequestBody(request, maxBytes) {
127
130
  const encoding = request.headers
128
131
  .get('content-encoding')
129
132
  ?.trim()
130
133
  .toLowerCase();
131
- if (encoding === undefined || encoding === '' || encoding === 'identity') {
132
- return new Uint8Array(await request.arrayBuffer());
133
- }
134
- if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
134
+ const identity = encoding === undefined || encoding === '' || encoding === 'identity';
135
+ if (!identity &&
136
+ (encoding !== 'gzip' || typeof DecompressionStream !== 'function')) {
135
137
  return undefined;
136
138
  }
137
139
  if (request.body === null) {
140
+ if (identity)
141
+ return new Uint8Array(0);
138
142
  throw syncError('sync.invalid_request', 'malformed gzip request body');
139
143
  }
140
- try {
141
- return new Uint8Array(await new Response(request.body.pipeThrough(new DecompressionStream('gzip'))).arrayBuffer());
144
+ const reader = (identity
145
+ ? request.body
146
+ : request.body.pipeThrough(new DecompressionStream('gzip'))).getReader();
147
+ const chunks = [];
148
+ let total = 0;
149
+ for (;;) {
150
+ let step;
151
+ try {
152
+ step = await reader.read();
153
+ }
154
+ catch (error) {
155
+ if (identity)
156
+ throw error;
157
+ throw syncError('sync.invalid_request', 'malformed gzip request body');
158
+ }
159
+ if (step.done)
160
+ break;
161
+ total += step.value.length;
162
+ if (total > maxBytes) {
163
+ await reader.cancel().catch(() => { });
164
+ throw syncError('sync.limit_exceeded', 'request body exceeds the server limit (§1.7)', JSON.stringify({ limit: 'maxRequestBytes', max: maxBytes }));
165
+ }
166
+ chunks.push(step.value);
142
167
  }
143
- catch {
144
- throw syncError('sync.invalid_request', 'malformed gzip request body');
168
+ const bytes = new Uint8Array(total);
169
+ let offset = 0;
170
+ for (const chunk of chunks) {
171
+ bytes.set(chunk, offset);
172
+ offset += chunk.length;
145
173
  }
174
+ return bytes;
146
175
  }
package/dist/context.d.ts CHANGED
@@ -64,6 +64,17 @@ export interface ServerLimits {
64
64
  readonly inlineSegmentMaxBytes: number;
65
65
  /** §6.11 declared-reference cascade cap per commit (reference default). */
66
66
  readonly maxCascadeOperationsPerCommit: number;
67
+ /**
68
+ * §1.7 cap on a request body's decoded bytes: an HTTP body after gzip
69
+ * decoding, and a socket round's assembled request (§8.7).
70
+ */
71
+ readonly maxRequestBytes: number;
72
+ /** §1.7 cap on `SUBSCRIPTION` frames per request. */
73
+ readonly maxSubscriptionsPerRequest: number;
74
+ /** §1.7 cap on requested scope values, summed over a subscription's keys. */
75
+ readonly maxScopeValuesPerSubscription: number;
76
+ /** Cap on concurrent registered-query watches per watch session. */
77
+ readonly maxWatchesPerSession: number;
67
78
  }
68
79
  export declare const DEFAULT_LIMITS: ServerLimits;
69
80
  /** Receives applied commits for realtime fanout (§8.2). */
@@ -202,6 +213,11 @@ export interface SyncRequestContext extends SyncServerConfig {
202
213
  readonly actorId: string;
203
214
  }
204
215
  export declare function clockOf(ctx: SyncServerConfig): () => number;
216
+ /**
217
+ * The configured limits over the defaults. A cap that is not a positive safe
218
+ * integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
219
+ * `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
220
+ */
205
221
  export declare function limitsOf(ctx: SyncServerConfig): ServerLimits;
206
222
  /** Refresh the registry after host authentication and return log continuity. */
207
223
  export declare function touchAuthenticatedPartition(ctx: SyncRequestContext): Promise<PartitionRegistryEntry>;
package/dist/context.js CHANGED
@@ -15,12 +15,32 @@ export const DEFAULT_LIMITS = {
15
15
  maxOperationsPerRequest: 500,
16
16
  inlineSegmentMaxBytes: 256 * 1024,
17
17
  maxCascadeOperationsPerCommit: 1_000,
18
+ // The size above which adapters already refuse to buffer a segment body.
19
+ maxRequestBytes: 16 * 1024 * 1024,
20
+ // One subscription per pushable operation: the existing per-request cap.
21
+ maxSubscriptionsPerRequest: 500,
22
+ // One default snapshot page (§4.2): a longer list wants a coarser scope.
23
+ maxScopeValuesPerSubscription: 1_000,
24
+ // Every commit reruns a session's affected watches one after another.
25
+ maxWatchesPerSession: 100,
18
26
  };
19
27
  export function clockOf(ctx) {
20
28
  return ctx.clock ?? Date.now;
21
29
  }
30
+ /**
31
+ * The configured limits over the defaults. A cap that is not a positive safe
32
+ * integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
33
+ * `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
34
+ */
22
35
  export function limitsOf(ctx) {
23
- return { ...DEFAULT_LIMITS, ...ctx.limits };
36
+ const limits = { ...DEFAULT_LIMITS, ...ctx.limits };
37
+ for (const [name, value] of Object.entries(limits)) {
38
+ const min = name === 'inlineSegmentMaxBytes' ? 0 : 1;
39
+ if (!Number.isSafeInteger(value) || value < min) {
40
+ throw new Error(`ServerLimits.${name} is out of range`);
41
+ }
42
+ }
43
+ return limits;
24
44
  }
25
45
  /** Refresh the registry after host authentication and return log continuity. */
26
46
  export function touchAuthenticatedPartition(ctx) {
@@ -1,48 +1,7 @@
1
- import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
2
- /**
3
- * Cloudflare D1 server storage for Workers deployments.
4
- *
5
- * D1 *is* SQLite exposed over an async, statement-at-a-time API
6
- * (`prepare(sql).bind(...).all()` / `.first()` / `.run()`, plus `batch([…])`
7
- * for atomic multi-statement writes). So this storage shares the schema and
8
- * value codecs with `SqliteServerStorage` via `sqlite-dialect.ts` — same
9
- * DDL, same `?` placeholders, same JSON-scope / base64-push-result
10
- * serialization — and differs only in the execution shape (async, no
11
- * interactive transaction).
12
- *
13
- * ## Transaction model (the one real divergence)
14
- *
15
- * D1 has **no interactive transaction**: you cannot `BEGIN`, run
16
- * conditional reads and writes on a pinned connection, then `COMMIT`. The
17
- * only atomic primitive is `db.batch([stmt, …])` — a set of statements
18
- * applied in one implicit transaction, all-or-nothing.
19
- *
20
- * The push handler (§6) reads first (conflict detection via `getRow`) and
21
- * only then writes (`upsertRow`/`deleteRow`/`appendCommit`/
22
- * `putPushResult`). `D1Transaction` exploits exactly that ordering:
23
- *
24
- * - reads (`getRow`) execute immediately against D1 (autocommit);
25
- * - writes are **buffered** as prepared statements;
26
- * - `commit()` flushes the whole buffer as one `db.batch(...)` — the same
27
- * atomicity §6.4 requires (a rejected op rolls back by simply never
28
- * flushing);
29
- * - `rollback()` drops the buffer unsent.
30
- *
31
- * `appendCommit` allocates the dense per-partition `commitSeq` (§2.1) by
32
- * reading `max_commit_seq` live and buffering the `+1` write. Under a single
33
- * Worker request this is exact. Two concurrent pushes to one partition need an
34
- * external serialization point (normally a per-partition Durable Object); a
35
- * realtime notifier alone does not serialize HTTP writes. Every deployment
36
- * that accepts D1 pushes MUST front same-partition sync rounds with a
37
- * coordinating primitive (a DO or a Queue). The adapter fails closed unless
38
- * that coordinator explicitly sets `pushApplySerialized`, because D1 cannot
39
- * provide the required pre-operation lock.
40
- * This mirrors PostgreSQL's per-partition row lock, achieved by placement
41
- * rather than a lock D1 does not expose.
42
- */
43
- import { type ScopeMap } from '@syncular/core';
44
- import type { CompiledSchema, CompiledTable } from './schema.js';
45
- import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
1
+ import type { CompiledSchema } from './schema.js';
2
+ import { type SqliteStatementSpec, SqliteFamilyStorage } from './sqlite-dialect.js';
3
+ import type { SqliteValue } from './sqlite-driver.js';
4
+ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, CommitPruneQuery, CommitPruneResult, ScopeValueHistoryQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint } from './storage.js';
46
5
  export interface D1PreparedStatement {
47
6
  bind(...values: unknown[]): D1PreparedStatement;
48
7
  first<T = Record<string, unknown>>(): Promise<T | null>;
@@ -71,13 +30,20 @@ export interface D1ServerStorageOptions {
71
30
  */
72
31
  readonly commitValidationSerialized?: boolean;
73
32
  }
74
- export declare class D1ServerStorage implements ServerStorage {
33
+ export declare class D1ServerStorage extends SqliteFamilyStorage implements ServerStorage {
75
34
  #private;
35
+ /**
36
+ * D1 caps bound parameters at 100 per statement; the relational upsert
37
+ * binds one per app column plus the five `_sync_*` meta columns.
38
+ */
39
+ protected readonly maxBindParams = 100;
76
40
  constructor(db: D1Database, options?: D1ServerStorageOptions);
41
+ protected all<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
42
+ /** Queue one read into the current turn's `db.batch` (see `#pendingReads`). */
43
+ protected guardedAll<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
44
+ protected atomic(statements: readonly SqliteStatementSpec[]): Promise<unknown[][]>;
77
45
  /** Apply the schema DDL (idempotent). Call once before use. */
78
46
  migrate(): Promise<void>;
79
- /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
80
- table(name: string): CompiledTable;
81
47
  ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
82
48
  /** Run once per Worker invocation. Resume an incomplete result in another invocation. */
83
49
  migrateSchema(schema: CompiledSchema, options?: {
@@ -86,64 +52,14 @@ export declare class D1ServerStorage implements ServerStorage {
86
52
  readonly complete: boolean;
87
53
  readonly statementsExecuted: number;
88
54
  }>;
89
- touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
90
- rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
91
- listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
92
55
  begin(partition: string): Promise<StorageTransaction>;
93
- getMaxCommitSeq(partition: string): Promise<number>;
94
56
  queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, _checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
95
- getPartitionLogEpoch(partition: string): Promise<string | undefined>;
96
- getHorizonSeq(partition: string): Promise<number>;
97
- setHorizonSeq(partition: string, seq: number): Promise<void>;
98
57
  pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
99
- getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
100
58
  readCheckpoints(_partition: string): Promise<StoredCheckpoint[]>;
101
59
  readServeGate(partition: string, _runningSchemaVersion: number): Promise<ServeGate>;
102
60
  declareCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
103
61
  activateCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _sources: readonly string[], _nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
104
62
  claimCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
105
63
  advanceCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _observedRows: number, _nowMs: number): Promise<boolean>;
106
- sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
107
- hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
108
- getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
109
- getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
110
- claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
111
- completeReaction(partition: string, idempotencyKey: string, leaseOwner: string, completedAtMs: number): Promise<boolean>;
112
- extendReactionLease(partition: string, idempotencyKey: string, leaseOwner: string, leaseExpiresAtMs: number): Promise<boolean>;
113
- failReaction(partition: string, idempotencyKey: string, update: ReactionFailureUpdate): Promise<boolean>;
114
- retryReaction(partition: string, idempotencyKey: string, nowMs: number): Promise<boolean>;
115
- getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
116
- listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
117
- pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
118
- latestScopedChangeSeq(partition: string, query: {
119
- readonly table: string;
120
- readonly scopeFilter: ScopeMap;
121
- }): Promise<number>;
122
- readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
123
- scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
124
- scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
125
- getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
126
- putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
127
- recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
128
- advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
129
- updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
130
- getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
131
- listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
132
- listRowsReferencingBlob(partition: string, blobId: string): Promise<{
133
- readonly table: string;
134
- readonly rowId: string;
135
- readonly scopes: Record<string, string>;
136
- }[]>;
137
- listReferencedBlobIds(partition: string): Promise<string[]>;
138
- listScopeValueRows(partition: string, query: ScopeValueQuery): Promise<string[]>;
139
64
  eraseScopeValueHistory(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
140
- listClientRecords(partition: string): Promise<ClientRecord[]>;
141
- listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
142
- getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
143
- listRowChanges(partition: string, query: RowChangesQuery): Promise<RowChangeRecord[]>;
144
- listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
145
- countRows(partition: string, table: string): Promise<number>;
146
- listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
147
- scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
148
- listPartitions(): Promise<string[]>;
149
65
  }