@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
|
@@ -138,38 +138,26 @@ export class BucketBlobStore {
|
|
|
138
138
|
};
|
|
139
139
|
const objectKey = this.objectKeyFor(partition, blobId);
|
|
140
140
|
// Idempotent re-put (same content-address ⇒ same key): count each distinct
|
|
141
|
-
// blob once
|
|
142
|
-
//
|
|
141
|
+
// blob once. A re-upload rewrites the object so its metadata carries the
|
|
142
|
+
// re-upload time (the sweep's grace clock, §5.9.2) and keeps a recorded
|
|
143
|
+
// media type.
|
|
143
144
|
const preexisting = await this.#bucket.head(objectKey);
|
|
144
145
|
const storedMeta = preexisting?.metadata[BLOB_META_KEY];
|
|
145
146
|
const existing = storedMeta === undefined
|
|
146
147
|
? undefined
|
|
147
148
|
: parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
|
|
148
|
-
|
|
149
|
-
(existing.mediaType !== undefined || mediaType === undefined)) {
|
|
150
|
-
return {
|
|
151
|
-
blobId,
|
|
152
|
-
partition,
|
|
153
|
-
byteLength: existing.byteLength,
|
|
154
|
-
...(existing.mediaType !== undefined
|
|
155
|
-
? { mediaType: existing.mediaType }
|
|
156
|
-
: {}),
|
|
157
|
-
createdAtMs: existing.createdAtMs,
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
// A re-upload that names the media type an earlier upload left out
|
|
161
|
-
// rewrites the object under the same precondition, keeping its age.
|
|
149
|
+
const storedType = existing?.mediaType ?? mediaType;
|
|
162
150
|
const meta = {
|
|
163
151
|
byteLength: bytes.length,
|
|
164
|
-
...(
|
|
165
|
-
createdAtMs: existing?.createdAtMs ?? nowMs,
|
|
152
|
+
...(storedType !== undefined ? { mediaType: storedType } : {}),
|
|
153
|
+
createdAtMs: Math.max(existing?.createdAtMs ?? nowMs, nowMs),
|
|
166
154
|
};
|
|
167
155
|
// Conditional write: of two concurrent uploads of the same blob exactly
|
|
168
|
-
// one lands
|
|
169
|
-
//
|
|
156
|
+
// one lands per pre-state and the blob counts once. The loser re-reads
|
|
157
|
+
// and writes over the winner.
|
|
170
158
|
const written = await this.#bucket.put(objectKey, {
|
|
171
159
|
body: bytes,
|
|
172
|
-
contentType:
|
|
160
|
+
contentType: storedType ?? 'application/octet-stream',
|
|
173
161
|
metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
|
|
174
162
|
...(preexisting === undefined
|
|
175
163
|
? { createOnly: true }
|
|
@@ -179,7 +167,11 @@ export class BucketBlobStore {
|
|
|
179
167
|
return this.put(partition, blobId, bytes, nowMs, mediaType);
|
|
180
168
|
if (preexisting === undefined)
|
|
181
169
|
await this.#bumpStats({ count: 1, bytes: bytes.length });
|
|
182
|
-
return {
|
|
170
|
+
return {
|
|
171
|
+
...record,
|
|
172
|
+
...(storedType !== undefined ? { mediaType: storedType } : {}),
|
|
173
|
+
createdAtMs: meta.createdAtMs,
|
|
174
|
+
};
|
|
183
175
|
}
|
|
184
176
|
async delete(partition, blobId) {
|
|
185
177
|
await this.#bucket.delete(this.objectKeyFor(partition, blobId));
|
|
@@ -191,35 +183,51 @@ export class BucketBlobStore {
|
|
|
191
183
|
return head !== undefined;
|
|
192
184
|
}
|
|
193
185
|
async get(partition, blobId) {
|
|
194
|
-
|
|
195
|
-
return undefined;
|
|
196
|
-
const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
|
|
186
|
+
const object = await this.#object(partition, blobId);
|
|
197
187
|
if (object === undefined)
|
|
198
188
|
return undefined;
|
|
199
189
|
const bytes = new Uint8Array(await object.arrayBuffer());
|
|
190
|
+
return {
|
|
191
|
+
record: this.#record(partition, blobId, object, bytes.length),
|
|
192
|
+
bytes,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
196
|
+
async open(partition, blobId) {
|
|
197
|
+
const object = await this.#object(partition, blobId);
|
|
198
|
+
if (object === undefined || object.body === null)
|
|
199
|
+
return undefined;
|
|
200
|
+
const record = this.#record(partition, blobId, object, object.size ?? 0);
|
|
201
|
+
return { record, body: object.body, byteLength: record.byteLength };
|
|
202
|
+
}
|
|
203
|
+
async #object(partition, blobId) {
|
|
204
|
+
if (!BLOB_ID_PATTERN.test(blobId))
|
|
205
|
+
return undefined;
|
|
206
|
+
return this.#bucket.get(this.objectKeyFor(partition, blobId));
|
|
207
|
+
}
|
|
208
|
+
/** An object without Syncular metadata was written by a presigned PUT. */
|
|
209
|
+
#record(partition, blobId, object, byteLength) {
|
|
200
210
|
const rawMeta = object.metadata[BLOB_META_KEY];
|
|
201
211
|
const meta = rawMeta !== undefined
|
|
202
212
|
? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
|
|
203
|
-
: { byteLength
|
|
213
|
+
: { byteLength, createdAtMs: 0 };
|
|
204
214
|
return {
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
createdAtMs: meta.createdAtMs,
|
|
211
|
-
},
|
|
212
|
-
bytes,
|
|
215
|
+
blobId,
|
|
216
|
+
partition,
|
|
217
|
+
byteLength: meta.byteLength,
|
|
218
|
+
...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
|
|
219
|
+
createdAtMs: meta.createdAtMs,
|
|
213
220
|
};
|
|
214
221
|
}
|
|
215
222
|
/**
|
|
216
223
|
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
|
|
217
|
-
* {partition}/` prefix, HEAD each object for its
|
|
218
|
-
*
|
|
219
|
-
*
|
|
220
|
-
*
|
|
224
|
+
* {partition}/` prefix, HEAD each object for its upload time (the metadata
|
|
225
|
+
* `createdAtMs`, else the backend's last-modified time; an object with
|
|
226
|
+
* neither is skipped), and DELETE the ones older than `olderThanMs` that
|
|
227
|
+
* `isReferenced` denies right before the delete. Returns the deleted
|
|
228
|
+
* blobIds. Off the hot path — a host-scheduled GC job.
|
|
221
229
|
*/
|
|
222
|
-
async sweepOrphans(partition, olderThanMs,
|
|
230
|
+
async sweepOrphans(partition, olderThanMs, isReferenced) {
|
|
223
231
|
const listPrefix = `${this.#prefix}blob/${partition}/`;
|
|
224
232
|
const swept = [];
|
|
225
233
|
let deletedBytes = 0;
|
|
@@ -235,20 +243,21 @@ export class BucketBlobStore {
|
|
|
235
243
|
const blobId = `sha256:${rest.slice('sha256/'.length)}`;
|
|
236
244
|
if (!BLOB_ID_PATTERN.test(blobId))
|
|
237
245
|
continue;
|
|
238
|
-
if (referencedBlobIds.has(blobId))
|
|
239
|
-
continue;
|
|
240
246
|
const head = await this.#bucket.head(key);
|
|
241
247
|
if (head === undefined)
|
|
242
248
|
continue;
|
|
243
249
|
const rawMeta = head.metadata[BLOB_META_KEY];
|
|
244
|
-
const
|
|
250
|
+
const uploadedAtMs = rawMeta !== undefined
|
|
245
251
|
? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
|
|
246
|
-
|
|
247
|
-
|
|
252
|
+
.createdAtMs
|
|
253
|
+
: head.lastModifiedMs;
|
|
254
|
+
if (uploadedAtMs === undefined || uploadedAtMs >= olderThanMs)
|
|
255
|
+
continue;
|
|
256
|
+
if (await isReferenced(blobId))
|
|
248
257
|
continue;
|
|
249
258
|
await this.#bucket.delete(key);
|
|
250
259
|
swept.push(blobId);
|
|
251
|
-
deletedBytes +=
|
|
260
|
+
deletedBytes += head.size ?? 0;
|
|
252
261
|
}
|
|
253
262
|
} while (cursor !== undefined);
|
|
254
263
|
if (swept.length > 0) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mergeSegmentRecord, publicationRecord, publicationsForRecord,
|
|
1
|
+
import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
|
|
2
2
|
import { sha256Hex } from './sigv4.js';
|
|
3
3
|
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
4
4
|
const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
@@ -311,25 +311,27 @@ export class BucketSegmentStore {
|
|
|
311
311
|
throw new Error(`${this.#label}: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`);
|
|
312
312
|
}
|
|
313
313
|
/**
|
|
314
|
-
* Write the immutable bytes object
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
314
|
+
* Write the immutable bytes object unless it is stored already. A HEAD
|
|
315
|
+
* comes first, so republishing a segment (every bootstrap pull of an
|
|
316
|
+
* unchanged window) moves no bytes. The write is create-only, so two first
|
|
317
|
+
* publishers cannot overwrite each other. A stored object under the same
|
|
318
|
+
* content address with a different size is the hash collision §5.1
|
|
319
|
+
* requires to fail loudly; equal size under one SHA-256 address is the same
|
|
320
|
+
* content.
|
|
318
321
|
*/
|
|
319
322
|
async #putBytes(segmentId, bytes) {
|
|
320
323
|
const key = this.objectKeyFor(segmentId);
|
|
321
|
-
const
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
const current = await this.#bucket.get(key);
|
|
329
|
-
if (current === undefined)
|
|
324
|
+
const head = await this.#bucket.head(key);
|
|
325
|
+
if (head === undefined &&
|
|
326
|
+
(await this.#bucket.put(key, {
|
|
327
|
+
body: bytes,
|
|
328
|
+
contentType: 'application/octet-stream',
|
|
329
|
+
createOnly: true,
|
|
330
|
+
}))) {
|
|
330
331
|
return;
|
|
331
|
-
|
|
332
|
-
|
|
332
|
+
}
|
|
333
|
+
const stored = head ?? (await this.#bucket.head(key));
|
|
334
|
+
if (stored !== undefined && stored.size !== bytes.length) {
|
|
333
335
|
throw new Error(`${this.#label}: content-address collision at ${segmentId} (§5.1)`);
|
|
334
336
|
}
|
|
335
337
|
}
|
|
@@ -40,6 +40,9 @@ export declare function encodeStream(body: ReadableStream<Uint8Array<ArrayBuffer
|
|
|
40
40
|
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
41
41
|
* servers MUST accept it). Returns `undefined` for any other declared
|
|
42
42
|
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
43
|
-
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
43
|
+
* A gzip body that does not decompress is a `sync.invalid_request`. One read
|
|
44
|
+
* loop counts the decoded bytes of identity and gzip bodies alike and stops
|
|
45
|
+
* with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
|
|
46
|
+
* small compressed body never inflates past the cap.
|
|
44
47
|
*/
|
|
45
|
-
export declare function decodeRequestBody(request: Request): Promise<Uint8Array | undefined>;
|
|
48
|
+
export declare function decodeRequestBody(request: Request, maxBytes: number): Promise<Uint8Array | undefined>;
|
package/dist/content-encoding.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* §1.3 transport compression for
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* §1.3 transport compression for the HTTP bodies the adapters serve —
|
|
3
|
+
* `<mount>/sync`, `<mount>/operations` and segment downloads (blob downloads
|
|
4
|
+
* stream identity, §5.9.5): negotiate `Content-Encoding` from the request's
|
|
5
5
|
* `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
|
|
6
6
|
* offers neither), feature-detecting the runtime's codecs, zero
|
|
7
7
|
* dependencies. Bun supplies zstd + gzip natively; every other runtime
|
|
@@ -121,26 +121,55 @@ export function encodeStream(body, acceptEncoding) {
|
|
|
121
121
|
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
122
122
|
* servers MUST accept it). Returns `undefined` for any other declared
|
|
123
123
|
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
124
|
-
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
124
|
+
* A gzip body that does not decompress is a `sync.invalid_request`. One read
|
|
125
|
+
* loop counts the decoded bytes of identity and gzip bodies alike and stops
|
|
126
|
+
* with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
|
|
127
|
+
* small compressed body never inflates past the cap.
|
|
125
128
|
*/
|
|
126
|
-
export async function decodeRequestBody(request) {
|
|
129
|
+
export async function decodeRequestBody(request, maxBytes) {
|
|
127
130
|
const encoding = request.headers
|
|
128
131
|
.get('content-encoding')
|
|
129
132
|
?.trim()
|
|
130
133
|
.toLowerCase();
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
|
|
134
|
+
const identity = encoding === undefined || encoding === '' || encoding === 'identity';
|
|
135
|
+
if (!identity &&
|
|
136
|
+
(encoding !== 'gzip' || typeof DecompressionStream !== 'function')) {
|
|
135
137
|
return undefined;
|
|
136
138
|
}
|
|
137
139
|
if (request.body === null) {
|
|
140
|
+
if (identity)
|
|
141
|
+
return new Uint8Array(0);
|
|
138
142
|
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
139
143
|
}
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
const reader = (identity
|
|
145
|
+
? request.body
|
|
146
|
+
: request.body.pipeThrough(new DecompressionStream('gzip'))).getReader();
|
|
147
|
+
const chunks = [];
|
|
148
|
+
let total = 0;
|
|
149
|
+
for (;;) {
|
|
150
|
+
let step;
|
|
151
|
+
try {
|
|
152
|
+
step = await reader.read();
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
if (identity)
|
|
156
|
+
throw error;
|
|
157
|
+
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
158
|
+
}
|
|
159
|
+
if (step.done)
|
|
160
|
+
break;
|
|
161
|
+
total += step.value.length;
|
|
162
|
+
if (total > maxBytes) {
|
|
163
|
+
await reader.cancel().catch(() => { });
|
|
164
|
+
throw syncError('sync.limit_exceeded', 'request body exceeds the server limit (§1.7)', JSON.stringify({ limit: 'maxRequestBytes', max: maxBytes }));
|
|
165
|
+
}
|
|
166
|
+
chunks.push(step.value);
|
|
142
167
|
}
|
|
143
|
-
|
|
144
|
-
|
|
168
|
+
const bytes = new Uint8Array(total);
|
|
169
|
+
let offset = 0;
|
|
170
|
+
for (const chunk of chunks) {
|
|
171
|
+
bytes.set(chunk, offset);
|
|
172
|
+
offset += chunk.length;
|
|
145
173
|
}
|
|
174
|
+
return bytes;
|
|
146
175
|
}
|
package/dist/context.d.ts
CHANGED
|
@@ -64,6 +64,17 @@ export interface ServerLimits {
|
|
|
64
64
|
readonly inlineSegmentMaxBytes: number;
|
|
65
65
|
/** §6.11 declared-reference cascade cap per commit (reference default). */
|
|
66
66
|
readonly maxCascadeOperationsPerCommit: number;
|
|
67
|
+
/**
|
|
68
|
+
* §1.7 cap on a request body's decoded bytes: an HTTP body after gzip
|
|
69
|
+
* decoding, and a socket round's assembled request (§8.7).
|
|
70
|
+
*/
|
|
71
|
+
readonly maxRequestBytes: number;
|
|
72
|
+
/** §1.7 cap on `SUBSCRIPTION` frames per request. */
|
|
73
|
+
readonly maxSubscriptionsPerRequest: number;
|
|
74
|
+
/** §1.7 cap on requested scope values, summed over a subscription's keys. */
|
|
75
|
+
readonly maxScopeValuesPerSubscription: number;
|
|
76
|
+
/** Cap on concurrent registered-query watches per watch session. */
|
|
77
|
+
readonly maxWatchesPerSession: number;
|
|
67
78
|
}
|
|
68
79
|
export declare const DEFAULT_LIMITS: ServerLimits;
|
|
69
80
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
@@ -202,6 +213,11 @@ export interface SyncRequestContext extends SyncServerConfig {
|
|
|
202
213
|
readonly actorId: string;
|
|
203
214
|
}
|
|
204
215
|
export declare function clockOf(ctx: SyncServerConfig): () => number;
|
|
216
|
+
/**
|
|
217
|
+
* The configured limits over the defaults. A cap that is not a positive safe
|
|
218
|
+
* integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
|
|
219
|
+
* `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
|
|
220
|
+
*/
|
|
205
221
|
export declare function limitsOf(ctx: SyncServerConfig): ServerLimits;
|
|
206
222
|
/** Refresh the registry after host authentication and return log continuity. */
|
|
207
223
|
export declare function touchAuthenticatedPartition(ctx: SyncRequestContext): Promise<PartitionRegistryEntry>;
|
package/dist/context.js
CHANGED
|
@@ -15,12 +15,32 @@ export const DEFAULT_LIMITS = {
|
|
|
15
15
|
maxOperationsPerRequest: 500,
|
|
16
16
|
inlineSegmentMaxBytes: 256 * 1024,
|
|
17
17
|
maxCascadeOperationsPerCommit: 1_000,
|
|
18
|
+
// The size above which adapters already refuse to buffer a segment body.
|
|
19
|
+
maxRequestBytes: 16 * 1024 * 1024,
|
|
20
|
+
// One subscription per pushable operation: the existing per-request cap.
|
|
21
|
+
maxSubscriptionsPerRequest: 500,
|
|
22
|
+
// One default snapshot page (§4.2): a longer list wants a coarser scope.
|
|
23
|
+
maxScopeValuesPerSubscription: 1_000,
|
|
24
|
+
// Every commit reruns a session's affected watches one after another.
|
|
25
|
+
maxWatchesPerSession: 100,
|
|
18
26
|
};
|
|
19
27
|
export function clockOf(ctx) {
|
|
20
28
|
return ctx.clock ?? Date.now;
|
|
21
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* The configured limits over the defaults. A cap that is not a positive safe
|
|
32
|
+
* integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
|
|
33
|
+
* `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
|
|
34
|
+
*/
|
|
22
35
|
export function limitsOf(ctx) {
|
|
23
|
-
|
|
36
|
+
const limits = { ...DEFAULT_LIMITS, ...ctx.limits };
|
|
37
|
+
for (const [name, value] of Object.entries(limits)) {
|
|
38
|
+
const min = name === 'inlineSegmentMaxBytes' ? 0 : 1;
|
|
39
|
+
if (!Number.isSafeInteger(value) || value < min) {
|
|
40
|
+
throw new Error(`ServerLimits.${name} is out of range`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return limits;
|
|
24
44
|
}
|
|
25
45
|
/** Refresh the registry after host authentication and return log continuity. */
|
|
26
46
|
export function touchAuthenticatedPartition(ctx) {
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -1,48 +1,7 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* D1 *is* SQLite exposed over an async, statement-at-a-time API
|
|
6
|
-
* (`prepare(sql).bind(...).all()` / `.first()` / `.run()`, plus `batch([…])`
|
|
7
|
-
* for atomic multi-statement writes). So this storage shares the schema and
|
|
8
|
-
* value codecs with `SqliteServerStorage` via `sqlite-dialect.ts` — same
|
|
9
|
-
* DDL, same `?` placeholders, same JSON-scope / base64-push-result
|
|
10
|
-
* serialization — and differs only in the execution shape (async, no
|
|
11
|
-
* interactive transaction).
|
|
12
|
-
*
|
|
13
|
-
* ## Transaction model (the one real divergence)
|
|
14
|
-
*
|
|
15
|
-
* D1 has **no interactive transaction**: you cannot `BEGIN`, run
|
|
16
|
-
* conditional reads and writes on a pinned connection, then `COMMIT`. The
|
|
17
|
-
* only atomic primitive is `db.batch([stmt, …])` — a set of statements
|
|
18
|
-
* applied in one implicit transaction, all-or-nothing.
|
|
19
|
-
*
|
|
20
|
-
* The push handler (§6) reads first (conflict detection via `getRow`) and
|
|
21
|
-
* only then writes (`upsertRow`/`deleteRow`/`appendCommit`/
|
|
22
|
-
* `putPushResult`). `D1Transaction` exploits exactly that ordering:
|
|
23
|
-
*
|
|
24
|
-
* - reads (`getRow`) execute immediately against D1 (autocommit);
|
|
25
|
-
* - writes are **buffered** as prepared statements;
|
|
26
|
-
* - `commit()` flushes the whole buffer as one `db.batch(...)` — the same
|
|
27
|
-
* atomicity §6.4 requires (a rejected op rolls back by simply never
|
|
28
|
-
* flushing);
|
|
29
|
-
* - `rollback()` drops the buffer unsent.
|
|
30
|
-
*
|
|
31
|
-
* `appendCommit` allocates the dense per-partition `commitSeq` (§2.1) by
|
|
32
|
-
* reading `max_commit_seq` live and buffering the `+1` write. Under a single
|
|
33
|
-
* Worker request this is exact. Two concurrent pushes to one partition need an
|
|
34
|
-
* external serialization point (normally a per-partition Durable Object); a
|
|
35
|
-
* realtime notifier alone does not serialize HTTP writes. Every deployment
|
|
36
|
-
* that accepts D1 pushes MUST front same-partition sync rounds with a
|
|
37
|
-
* coordinating primitive (a DO or a Queue). The adapter fails closed unless
|
|
38
|
-
* that coordinator explicitly sets `pushApplySerialized`, because D1 cannot
|
|
39
|
-
* provide the required pre-operation lock.
|
|
40
|
-
* This mirrors PostgreSQL's per-partition row lock, achieved by placement
|
|
41
|
-
* rather than a lock D1 does not expose.
|
|
42
|
-
*/
|
|
43
|
-
import { type ScopeMap } from '@syncular/core';
|
|
44
|
-
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
45
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
1
|
+
import type { CompiledSchema } from './schema.js';
|
|
2
|
+
import { type SqliteStatementSpec, SqliteFamilyStorage } from './sqlite-dialect.js';
|
|
3
|
+
import type { SqliteValue } from './sqlite-driver.js';
|
|
4
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, CommitPruneQuery, CommitPruneResult, ScopeValueHistoryQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint } from './storage.js';
|
|
46
5
|
export interface D1PreparedStatement {
|
|
47
6
|
bind(...values: unknown[]): D1PreparedStatement;
|
|
48
7
|
first<T = Record<string, unknown>>(): Promise<T | null>;
|
|
@@ -71,13 +30,20 @@ export interface D1ServerStorageOptions {
|
|
|
71
30
|
*/
|
|
72
31
|
readonly commitValidationSerialized?: boolean;
|
|
73
32
|
}
|
|
74
|
-
export declare class D1ServerStorage implements ServerStorage {
|
|
33
|
+
export declare class D1ServerStorage extends SqliteFamilyStorage implements ServerStorage {
|
|
75
34
|
#private;
|
|
35
|
+
/**
|
|
36
|
+
* D1 caps bound parameters at 100 per statement; the relational upsert
|
|
37
|
+
* binds one per app column plus the five `_sync_*` meta columns.
|
|
38
|
+
*/
|
|
39
|
+
protected readonly maxBindParams = 100;
|
|
76
40
|
constructor(db: D1Database, options?: D1ServerStorageOptions);
|
|
41
|
+
protected all<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
|
|
42
|
+
/** Queue one read into the current turn's `db.batch` (see `#pendingReads`). */
|
|
43
|
+
protected guardedAll<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
|
|
44
|
+
protected atomic(statements: readonly SqliteStatementSpec[]): Promise<unknown[][]>;
|
|
77
45
|
/** Apply the schema DDL (idempotent). Call once before use. */
|
|
78
46
|
migrate(): Promise<void>;
|
|
79
|
-
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
80
|
-
table(name: string): CompiledTable;
|
|
81
47
|
ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
|
|
82
48
|
/** Run once per Worker invocation. Resume an incomplete result in another invocation. */
|
|
83
49
|
migrateSchema(schema: CompiledSchema, options?: {
|
|
@@ -86,64 +52,14 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
86
52
|
readonly complete: boolean;
|
|
87
53
|
readonly statementsExecuted: number;
|
|
88
54
|
}>;
|
|
89
|
-
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
90
|
-
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
91
|
-
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
92
55
|
begin(partition: string): Promise<StorageTransaction>;
|
|
93
|
-
getMaxCommitSeq(partition: string): Promise<number>;
|
|
94
56
|
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, _checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
|
|
95
|
-
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
96
|
-
getHorizonSeq(partition: string): Promise<number>;
|
|
97
|
-
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
98
57
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
99
|
-
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
100
58
|
readCheckpoints(_partition: string): Promise<StoredCheckpoint[]>;
|
|
101
59
|
readServeGate(partition: string, _runningSchemaVersion: number): Promise<ServeGate>;
|
|
102
60
|
declareCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
|
|
103
61
|
activateCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _sources: readonly string[], _nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
|
|
104
62
|
claimCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
|
|
105
63
|
advanceCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _observedRows: number, _nowMs: number): Promise<boolean>;
|
|
106
|
-
sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
|
|
107
|
-
hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
|
|
108
|
-
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
109
|
-
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
110
|
-
claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
|
|
111
|
-
completeReaction(partition: string, idempotencyKey: string, leaseOwner: string, completedAtMs: number): Promise<boolean>;
|
|
112
|
-
extendReactionLease(partition: string, idempotencyKey: string, leaseOwner: string, leaseExpiresAtMs: number): Promise<boolean>;
|
|
113
|
-
failReaction(partition: string, idempotencyKey: string, update: ReactionFailureUpdate): Promise<boolean>;
|
|
114
|
-
retryReaction(partition: string, idempotencyKey: string, nowMs: number): Promise<boolean>;
|
|
115
|
-
getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
|
|
116
|
-
listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
|
|
117
|
-
pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
118
|
-
latestScopedChangeSeq(partition: string, query: {
|
|
119
|
-
readonly table: string;
|
|
120
|
-
readonly scopeFilter: ScopeMap;
|
|
121
|
-
}): Promise<number>;
|
|
122
|
-
readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
|
|
123
|
-
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
124
|
-
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
125
|
-
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
126
|
-
putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
|
|
127
|
-
recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
|
|
128
|
-
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
129
|
-
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
130
|
-
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
131
|
-
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
132
|
-
listRowsReferencingBlob(partition: string, blobId: string): Promise<{
|
|
133
|
-
readonly table: string;
|
|
134
|
-
readonly rowId: string;
|
|
135
|
-
readonly scopes: Record<string, string>;
|
|
136
|
-
}[]>;
|
|
137
|
-
listReferencedBlobIds(partition: string): Promise<string[]>;
|
|
138
|
-
listScopeValueRows(partition: string, query: ScopeValueQuery): Promise<string[]>;
|
|
139
64
|
eraseScopeValueHistory(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
|
|
140
|
-
listClientRecords(partition: string): Promise<ClientRecord[]>;
|
|
141
|
-
listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
142
|
-
getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
|
|
143
|
-
listRowChanges(partition: string, query: RowChangesQuery): Promise<RowChangeRecord[]>;
|
|
144
|
-
listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
|
|
145
|
-
countRows(partition: string, table: string): Promise<number>;
|
|
146
|
-
listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
147
|
-
scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
|
|
148
|
-
listPartitions(): Promise<string[]>;
|
|
149
65
|
}
|