@syncular/server 0.32.1 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +3 -3
- package/dist/bucket-blob-store.d.ts +77 -0
- package/dist/bucket-blob-store.js +266 -0
- package/dist/bucket-segment-store.d.ts +77 -0
- package/dist/bucket-segment-store.js +445 -0
- package/dist/context.d.ts +7 -0
- package/dist/d1-storage.js +2 -2
- package/dist/handler.js +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/object-bucket.d.ts +78 -0
- package/dist/object-bucket.js +18 -0
- package/dist/operations-realtime.js +3 -0
- package/dist/operations.d.ts +20 -3
- package/dist/operations.js +23 -3
- package/dist/realtime.d.ts +11 -0
- package/dist/realtime.js +25 -58
- package/dist/relational-rows.d.ts +6 -3
- package/dist/relational-rows.js +8 -9
- package/dist/s3-blob-store.d.ts +9 -64
- package/dist/s3-blob-store.js +25 -359
- package/dist/s3-object-bucket.d.ts +39 -0
- package/dist/s3-object-bucket.js +164 -0
- package/dist/s3-segment-store.d.ts +9 -61
- package/dist/s3-segment-store.js +25 -503
- package/dist/sqlite-storage.js +3 -8
- package/package.json +2 -2
- package/src/blob-handlers.ts +4 -4
- package/src/bucket-blob-store.ts +351 -0
- package/src/bucket-segment-store.ts +582 -0
- package/src/context.ts +10 -0
- package/src/d1-storage.ts +3 -3
- package/src/handler.ts +11 -0
- package/src/index.ts +4 -0
- package/src/object-bucket.ts +83 -0
- package/src/operations-realtime.ts +7 -0
- package/src/operations.ts +52 -5
- package/src/realtime.ts +29 -58
- package/src/relational-rows.ts +8 -11
- package/src/s3-blob-store.ts +25 -466
- package/src/s3-object-bucket.ts +233 -0
- package/src/s3-segment-store.ts +25 -646
- package/src/sqlite-storage.ts +3 -10
|
@@ -0,0 +1,582 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content-addressed segment store over the `ObjectBucket` seam (§5.1 cache
|
|
3
|
+
* semantics) — the store logic shared by `S3SegmentStore` (S3 REST + SigV4)
|
|
4
|
+
* and `R2SegmentStore` (`@syncular/server-workers`, the R2 binding).
|
|
5
|
+
*
|
|
6
|
+
* Key layout (deterministic — every lookup is a GET/HEAD, never a LIST):
|
|
7
|
+
*
|
|
8
|
+
* {keyPrefix}seg/sha256/{hex}
|
|
9
|
+
* The segment bytes, verbatim and immutable (the object body MUST be
|
|
10
|
+
* exactly the content-addressed bytes so a presigned GET or the inline
|
|
11
|
+
* download route serves them directly and the client's §5.1 hash check
|
|
12
|
+
* passes).
|
|
13
|
+
*
|
|
14
|
+
* {keyPrefix}rec/sha256/{hex}.json
|
|
15
|
+
* The `SegmentRecord` (§5.1). It is the one MUTABLE object of the two:
|
|
16
|
+
* the scope-digest set is a union that every publication extends, so
|
|
17
|
+
* the merge is a read-modify-write. Keeping the record in the body (not
|
|
18
|
+
* in the bytes object's user metadata) is what makes that write
|
|
19
|
+
* conditional — a body ETag changes when the union changes, while the
|
|
20
|
+
* bytes ETag is a content hash and is identical for identical bytes. A
|
|
21
|
+
* failed precondition re-reads and merges again, so no concurrent
|
|
22
|
+
* publication can drop another one's digest. `get` therefore reads two
|
|
23
|
+
* objects.
|
|
24
|
+
*
|
|
25
|
+
* {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
|
|
26
|
+
* Whole-table reuse pointer (§5.3): written only when
|
|
27
|
+
* `rowCursor === null`, body = the record JSON. The reuse key is the
|
|
28
|
+
* canonical JSON array `[partition, logEpoch, table, schemaVersion,
|
|
29
|
+
* mediaType, scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a
|
|
30
|
+
* HEAD to confirm the segment object itself still exists — lifecycle GC
|
|
31
|
+
* may remove objects independently of pointers).
|
|
32
|
+
*
|
|
33
|
+
* Records written before the split kept the record in the bytes object's
|
|
34
|
+
* user metadata `syncular-record`; `get` still reads that shape and the next
|
|
35
|
+
* `put` materializes the record object from it, so an upgrade loses no cache
|
|
36
|
+
* entry.
|
|
37
|
+
*
|
|
38
|
+
* TTL mapping: expiry is **store-side and authoritative** — `expiresAtMs`
|
|
39
|
+
* (put-time + `ttlMs`, default 24 h) is recorded in the record and the
|
|
40
|
+
* pointer; `get` returns expired records so §5.5 can answer
|
|
41
|
+
* `sync.segment_expired`, and `find` filters them out itself. Bucket
|
|
42
|
+
* lifecycle expiration is garbage collection only: configure it comfortably
|
|
43
|
+
* ABOVE `ttlMs` (e.g. 2 days for the 24 h default) so clients normally see
|
|
44
|
+
* the precise, retryable `sync.segment_expired` and hit `sync.not_found` only
|
|
45
|
+
* long after. Never set lifecycle below `ttlMs`.
|
|
46
|
+
*/
|
|
47
|
+
import type { ObjectBucket } from './object-bucket';
|
|
48
|
+
import type {
|
|
49
|
+
SegmentFindKey,
|
|
50
|
+
SegmentMetadata,
|
|
51
|
+
SegmentPublication,
|
|
52
|
+
SegmentRecord,
|
|
53
|
+
SegmentStore,
|
|
54
|
+
SegmentStoreStats,
|
|
55
|
+
SegmentStream,
|
|
56
|
+
} from './segment-store';
|
|
57
|
+
import {
|
|
58
|
+
mergeSegmentRecord,
|
|
59
|
+
publicationRecord,
|
|
60
|
+
publicationsForRecord,
|
|
61
|
+
segmentBytesEqual,
|
|
62
|
+
segmentIdFor,
|
|
63
|
+
selectSegmentPublication,
|
|
64
|
+
} from './segment-store';
|
|
65
|
+
import { sha256Hex } from './sigv4';
|
|
66
|
+
|
|
67
|
+
export interface BucketSegmentStoreOptions {
|
|
68
|
+
/** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
|
|
69
|
+
readonly keyPrefix?: string | undefined;
|
|
70
|
+
/** Segment TTL (§5.1 cache semantics). Default 24 h. */
|
|
71
|
+
readonly ttlMs?: number | undefined;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
75
|
+
const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
76
|
+
/** Legacy record carried in the bytes object's user metadata. */
|
|
77
|
+
const LEGACY_RECORD_META_KEY = 'syncular-record';
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Stats accumulator (§ "S3 stats" in the README). The store keeps a small
|
|
81
|
+
* counter object under a fixed key, updated read-modify-write on `put`. It
|
|
82
|
+
* exists because a LIST-free store (every lookup is a GET/HEAD) has no cheap
|
|
83
|
+
* way to count objects on demand — so `stats()` reads the accumulator instead
|
|
84
|
+
* of enumerating the bucket.
|
|
85
|
+
*
|
|
86
|
+
* The write is guarded by an ETag compare-and-swap (`ifMatch`, or `createOnly`
|
|
87
|
+
* to create): the loop re-reads and retries on a failed precondition, so
|
|
88
|
+
* concurrent writers do not silently clobber each other's increment. The
|
|
89
|
+
* counters are still APPROXIMATE — a crash between the segment PUT and the
|
|
90
|
+
* accumulator CAS, or a writer against a backend that ignores conditional
|
|
91
|
+
* writes, can drift them — so the admin surface labels these stats
|
|
92
|
+
* `approximate: true`. They are a health gauge, not an invoice.
|
|
93
|
+
*/
|
|
94
|
+
const STATS_KEY_SUFFIX = 'stats/segments.json';
|
|
95
|
+
const CAS_ATTEMPTS = 16;
|
|
96
|
+
|
|
97
|
+
interface StatsAccumulator {
|
|
98
|
+
count: number;
|
|
99
|
+
bytes: number;
|
|
100
|
+
rowsSegments: number;
|
|
101
|
+
sqliteSegments: number;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function emptyStats(): StatsAccumulator {
|
|
105
|
+
return { count: 0, bytes: 0, rowsSegments: 0, sqliteSegments: 0 };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function parseStatsJson(json: string): StatsAccumulator {
|
|
109
|
+
const parsed = JSON.parse(json) as Partial<StatsAccumulator>;
|
|
110
|
+
return {
|
|
111
|
+
count: Number(parsed.count ?? 0),
|
|
112
|
+
bytes: Number(parsed.bytes ?? 0),
|
|
113
|
+
rowsSegments: Number(parsed.rowsSegments ?? 0),
|
|
114
|
+
sqliteSegments: Number(parsed.sqliteSegments ?? 0),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Runtime-neutral base64url: `Buffer` is not present on
|
|
119
|
+
// Cloudflare Workers without `nodejs_compat`, so a legacy object-metadata
|
|
120
|
+
// record header is decoded with `atob`, available in every runtime.
|
|
121
|
+
function base64urlToUtf8(value: string): string {
|
|
122
|
+
const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
|
|
123
|
+
const bytes = new Uint8Array(binary.length);
|
|
124
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
125
|
+
return new TextDecoder().decode(bytes);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function recordToJson(record: SegmentRecord): string {
|
|
129
|
+
return JSON.stringify(record);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function isMediaType(value: unknown): value is 'rows' | 'sqlite' {
|
|
133
|
+
return value === 'rows' || value === 'sqlite';
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** One publication inside a stored record (or a legacy materialized one). */
|
|
137
|
+
function parsePublication(
|
|
138
|
+
value: unknown,
|
|
139
|
+
source: string,
|
|
140
|
+
label: string,
|
|
141
|
+
): SegmentPublication {
|
|
142
|
+
if (typeof value !== 'object' || value === null) {
|
|
143
|
+
throw new Error(`${label}: corrupt publication in ${source}`);
|
|
144
|
+
}
|
|
145
|
+
const p = value as Record<string, unknown>;
|
|
146
|
+
const strings = ['partition', 'logEpoch', 'table', 'scopeDigest'] as const;
|
|
147
|
+
const numbers = [
|
|
148
|
+
'schemaVersion',
|
|
149
|
+
'asOfCommitSeq',
|
|
150
|
+
'rowCount',
|
|
151
|
+
'byteLength',
|
|
152
|
+
'createdAtMs',
|
|
153
|
+
'expiresAtMs',
|
|
154
|
+
] as const;
|
|
155
|
+
for (const field of strings) {
|
|
156
|
+
if (typeof p[field] !== 'string') {
|
|
157
|
+
throw new Error(`${label}: bad publication field ${field} in ${source}`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
for (const field of numbers) {
|
|
161
|
+
if (typeof p[field] !== 'number') {
|
|
162
|
+
throw new Error(`${label}: bad publication field ${field} in ${source}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (!isMediaType(p.mediaType)) {
|
|
166
|
+
throw new Error(`${label}: bad publication mediaType in ${source}`);
|
|
167
|
+
}
|
|
168
|
+
const cursorOk = (v: unknown): v is string | null =>
|
|
169
|
+
v === null || typeof v === 'string';
|
|
170
|
+
if (!cursorOk(p.rowCursor) || !cursorOk(p.nextRowCursor)) {
|
|
171
|
+
throw new Error(`${label}: bad publication cursor in ${source}`);
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
partition: p.partition as string,
|
|
175
|
+
logEpoch: p.logEpoch as string,
|
|
176
|
+
table: p.table as string,
|
|
177
|
+
schemaVersion: p.schemaVersion as number,
|
|
178
|
+
mediaType: p.mediaType,
|
|
179
|
+
scopeDigest: p.scopeDigest as string,
|
|
180
|
+
asOfCommitSeq: p.asOfCommitSeq as number,
|
|
181
|
+
rowCount: p.rowCount as number,
|
|
182
|
+
rowCursor: p.rowCursor,
|
|
183
|
+
nextRowCursor: p.nextRowCursor,
|
|
184
|
+
byteLength: p.byteLength as number,
|
|
185
|
+
createdAtMs: p.createdAtMs as number,
|
|
186
|
+
expiresAtMs: p.expiresAtMs as number,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function parseRecordJson(
|
|
191
|
+
json: string,
|
|
192
|
+
source: string,
|
|
193
|
+
label: string,
|
|
194
|
+
): SegmentRecord {
|
|
195
|
+
let parsed: unknown;
|
|
196
|
+
try {
|
|
197
|
+
parsed = JSON.parse(json);
|
|
198
|
+
} catch {
|
|
199
|
+
throw new Error(`${label}: corrupt record JSON in ${source}`);
|
|
200
|
+
}
|
|
201
|
+
if (typeof parsed !== 'object' || parsed === null) {
|
|
202
|
+
throw new Error(`${label}: corrupt record in ${source}`);
|
|
203
|
+
}
|
|
204
|
+
const r = parsed as Record<string, unknown>;
|
|
205
|
+
const strings = [
|
|
206
|
+
'segmentId',
|
|
207
|
+
'partition',
|
|
208
|
+
'logEpoch',
|
|
209
|
+
'table',
|
|
210
|
+
'scopeDigest',
|
|
211
|
+
] as const;
|
|
212
|
+
const numbers = [
|
|
213
|
+
'schemaVersion',
|
|
214
|
+
'asOfCommitSeq',
|
|
215
|
+
'rowCount',
|
|
216
|
+
'byteLength',
|
|
217
|
+
'createdAtMs',
|
|
218
|
+
'expiresAtMs',
|
|
219
|
+
] as const;
|
|
220
|
+
for (const field of strings) {
|
|
221
|
+
if (typeof r[field] !== 'string') {
|
|
222
|
+
throw new Error(`${label}: bad record field ${field} in ${source}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
for (const field of numbers) {
|
|
226
|
+
if (typeof r[field] !== 'number') {
|
|
227
|
+
throw new Error(`${label}: bad record field ${field} in ${source}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (!isMediaType(r.mediaType)) {
|
|
231
|
+
throw new Error(`${label}: bad record mediaType in ${source}`);
|
|
232
|
+
}
|
|
233
|
+
const cursorOk = (v: unknown): v is string | null =>
|
|
234
|
+
v === null || typeof v === 'string';
|
|
235
|
+
if (!cursorOk(r.rowCursor) || !cursorOk(r.nextRowCursor)) {
|
|
236
|
+
throw new Error(`${label}: bad record cursor in ${source}`);
|
|
237
|
+
}
|
|
238
|
+
const scopeDigest = r.scopeDigest as string;
|
|
239
|
+
// Records written before §5.1 recorded the digest set carry only the
|
|
240
|
+
// primary digest; the object is a cache entry with a TTL, so reading it
|
|
241
|
+
// back as a one-digest record is correct, not a degraded path.
|
|
242
|
+
const rawDigests = r.scopeDigests;
|
|
243
|
+
if (
|
|
244
|
+
rawDigests !== undefined &&
|
|
245
|
+
(!Array.isArray(rawDigests) ||
|
|
246
|
+
rawDigests.some((digest) => typeof digest !== 'string'))
|
|
247
|
+
) {
|
|
248
|
+
throw new Error(`${label}: bad record scopeDigests in ${source}`);
|
|
249
|
+
}
|
|
250
|
+
const rawPublications = r.publications;
|
|
251
|
+
if (rawPublications !== undefined && !Array.isArray(rawPublications)) {
|
|
252
|
+
throw new Error(`${label}: bad record publications in ${source}`);
|
|
253
|
+
}
|
|
254
|
+
const top = {
|
|
255
|
+
segmentId: r.segmentId as string,
|
|
256
|
+
partition: r.partition as string,
|
|
257
|
+
logEpoch: r.logEpoch as string,
|
|
258
|
+
table: r.table as string,
|
|
259
|
+
schemaVersion: r.schemaVersion as number,
|
|
260
|
+
mediaType: r.mediaType,
|
|
261
|
+
scopeDigest,
|
|
262
|
+
scopeDigests: [
|
|
263
|
+
scopeDigest,
|
|
264
|
+
...((rawDigests ?? []) as string[]).filter(
|
|
265
|
+
(digest) => digest !== scopeDigest,
|
|
266
|
+
),
|
|
267
|
+
],
|
|
268
|
+
asOfCommitSeq: r.asOfCommitSeq as number,
|
|
269
|
+
rowCount: r.rowCount as number,
|
|
270
|
+
rowCursor: r.rowCursor,
|
|
271
|
+
nextRowCursor: r.nextRowCursor,
|
|
272
|
+
byteLength: r.byteLength as number,
|
|
273
|
+
createdAtMs: r.createdAtMs as number,
|
|
274
|
+
expiresAtMs: r.expiresAtMs as number,
|
|
275
|
+
};
|
|
276
|
+
return {
|
|
277
|
+
...top,
|
|
278
|
+
// A record from before per-publication provenance, or a legacy record
|
|
279
|
+
// whose union came from object metadata, materializes its publications
|
|
280
|
+
// from the merged top-level context — the old single-record read.
|
|
281
|
+
publications:
|
|
282
|
+
rawPublications === undefined
|
|
283
|
+
? publicationsForRecord(top)
|
|
284
|
+
: rawPublications.map((value) =>
|
|
285
|
+
parsePublication(value, source, label),
|
|
286
|
+
),
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export class BucketSegmentStore implements SegmentStore {
|
|
291
|
+
readonly #bucket: ObjectBucket;
|
|
292
|
+
readonly #ttlMs: number;
|
|
293
|
+
readonly #prefix: string;
|
|
294
|
+
/** Error prefix: the concrete store class the host constructed. */
|
|
295
|
+
readonly #label: string;
|
|
296
|
+
|
|
297
|
+
constructor(bucket: ObjectBucket, options?: BucketSegmentStoreOptions) {
|
|
298
|
+
this.#bucket = bucket;
|
|
299
|
+
this.#ttlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
|
|
300
|
+
this.#prefix = options?.keyPrefix ?? '';
|
|
301
|
+
this.#label = this.constructor.name;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
|
|
305
|
+
objectKeyFor(segmentId: string): string {
|
|
306
|
+
return `${this.#prefix}seg/${segmentId.replace(':', '/')}`;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** `{keyPrefix}rec/sha256/{hex}.json` — the §5.1 record (§5.1 union). */
|
|
310
|
+
#recordKeyFor(segmentId: string): string {
|
|
311
|
+
return `${this.#prefix}rec/${segmentId.replace(':', '/')}.json`;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async #findKeyFor(key: SegmentFindKey): Promise<string> {
|
|
315
|
+
const canonical = JSON.stringify([
|
|
316
|
+
key.partition,
|
|
317
|
+
key.logEpoch,
|
|
318
|
+
key.table,
|
|
319
|
+
key.schemaVersion,
|
|
320
|
+
key.mediaType,
|
|
321
|
+
key.scopeDigest,
|
|
322
|
+
key.asOfCommitSeq,
|
|
323
|
+
]);
|
|
324
|
+
return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
#statsKey(): string {
|
|
328
|
+
return `${this.#prefix}${STATS_KEY_SUFFIX}`;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** Read the stats accumulator + its ETag (for the CAS write), or none. */
|
|
332
|
+
async #readStats(): Promise<{
|
|
333
|
+
stats: StatsAccumulator;
|
|
334
|
+
etag: string | undefined;
|
|
335
|
+
}> {
|
|
336
|
+
const object = await this.#bucket.get(this.#statsKey());
|
|
337
|
+
if (object === undefined) {
|
|
338
|
+
return { stats: emptyStats(), etag: undefined };
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
stats: parseStatsJson(await object.text()),
|
|
342
|
+
etag: object.etag,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Conditionally write the stats accumulator. `etag` present ⇒ `ifMatch`
|
|
348
|
+
* (update the object we read); absent ⇒ `createOnly` (create only). Returns
|
|
349
|
+
* `false` on a failed precondition so the caller retries.
|
|
350
|
+
*/
|
|
351
|
+
async #writeStatsCas(
|
|
352
|
+
stats: StatsAccumulator,
|
|
353
|
+
etag: string | undefined,
|
|
354
|
+
): Promise<boolean> {
|
|
355
|
+
return this.#bucket.put(this.#statsKey(), {
|
|
356
|
+
body: new TextEncoder().encode(JSON.stringify(stats)),
|
|
357
|
+
contentType: 'application/json',
|
|
358
|
+
...(etag === undefined ? { createOnly: true } : { ifMatch: etag }),
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Fold `delta` into the stats accumulator under an ETag compare-and-swap
|
|
364
|
+
* retry loop. Best-effort: exhausting the retry budget leaves the counters
|
|
365
|
+
* drifted (they are documented APPROXIMATE) rather than failing the `put`
|
|
366
|
+
* that already stored the bytes.
|
|
367
|
+
*/
|
|
368
|
+
async #bumpStats(delta: StatsAccumulator): Promise<void> {
|
|
369
|
+
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
370
|
+
const { stats, etag } = await this.#readStats();
|
|
371
|
+
const next: StatsAccumulator = {
|
|
372
|
+
count: stats.count + delta.count,
|
|
373
|
+
bytes: stats.bytes + delta.bytes,
|
|
374
|
+
rowsSegments: stats.rowsSegments + delta.rowsSegments,
|
|
375
|
+
sqliteSegments: stats.sqliteSegments + delta.sqliteSegments,
|
|
376
|
+
};
|
|
377
|
+
if (await this.#writeStatsCas(next, etag)) return;
|
|
378
|
+
}
|
|
379
|
+
// Lost the CAS race `CAS_ATTEMPTS` times running — leave the
|
|
380
|
+
// counters as-is. `stats()` stays APPROXIMATE by contract.
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
async put(
|
|
384
|
+
metadata: SegmentMetadata,
|
|
385
|
+
bytes: Uint8Array,
|
|
386
|
+
nowMs: number,
|
|
387
|
+
): Promise<SegmentRecord> {
|
|
388
|
+
const segmentId = await segmentIdFor(bytes);
|
|
389
|
+
await this.#putBytes(segmentId, bytes);
|
|
390
|
+
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
391
|
+
// The id IS the content address, so an existing record means a second
|
|
392
|
+
// scope published the same bytes: read it back and merge the digest set
|
|
393
|
+
// (§5.1).
|
|
394
|
+
const preexisting = await this.#readRecord(segmentId);
|
|
395
|
+
const record = mergeSegmentRecord(
|
|
396
|
+
preexisting?.record,
|
|
397
|
+
metadata,
|
|
398
|
+
segmentId,
|
|
399
|
+
bytes.length,
|
|
400
|
+
nowMs,
|
|
401
|
+
this.#ttlMs,
|
|
402
|
+
);
|
|
403
|
+
const recordJson = recordToJson(record);
|
|
404
|
+
// The digest set is monotone (§5.1) and this is a read-modify-write: a
|
|
405
|
+
// concurrent publisher of the same bytes computed its own union from the
|
|
406
|
+
// same pre-state, so an unconditional PUT would drop one of the digests
|
|
407
|
+
// and make that scope's download `sync.forbidden`. The write is
|
|
408
|
+
// conditional on the record body that was read — the record ETag is the
|
|
409
|
+
// body hash, so it changes whenever the union does — and a failed
|
|
410
|
+
// precondition re-reads and merges again. In-process dedupe cannot cover
|
|
411
|
+
// this: two scopes whose rows are byte-identical are two builds, and two
|
|
412
|
+
// server instances share one bucket.
|
|
413
|
+
const written = await this.#bucket.put(this.#recordKeyFor(segmentId), {
|
|
414
|
+
body: new TextEncoder().encode(recordJson),
|
|
415
|
+
contentType: 'application/json',
|
|
416
|
+
...(preexisting?.etag === undefined
|
|
417
|
+
? { createOnly: true }
|
|
418
|
+
: { ifMatch: preexisting.etag }),
|
|
419
|
+
});
|
|
420
|
+
if (!written) continue;
|
|
421
|
+
if (preexisting === undefined) {
|
|
422
|
+
await this.#bumpStats({
|
|
423
|
+
count: 1,
|
|
424
|
+
bytes: bytes.length,
|
|
425
|
+
rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
|
|
426
|
+
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
if (metadata.rowCursor === null) {
|
|
430
|
+
await this.#bucket.put(await this.#findKeyFor(metadata), {
|
|
431
|
+
body: new TextEncoder().encode(recordJson),
|
|
432
|
+
contentType: 'application/json',
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
return record;
|
|
436
|
+
}
|
|
437
|
+
throw new Error(
|
|
438
|
+
`${this.#label}: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`,
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* Write the immutable bytes object. Create-only, so a second publisher of
|
|
444
|
+
* identical bytes is a no-op and a second publisher of DIFFERENT bytes under
|
|
445
|
+
* the same content address is the hash collision §5.1 requires to fail
|
|
446
|
+
* loudly instead of overwriting.
|
|
447
|
+
*/
|
|
448
|
+
async #putBytes(segmentId: string, bytes: Uint8Array): Promise<void> {
|
|
449
|
+
const key = this.objectKeyFor(segmentId);
|
|
450
|
+
const written = await this.#bucket.put(key, {
|
|
451
|
+
body: bytes,
|
|
452
|
+
contentType: 'application/octet-stream',
|
|
453
|
+
createOnly: true,
|
|
454
|
+
});
|
|
455
|
+
if (written) return;
|
|
456
|
+
const current = await this.#bucket.get(key);
|
|
457
|
+
if (current === undefined) return;
|
|
458
|
+
const stored = new Uint8Array(await current.arrayBuffer());
|
|
459
|
+
if (!segmentBytesEqual(stored, bytes)) {
|
|
460
|
+
throw new Error(
|
|
461
|
+
`${this.#label}: content-address collision at ${segmentId} (§5.1)`,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Read the record object plus the ETag of the body that was read, so the
|
|
468
|
+
* union write can be made conditional on exactly that pre-state (§5.1).
|
|
469
|
+
*
|
|
470
|
+
* A record written before the record/bytes split rides in the bytes object's
|
|
471
|
+
* user metadata; it is read here and its union is materialized into a record
|
|
472
|
+
* object by the next `put`. That shape carries no usable ETag for the union
|
|
473
|
+
* (the bytes ETag is a content hash), which is exactly why it was replaced.
|
|
474
|
+
* A bytes object with neither a record object nor the legacy metadata is an
|
|
475
|
+
* entry whose record is missing — a cache miss, not corruption, so `get`
|
|
476
|
+
* reports it as absent and the client re-pulls.
|
|
477
|
+
*/
|
|
478
|
+
async #readRecord(
|
|
479
|
+
segmentId: string,
|
|
480
|
+
): Promise<{ record: SegmentRecord; etag: string | undefined } | undefined> {
|
|
481
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
|
|
482
|
+
const object = await this.#bucket.get(this.#recordKeyFor(segmentId));
|
|
483
|
+
if (object !== undefined) {
|
|
484
|
+
return {
|
|
485
|
+
record: parseRecordJson(
|
|
486
|
+
await object.text(),
|
|
487
|
+
'segment record',
|
|
488
|
+
this.#label,
|
|
489
|
+
),
|
|
490
|
+
etag: object.etag,
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
const legacy = await this.#bucket.get(this.objectKeyFor(segmentId));
|
|
494
|
+
if (legacy === undefined) return undefined;
|
|
495
|
+
const meta = legacy.metadata[LEGACY_RECORD_META_KEY];
|
|
496
|
+
await legacy.arrayBuffer();
|
|
497
|
+
if (meta === undefined) return undefined;
|
|
498
|
+
return {
|
|
499
|
+
record: parseRecordJson(base64urlToUtf8(meta), segmentId, this.#label),
|
|
500
|
+
etag: undefined,
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
async get(
|
|
505
|
+
segmentId: string,
|
|
506
|
+
): Promise<{ record: SegmentRecord; bytes: Uint8Array } | undefined> {
|
|
507
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
|
|
508
|
+
const record = await this.#readRecord(segmentId);
|
|
509
|
+
if (record === undefined) return undefined;
|
|
510
|
+
const object = await this.#bucket.get(this.objectKeyFor(segmentId));
|
|
511
|
+
if (object === undefined) return undefined;
|
|
512
|
+
return {
|
|
513
|
+
record: record.record,
|
|
514
|
+
bytes: new Uint8Array(await object.arrayBuffer()),
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
519
|
+
async open(segmentId: string): Promise<SegmentStream | undefined> {
|
|
520
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
|
|
521
|
+
const record = await this.#readRecord(segmentId);
|
|
522
|
+
if (record === undefined) return undefined;
|
|
523
|
+
const object = await this.#bucket.get(this.objectKeyFor(segmentId));
|
|
524
|
+
if (object === undefined || object.body === null) return undefined;
|
|
525
|
+
return {
|
|
526
|
+
record: record.record,
|
|
527
|
+
body: object.body,
|
|
528
|
+
byteLength: record.record.byteLength,
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
async find(
|
|
533
|
+
key: SegmentFindKey,
|
|
534
|
+
nowMs: number,
|
|
535
|
+
): Promise<SegmentRecord | undefined> {
|
|
536
|
+
const pointer = await this.#bucket.get(await this.#findKeyFor(key));
|
|
537
|
+
if (pointer === undefined) return undefined;
|
|
538
|
+
const record = parseRecordJson(
|
|
539
|
+
await pointer.text(),
|
|
540
|
+
'reuse pointer',
|
|
541
|
+
this.#label,
|
|
542
|
+
);
|
|
543
|
+
// The pointer is keyed by the full context, but the record body is the
|
|
544
|
+
// whole merged entry: select the publication the key names, never the
|
|
545
|
+
// top-level fields or the digest union (§5.1).
|
|
546
|
+
const publication = selectSegmentPublication(record, {
|
|
547
|
+
partition: key.partition,
|
|
548
|
+
logEpoch: key.logEpoch,
|
|
549
|
+
table: key.table,
|
|
550
|
+
schemaVersion: key.schemaVersion,
|
|
551
|
+
mediaType: key.mediaType,
|
|
552
|
+
scopeDigest: key.scopeDigest,
|
|
553
|
+
asOfCommitSeq: key.asOfCommitSeq,
|
|
554
|
+
rowCursor: null,
|
|
555
|
+
nowMs,
|
|
556
|
+
});
|
|
557
|
+
if (publication === undefined) return undefined;
|
|
558
|
+
// Lifecycle GC may have removed the object while the pointer survived.
|
|
559
|
+
const head = await this.#bucket.head(this.objectKeyFor(record.segmentId));
|
|
560
|
+
if (head === undefined) return undefined;
|
|
561
|
+
return publicationRecord(record, publication);
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Store-wide counters from the pointer-object accumulator (never a LIST).
|
|
566
|
+
* Marked `approximate: true`: the counters are maintained by a best-effort
|
|
567
|
+
* ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
|
|
568
|
+
* and the accumulator write, or lifecycle GC that deletes objects the
|
|
569
|
+
* accumulator still counts, can drift them. Exact within a single writer;
|
|
570
|
+
* a health gauge under concurrency. See the README "S3 stats".
|
|
571
|
+
*/
|
|
572
|
+
async stats(): Promise<SegmentStoreStats> {
|
|
573
|
+
const { stats } = await this.#readStats();
|
|
574
|
+
return {
|
|
575
|
+
count: stats.count,
|
|
576
|
+
bytes: stats.bytes,
|
|
577
|
+
rowsSegments: stats.rowsSegments,
|
|
578
|
+
sqliteSegments: stats.sqliteSegments,
|
|
579
|
+
approximate: true,
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
}
|
package/src/context.ts
CHANGED
|
@@ -93,6 +93,16 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
93
93
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
94
94
|
export interface RealtimeNotifier {
|
|
95
95
|
notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
|
|
96
|
+
/**
|
|
97
|
+
* A pull changed the client's persisted subscription list (§8.7). A hub
|
|
98
|
+
* re-registers that client's live sockets, so a client that runs its
|
|
99
|
+
* rounds over HTTP while a socket is open receives deltas for the new
|
|
100
|
+
* subscriptions.
|
|
101
|
+
*/
|
|
102
|
+
notifySubscriptions?(
|
|
103
|
+
partition: string,
|
|
104
|
+
clientId: string,
|
|
105
|
+
): void | Promise<void>;
|
|
96
106
|
}
|
|
97
107
|
|
|
98
108
|
export interface SyncServerConfig {
|
package/src/d1-storage.ts
CHANGED
|
@@ -54,7 +54,7 @@ import {
|
|
|
54
54
|
assertPhysicalColumns,
|
|
55
55
|
assertStoredLayouts,
|
|
56
56
|
commitWindowPageSql,
|
|
57
|
-
|
|
57
|
+
LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL,
|
|
58
58
|
deleteRowSql,
|
|
59
59
|
deleteSqliteRowScopesSql,
|
|
60
60
|
dropTableDdl,
|
|
@@ -2189,8 +2189,8 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2189
2189
|
if (firstVariable === undefined || values.length === 0) return 0;
|
|
2190
2190
|
// Concurrent reads of a pull share one `db.batch` round trip.
|
|
2191
2191
|
const [row] = await this.#batchedAll<{ seq: number | null }>(
|
|
2192
|
-
|
|
2193
|
-
|
|
2192
|
+
LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL,
|
|
2193
|
+
[partition, query.table, firstVariable, JSON.stringify(values)],
|
|
2194
2194
|
);
|
|
2195
2195
|
return row?.seq ?? 0;
|
|
2196
2196
|
}
|
package/src/handler.ts
CHANGED
|
@@ -717,6 +717,17 @@ async function* streamResponse(
|
|
|
717
717
|
updatedAtMs: clockOf(ctx)(),
|
|
718
718
|
subscriptions,
|
|
719
719
|
});
|
|
720
|
+
if (
|
|
721
|
+
ctx.realtime?.notifySubscriptions !== undefined &&
|
|
722
|
+
JSON.stringify(
|
|
723
|
+
(previous?.subscriptions ?? []).map((s) => [s.id, s.table, s.scopes]),
|
|
724
|
+
) !== JSON.stringify(subscriptions.map((s) => [s.id, s.table, s.scopes]))
|
|
725
|
+
) {
|
|
726
|
+
await ctx.realtime.notifySubscriptions(
|
|
727
|
+
ctx.partition,
|
|
728
|
+
plan.header.clientId,
|
|
729
|
+
);
|
|
730
|
+
}
|
|
720
731
|
} catch (error) {
|
|
721
732
|
if (error instanceof SyncError) {
|
|
722
733
|
if (report !== undefined) {
|
package/src/index.ts
CHANGED
|
@@ -11,6 +11,8 @@ export * from './admin';
|
|
|
11
11
|
export * from './authoritative-query';
|
|
12
12
|
export * from './blob-handlers';
|
|
13
13
|
export * from './blob-store';
|
|
14
|
+
export * from './bucket-blob-store';
|
|
15
|
+
export * from './bucket-segment-store';
|
|
14
16
|
export * from './content-encoding';
|
|
15
17
|
export * from './context';
|
|
16
18
|
export * from './crdt-merger';
|
|
@@ -65,8 +67,10 @@ export {
|
|
|
65
67
|
} from './reactions';
|
|
66
68
|
export * from './realtime';
|
|
67
69
|
export * from './restore';
|
|
70
|
+
export * from './object-bucket';
|
|
68
71
|
export * from './relational-rows';
|
|
69
72
|
export * from './s3-blob-store';
|
|
73
|
+
export * from './s3-object-bucket';
|
|
70
74
|
export * from './s3-segment-store';
|
|
71
75
|
export * from './schema';
|
|
72
76
|
export * from './scopes';
|