@syncular/server 0.32.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +3 -3
- package/dist/bucket-blob-store.d.ts +77 -0
- package/dist/bucket-blob-store.js +266 -0
- package/dist/bucket-segment-store.d.ts +77 -0
- package/dist/bucket-segment-store.js +445 -0
- package/dist/context.d.ts +7 -0
- package/dist/d1-storage.js +8 -2
- package/dist/handler.js +21 -7
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/object-bucket.d.ts +78 -0
- package/dist/object-bucket.js +18 -0
- package/dist/operations-realtime.js +3 -0
- package/dist/operations.js +4 -4
- package/dist/push.js +28 -7
- package/dist/realtime.d.ts +11 -0
- package/dist/realtime.js +38 -56
- package/dist/s3-blob-store.d.ts +9 -64
- package/dist/s3-blob-store.js +25 -359
- package/dist/s3-object-bucket.d.ts +39 -0
- package/dist/s3-object-bucket.js +164 -0
- package/dist/s3-segment-store.d.ts +9 -61
- package/dist/s3-segment-store.js +25 -503
- package/dist/seed.js +4 -3
- package/package.json +2 -2
- package/src/blob-handlers.ts +4 -4
- package/src/bucket-blob-store.ts +351 -0
- package/src/bucket-segment-store.ts +582 -0
- package/src/context.ts +10 -0
- package/src/d1-storage.ts +12 -2
- package/src/handler.ts +38 -10
- package/src/index.ts +4 -0
- package/src/object-bucket.ts +83 -0
- package/src/operations-realtime.ts +7 -0
- package/src/operations.ts +8 -4
- package/src/push.ts +40 -9
- package/src/realtime.ts +45 -55
- package/src/s3-blob-store.ts +25 -466
- package/src/s3-object-bucket.ts +233 -0
- package/src/s3-segment-store.ts +25 -646
- package/src/seed.ts +4 -3
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentBytesEqual, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
|
|
2
|
+
import { sha256Hex } from './sigv4.js';
|
|
3
|
+
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
4
|
+
const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
5
|
+
/** Legacy record carried in the bytes object's user metadata. */
|
|
6
|
+
const LEGACY_RECORD_META_KEY = 'syncular-record';
|
|
7
|
+
/**
|
|
8
|
+
* Stats accumulator (§ "S3 stats" in the README). The store keeps a small
|
|
9
|
+
* counter object under a fixed key, updated read-modify-write on `put`. It
|
|
10
|
+
* exists because a LIST-free store (every lookup is a GET/HEAD) has no cheap
|
|
11
|
+
* way to count objects on demand — so `stats()` reads the accumulator instead
|
|
12
|
+
* of enumerating the bucket.
|
|
13
|
+
*
|
|
14
|
+
* The write is guarded by an ETag compare-and-swap (`ifMatch`, or `createOnly`
|
|
15
|
+
* to create): the loop re-reads and retries on a failed precondition, so
|
|
16
|
+
* concurrent writers do not silently clobber each other's increment. The
|
|
17
|
+
* counters are still APPROXIMATE — a crash between the segment PUT and the
|
|
18
|
+
* accumulator CAS, or a writer against a backend that ignores conditional
|
|
19
|
+
* writes, can drift them — so the admin surface labels these stats
|
|
20
|
+
* `approximate: true`. They are a health gauge, not an invoice.
|
|
21
|
+
*/
|
|
22
|
+
const STATS_KEY_SUFFIX = 'stats/segments.json';
|
|
23
|
+
const CAS_ATTEMPTS = 16;
|
|
24
|
+
function emptyStats() {
|
|
25
|
+
return { count: 0, bytes: 0, rowsSegments: 0, sqliteSegments: 0 };
|
|
26
|
+
}
|
|
27
|
+
function parseStatsJson(json) {
|
|
28
|
+
const parsed = JSON.parse(json);
|
|
29
|
+
return {
|
|
30
|
+
count: Number(parsed.count ?? 0),
|
|
31
|
+
bytes: Number(parsed.bytes ?? 0),
|
|
32
|
+
rowsSegments: Number(parsed.rowsSegments ?? 0),
|
|
33
|
+
sqliteSegments: Number(parsed.sqliteSegments ?? 0),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
// Runtime-neutral base64url: `Buffer` is not present on
|
|
37
|
+
// Cloudflare Workers without `nodejs_compat`, so a legacy object-metadata
|
|
38
|
+
// record header is decoded with `atob`, available in every runtime.
|
|
39
|
+
function base64urlToUtf8(value) {
|
|
40
|
+
const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
|
|
41
|
+
const bytes = new Uint8Array(binary.length);
|
|
42
|
+
for (let i = 0; i < binary.length; i++)
|
|
43
|
+
bytes[i] = binary.charCodeAt(i);
|
|
44
|
+
return new TextDecoder().decode(bytes);
|
|
45
|
+
}
|
|
46
|
+
function recordToJson(record) {
|
|
47
|
+
return JSON.stringify(record);
|
|
48
|
+
}
|
|
49
|
+
function isMediaType(value) {
|
|
50
|
+
return value === 'rows' || value === 'sqlite';
|
|
51
|
+
}
|
|
52
|
+
/** One publication inside a stored record (or a legacy materialized one). */
|
|
53
|
+
function parsePublication(value, source, label) {
|
|
54
|
+
if (typeof value !== 'object' || value === null) {
|
|
55
|
+
throw new Error(`${label}: corrupt publication in ${source}`);
|
|
56
|
+
}
|
|
57
|
+
const p = value;
|
|
58
|
+
const strings = ['partition', 'logEpoch', 'table', 'scopeDigest'];
|
|
59
|
+
const numbers = [
|
|
60
|
+
'schemaVersion',
|
|
61
|
+
'asOfCommitSeq',
|
|
62
|
+
'rowCount',
|
|
63
|
+
'byteLength',
|
|
64
|
+
'createdAtMs',
|
|
65
|
+
'expiresAtMs',
|
|
66
|
+
];
|
|
67
|
+
for (const field of strings) {
|
|
68
|
+
if (typeof p[field] !== 'string') {
|
|
69
|
+
throw new Error(`${label}: bad publication field ${field} in ${source}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
for (const field of numbers) {
|
|
73
|
+
if (typeof p[field] !== 'number') {
|
|
74
|
+
throw new Error(`${label}: bad publication field ${field} in ${source}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
if (!isMediaType(p.mediaType)) {
|
|
78
|
+
throw new Error(`${label}: bad publication mediaType in ${source}`);
|
|
79
|
+
}
|
|
80
|
+
const cursorOk = (v) => v === null || typeof v === 'string';
|
|
81
|
+
if (!cursorOk(p.rowCursor) || !cursorOk(p.nextRowCursor)) {
|
|
82
|
+
throw new Error(`${label}: bad publication cursor in ${source}`);
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
partition: p.partition,
|
|
86
|
+
logEpoch: p.logEpoch,
|
|
87
|
+
table: p.table,
|
|
88
|
+
schemaVersion: p.schemaVersion,
|
|
89
|
+
mediaType: p.mediaType,
|
|
90
|
+
scopeDigest: p.scopeDigest,
|
|
91
|
+
asOfCommitSeq: p.asOfCommitSeq,
|
|
92
|
+
rowCount: p.rowCount,
|
|
93
|
+
rowCursor: p.rowCursor,
|
|
94
|
+
nextRowCursor: p.nextRowCursor,
|
|
95
|
+
byteLength: p.byteLength,
|
|
96
|
+
createdAtMs: p.createdAtMs,
|
|
97
|
+
expiresAtMs: p.expiresAtMs,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function parseRecordJson(json, source, label) {
|
|
101
|
+
let parsed;
|
|
102
|
+
try {
|
|
103
|
+
parsed = JSON.parse(json);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
throw new Error(`${label}: corrupt record JSON in ${source}`);
|
|
107
|
+
}
|
|
108
|
+
if (typeof parsed !== 'object' || parsed === null) {
|
|
109
|
+
throw new Error(`${label}: corrupt record in ${source}`);
|
|
110
|
+
}
|
|
111
|
+
const r = parsed;
|
|
112
|
+
const strings = [
|
|
113
|
+
'segmentId',
|
|
114
|
+
'partition',
|
|
115
|
+
'logEpoch',
|
|
116
|
+
'table',
|
|
117
|
+
'scopeDigest',
|
|
118
|
+
];
|
|
119
|
+
const numbers = [
|
|
120
|
+
'schemaVersion',
|
|
121
|
+
'asOfCommitSeq',
|
|
122
|
+
'rowCount',
|
|
123
|
+
'byteLength',
|
|
124
|
+
'createdAtMs',
|
|
125
|
+
'expiresAtMs',
|
|
126
|
+
];
|
|
127
|
+
for (const field of strings) {
|
|
128
|
+
if (typeof r[field] !== 'string') {
|
|
129
|
+
throw new Error(`${label}: bad record field ${field} in ${source}`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
for (const field of numbers) {
|
|
133
|
+
if (typeof r[field] !== 'number') {
|
|
134
|
+
throw new Error(`${label}: bad record field ${field} in ${source}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (!isMediaType(r.mediaType)) {
|
|
138
|
+
throw new Error(`${label}: bad record mediaType in ${source}`);
|
|
139
|
+
}
|
|
140
|
+
const cursorOk = (v) => v === null || typeof v === 'string';
|
|
141
|
+
if (!cursorOk(r.rowCursor) || !cursorOk(r.nextRowCursor)) {
|
|
142
|
+
throw new Error(`${label}: bad record cursor in ${source}`);
|
|
143
|
+
}
|
|
144
|
+
const scopeDigest = r.scopeDigest;
|
|
145
|
+
// Records written before §5.1 recorded the digest set carry only the
|
|
146
|
+
// primary digest; the object is a cache entry with a TTL, so reading it
|
|
147
|
+
// back as a one-digest record is correct, not a degraded path.
|
|
148
|
+
const rawDigests = r.scopeDigests;
|
|
149
|
+
if (rawDigests !== undefined &&
|
|
150
|
+
(!Array.isArray(rawDigests) ||
|
|
151
|
+
rawDigests.some((digest) => typeof digest !== 'string'))) {
|
|
152
|
+
throw new Error(`${label}: bad record scopeDigests in ${source}`);
|
|
153
|
+
}
|
|
154
|
+
const rawPublications = r.publications;
|
|
155
|
+
if (rawPublications !== undefined && !Array.isArray(rawPublications)) {
|
|
156
|
+
throw new Error(`${label}: bad record publications in ${source}`);
|
|
157
|
+
}
|
|
158
|
+
const top = {
|
|
159
|
+
segmentId: r.segmentId,
|
|
160
|
+
partition: r.partition,
|
|
161
|
+
logEpoch: r.logEpoch,
|
|
162
|
+
table: r.table,
|
|
163
|
+
schemaVersion: r.schemaVersion,
|
|
164
|
+
mediaType: r.mediaType,
|
|
165
|
+
scopeDigest,
|
|
166
|
+
scopeDigests: [
|
|
167
|
+
scopeDigest,
|
|
168
|
+
...(rawDigests ?? []).filter((digest) => digest !== scopeDigest),
|
|
169
|
+
],
|
|
170
|
+
asOfCommitSeq: r.asOfCommitSeq,
|
|
171
|
+
rowCount: r.rowCount,
|
|
172
|
+
rowCursor: r.rowCursor,
|
|
173
|
+
nextRowCursor: r.nextRowCursor,
|
|
174
|
+
byteLength: r.byteLength,
|
|
175
|
+
createdAtMs: r.createdAtMs,
|
|
176
|
+
expiresAtMs: r.expiresAtMs,
|
|
177
|
+
};
|
|
178
|
+
return {
|
|
179
|
+
...top,
|
|
180
|
+
// A record from before per-publication provenance, or a legacy record
|
|
181
|
+
// whose union came from object metadata, materializes its publications
|
|
182
|
+
// from the merged top-level context — the old single-record read.
|
|
183
|
+
publications: rawPublications === undefined
|
|
184
|
+
? publicationsForRecord(top)
|
|
185
|
+
: rawPublications.map((value) => parsePublication(value, source, label)),
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
export class BucketSegmentStore {
|
|
189
|
+
#bucket;
|
|
190
|
+
#ttlMs;
|
|
191
|
+
#prefix;
|
|
192
|
+
/** Error prefix: the concrete store class the host constructed. */
|
|
193
|
+
#label;
|
|
194
|
+
constructor(bucket, options) {
|
|
195
|
+
this.#bucket = bucket;
|
|
196
|
+
this.#ttlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
|
|
197
|
+
this.#prefix = options?.keyPrefix ?? '';
|
|
198
|
+
this.#label = this.constructor.name;
|
|
199
|
+
}
|
|
200
|
+
/** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
|
|
201
|
+
objectKeyFor(segmentId) {
|
|
202
|
+
return `${this.#prefix}seg/${segmentId.replace(':', '/')}`;
|
|
203
|
+
}
|
|
204
|
+
/** `{keyPrefix}rec/sha256/{hex}.json` — the §5.1 record (§5.1 union). */
|
|
205
|
+
#recordKeyFor(segmentId) {
|
|
206
|
+
return `${this.#prefix}rec/${segmentId.replace(':', '/')}.json`;
|
|
207
|
+
}
|
|
208
|
+
async #findKeyFor(key) {
|
|
209
|
+
const canonical = JSON.stringify([
|
|
210
|
+
key.partition,
|
|
211
|
+
key.logEpoch,
|
|
212
|
+
key.table,
|
|
213
|
+
key.schemaVersion,
|
|
214
|
+
key.mediaType,
|
|
215
|
+
key.scopeDigest,
|
|
216
|
+
key.asOfCommitSeq,
|
|
217
|
+
]);
|
|
218
|
+
return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
|
|
219
|
+
}
|
|
220
|
+
#statsKey() {
|
|
221
|
+
return `${this.#prefix}${STATS_KEY_SUFFIX}`;
|
|
222
|
+
}
|
|
223
|
+
/** Read the stats accumulator + its ETag (for the CAS write), or none. */
|
|
224
|
+
async #readStats() {
|
|
225
|
+
const object = await this.#bucket.get(this.#statsKey());
|
|
226
|
+
if (object === undefined) {
|
|
227
|
+
return { stats: emptyStats(), etag: undefined };
|
|
228
|
+
}
|
|
229
|
+
return {
|
|
230
|
+
stats: parseStatsJson(await object.text()),
|
|
231
|
+
etag: object.etag,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Conditionally write the stats accumulator. `etag` present ⇒ `ifMatch`
|
|
236
|
+
* (update the object we read); absent ⇒ `createOnly` (create only). Returns
|
|
237
|
+
* `false` on a failed precondition so the caller retries.
|
|
238
|
+
*/
|
|
239
|
+
async #writeStatsCas(stats, etag) {
|
|
240
|
+
return this.#bucket.put(this.#statsKey(), {
|
|
241
|
+
body: new TextEncoder().encode(JSON.stringify(stats)),
|
|
242
|
+
contentType: 'application/json',
|
|
243
|
+
...(etag === undefined ? { createOnly: true } : { ifMatch: etag }),
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Fold `delta` into the stats accumulator under an ETag compare-and-swap
|
|
248
|
+
* retry loop. Best-effort: exhausting the retry budget leaves the counters
|
|
249
|
+
* drifted (they are documented APPROXIMATE) rather than failing the `put`
|
|
250
|
+
* that already stored the bytes.
|
|
251
|
+
*/
|
|
252
|
+
async #bumpStats(delta) {
|
|
253
|
+
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
254
|
+
const { stats, etag } = await this.#readStats();
|
|
255
|
+
const next = {
|
|
256
|
+
count: stats.count + delta.count,
|
|
257
|
+
bytes: stats.bytes + delta.bytes,
|
|
258
|
+
rowsSegments: stats.rowsSegments + delta.rowsSegments,
|
|
259
|
+
sqliteSegments: stats.sqliteSegments + delta.sqliteSegments,
|
|
260
|
+
};
|
|
261
|
+
if (await this.#writeStatsCas(next, etag))
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
// Lost the CAS race `CAS_ATTEMPTS` times running — leave the
|
|
265
|
+
// counters as-is. `stats()` stays APPROXIMATE by contract.
|
|
266
|
+
}
|
|
267
|
+
async put(metadata, bytes, nowMs) {
|
|
268
|
+
const segmentId = await segmentIdFor(bytes);
|
|
269
|
+
await this.#putBytes(segmentId, bytes);
|
|
270
|
+
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
271
|
+
// The id IS the content address, so an existing record means a second
|
|
272
|
+
// scope published the same bytes: read it back and merge the digest set
|
|
273
|
+
// (§5.1).
|
|
274
|
+
const preexisting = await this.#readRecord(segmentId);
|
|
275
|
+
const record = mergeSegmentRecord(preexisting?.record, metadata, segmentId, bytes.length, nowMs, this.#ttlMs);
|
|
276
|
+
const recordJson = recordToJson(record);
|
|
277
|
+
// The digest set is monotone (§5.1) and this is a read-modify-write: a
|
|
278
|
+
// concurrent publisher of the same bytes computed its own union from the
|
|
279
|
+
// same pre-state, so an unconditional PUT would drop one of the digests
|
|
280
|
+
// and make that scope's download `sync.forbidden`. The write is
|
|
281
|
+
// conditional on the record body that was read — the record ETag is the
|
|
282
|
+
// body hash, so it changes whenever the union does — and a failed
|
|
283
|
+
// precondition re-reads and merges again. In-process dedupe cannot cover
|
|
284
|
+
// this: two scopes whose rows are byte-identical are two builds, and two
|
|
285
|
+
// server instances share one bucket.
|
|
286
|
+
const written = await this.#bucket.put(this.#recordKeyFor(segmentId), {
|
|
287
|
+
body: new TextEncoder().encode(recordJson),
|
|
288
|
+
contentType: 'application/json',
|
|
289
|
+
...(preexisting?.etag === undefined
|
|
290
|
+
? { createOnly: true }
|
|
291
|
+
: { ifMatch: preexisting.etag }),
|
|
292
|
+
});
|
|
293
|
+
if (!written)
|
|
294
|
+
continue;
|
|
295
|
+
if (preexisting === undefined) {
|
|
296
|
+
await this.#bumpStats({
|
|
297
|
+
count: 1,
|
|
298
|
+
bytes: bytes.length,
|
|
299
|
+
rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
|
|
300
|
+
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
if (metadata.rowCursor === null) {
|
|
304
|
+
await this.#bucket.put(await this.#findKeyFor(metadata), {
|
|
305
|
+
body: new TextEncoder().encode(recordJson),
|
|
306
|
+
contentType: 'application/json',
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
return record;
|
|
310
|
+
}
|
|
311
|
+
throw new Error(`${this.#label}: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`);
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Write the immutable bytes object. Create-only, so a second publisher of
|
|
315
|
+
* identical bytes is a no-op and a second publisher of DIFFERENT bytes under
|
|
316
|
+
* the same content address is the hash collision §5.1 requires to fail
|
|
317
|
+
* loudly instead of overwriting.
|
|
318
|
+
*/
|
|
319
|
+
async #putBytes(segmentId, bytes) {
|
|
320
|
+
const key = this.objectKeyFor(segmentId);
|
|
321
|
+
const written = await this.#bucket.put(key, {
|
|
322
|
+
body: bytes,
|
|
323
|
+
contentType: 'application/octet-stream',
|
|
324
|
+
createOnly: true,
|
|
325
|
+
});
|
|
326
|
+
if (written)
|
|
327
|
+
return;
|
|
328
|
+
const current = await this.#bucket.get(key);
|
|
329
|
+
if (current === undefined)
|
|
330
|
+
return;
|
|
331
|
+
const stored = new Uint8Array(await current.arrayBuffer());
|
|
332
|
+
if (!segmentBytesEqual(stored, bytes)) {
|
|
333
|
+
throw new Error(`${this.#label}: content-address collision at ${segmentId} (§5.1)`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Read the record object plus the ETag of the body that was read, so the
|
|
338
|
+
* union write can be made conditional on exactly that pre-state (§5.1).
|
|
339
|
+
*
|
|
340
|
+
* A record written before the record/bytes split rides in the bytes object's
|
|
341
|
+
* user metadata; it is read here and its union is materialized into a record
|
|
342
|
+
* object by the next `put`. That shape carries no usable ETag for the union
|
|
343
|
+
* (the bytes ETag is a content hash), which is exactly why it was replaced.
|
|
344
|
+
* A bytes object with neither a record object nor the legacy metadata is an
|
|
345
|
+
* entry whose record is missing — a cache miss, not corruption, so `get`
|
|
346
|
+
* reports it as absent and the client re-pulls.
|
|
347
|
+
*/
|
|
348
|
+
async #readRecord(segmentId) {
|
|
349
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId))
|
|
350
|
+
return undefined;
|
|
351
|
+
const object = await this.#bucket.get(this.#recordKeyFor(segmentId));
|
|
352
|
+
if (object !== undefined) {
|
|
353
|
+
return {
|
|
354
|
+
record: parseRecordJson(await object.text(), 'segment record', this.#label),
|
|
355
|
+
etag: object.etag,
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
const legacy = await this.#bucket.get(this.objectKeyFor(segmentId));
|
|
359
|
+
if (legacy === undefined)
|
|
360
|
+
return undefined;
|
|
361
|
+
const meta = legacy.metadata[LEGACY_RECORD_META_KEY];
|
|
362
|
+
await legacy.arrayBuffer();
|
|
363
|
+
if (meta === undefined)
|
|
364
|
+
return undefined;
|
|
365
|
+
return {
|
|
366
|
+
record: parseRecordJson(base64urlToUtf8(meta), segmentId, this.#label),
|
|
367
|
+
etag: undefined,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
async get(segmentId) {
|
|
371
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId))
|
|
372
|
+
return undefined;
|
|
373
|
+
const record = await this.#readRecord(segmentId);
|
|
374
|
+
if (record === undefined)
|
|
375
|
+
return undefined;
|
|
376
|
+
const object = await this.#bucket.get(this.objectKeyFor(segmentId));
|
|
377
|
+
if (object === undefined)
|
|
378
|
+
return undefined;
|
|
379
|
+
return {
|
|
380
|
+
record: record.record,
|
|
381
|
+
bytes: new Uint8Array(await object.arrayBuffer()),
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
385
|
+
async open(segmentId) {
|
|
386
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId))
|
|
387
|
+
return undefined;
|
|
388
|
+
const record = await this.#readRecord(segmentId);
|
|
389
|
+
if (record === undefined)
|
|
390
|
+
return undefined;
|
|
391
|
+
const object = await this.#bucket.get(this.objectKeyFor(segmentId));
|
|
392
|
+
if (object === undefined || object.body === null)
|
|
393
|
+
return undefined;
|
|
394
|
+
return {
|
|
395
|
+
record: record.record,
|
|
396
|
+
body: object.body,
|
|
397
|
+
byteLength: record.record.byteLength,
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
async find(key, nowMs) {
|
|
401
|
+
const pointer = await this.#bucket.get(await this.#findKeyFor(key));
|
|
402
|
+
if (pointer === undefined)
|
|
403
|
+
return undefined;
|
|
404
|
+
const record = parseRecordJson(await pointer.text(), 'reuse pointer', this.#label);
|
|
405
|
+
// The pointer is keyed by the full context, but the record body is the
|
|
406
|
+
// whole merged entry: select the publication the key names, never the
|
|
407
|
+
// top-level fields or the digest union (§5.1).
|
|
408
|
+
const publication = selectSegmentPublication(record, {
|
|
409
|
+
partition: key.partition,
|
|
410
|
+
logEpoch: key.logEpoch,
|
|
411
|
+
table: key.table,
|
|
412
|
+
schemaVersion: key.schemaVersion,
|
|
413
|
+
mediaType: key.mediaType,
|
|
414
|
+
scopeDigest: key.scopeDigest,
|
|
415
|
+
asOfCommitSeq: key.asOfCommitSeq,
|
|
416
|
+
rowCursor: null,
|
|
417
|
+
nowMs,
|
|
418
|
+
});
|
|
419
|
+
if (publication === undefined)
|
|
420
|
+
return undefined;
|
|
421
|
+
// Lifecycle GC may have removed the object while the pointer survived.
|
|
422
|
+
const head = await this.#bucket.head(this.objectKeyFor(record.segmentId));
|
|
423
|
+
if (head === undefined)
|
|
424
|
+
return undefined;
|
|
425
|
+
return publicationRecord(record, publication);
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Store-wide counters from the pointer-object accumulator (never a LIST).
|
|
429
|
+
* Marked `approximate: true`: the counters are maintained by a best-effort
|
|
430
|
+
* ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
|
|
431
|
+
* and the accumulator write, or lifecycle GC that deletes objects the
|
|
432
|
+
* accumulator still counts, can drift them. Exact within a single writer;
|
|
433
|
+
* a health gauge under concurrency. See the README "S3 stats".
|
|
434
|
+
*/
|
|
435
|
+
async stats() {
|
|
436
|
+
const { stats } = await this.#readStats();
|
|
437
|
+
return {
|
|
438
|
+
count: stats.count,
|
|
439
|
+
bytes: stats.bytes,
|
|
440
|
+
rowsSegments: stats.rowsSegments,
|
|
441
|
+
sqliteSegments: stats.sqliteSegments,
|
|
442
|
+
approximate: true,
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
}
|
package/dist/context.d.ts
CHANGED
|
@@ -69,6 +69,13 @@ export declare const DEFAULT_LIMITS: ServerLimits;
|
|
|
69
69
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
70
70
|
export interface RealtimeNotifier {
|
|
71
71
|
notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
|
|
72
|
+
/**
|
|
73
|
+
* A pull changed the client's persisted subscription list (§8.7). A hub
|
|
74
|
+
* re-registers that client's live sockets, so a client that runs its
|
|
75
|
+
* rounds over HTTP while a socket is open receives deltas for the new
|
|
76
|
+
* subscriptions.
|
|
77
|
+
*/
|
|
78
|
+
notifySubscriptions?(partition: string, clientId: string): void | Promise<void>;
|
|
72
79
|
}
|
|
73
80
|
export interface SyncServerConfig {
|
|
74
81
|
readonly schema: ServerSchema;
|
package/dist/d1-storage.js
CHANGED
|
@@ -144,6 +144,7 @@ class D1Transaction {
|
|
|
144
144
|
* `tbl\u0000rowId` → the pending state, consulted by `getRow` before D1.
|
|
145
145
|
*/
|
|
146
146
|
#pending = new Map();
|
|
147
|
+
#clearedTombstones = new Set();
|
|
147
148
|
constructor(db, partition, resolveTable, prepareQuery, pushApplySerialized) {
|
|
148
149
|
this.#db = db;
|
|
149
150
|
this.#partition = partition;
|
|
@@ -175,6 +176,9 @@ class D1Transaction {
|
|
|
175
176
|
}
|
|
176
177
|
async getTombstoneSeq(table, rowId) {
|
|
177
178
|
this.#assertOpen();
|
|
179
|
+
if (this.#clearedTombstones.has(_a.#key(table, rowId))) {
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
178
182
|
// Persisted read, no #pending overlay: a delete buffered by this commit
|
|
179
183
|
// records its tombstone at appendCommit, matching the SQLite/Postgres
|
|
180
184
|
// adapters where the tombstone insert also runs at commit append.
|
|
@@ -186,6 +190,7 @@ class D1Transaction {
|
|
|
186
190
|
}
|
|
187
191
|
async clearTombstone(table, rowId) {
|
|
188
192
|
this.#assertOpen();
|
|
193
|
+
this.#clearedTombstones.add(_a.#key(table, rowId));
|
|
189
194
|
this.#buffer_('DELETE FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?', [this.#partition, table, rowId]);
|
|
190
195
|
}
|
|
191
196
|
async getPushResult(clientId, clientCommitId) {
|
|
@@ -250,7 +255,7 @@ class D1Transaction {
|
|
|
250
255
|
}
|
|
251
256
|
}
|
|
252
257
|
return [...rows.values()]
|
|
253
|
-
.sort((left, right) => left.rowId.
|
|
258
|
+
.sort((left, right) => left.rowId < right.rowId ? -1 : left.rowId > right.rowId ? 1 : 0)
|
|
254
259
|
.slice(0, query.limit);
|
|
255
260
|
}
|
|
256
261
|
async scanRowsByIndex(query) {
|
|
@@ -287,7 +292,7 @@ class D1Transaction {
|
|
|
287
292
|
rows.set(rowId, pending.row);
|
|
288
293
|
}
|
|
289
294
|
return [...rows.values()]
|
|
290
|
-
.sort((left, right) => left.rowId.
|
|
295
|
+
.sort((left, right) => left.rowId < right.rowId ? -1 : left.rowId > right.rowId ? 1 : 0)
|
|
291
296
|
.slice(0, query.limit);
|
|
292
297
|
}
|
|
293
298
|
async queryAuthoritative(query) {
|
|
@@ -334,6 +339,7 @@ class D1Transaction {
|
|
|
334
339
|
}
|
|
335
340
|
this.#buffer.length = checkpoint;
|
|
336
341
|
this.#pending.clear();
|
|
342
|
+
this.#clearedTombstones.clear();
|
|
337
343
|
this.#maxCommitSeq = undefined;
|
|
338
344
|
this.#applicationOpIndexes.clear();
|
|
339
345
|
await this.putPushResult(clientId, clientCommitId, result);
|
package/dist/handler.js
CHANGED
|
@@ -341,25 +341,35 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
341
341
|
}
|
|
342
342
|
try {
|
|
343
343
|
// Push half (§6): one PUSH_RESULT per PUSH_COMMIT, in request order.
|
|
344
|
+
const currentSchema = compileSchema(ctx.schema);
|
|
344
345
|
let acceptedThrough = 0;
|
|
345
346
|
for (const push of plan.pushes) {
|
|
346
|
-
const processed = await processPushCommitWithTrace(ctx,
|
|
347
|
+
const processed = await processPushCommitWithTrace(ctx, currentSchema, plan.resolved, plan.header.clientId, push, schema);
|
|
347
348
|
acceptedThrough = Math.max(acceptedThrough, processed.frame.commitSeq ?? 0);
|
|
348
349
|
const frame = {
|
|
349
350
|
...processed.frame,
|
|
350
351
|
results: processed.frame.results.map((result) => {
|
|
351
|
-
if (result.status !== 'conflict')
|
|
352
|
+
if (result.status !== 'conflict' || schema === currentSchema) {
|
|
352
353
|
return result;
|
|
354
|
+
}
|
|
353
355
|
const operation = push.operations[result.opIndex];
|
|
356
|
+
// §2.3: a replay returns the persisted result without rebuilding
|
|
357
|
+
// operations, so a reused clientCommitId with changed operations
|
|
358
|
+
// leaves no table to project the stored conflict row through.
|
|
359
|
+
if (operation === undefined) {
|
|
360
|
+
throw syncError('sync.invalid_request', 'replayed clientCommitId carries changed operations', JSON.stringify({
|
|
361
|
+
clientCommitId: push.clientCommitId,
|
|
362
|
+
opIndex: result.opIndex,
|
|
363
|
+
}));
|
|
364
|
+
}
|
|
354
365
|
const target = schema.tables.get(operation.table);
|
|
355
|
-
const
|
|
366
|
+
const bitmapBytes = Math.ceil(target.columns.length / 8);
|
|
356
367
|
return {
|
|
357
368
|
...result,
|
|
358
|
-
serverRow: projectRowPayload(
|
|
369
|
+
serverRow: projectRowPayload(currentSchema.tables.get(operation.table), target, result.serverRow),
|
|
359
370
|
conflictColumns: result.conflictColumns
|
|
360
|
-
.slice(0,
|
|
361
|
-
.map((byte, i) => i ===
|
|
362
|
-
target.columns.length % 8 !== 0
|
|
371
|
+
.slice(0, bitmapBytes)
|
|
372
|
+
.map((byte, i) => i === bitmapBytes - 1 && target.columns.length % 8 !== 0
|
|
363
373
|
? byte & ((1 << (target.columns.length % 8)) - 1)
|
|
364
374
|
: byte),
|
|
365
375
|
};
|
|
@@ -474,6 +484,10 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
474
484
|
updatedAtMs: clockOf(ctx)(),
|
|
475
485
|
subscriptions,
|
|
476
486
|
});
|
|
487
|
+
if (ctx.realtime?.notifySubscriptions !== undefined &&
|
|
488
|
+
JSON.stringify((previous?.subscriptions ?? []).map((s) => [s.id, s.table, s.scopes])) !== JSON.stringify(subscriptions.map((s) => [s.id, s.table, s.scopes]))) {
|
|
489
|
+
await ctx.realtime.notifySubscriptions(ctx.partition, plan.header.clientId);
|
|
490
|
+
}
|
|
477
491
|
}
|
|
478
492
|
catch (error) {
|
|
479
493
|
if (error instanceof SyncError) {
|
package/dist/index.d.ts
CHANGED
|
@@ -11,6 +11,8 @@ export * from './admin.js';
|
|
|
11
11
|
export * from './authoritative-query.js';
|
|
12
12
|
export * from './blob-handlers.js';
|
|
13
13
|
export * from './blob-store.js';
|
|
14
|
+
export * from './bucket-blob-store.js';
|
|
15
|
+
export * from './bucket-segment-store.js';
|
|
14
16
|
export * from './content-encoding.js';
|
|
15
17
|
export * from './context.js';
|
|
16
18
|
export * from './crdt-merger.js';
|
|
@@ -33,8 +35,10 @@ export * from './readiness.js';
|
|
|
33
35
|
export { DEFAULT_REACTION_INITIAL_BACKOFF_MS, DEFAULT_REACTION_LEASE_MS, DEFAULT_REACTION_MAX_ATTEMPTS, DEFAULT_REACTION_MAX_BACKOFF_MS, DEFAULT_REACTION_RETENTION, MAX_REACTION_FAILURE_DETAILS_BYTES, MAX_REACTION_PAYLOAD_BYTES, MAX_REACTIONS_PER_COMMIT, PermanentReactionError, pruneReactions, ReactionRunner, reactionIdempotencyKey, retryDeadLetterReaction, RetryableReactionError, type PlannedReaction, type PruneReactionsOptions, type ReactionHandler, type ReactionHandlerInput, type ReactionHandlers, type ReactionPlan, type ReactionPlanner, type ReactionPlannerInput, type ReactionPruneResult, type ReactionRetentionPolicy, type ReactionRunnerOptions, type ReactionRunResult, type ReactionTypeMap, } from './reactions.js';
|
|
34
36
|
export * from './realtime.js';
|
|
35
37
|
export * from './restore.js';
|
|
38
|
+
export * from './object-bucket.js';
|
|
36
39
|
export * from './relational-rows.js';
|
|
37
40
|
export * from './s3-blob-store.js';
|
|
41
|
+
export * from './s3-object-bucket.js';
|
|
38
42
|
export * from './s3-segment-store.js';
|
|
39
43
|
export * from './schema.js';
|
|
40
44
|
export * from './scopes.js';
|
package/dist/index.js
CHANGED
|
@@ -11,6 +11,8 @@ export * from './admin.js';
|
|
|
11
11
|
export * from './authoritative-query.js';
|
|
12
12
|
export * from './blob-handlers.js';
|
|
13
13
|
export * from './blob-store.js';
|
|
14
|
+
export * from './bucket-blob-store.js';
|
|
15
|
+
export * from './bucket-segment-store.js';
|
|
14
16
|
export * from './content-encoding.js';
|
|
15
17
|
export * from './context.js';
|
|
16
18
|
export * from './crdt-merger.js';
|
|
@@ -37,8 +39,10 @@ export * from './readiness.js';
|
|
|
37
39
|
export { DEFAULT_REACTION_INITIAL_BACKOFF_MS, DEFAULT_REACTION_LEASE_MS, DEFAULT_REACTION_MAX_ATTEMPTS, DEFAULT_REACTION_MAX_BACKOFF_MS, DEFAULT_REACTION_RETENTION, MAX_REACTION_FAILURE_DETAILS_BYTES, MAX_REACTION_PAYLOAD_BYTES, MAX_REACTIONS_PER_COMMIT, PermanentReactionError, pruneReactions, ReactionRunner, reactionIdempotencyKey, retryDeadLetterReaction, RetryableReactionError, } from './reactions.js';
|
|
38
40
|
export * from './realtime.js';
|
|
39
41
|
export * from './restore.js';
|
|
42
|
+
export * from './object-bucket.js';
|
|
40
43
|
export * from './relational-rows.js';
|
|
41
44
|
export * from './s3-blob-store.js';
|
|
45
|
+
export * from './s3-object-bucket.js';
|
|
42
46
|
export * from './s3-segment-store.js';
|
|
43
47
|
export * from './schema.js';
|
|
44
48
|
export * from './scopes.js';
|