@syncular/server 0.32.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +3 -3
- package/dist/bucket-blob-store.d.ts +77 -0
- package/dist/bucket-blob-store.js +266 -0
- package/dist/bucket-segment-store.d.ts +77 -0
- package/dist/bucket-segment-store.js +445 -0
- package/dist/context.d.ts +7 -0
- package/dist/d1-storage.js +8 -2
- package/dist/handler.js +21 -7
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/object-bucket.d.ts +78 -0
- package/dist/object-bucket.js +18 -0
- package/dist/operations-realtime.js +3 -0
- package/dist/operations.js +4 -4
- package/dist/push.js +28 -7
- package/dist/realtime.d.ts +11 -0
- package/dist/realtime.js +38 -56
- package/dist/s3-blob-store.d.ts +9 -64
- package/dist/s3-blob-store.js +25 -359
- package/dist/s3-object-bucket.d.ts +39 -0
- package/dist/s3-object-bucket.js +164 -0
- package/dist/s3-segment-store.d.ts +9 -61
- package/dist/s3-segment-store.js +25 -503
- package/dist/seed.js +4 -3
- package/package.json +2 -2
- package/src/blob-handlers.ts +4 -4
- package/src/bucket-blob-store.ts +351 -0
- package/src/bucket-segment-store.ts +582 -0
- package/src/context.ts +10 -0
- package/src/d1-storage.ts +12 -2
- package/src/handler.ts +38 -10
- package/src/index.ts +4 -0
- package/src/object-bucket.ts +83 -0
- package/src/operations-realtime.ts +7 -0
- package/src/operations.ts +8 -4
- package/src/push.ts +40 -9
- package/src/realtime.ts +45 -55
- package/src/s3-blob-store.ts +25 -466
- package/src/s3-object-bucket.ts +233 -0
- package/src/s3-segment-store.ts +25 -646
- package/src/seed.ts +4 -3
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The object-bucket seam: the handful of object-store operations the durable
|
|
3
|
+
* segment store (§5.1) and blob store (§5.9.2) actually perform, expressed
|
|
4
|
+
* without an HTTP transport.
|
|
5
|
+
*
|
|
6
|
+
* Two adapters implement it — `S3ObjectBucket` (`s3-object-bucket.ts`,
|
|
7
|
+
* hand-rolled SigV4 over `fetch`) and the R2 binding adapter in
|
|
8
|
+
* `@syncular/server-workers` (`r2-stores.ts`, the Worker's `env.BUCKET`) — so
|
|
9
|
+
* one implementation of the store logic (`bucket-segment-store.ts`,
|
|
10
|
+
* `bucket-blob-store.ts`) runs over both. The key layouts are the same on
|
|
11
|
+
* either transport, so a bucket can be served through the S3 API for one host
|
|
12
|
+
* and the binding for another.
|
|
13
|
+
*
|
|
14
|
+
* Everything here is structural: an adapter may be any object with these
|
|
15
|
+
* methods, and Cloudflare's `R2Bucket` is assignable to the declaration in
|
|
16
|
+
* `r2-stores.ts` without a dependency on `@cloudflare/workers-types`.
|
|
17
|
+
*/
|
|
18
|
+
/** One stored object, readable once. Mirrors `R2ObjectBody`. */
|
|
19
|
+
export interface BucketObject {
|
|
20
|
+
/** The body, streamed; `null` only if the backend served none. */
|
|
21
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>> | null;
|
|
22
|
+
/** Backend ETag of the stored object (opaque, used as a CAS token). */
|
|
23
|
+
readonly etag: string | undefined;
|
|
24
|
+
/** Stored size in bytes, when the backend reports one. */
|
|
25
|
+
readonly size: number | undefined;
|
|
26
|
+
/** User metadata: the `x-amz-meta-*` headers S3-side, `customMetadata` on R2. */
|
|
27
|
+
readonly metadata: Readonly<Record<string, string>>;
|
|
28
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
29
|
+
text(): Promise<string>;
|
|
30
|
+
}
|
|
31
|
+
/** One stored object's description without its body. Mirrors `R2Object`. */
|
|
32
|
+
export interface BucketObjectHead {
|
|
33
|
+
readonly etag: string | undefined;
|
|
34
|
+
readonly size: number | undefined;
|
|
35
|
+
readonly metadata: Readonly<Record<string, string>>;
|
|
36
|
+
}
|
|
37
|
+
export interface BucketWriteOptions {
|
|
38
|
+
readonly body: Uint8Array;
|
|
39
|
+
readonly contentType: string;
|
|
40
|
+
/** User metadata; values must be ASCII (both backends require it). */
|
|
41
|
+
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
42
|
+
/**
|
|
43
|
+
* Create-only write (`If-None-Match: *`, R2 `etagDoesNotMatch: '*'`). The
|
|
44
|
+
* content-addressed bytes object is written this way so a second publisher
|
|
45
|
+
* of identical bytes is a no-op and a hash collision cannot overwrite.
|
|
46
|
+
*/
|
|
47
|
+
readonly createOnly?: boolean | undefined;
|
|
48
|
+
/**
|
|
49
|
+
* Compare-and-swap write (`If-Match`, R2 `etagMatches`): the write lands
|
|
50
|
+
* only if the stored ETag still equals this. The read-modify-write of the
|
|
51
|
+
* segment record and of the stats accumulator depend on it.
|
|
52
|
+
*/
|
|
53
|
+
readonly ifMatch?: string | undefined;
|
|
54
|
+
}
|
|
55
|
+
/** One page of keys under a prefix. `cursor` is set when more remain. */
|
|
56
|
+
export interface BucketListPage {
|
|
57
|
+
readonly keys: readonly string[];
|
|
58
|
+
readonly cursor: string | undefined;
|
|
59
|
+
}
|
|
60
|
+
export interface ObjectBucket {
|
|
61
|
+
/** The object plus its bytes, or `undefined` when the key is absent. */
|
|
62
|
+
get(key: string): Promise<BucketObject | undefined>;
|
|
63
|
+
head(key: string): Promise<BucketObjectHead | undefined>;
|
|
64
|
+
/**
|
|
65
|
+
* Write one object. Returns `false` when the `createOnly` / `ifMatch`
|
|
66
|
+
* precondition failed, so the caller can re-read and retry; a backend that
|
|
67
|
+
* ignores conditional writes would instead silently lose a concurrent
|
|
68
|
+
* update, which is why the return value is not optional.
|
|
69
|
+
*/
|
|
70
|
+
put(key: string, options: BucketWriteOptions): Promise<boolean>;
|
|
71
|
+
/** Delete one object. Deleting an absent key succeeds (idempotent). */
|
|
72
|
+
delete(key: string): Promise<void>;
|
|
73
|
+
/**
|
|
74
|
+
* List keys under a prefix, one page at a time. The blob orphan sweep
|
|
75
|
+
* (§5.9.2) is the only caller: everything on the hot path is a GET/HEAD.
|
|
76
|
+
*/
|
|
77
|
+
list(prefix: string, cursor?: string): Promise<BucketListPage>;
|
|
78
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The object-bucket seam: the handful of object-store operations the durable
|
|
3
|
+
* segment store (§5.1) and blob store (§5.9.2) actually perform, expressed
|
|
4
|
+
* without an HTTP transport.
|
|
5
|
+
*
|
|
6
|
+
* Two adapters implement it — `S3ObjectBucket` (`s3-object-bucket.ts`,
|
|
7
|
+
* hand-rolled SigV4 over `fetch`) and the R2 binding adapter in
|
|
8
|
+
* `@syncular/server-workers` (`r2-stores.ts`, the Worker's `env.BUCKET`) — so
|
|
9
|
+
* one implementation of the store logic (`bucket-segment-store.ts`,
|
|
10
|
+
* `bucket-blob-store.ts`) runs over both. The key layouts are the same on
|
|
11
|
+
* either transport, so a bucket can be served through the S3 API for one host
|
|
12
|
+
* and the binding for another.
|
|
13
|
+
*
|
|
14
|
+
* Everything here is structural: an adapter may be any object with these
|
|
15
|
+
* methods, and Cloudflare's `R2Bucket` is assignable to the declaration in
|
|
16
|
+
* `r2-stores.ts` without a dependency on `@cloudflare/workers-types`.
|
|
17
|
+
*/
|
|
18
|
+
export {};
|
|
@@ -195,5 +195,8 @@ export function composeRealtimeNotifiers(...notifiers) {
|
|
|
195
195
|
notifyCommit: async (partition, commit) => {
|
|
196
196
|
await Promise.all(notifiers.map((notifier) => notifier.notifyCommit(partition, commit)));
|
|
197
197
|
},
|
|
198
|
+
notifySubscriptions: async (partition, clientId) => {
|
|
199
|
+
await Promise.all(notifiers.map((notifier) => notifier.notifySubscriptions?.(partition, clientId)));
|
|
200
|
+
},
|
|
198
201
|
};
|
|
199
202
|
}
|
package/dist/operations.js
CHANGED
|
@@ -4,7 +4,7 @@ import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPar
|
|
|
4
4
|
import { reportError, SyncError, syncError } from './errors.js';
|
|
5
5
|
import { processPushOperationsWithTrace } from './push.js';
|
|
6
6
|
import { compileSchema } from './schema.js';
|
|
7
|
-
import { authorizeWrite } from './scopes.js';
|
|
7
|
+
import { authorizeWrite, renderScopeValue, } from './scopes.js';
|
|
8
8
|
import { transactionQuery } from './storage-query.js';
|
|
9
9
|
import { toValidateRow } from './validate.js';
|
|
10
10
|
function scopeAllowed(allowed, variable, value) {
|
|
@@ -265,9 +265,9 @@ function commandOperations(mutations, schema) {
|
|
|
265
265
|
}
|
|
266
266
|
return mutation.values[column.name] ?? null;
|
|
267
267
|
});
|
|
268
|
-
const rowId = values[table.primaryKeyIndex];
|
|
269
|
-
if (
|
|
270
|
-
throw syncError('operation.invalid_request', 'command upsert requires a non-empty
|
|
268
|
+
const rowId = renderScopeValue(values[table.primaryKeyIndex]);
|
|
269
|
+
if (rowId === undefined) {
|
|
270
|
+
throw syncError('operation.invalid_request', 'command upsert requires a non-empty primary key');
|
|
271
271
|
}
|
|
272
272
|
return {
|
|
273
273
|
table: table.name,
|
package/dist/push.js
CHANGED
|
@@ -298,6 +298,7 @@ async function applyOperation(tx, schema, wireSchema, resolved, op, opIndex, blo
|
|
|
298
298
|
for (let i = 0; i < table.columns.length; i++) {
|
|
299
299
|
if (present[i] &&
|
|
300
300
|
!crdtIndexes.has(i) &&
|
|
301
|
+
i !== table.primaryKeyIndex &&
|
|
301
302
|
effectiveVersion(i) > op.baseVersion) {
|
|
302
303
|
marked.push(i);
|
|
303
304
|
}
|
|
@@ -602,7 +603,16 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
|
|
|
602
603
|
const changes = [];
|
|
603
604
|
const operations = [];
|
|
604
605
|
const originByIndex = [];
|
|
606
|
+
const finalByRow = new Map();
|
|
605
607
|
for (const write of staged) {
|
|
608
|
+
const rowKey = `${write.table.name}\u0000${write.rowId}`;
|
|
609
|
+
const existing = finalByRow.get(rowKey);
|
|
610
|
+
finalByRow.set(rowKey, {
|
|
611
|
+
first: existing?.first ?? write,
|
|
612
|
+
last: write,
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
for (const { last: write } of finalByRow.values()) {
|
|
606
616
|
if (write.op !== 'upsert' || write.values === undefined)
|
|
607
617
|
continue;
|
|
608
618
|
for (const reference of write.table.references) {
|
|
@@ -636,9 +646,9 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
|
|
|
636
646
|
const scheduled = new Set();
|
|
637
647
|
const processed = new Set();
|
|
638
648
|
const queue = [];
|
|
639
|
-
for (const
|
|
640
|
-
if (
|
|
641
|
-
schedule(
|
|
649
|
+
for (const { first, last } of finalByRow.values()) {
|
|
650
|
+
if (last.op === 'delete' && first.stored !== undefined) {
|
|
651
|
+
schedule(last.table, last.rowId, last.opIndex, scheduled, queue);
|
|
642
652
|
}
|
|
643
653
|
}
|
|
644
654
|
for (let cursor = 0; cursor < queue.length; cursor++) {
|
|
@@ -649,6 +659,12 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
|
|
|
649
659
|
if (processed.has(key))
|
|
650
660
|
continue;
|
|
651
661
|
processed.add(key);
|
|
662
|
+
const parentPkColumn = item.table.columns[item.table.primaryKeyIndex];
|
|
663
|
+
const parentKeyValue = parentPkColumn?.type === 'integer'
|
|
664
|
+
? Number(item.rowId)
|
|
665
|
+
: parentPkColumn?.type === 'boolean'
|
|
666
|
+
? item.rowId === 'true'
|
|
667
|
+
: item.rowId;
|
|
652
668
|
for (const reference of item.table.referencedBy) {
|
|
653
669
|
if (tx.scanRowsByIndex === undefined) {
|
|
654
670
|
throw new Error('storage transaction does not support declared-reference enforcement (scanRowsByIndex)');
|
|
@@ -659,7 +675,7 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
|
|
|
659
675
|
const page = await tx.scanRowsByIndex({
|
|
660
676
|
table: reference.table,
|
|
661
677
|
index: reference.index,
|
|
662
|
-
values: [
|
|
678
|
+
values: [parentKeyValue],
|
|
663
679
|
afterRowId,
|
|
664
680
|
limit: 1_000,
|
|
665
681
|
});
|
|
@@ -684,15 +700,20 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
|
|
|
684
700
|
throw new Error(`unreachable: reference child table ${reference.table} is not compiled`);
|
|
685
701
|
}
|
|
686
702
|
for (const childRow of children) {
|
|
703
|
+
if (reference.onDelete === 'CASCADE') {
|
|
704
|
+
if (!schedule(child, childRow.rowId, item.originOpIndex, scheduled, queue)) {
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
else if (scheduled.has(`${child.name}\u0000${childRow.rowId}`)) {
|
|
709
|
+
continue;
|
|
710
|
+
}
|
|
687
711
|
if (originByIndex.length >= cascadeLimit) {
|
|
688
712
|
return {
|
|
689
713
|
kind: 'terminate',
|
|
690
714
|
record: referenceRejection(item.originOpIndex, 'declared-reference cascade exceeded the per-commit operation cap (§6.11)', { reason: 'cascade_limit' }),
|
|
691
715
|
};
|
|
692
716
|
}
|
|
693
|
-
if (!schedule(child, childRow.rowId, item.originOpIndex, scheduled, queue)) {
|
|
694
|
-
continue;
|
|
695
|
-
}
|
|
696
717
|
const syntheticIndex = clientOpCount + originByIndex.length;
|
|
697
718
|
const storedValues = decodeRow(child.columns, childRow.payload);
|
|
698
719
|
if (reference.onDelete === 'CASCADE') {
|
package/dist/realtime.d.ts
CHANGED
|
@@ -162,6 +162,11 @@ export declare class RealtimeSession {
|
|
|
162
162
|
* deltas immediately; it also survives a newer round-end refresh that
|
|
163
163
|
* failed, because that round applied nothing.
|
|
164
164
|
*/
|
|
165
|
+
/**
|
|
166
|
+
* A pull outside this connection changed the client's subscriptions. A
|
|
167
|
+
* round in flight on the connection reloads them at its end anyway.
|
|
168
|
+
*/
|
|
169
|
+
refreshAfterPull(): Promise<void>;
|
|
165
170
|
refreshRegistrations(failClosed: boolean): Promise<void>;
|
|
166
171
|
sendHeartbeat(): void;
|
|
167
172
|
sendWake(reason: WakeReason): void;
|
|
@@ -230,6 +235,12 @@ export declare class RealtimeHub {
|
|
|
230
235
|
disconnect(session: RealtimeSession): void;
|
|
231
236
|
/** RealtimeNotifier: fan an applied commit out to matching sessions. */
|
|
232
237
|
notifyCommit(partition: string, commit: StoredCommit): Promise<void>;
|
|
238
|
+
/**
|
|
239
|
+
* RealtimeNotifier: a pull changed a client's subscription list (§8.7).
|
|
240
|
+
* Its sockets reload their registrations with the round-end semantics, so
|
|
241
|
+
* an unreadable record keeps the previous registrations.
|
|
242
|
+
*/
|
|
243
|
+
notifySubscriptions(partition: string, clientId: string): Promise<void>;
|
|
233
244
|
/**
|
|
234
245
|
* Host-initiated, fail-closed registration refresh (§8.7). A host calls this
|
|
235
246
|
* after it changes an actor's membership or connection state, so the
|
package/dist/realtime.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* one handler, two framings); the only JSON data-plane server event is
|
|
17
17
|
* the `sync` wake-up (§8.3).
|
|
18
18
|
*/
|
|
19
|
-
import { DecodeError, decodeMessage, encodeMessage, encodePresenceError, encodePresenceFanout, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimePresencePublish, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
19
|
+
import { DecodeError, decodeMessage, encodeMessage, encodePresenceError, encodePresenceFanout, isSupportedProtocolWireVersion, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimePresencePublish, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
20
20
|
import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
21
21
|
import { adapterSyncError, reportError, SyncError, syncError, } from './errors.js';
|
|
22
22
|
import { emitEvent } from './events.js';
|
|
@@ -160,6 +160,8 @@ export class RealtimeSession {
|
|
|
160
160
|
#registrationGeneration = 0;
|
|
161
161
|
/** Generation of the most recent refresh whose result was applied. */
|
|
162
162
|
#registrationApplied = 0;
|
|
163
|
+
/** Highest sequence that required a catch-up wake for this session. */
|
|
164
|
+
#wakeRequiredThroughSeq;
|
|
163
165
|
constructor(hub, options, registrations, cursor, latestSeq, clock, maxDeltaBytes, storage, logEpoch, wireVersion, events) {
|
|
164
166
|
this.sessionId = crypto.randomUUID();
|
|
165
167
|
this.partition = options.partition;
|
|
@@ -170,6 +172,7 @@ export class RealtimeSession {
|
|
|
170
172
|
this.cursor = cursor;
|
|
171
173
|
this.lastKnownSeq = latestSeq;
|
|
172
174
|
this.wakePending = cursor < latestSeq;
|
|
175
|
+
this.#wakeRequiredThroughSeq = cursor < latestSeq ? latestSeq : 0;
|
|
173
176
|
this.registrations = registrations;
|
|
174
177
|
this.openedAtMs = clock();
|
|
175
178
|
this.#send = options.send;
|
|
@@ -208,10 +211,10 @@ export class RealtimeSession {
|
|
|
208
211
|
// proves those commits no longer need socket notifications, so the next
|
|
209
212
|
// notification must be adjacent to the acknowledged cursor.
|
|
210
213
|
this.lastKnownSeq = Math.max(this.lastKnownSeq, this.cursor);
|
|
211
|
-
if (this.cursor >= this
|
|
214
|
+
if (this.cursor >= this.#wakeRequiredThroughSeq)
|
|
212
215
|
this.wakePending = false;
|
|
213
216
|
// §8.2: acks update the client cursor record without an HTTP pull.
|
|
214
|
-
this.#persistCursor();
|
|
217
|
+
this.#persistCursor(cursor);
|
|
215
218
|
}
|
|
216
219
|
/**
|
|
217
220
|
* Resolve once every queued control-plane write has settled (§8.2). A host
|
|
@@ -413,7 +416,7 @@ export class RealtimeSession {
|
|
|
413
416
|
};
|
|
414
417
|
try {
|
|
415
418
|
const requestedWireVersion = (requestBytes[4] ?? 0) | ((requestBytes[5] ?? 0) << 8);
|
|
416
|
-
if (requestedWireVersion
|
|
419
|
+
if (isSupportedProtocolWireVersion(requestedWireVersion)) {
|
|
417
420
|
this.wireVersion = requestedWireVersion;
|
|
418
421
|
}
|
|
419
422
|
let stream;
|
|
@@ -507,6 +510,15 @@ export class RealtimeSession {
|
|
|
507
510
|
* deltas immediately; it also survives a newer round-end refresh that
|
|
508
511
|
* failed, because that round applied nothing.
|
|
509
512
|
*/
|
|
513
|
+
/**
|
|
514
|
+
* A pull outside this connection changed the client's subscriptions. A
|
|
515
|
+
* round in flight on the connection reloads them at its end anyway.
|
|
516
|
+
*/
|
|
517
|
+
async refreshAfterPull() {
|
|
518
|
+
if (this.#activeRound !== undefined)
|
|
519
|
+
return;
|
|
520
|
+
await this.refreshRegistrations(false);
|
|
521
|
+
}
|
|
510
522
|
async refreshRegistrations(failClosed) {
|
|
511
523
|
const generation = ++this.#registrationGeneration;
|
|
512
524
|
let registrations;
|
|
@@ -537,12 +549,12 @@ export class RealtimeSession {
|
|
|
537
549
|
this.#closeSocket?.();
|
|
538
550
|
this.#hub.disconnect(this);
|
|
539
551
|
}
|
|
540
|
-
#persistCursor() {
|
|
552
|
+
#persistCursor(cursor) {
|
|
541
553
|
// §8.2: start the write now and keep it on the control tail so `drain()`
|
|
542
554
|
// can await it. Two acks may write concurrently: storage folds them with
|
|
543
555
|
// `MAX(cursor, ?)`, so neither order loses the later cursor.
|
|
544
556
|
const write = (async () => {
|
|
545
|
-
await this.#storage.advanceClientCursor(this.partition, this.clientId, this.actorId, this.logEpoch,
|
|
557
|
+
await this.#storage.advanceClientCursor(this.partition, this.clientId, this.actorId, this.logEpoch, cursor, this.#clock());
|
|
546
558
|
})();
|
|
547
559
|
write.catch((error) => {
|
|
548
560
|
// Best-effort for the connection (the next pull repairs the cursor);
|
|
@@ -571,6 +583,7 @@ export class RealtimeSession {
|
|
|
571
583
|
}
|
|
572
584
|
sendWake(reason) {
|
|
573
585
|
this.wakePending = true;
|
|
586
|
+
this.#wakeRequiredThroughSeq = Math.max(this.#wakeRequiredThroughSeq, this.lastKnownSeq);
|
|
574
587
|
this.#sendSafe(JSON.stringify({
|
|
575
588
|
event: 'sync',
|
|
576
589
|
data: {
|
|
@@ -841,58 +854,13 @@ export class RealtimeHub {
|
|
|
841
854
|
// segment store the HTTP binding uses — never a degraded round.
|
|
842
855
|
throw syncError('sync.invalid_request', 'socket sync rounds require a segment store on the realtime hub (§8.7)');
|
|
843
856
|
}
|
|
857
|
+
// The whole config, so a new `SyncServerConfig` field reaches socket and
|
|
858
|
+
// hub-served HTTP rounds without being listed here.
|
|
844
859
|
return {
|
|
860
|
+
...this.#config,
|
|
861
|
+
segments,
|
|
845
862
|
partition: identity.partition,
|
|
846
863
|
actorId: identity.actorId,
|
|
847
|
-
schema: this.#config.schema,
|
|
848
|
-
...(this.#config.schemaWindow !== undefined
|
|
849
|
-
? { schemaWindow: this.#config.schemaWindow }
|
|
850
|
-
: {}),
|
|
851
|
-
storage: this.#config.storage,
|
|
852
|
-
segments,
|
|
853
|
-
resolveScopes: this.#config.resolveScopes,
|
|
854
|
-
...(this.#config.blobs !== undefined
|
|
855
|
-
? { blobs: this.#config.blobs }
|
|
856
|
-
: {}),
|
|
857
|
-
...(this.#config.maxBlobBytes !== undefined
|
|
858
|
-
? { maxBlobBytes: this.#config.maxBlobBytes }
|
|
859
|
-
: {}),
|
|
860
|
-
...(this.#config.crdtMergers !== undefined
|
|
861
|
-
? { crdtMergers: this.#config.crdtMergers }
|
|
862
|
-
: {}),
|
|
863
|
-
...(this.#config.validators !== undefined
|
|
864
|
-
? { validators: this.#config.validators }
|
|
865
|
-
: {}),
|
|
866
|
-
...(this.#config.commitValidator !== undefined
|
|
867
|
-
? { commitValidator: this.#config.commitValidator }
|
|
868
|
-
: {}),
|
|
869
|
-
...(this.#config.clock !== undefined
|
|
870
|
-
? { clock: this.#config.clock }
|
|
871
|
-
: {}),
|
|
872
|
-
...(this.#config.limits !== undefined
|
|
873
|
-
? { limits: this.#config.limits }
|
|
874
|
-
: {}),
|
|
875
|
-
...(this.#config.signedUrls !== undefined
|
|
876
|
-
? { signedUrls: this.#config.signedUrls }
|
|
877
|
-
: {}),
|
|
878
|
-
...(this.#config.blobSignedUrls !== undefined
|
|
879
|
-
? { blobSignedUrls: this.#config.blobSignedUrls }
|
|
880
|
-
: {}),
|
|
881
|
-
...(this.#config.blobUploadUrls !== undefined
|
|
882
|
-
? { blobUploadUrls: this.#config.blobUploadUrls }
|
|
883
|
-
: {}),
|
|
884
|
-
...(this.#config.sqliteImageBuilder !== undefined
|
|
885
|
-
? { sqliteImageBuilder: this.#config.sqliteImageBuilder }
|
|
886
|
-
: {}),
|
|
887
|
-
...(this.#config.leases !== undefined
|
|
888
|
-
? { leases: this.#config.leases }
|
|
889
|
-
: {}),
|
|
890
|
-
...(this.#config.events !== undefined
|
|
891
|
-
? { events: this.#config.events }
|
|
892
|
-
: {}),
|
|
893
|
-
...(this.#config.onError !== undefined
|
|
894
|
-
? { onError: this.#config.onError }
|
|
895
|
-
: {}),
|
|
896
864
|
realtime: this,
|
|
897
865
|
};
|
|
898
866
|
}
|
|
@@ -929,7 +897,9 @@ export class RealtimeHub {
|
|
|
929
897
|
const registrations = await this.loadRegistrations(options.partition, options.actorId, options.clientId);
|
|
930
898
|
const latestSeq = await storage.getMaxCommitSeq(options.partition);
|
|
931
899
|
const cursor = record?.cursor ?? -1;
|
|
932
|
-
const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage, registry.logEpoch, record
|
|
900
|
+
const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage, registry.logEpoch, record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
|
|
901
|
+
? record.wireVersion
|
|
902
|
+
: PROTOCOL_WIRE_VERSION, this.#config.events);
|
|
933
903
|
this.#sessions.add(session);
|
|
934
904
|
const helloResult = options.send(JSON.stringify({
|
|
935
905
|
event: 'hello',
|
|
@@ -997,6 +967,18 @@ export class RealtimeHub {
|
|
|
997
967
|
session.deliverCommit(commit);
|
|
998
968
|
}
|
|
999
969
|
}
|
|
970
|
+
/**
|
|
971
|
+
* RealtimeNotifier: a pull changed a client's subscription list (§8.7).
|
|
972
|
+
* Its sockets reload their registrations with the round-end semantics, so
|
|
973
|
+
* an unreadable record keeps the previous registrations.
|
|
974
|
+
*/
|
|
975
|
+
async notifySubscriptions(partition, clientId) {
|
|
976
|
+
for (const session of [...this.#sessions]) {
|
|
977
|
+
if (session.partition !== partition || session.clientId !== clientId)
|
|
978
|
+
continue;
|
|
979
|
+
await session.refreshAfterPull();
|
|
980
|
+
}
|
|
981
|
+
}
|
|
1000
982
|
/**
|
|
1001
983
|
* Host-initiated, fail-closed registration refresh (§8.7). A host calls this
|
|
1002
984
|
* after it changes an actor's membership or connection state, so the
|
package/dist/s3-blob-store.d.ts
CHANGED
|
@@ -1,37 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* S3-compatible blob store (SPEC.md §5.9.2 durable objects, §5.9.5 delegated
|
|
3
|
-
* presign) — AWS S3, Cloudflare R2, MinIO. The blob twin of
|
|
4
|
-
* `s3-segment-store.ts
|
|
5
|
-
* content-addressed key layout, same LIST-free-on-the-hot-path CAS stats
|
|
6
|
-
* accumulator. Zero dependencies.
|
|
3
|
+
* presign) — AWS S3, Cloudflare R2's S3 endpoint, MinIO. The blob twin of
|
|
4
|
+
* `s3-segment-store.ts`.
|
|
7
5
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* client's §5.9.1 hash check passes — blobs are never re-compressed at
|
|
14
|
-
* rest, §5.8/§5.9.5). The `byteLength` + optional `mediaType` ride along
|
|
15
|
-
* as object user metadata `x-amz-meta-syncular-blob` = base64url(JSON), so
|
|
16
|
-
* `get` is a single GET. `createdAtMs` is stored there too so the orphan
|
|
17
|
-
* sweep (§5.9.2) can read the upload age without a separate index.
|
|
18
|
-
*
|
|
19
|
-
* DURABILITY vs SEGMENT TTL — the honest interface difference (§5.9.2): a
|
|
20
|
-
* blob referenced by a live row must stay downloadable INDEFINITELY. So,
|
|
21
|
-
* unlike `S3SegmentStore`, this store writes **no `expiresAtMs`, no S3
|
|
22
|
-
* lifecycle-expiration mapping, and no TTL config**. Expiry is
|
|
23
|
-
* REFERENCE-driven, not time-driven: reclamation is the host-scheduled
|
|
24
|
-
* `sweepOrphanBlobs` (blob-store.ts) deleting only objects no live row
|
|
25
|
-
* references after a grace period — never a bucket lifecycle rule. Do NOT
|
|
26
|
-
* put a lifecycle expiration on the `blob/` prefix; it would delete
|
|
27
|
-
* still-referenced attachments.
|
|
28
|
-
*
|
|
29
|
-
* The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages
|
|
30
|
-
* `ListObjectsV2` over the `blob/` prefix (an admin/GC path, off the hot
|
|
31
|
-
* path) to find candidates, reads each object's `createdAtMs` from its
|
|
32
|
-
* metadata for the grace check, and DELETEs the unreferenced-and-old ones.
|
|
6
|
+
* The store logic (content-addressed key layout, idempotent re-put, orphan
|
|
7
|
+
* sweep, CAS stats accumulator) is `BucketBlobStore`, shared with
|
|
8
|
+
* `R2BlobStore` in `@syncular/server-workers`; this class adds the S3 REST
|
|
9
|
+
* transport (`S3ObjectBucket`) and the SigV4 presign methods, which a bucket
|
|
10
|
+
* binding cannot mint.
|
|
33
11
|
*/
|
|
34
|
-
import
|
|
12
|
+
import { BucketBlobStore } from './bucket-blob-store.js';
|
|
35
13
|
import type { BlobPresignConfig, BlobUploadPresignConfig, SegmentUrlIssue } from './signed-url.js';
|
|
36
14
|
export interface S3BlobStoreConfig {
|
|
37
15
|
/**
|
|
@@ -52,42 +30,9 @@ export interface S3BlobStoreConfig {
|
|
|
52
30
|
/** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
|
|
53
31
|
readonly keyPrefix?: string;
|
|
54
32
|
}
|
|
55
|
-
export declare class S3BlobStore
|
|
33
|
+
export declare class S3BlobStore extends BucketBlobStore {
|
|
56
34
|
#private;
|
|
57
35
|
constructor(config: S3BlobStoreConfig);
|
|
58
|
-
/**
|
|
59
|
-
* `{keyPrefix}blob/{partition}/sha256/{hex}` — the §5.9.5 "signed key embeds
|
|
60
|
-
* the blobId". Blobs are content-addressed, but download authorization and
|
|
61
|
-
* the reference index are per-partition, so the same bytes uploaded under
|
|
62
|
-
* two partitions are two objects (a partition MUST NOT read another's
|
|
63
|
-
* attachment by guessing a content address). The partition is folded into
|
|
64
|
-
* the key as a path segment.
|
|
65
|
-
*/
|
|
66
|
-
objectKeyFor(partition: string, blobId: string): string;
|
|
67
|
-
put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
|
|
68
|
-
has(partition: string, blobId: string): Promise<boolean>;
|
|
69
|
-
get(partition: string, blobId: string): Promise<{
|
|
70
|
-
record: BlobRecord;
|
|
71
|
-
bytes: Uint8Array;
|
|
72
|
-
} | undefined>;
|
|
73
|
-
/**
|
|
74
|
-
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page `ListObjectsV2`
|
|
75
|
-
* over `{prefix}blob/{partition}/`, HEAD each object for its `createdAtMs`,
|
|
76
|
-
* and DELETE the ones both older than `olderThanMs` AND absent from
|
|
77
|
-
* `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
|
|
78
|
-
* referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
|
|
79
|
-
*/
|
|
80
|
-
sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
|
|
81
|
-
/**
|
|
82
|
-
* Store-wide counters from the pointer-object accumulator (never a LIST on
|
|
83
|
-
* the read path). `approximate: true` for the same reasons `S3SegmentStore`
|
|
84
|
-
* documents: best-effort ETag-CAS maintenance and sweep deletes that may
|
|
85
|
-
* miss the decrement. Exact within a single writer; a health gauge under
|
|
86
|
-
* concurrency. Note this is store-WIDE, not partition-scoped, so a shared
|
|
87
|
-
* bucket reports the whole bucket — the parameter is accepted for interface
|
|
88
|
-
* parity but the accumulator is one object per store.
|
|
89
|
-
*/
|
|
90
|
-
stats(_partition: string): Promise<BlobStoreStats>;
|
|
91
36
|
/**
|
|
92
37
|
* SigV4 presigned GET for a blob object (§5.9.5 delegated presign). The
|
|
93
38
|
* signed key embeds the `blobId`; `urlExpiresAtMs` is the provider-enforced
|