@syncular/server 0.95.0 → 0.97.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/context.d.ts +4 -16
- package/dist/context.js +0 -3
- package/dist/d1-storage.js +1 -1
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +36 -0
- package/dist/frame-bytes.js +1 -0
- package/dist/handler.d.ts +20 -2
- package/dist/handler.js +199 -59
- package/dist/postgres-storage.d.ts +3 -2
- package/dist/postgres-storage.js +18 -2
- package/dist/pull.d.ts +2 -2
- package/dist/pull.js +22 -16
- package/dist/push.js +41 -77
- package/dist/realtime.d.ts +5 -11
- package/dist/realtime.js +10 -23
- package/dist/relational-rows.d.ts +23 -13
- package/dist/relational-rows.js +49 -30
- package/dist/schema-window.js +2 -2
- package/dist/schema.d.ts +2 -2
- package/dist/schema.js +2 -2
- package/dist/segment-download.js +11 -4
- package/dist/sqlite-dialect.d.ts +3 -1
- package/dist/sqlite-dialect.js +37 -6
- package/dist/storage-query.d.ts +5 -2
- package/dist/storage-query.js +7 -4
- package/dist/storage.d.ts +12 -0
- package/dist/validate.d.ts +21 -39
- package/dist/validate.js +4 -3
- package/package.json +2 -2
- package/src/context.ts +4 -23
- package/src/d1-storage.ts +0 -1
- package/src/errors.ts +46 -0
- package/src/frame-bytes.ts +1 -0
- package/src/handler.ts +257 -73
- package/src/postgres-storage.ts +32 -6
- package/src/pull.ts +50 -35
- package/src/push.ts +51 -129
- package/src/realtime.ts +10 -22
- package/src/relational-rows.ts +51 -30
- package/src/schema-window.ts +2 -2
- package/src/schema.ts +4 -4
- package/src/segment-download.ts +19 -12
- package/src/sqlite-dialect.ts +47 -7
- package/src/storage-query.ts +7 -4
- package/src/storage.ts +13 -0
- package/src/validate.ts +21 -48
package/README.md
CHANGED
|
@@ -195,20 +195,21 @@ fail at construction. Diagnostic prose stays in `message`; apps should map
|
|
|
195
195
|
the stable code/details to localized copy instead of displaying that message.
|
|
196
196
|
|
|
197
197
|
Validators are per-operation. A validator must not mutate the row it receives;
|
|
198
|
-
a value only the server decides belongs in a server-owned column,
|
|
199
|
-
|
|
198
|
+
a value only the server decides belongs in a server-owned column, whose value
|
|
199
|
+
the validator returns
|
|
200
200
|
([Server-owned columns](https://syncular.dev/guide-server/#server-owned-columns)).
|
|
201
201
|
|
|
202
202
|
A validator that decides from other rows (a membership, a role, a grant) calls
|
|
203
|
-
`context.queryAuthoritative` with the request `storage.queryAuthoritative`
|
|
203
|
+
`context.read.queryAuthoritative` with the request `storage.queryAuthoritative`
|
|
204
204
|
takes, built from a generated query descriptor. The storage binds every
|
|
205
205
|
relation to the commit's partition and runs the statement on the push
|
|
206
206
|
transaction's connection, so the read sees the operations staged before this
|
|
207
207
|
one and needs no second pool connection. Never call
|
|
208
208
|
`storage.queryAuthoritative` from a validator: on SQLite and PGlite it waits
|
|
209
209
|
for the push transaction and the push never completes, and on a pool it reads
|
|
210
|
-
committed state.
|
|
211
|
-
remote command context expose the same
|
|
210
|
+
committed state. `context.read` is the whole-commit candidate-state reader;
|
|
211
|
+
the reaction planner reader and the remote command context expose the same
|
|
212
|
+
method. See
|
|
212
213
|
[Read a generated query inside a push transaction](https://syncular.dev/guide-remote-operations/#read-a-generated-query-inside-a-push-transaction).
|
|
213
214
|
|
|
214
215
|
For a multi-row or multi-table invariant, install `commitValidator`. It runs
|
package/dist/context.d.ts
CHANGED
|
@@ -16,7 +16,7 @@ import type { SegmentStore } from './segment-store.js';
|
|
|
16
16
|
import type { BlobPresignConfig, BlobUploadPresignConfig, SegmentUrlConfig } from './signed-url.js';
|
|
17
17
|
import type { SqliteImageBuilder } from './sqlite-image.js';
|
|
18
18
|
import type { CheckpointDeclaration, PartitionTouch, ServerStorage, StoredCommit } from './storage.js';
|
|
19
|
-
import type { CommitValidator,
|
|
19
|
+
import type { CommitValidator, ValidatorRegistry } from './validate.js';
|
|
20
20
|
/** SSP2 body content type (§1.1). */
|
|
21
21
|
export declare const SSP2_CONTENT_TYPE = "application/vnd.syncular.sync.v2";
|
|
22
22
|
/** Internal idempotency namespace. Ordinary SSP2 client IDs cannot use it. */
|
|
@@ -102,12 +102,6 @@ export interface ServerLimits {
|
|
|
102
102
|
* bound gets an image of its own.
|
|
103
103
|
*/
|
|
104
104
|
readonly maxImageSegmentBytes: number;
|
|
105
|
-
/**
|
|
106
|
-
* Segment puts one pull keeps in flight. The pull scans and encodes its
|
|
107
|
-
* pages in order and stores up to this many at once, so it holds this
|
|
108
|
-
* many encoded segments in memory.
|
|
109
|
-
*/
|
|
110
|
-
readonly maxConcurrentSegmentPuts: number;
|
|
111
105
|
}
|
|
112
106
|
export declare const DEFAULT_LIMITS: ServerLimits;
|
|
113
107
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
@@ -155,19 +149,13 @@ export interface SyncServerConfig {
|
|
|
155
149
|
* builds no validator context — zero cost. Present ⇒ each push operation
|
|
156
150
|
* on a listed table runs its validator after decode + §3.4 scope
|
|
157
151
|
* authorization, inside the commit transaction; a throw rejects the
|
|
158
|
-
* whole commit atomically (§6.4) with the host code (§6.3).
|
|
152
|
+
* whole commit atomically (§6.4) with the host code (§6.3). On an insert
|
|
153
|
+
* or update the validator may return values of the table's server-owned
|
|
154
|
+
* columns, which are stored and fanned out in the same commit. Kept a host
|
|
159
155
|
* callback (like `resolveScopes`) — business rules live in the host
|
|
160
156
|
* process, never on the wire.
|
|
161
157
|
*/
|
|
162
158
|
readonly validators?: ValidatorRegistry;
|
|
163
|
-
/**
|
|
164
|
-
* §6.12 per-table server column hooks (`table` → hook). Each accepted
|
|
165
|
-
* upsert on a listed table runs its hook inside the push transaction,
|
|
166
|
-
* after §3.4 authorization and before §6.7 validation; the returned values
|
|
167
|
-
* of the table's server-owned columns are stored and fanned out in the
|
|
168
|
-
* same commit.
|
|
169
|
-
*/
|
|
170
|
-
readonly serverColumns?: ServerColumnsRegistry;
|
|
171
159
|
/**
|
|
172
160
|
* §6.8 whole-commit validator. When present, the storage serializes this
|
|
173
161
|
* partition before operation reads/writes, then invokes the callback once
|
package/dist/context.js
CHANGED
|
@@ -25,9 +25,6 @@ export const DEFAULT_LIMITS = {
|
|
|
25
25
|
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
26
26
|
// §5.3 DECISION (2026-11-17): a client holds about three images at once.
|
|
27
27
|
maxImageSegmentBytes: 32 * 1024 * 1024,
|
|
28
|
-
// A Workers invocation holds six connections open at once, and an R2 put
|
|
29
|
-
// uses one; past six the runtime queues the put.
|
|
30
|
-
maxConcurrentSegmentPuts: 6,
|
|
31
28
|
};
|
|
32
29
|
/** §1.1: `authenticate` returned a context longer than 256 UTF-8 bytes. */
|
|
33
30
|
export class AuthContextTooLargeError extends Error {
|
package/dist/d1-storage.js
CHANGED
|
@@ -245,7 +245,7 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
245
245
|
while (values.length > 0 && persisted.length < persistedLimit) {
|
|
246
246
|
const { results: records } = await this.#db
|
|
247
247
|
.prepare(sql)
|
|
248
|
-
.bind(this.partition, query.table, variable, JSON.stringify(values), afterRowId, batchSize
|
|
248
|
+
.bind(this.partition, query.table, variable, JSON.stringify(values), afterRowId, batchSize)
|
|
249
249
|
.all();
|
|
250
250
|
if (records.length === 0)
|
|
251
251
|
break;
|
package/dist/errors.d.ts
CHANGED
|
@@ -34,6 +34,17 @@ export declare function errorBody(error: SyncError): {
|
|
|
34
34
|
message: string;
|
|
35
35
|
details?: unknown;
|
|
36
36
|
};
|
|
37
|
+
/**
|
|
38
|
+
* HTTP headers of a §10.1 error answer: `sync.segment_store_unavailable`
|
|
39
|
+
* carries a fixed `Retry-After: 5` (§10.2).
|
|
40
|
+
*/
|
|
41
|
+
export declare function errorHeaders(error: SyncError): Record<string, string>;
|
|
42
|
+
/**
|
|
43
|
+
* The rejection handler of a `SegmentStore` call (§5.4, §5.5): the store's
|
|
44
|
+
* error goes to `onError`, and the request fails with the retryable
|
|
45
|
+
* `sync.segment_store_unavailable`, its `details.operation` naming the call.
|
|
46
|
+
*/
|
|
47
|
+
export declare function segmentStoreFailure(onError: SyncularErrorHandler | undefined, route: 'sync' | 'segments', operation: 'put' | 'find' | 'get' | 'open'): (error: unknown) => never;
|
|
37
48
|
/** The adapter surface that caught an exception reported through `onError`. */
|
|
38
49
|
export type SyncularErrorRoute = 'sync' | 'operations' | 'segments' | 'blobs' | 'admin' | 'realtime';
|
|
39
50
|
/**
|
package/dist/errors.js
CHANGED
|
@@ -226,6 +226,22 @@ export const ERROR_CATALOG = {
|
|
|
226
226
|
recommendedAction: 'forceResync',
|
|
227
227
|
httpStatus: 404,
|
|
228
228
|
},
|
|
229
|
+
// §1.6: a response still building at the 300 s build deadline. In-band
|
|
230
|
+
// after its header frames, HTTP 503 before them.
|
|
231
|
+
'sync.segment_build_timeout': {
|
|
232
|
+
category: 'internal',
|
|
233
|
+
retryable: true,
|
|
234
|
+
recommendedAction: 'retryLater',
|
|
235
|
+
httpStatus: 503,
|
|
236
|
+
},
|
|
237
|
+
// §5.4/§5.5: a SegmentStore call rejected (an object store outage). The
|
|
238
|
+
// HTTP answer carries `Retry-After: 5` (`errorHeaders`).
|
|
239
|
+
'sync.segment_store_unavailable': {
|
|
240
|
+
category: 'internal',
|
|
241
|
+
retryable: true,
|
|
242
|
+
recommendedAction: 'retryLater',
|
|
243
|
+
httpStatus: 503,
|
|
244
|
+
},
|
|
229
245
|
'sync.segment_expired': {
|
|
230
246
|
category: 'not-found',
|
|
231
247
|
retryable: true,
|
|
@@ -333,6 +349,26 @@ export function errorBody(error) {
|
|
|
333
349
|
: {}),
|
|
334
350
|
};
|
|
335
351
|
}
|
|
352
|
+
/**
|
|
353
|
+
* HTTP headers of a §10.1 error answer: `sync.segment_store_unavailable`
|
|
354
|
+
* carries a fixed `Retry-After: 5` (§10.2).
|
|
355
|
+
*/
|
|
356
|
+
export function errorHeaders(error) {
|
|
357
|
+
return error.code === 'sync.segment_store_unavailable'
|
|
358
|
+
? { 'retry-after': '5' }
|
|
359
|
+
: {};
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* The rejection handler of a `SegmentStore` call (§5.4, §5.5): the store's
|
|
363
|
+
* error goes to `onError`, and the request fails with the retryable
|
|
364
|
+
* `sync.segment_store_unavailable`, its `details.operation` naming the call.
|
|
365
|
+
*/
|
|
366
|
+
export function segmentStoreFailure(onError, route, operation) {
|
|
367
|
+
return (error) => {
|
|
368
|
+
reportError(error, onError, route);
|
|
369
|
+
throw new SyncError('sync.segment_store_unavailable', 'the segment store failed', JSON.stringify({ operation }));
|
|
370
|
+
};
|
|
371
|
+
}
|
|
336
372
|
/**
|
|
337
373
|
* The protocol error for an exception an adapter caught. A `SyncError`
|
|
338
374
|
* passes through unchanged. Anything else goes to `onError` and answers
|
package/dist/frame-bytes.js
CHANGED
|
@@ -53,6 +53,7 @@ function wrapperFor(frame) {
|
|
|
53
53
|
switch (frame.type) {
|
|
54
54
|
case 'RESP_HEADER':
|
|
55
55
|
return { frames: [frame], index: 0 };
|
|
56
|
+
case 'KEEPALIVE':
|
|
56
57
|
case 'LEASE':
|
|
57
58
|
// §7.3.2: LEASE rides immediately after RESP_HEADER.
|
|
58
59
|
return { frames: [stubHeader(), frame], index: 1 };
|
package/dist/handler.d.ts
CHANGED
|
@@ -1,9 +1,27 @@
|
|
|
1
1
|
import type { SyncRequestContext } from './context.js';
|
|
2
|
+
/**
|
|
3
|
+
* A response's chunks, one per encoded frame (§1.4). `live` turns true when
|
|
4
|
+
* the response goes out before it is built: its header frames and
|
|
5
|
+
* `KEEPALIVE` frames (§1.6) precede the rest, and a later failure is an
|
|
6
|
+
* in-band `ERROR`.
|
|
7
|
+
*/
|
|
8
|
+
export interface SyncResponseStream extends AsyncIterable<Uint8Array> {
|
|
9
|
+
readonly live: boolean;
|
|
10
|
+
}
|
|
2
11
|
/**
|
|
3
12
|
* Decode and validate the request, then return the streaming response.
|
|
4
13
|
* Request-validation failures (§1.7) throw `SyncError` here, before any
|
|
5
|
-
* bytes are produced; adapters map them to HTTP JSON errors (§1.1).
|
|
14
|
+
* bytes are produced; adapters map them to HTTP JSON errors (§1.1). The
|
|
15
|
+
* first chunk arrives once the response is built or its first `KEEPALIVE`
|
|
16
|
+
* is due (§1.6); a failure before then throws from it, before any byte.
|
|
17
|
+
*/
|
|
18
|
+
export declare function createSyncResponseStream(bytes: Uint8Array, ctx: SyncRequestContext): Promise<SyncResponseStream>;
|
|
19
|
+
/**
|
|
20
|
+
* The `POST /sync` body (§1.1): the whole response when it was built before
|
|
21
|
+
* its first `KEEPALIVE` was due, otherwise a stream that carries the header
|
|
22
|
+
* frames and `KEEPALIVE` frames first (§1.6). Adapters compress only the
|
|
23
|
+
* whole response (§1.3).
|
|
6
24
|
*/
|
|
7
|
-
export declare function
|
|
25
|
+
export declare function createSyncResponseBody(bytes: Uint8Array, ctx: SyncRequestContext): Promise<Uint8Array | ReadableStream<Uint8Array>>;
|
|
8
26
|
/** Byte-concatenating wrapper over the streaming core (§1.4). */
|
|
9
27
|
export declare function handleSyncRequest(bytes: Uint8Array, ctx: SyncRequestContext): Promise<Uint8Array>;
|
package/dist/handler.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { DecodeError, decodeRequest, echoDigest, registrationDigest, unitId, } from '@syncular/core';
|
|
13
13
|
import { clockOf, limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
14
|
-
import { reportError, SyncError, syncError } from './errors.js';
|
|
14
|
+
import { adapterSyncError, reportError, SyncError, syncError } from './errors.js';
|
|
15
15
|
import { emitEvent, } from './events.js';
|
|
16
16
|
import { END_FRAME_BYTES, encodeResponseFrame, responseEnvelopeHeader, } from './frame-bytes.js';
|
|
17
17
|
import { ACCEPT_EXTERNAL_ROWS, ACCEPT_INLINE_ROWS, bootstrapSegments, clampPullLimits, commitFrame, coveredTables, planBootstrap, planUnit, } from './pull.js';
|
|
@@ -558,7 +558,8 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
558
558
|
}
|
|
559
559
|
}
|
|
560
560
|
else {
|
|
561
|
-
const
|
|
561
|
+
const horizons = await ctx.storage.getHorizons(ctx.partition);
|
|
562
|
+
const { seq: horizonSeq, tables: tableHorizons } = horizons;
|
|
562
563
|
const R = pull.header.cursor;
|
|
563
564
|
const echo = echoDigest(units.map(({ plan: unit }) => ({
|
|
564
565
|
id: unit.id,
|
|
@@ -572,14 +573,25 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
572
573
|
const liveUnits = units.some((unit) => unit.progress === undefined && unit.plan.status === 'active');
|
|
573
574
|
// §4.6 replica reset: a live unit with R outside [horizon, max].
|
|
574
575
|
let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
|
|
575
|
-
|
|
576
|
+
// §4.6 unit reset: `cursor` lies below the table horizon of one of
|
|
577
|
+
// the unit's active pairs outside its RESTRICT tables.
|
|
578
|
+
const belowTable = (index, cursor, horizons) => {
|
|
579
|
+
const { plan: unit, restrict } = units[index];
|
|
580
|
+
return [...unit.pairs.keys()].some((table) => cursor < (horizons.get(table) ?? -1) &&
|
|
581
|
+
restrict?.tables.includes(table) !== true);
|
|
582
|
+
};
|
|
583
|
+
const states = units.map(({ plan: unit, progress, restrict }, index) => {
|
|
576
584
|
if (unit.status === 'revoked')
|
|
577
585
|
return { kind: 'revoked' };
|
|
578
586
|
if (progress === undefined)
|
|
579
|
-
return
|
|
587
|
+
return !liveReset && belowTable(index, R, tableHorizons)
|
|
588
|
+
? { kind: 'reset' }
|
|
589
|
+
: { kind: 'live' };
|
|
580
590
|
// §4.6: a kept cursor outside [horizon, max] cannot join.
|
|
581
591
|
if (restrict !== undefined)
|
|
582
|
-
return restrict.cursor < horizonSeq ||
|
|
592
|
+
return restrict.cursor < horizonSeq ||
|
|
593
|
+
restrict.cursor > maxSeq ||
|
|
594
|
+
belowTable(index, restrict.cursor, tableHorizons)
|
|
583
595
|
? { kind: 'reset' }
|
|
584
596
|
: {
|
|
585
597
|
kind: 'bootstrap',
|
|
@@ -592,7 +604,8 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
592
604
|
if (progress.bootstrapState === undefined &&
|
|
593
605
|
progress.cursor >= 0 &&
|
|
594
606
|
progress.cursor <= maxSeq) {
|
|
595
|
-
return progress.cursor < horizonSeq
|
|
607
|
+
return progress.cursor < horizonSeq ||
|
|
608
|
+
belowTable(index, progress.cursor, tableHorizons)
|
|
596
609
|
? { kind: 'reset' }
|
|
597
610
|
: { kind: 'post', cursor: progress.cursor };
|
|
598
611
|
}
|
|
@@ -619,7 +632,6 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
619
632
|
// pull holds the encoded bytes of the puts in flight.
|
|
620
633
|
const puts = [];
|
|
621
634
|
const inFlight = new Set();
|
|
622
|
-
const maxPuts = limitsOf(ctx).maxConcurrentSegmentPuts;
|
|
623
635
|
let failed = false;
|
|
624
636
|
for (let next = 0; next < bootstrapping.length;) {
|
|
625
637
|
const window = [];
|
|
@@ -645,7 +657,7 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
645
657
|
...unit,
|
|
646
658
|
tables: unit.tables.filter((table) => only.has(table.name)),
|
|
647
659
|
pairs: new Map([...unit.pairs].filter(([name]) => only.has(name))),
|
|
648
|
-
}, progress?.bootstrapState, maxSeq,
|
|
660
|
+
}, progress?.bootstrapState, maxSeq, horizons, plan.logEpoch, restrict?.cursor),
|
|
649
661
|
};
|
|
650
662
|
});
|
|
651
663
|
for (const { index, plan: bootstrap } of batch) {
|
|
@@ -656,7 +668,9 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
656
668
|
const frame = step.value;
|
|
657
669
|
if (frame.type === 'PENDING_REF') {
|
|
658
670
|
// A rows segment's put runs while later pages are scanned;
|
|
659
|
-
// the next page waits while
|
|
671
|
+
// the next page waits while six puts are in flight: a
|
|
672
|
+
// Workers invocation holds six connections open at once and
|
|
673
|
+
// an R2 put uses one, so past six the runtime queues the put.
|
|
660
674
|
const settled = frame.ref.then(() => void inFlight.delete(settled), () => {
|
|
661
675
|
failed = true;
|
|
662
676
|
inFlight.delete(settled);
|
|
@@ -664,7 +678,7 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
664
678
|
inFlight.add(settled);
|
|
665
679
|
puts.push(frame.ref);
|
|
666
680
|
frames.push(frame.ref);
|
|
667
|
-
while (inFlight.size >=
|
|
681
|
+
while (inFlight.size >= 6) {
|
|
668
682
|
await Promise.race(inFlight);
|
|
669
683
|
if (failed)
|
|
670
684
|
break;
|
|
@@ -709,10 +723,22 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
709
723
|
let live;
|
|
710
724
|
for (;;) {
|
|
711
725
|
const scanned = units.flatMap(({ plan: unit }, index) => {
|
|
712
|
-
const
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
726
|
+
const bounded = bound(states[index]);
|
|
727
|
+
if (bounded === undefined)
|
|
728
|
+
return [];
|
|
729
|
+
// §4.5: a table reaches the unit only above its table horizon,
|
|
730
|
+
// which lies above the bound only for a RESTRICT table.
|
|
731
|
+
const byBound = new Map();
|
|
732
|
+
for (const [table, scopes] of unit.pairs) {
|
|
733
|
+
const afterSeq = Math.max(bounded, tableHorizons.get(table) ?? -1);
|
|
734
|
+
byBound.set(afterSeq, (byBound.get(afterSeq) ?? new Map()).set(table, scopes));
|
|
735
|
+
}
|
|
736
|
+
return [...byBound].map(([afterSeq, tables]) => ({
|
|
737
|
+
index,
|
|
738
|
+
scopes: unit.scopes,
|
|
739
|
+
afterSeq,
|
|
740
|
+
tables,
|
|
741
|
+
}));
|
|
716
742
|
});
|
|
717
743
|
if (liveReset || scanned.length === 0)
|
|
718
744
|
break;
|
|
@@ -735,18 +761,24 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
735
761
|
live = undefined;
|
|
736
762
|
continue;
|
|
737
763
|
}
|
|
738
|
-
const recheck = await ctx.storage.
|
|
764
|
+
const { seq: recheck, tables: recheckTables } = await ctx.storage.getHorizons(ctx.partition);
|
|
739
765
|
let expired = false;
|
|
740
766
|
for (const [index, state] of states.entries()) {
|
|
741
|
-
// §4.6: a restricted bootstrap's kept cursor is rechecked too
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
if (afterSeq === undefined ||
|
|
767
|
+
// §4.6: a restricted bootstrap's kept cursor is rechecked too,
|
|
768
|
+
// and a table horizon that rose during the round resets it: its
|
|
769
|
+
// segments were read before the re-key.
|
|
770
|
+
const restricted = state.kind === 'bootstrap' && state.restricted;
|
|
771
|
+
const afterSeq = bound(state) ?? (restricted ? state.pin : undefined);
|
|
772
|
+
if (afterSeq === undefined ||
|
|
773
|
+
(afterSeq >= recheck &&
|
|
774
|
+
!belowTable(index, afterSeq, recheckTables) &&
|
|
775
|
+
!(restricted &&
|
|
776
|
+
[...units[index].plan.pairs.keys()].some((table) => (recheckTables.get(table) ?? -1) >
|
|
777
|
+
(tableHorizons.get(table) ?? -1)))))
|
|
747
778
|
continue;
|
|
748
779
|
expired = true;
|
|
749
|
-
|
|
780
|
+
// A live unit below a table horizon alone takes a unit reset.
|
|
781
|
+
if (state.kind === 'live' && afterSeq < recheck)
|
|
750
782
|
liveReset = true;
|
|
751
783
|
else
|
|
752
784
|
states[index] = { kind: 'reset' };
|
|
@@ -932,20 +964,15 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
932
964
|
}
|
|
933
965
|
}
|
|
934
966
|
catch (error) {
|
|
935
|
-
|
|
967
|
+
// §5.4: a segment store failure leaves the build, so `createStreamCore`
|
|
968
|
+
// answers it as HTTP 503 before the header frames and in-band after them.
|
|
969
|
+
if (error instanceof SyncError &&
|
|
970
|
+
error.code !== 'sync.segment_store_unavailable') {
|
|
936
971
|
report.outcome = 'error';
|
|
937
972
|
report.errorCode = error.code;
|
|
938
973
|
await recordRoundFailure(ctx, clientId, error);
|
|
939
974
|
// §1.6: in-band ERROR, then END, nothing else.
|
|
940
|
-
yield
|
|
941
|
-
type: 'ERROR',
|
|
942
|
-
code: error.code,
|
|
943
|
-
message: error.message,
|
|
944
|
-
category: error.category,
|
|
945
|
-
retryable: error.retryable,
|
|
946
|
-
recommendedAction: error.recommendedAction,
|
|
947
|
-
...(error.details !== undefined ? { details: error.details } : {}),
|
|
948
|
-
}, plan.wireVersion);
|
|
975
|
+
yield errorFrameBytes(error, plan.wireVersion);
|
|
949
976
|
yield END_FRAME_BYTES;
|
|
950
977
|
return;
|
|
951
978
|
}
|
|
@@ -953,10 +980,24 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
953
980
|
}
|
|
954
981
|
yield END_FRAME_BYTES;
|
|
955
982
|
}
|
|
983
|
+
/** The in-band `ERROR` frame of a failure (§1.6). */
|
|
984
|
+
function errorFrameBytes(error, wireVersion) {
|
|
985
|
+
return encodeResponseFrame({
|
|
986
|
+
type: 'ERROR',
|
|
987
|
+
code: error.code,
|
|
988
|
+
message: error.message,
|
|
989
|
+
category: error.category,
|
|
990
|
+
retryable: error.retryable,
|
|
991
|
+
recommendedAction: error.recommendedAction,
|
|
992
|
+
...(error.details !== undefined ? { details: error.details } : {}),
|
|
993
|
+
}, wireVersion);
|
|
994
|
+
}
|
|
956
995
|
/**
|
|
957
996
|
* Decode and validate the request, then return the streaming response.
|
|
958
997
|
* Request-validation failures (§1.7) throw `SyncError` here, before any
|
|
959
|
-
* bytes are produced; adapters map them to HTTP JSON errors (§1.1).
|
|
998
|
+
* bytes are produced; adapters map them to HTTP JSON errors (§1.1). The
|
|
999
|
+
* first chunk arrives once the response is built or its first `KEEPALIVE`
|
|
1000
|
+
* is due (§1.6); a failure before then throws from it, before any byte.
|
|
960
1001
|
*/
|
|
961
1002
|
export async function createSyncResponseStream(bytes, ctx) {
|
|
962
1003
|
const sink = ctx.events;
|
|
@@ -1093,19 +1134,24 @@ async function createStreamCore(bytes, ctx, events, startedAtMs = 0) {
|
|
|
1093
1134
|
await recordRoundFailureCode(ctx, plan.header.clientId, plan.roundFailure.code);
|
|
1094
1135
|
}
|
|
1095
1136
|
const report = { outcome: 'ok' };
|
|
1096
|
-
// RFC 0007 read-verify-refuse for the streamed read path: the
|
|
1097
|
-
//
|
|
1098
|
-
//
|
|
1099
|
-
//
|
|
1100
|
-
//
|
|
1137
|
+
// RFC 0007 read-verify-refuse for the streamed read path: the build
|
|
1138
|
+
// buffers the whole response (its data reads), then compares the gate token
|
|
1139
|
+
// across those reads. No push result or section escapes before the
|
|
1140
|
+
// verification, and the buffered size is bounded by the request's pull
|
|
1141
|
+
// limits and the server's inline segment cap. Every request verifies,
|
|
1101
1142
|
// including a mixed push+pull: a write that already applied when the token
|
|
1102
1143
|
// changed is durable and replayable under the same commit id / idempotency
|
|
1103
1144
|
// key, so refusing the request is safe (proved in the serve-gate lane).
|
|
1104
|
-
const
|
|
1105
|
-
|
|
1145
|
+
const buffered = [];
|
|
1146
|
+
// Set at the build deadline: a build that finishes later stops before the
|
|
1147
|
+
// serve-gate read and persists nothing.
|
|
1148
|
+
let abandoned = false;
|
|
1149
|
+
const built = (async () => {
|
|
1106
1150
|
for await (const chunk of streamResponse(plan, ctx, served, report)) {
|
|
1107
1151
|
buffered.push(chunk);
|
|
1108
1152
|
}
|
|
1153
|
+
if (abandoned)
|
|
1154
|
+
return false;
|
|
1109
1155
|
const after = await ctx.storage.readServeGate(ctx.partition, schema.version);
|
|
1110
1156
|
const changed = serveGateTokenChanged(gate, after);
|
|
1111
1157
|
const afterRefusal = serveGateRefusal(after, schema.version, ctx.checkpoints);
|
|
@@ -1114,13 +1160,117 @@ async function createStreamCore(bytes, ctx, events, startedAtMs = 0) {
|
|
|
1114
1160
|
if (changed !== undefined)
|
|
1115
1161
|
throw serveNotReadyError(changed);
|
|
1116
1162
|
await report.persist?.();
|
|
1117
|
-
|
|
1118
|
-
|
|
1163
|
+
return true;
|
|
1164
|
+
})();
|
|
1165
|
+
// The races below observe the failure; this keeps an abandoned stream's
|
|
1166
|
+
// failure from surfacing as unhandled.
|
|
1167
|
+
built.catch(() => { });
|
|
1168
|
+
// §1.6 header frames: the envelope header, RESP_HEADER and its LEASE.
|
|
1169
|
+
const headerChunks = plan.leaseToEmit !== undefined ? 3 : 2;
|
|
1170
|
+
let live = false;
|
|
1171
|
+
const verified = async function* () {
|
|
1172
|
+
let timer;
|
|
1173
|
+
let elapsedMs = 0;
|
|
1174
|
+
try {
|
|
1175
|
+
for (;;) {
|
|
1176
|
+
let done;
|
|
1177
|
+
try {
|
|
1178
|
+
done = await Promise.race([
|
|
1179
|
+
built,
|
|
1180
|
+
// §1.6: every 10 s, a third of the native client's 30 s socket
|
|
1181
|
+
// round silence bound (§1.8).
|
|
1182
|
+
new Promise((resolve) => {
|
|
1183
|
+
timer = setTimeout(() => resolve(false), 10_000);
|
|
1184
|
+
}),
|
|
1185
|
+
]);
|
|
1186
|
+
if (!done)
|
|
1187
|
+
elapsedMs += 10_000;
|
|
1188
|
+
// §1.6 build deadline: 300 s, 2.5 times the slowest cold build
|
|
1189
|
+
// measured in production (119.7 s for 67 segment puts). A build
|
|
1190
|
+
// still running then is abandoned like a failed put.
|
|
1191
|
+
if (elapsedMs === 300_000) {
|
|
1192
|
+
abandoned = true;
|
|
1193
|
+
throw syncError('sync.segment_build_timeout', 'the response was not built within the server build deadline');
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
catch (thrown) {
|
|
1197
|
+
if (!live)
|
|
1198
|
+
throw thrown;
|
|
1199
|
+
// The header frames went out: the failure is in-band (§1.6), and
|
|
1200
|
+
// one outside the catalog answers `sync.internal_error`.
|
|
1201
|
+
const error = adapterSyncError(thrown, ctx.onError, 'sync', ctx.mapError);
|
|
1202
|
+
report.outcome = 'error';
|
|
1203
|
+
report.errorCode = error.code;
|
|
1204
|
+
await recordRoundFailure(ctx, plan.header.clientId, error);
|
|
1205
|
+
yield errorFrameBytes(error, plan.wireVersion);
|
|
1206
|
+
yield END_FRAME_BYTES;
|
|
1207
|
+
return;
|
|
1208
|
+
}
|
|
1209
|
+
clearTimeout(timer);
|
|
1210
|
+
if (done)
|
|
1211
|
+
break;
|
|
1212
|
+
if (!live && buffered.length < headerChunks)
|
|
1213
|
+
continue;
|
|
1214
|
+
if (!live) {
|
|
1215
|
+
live = true;
|
|
1216
|
+
yield* buffered.slice(0, headerChunks);
|
|
1217
|
+
}
|
|
1218
|
+
yield encodeResponseFrame({ type: 'KEEPALIVE' }, plan.wireVersion);
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
finally {
|
|
1222
|
+
clearTimeout(timer);
|
|
1223
|
+
}
|
|
1224
|
+
yield* buffered.slice(live ? headerChunks : 0);
|
|
1119
1225
|
};
|
|
1120
|
-
|
|
1121
|
-
|
|
1226
|
+
const stream = recordedStream(events === undefined
|
|
1227
|
+
? verified()
|
|
1228
|
+
: instrumentedStream(verified(), ctx, events, plan, report, bytes.length, startedAtMs), ctx, plan.header.clientId);
|
|
1229
|
+
return {
|
|
1230
|
+
get live() {
|
|
1231
|
+
return live;
|
|
1232
|
+
},
|
|
1233
|
+
[Symbol.asyncIterator]: () => stream,
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* The `POST /sync` body (§1.1): the whole response when it was built before
|
|
1238
|
+
* its first `KEEPALIVE` was due, otherwise a stream that carries the header
|
|
1239
|
+
* frames and `KEEPALIVE` frames first (§1.6). Adapters compress only the
|
|
1240
|
+
* whole response (§1.3).
|
|
1241
|
+
*/
|
|
1242
|
+
export async function createSyncResponseBody(bytes, ctx) {
|
|
1243
|
+
const stream = await createSyncResponseStream(bytes, ctx);
|
|
1244
|
+
const iterator = stream[Symbol.asyncIterator]();
|
|
1245
|
+
const first = await iterator.next();
|
|
1246
|
+
if (stream.live) {
|
|
1247
|
+
return new ReadableStream({
|
|
1248
|
+
start(controller) {
|
|
1249
|
+
if (!first.done)
|
|
1250
|
+
controller.enqueue(first.value);
|
|
1251
|
+
},
|
|
1252
|
+
async pull(controller) {
|
|
1253
|
+
const step = await iterator.next();
|
|
1254
|
+
if (step.done)
|
|
1255
|
+
controller.close();
|
|
1256
|
+
else
|
|
1257
|
+
controller.enqueue(step.value);
|
|
1258
|
+
},
|
|
1259
|
+
async cancel() {
|
|
1260
|
+
await iterator.return?.();
|
|
1261
|
+
},
|
|
1262
|
+
});
|
|
1122
1263
|
}
|
|
1123
|
-
|
|
1264
|
+
const chunks = first.done ? [] : [first.value];
|
|
1265
|
+
for (let step = await iterator.next(); !step.done; step = await iterator.next())
|
|
1266
|
+
chunks.push(step.value);
|
|
1267
|
+
const out = new Uint8Array(chunks.reduce((sum, chunk) => sum + chunk.length, 0));
|
|
1268
|
+
let offset = 0;
|
|
1269
|
+
for (const chunk of chunks) {
|
|
1270
|
+
out.set(chunk, offset);
|
|
1271
|
+
offset += chunk.length;
|
|
1272
|
+
}
|
|
1273
|
+
return out;
|
|
1124
1274
|
}
|
|
1125
1275
|
/** Counts response bytes and emits `request.handled` when the stream
|
|
1126
1276
|
* finishes — including a thrown host failure mid-stream (fail loud). */
|
|
@@ -1163,18 +1313,8 @@ async function* instrumentedStream(stream, ctx, events, plan, report, bytesIn, s
|
|
|
1163
1313
|
}
|
|
1164
1314
|
/** Byte-concatenating wrapper over the streaming core (§1.4). */
|
|
1165
1315
|
export async function handleSyncRequest(bytes, ctx) {
|
|
1166
|
-
const
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
chunks.push(chunk);
|
|
1171
|
-
total += chunk.length;
|
|
1172
|
-
}
|
|
1173
|
-
const out = new Uint8Array(total);
|
|
1174
|
-
let offset = 0;
|
|
1175
|
-
for (const chunk of chunks) {
|
|
1176
|
-
out.set(chunk, offset);
|
|
1177
|
-
offset += chunk.length;
|
|
1178
|
-
}
|
|
1179
|
-
return out;
|
|
1316
|
+
const body = await createSyncResponseBody(bytes, ctx);
|
|
1317
|
+
return body instanceof Uint8Array
|
|
1318
|
+
? body
|
|
1319
|
+
: new Uint8Array(await new Response(body).arrayBuffer());
|
|
1180
1320
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
1
|
+
import type { CommitPruneQuery, CommitPruneResult, PartitionHorizons } from './storage.js';
|
|
2
2
|
/**
|
|
3
3
|
* Postgres server storage: the production database path.
|
|
4
4
|
*
|
|
@@ -74,7 +74,7 @@ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDec
|
|
|
74
74
|
* set, §5.9.5) as an index range, never a scan. `postgres-explain
|
|
75
75
|
* .test.ts` asserts an `Index` node here too.
|
|
76
76
|
*/
|
|
77
|
-
export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0,\n delete_floor BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_change_scopes_by_key\n ON sync_change_scopes(partition, var, value, commit_seq);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n outbox_id TEXT, client_seq BIGINT,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS outbox_id TEXT;\nALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS client_seq BIGINT;\nCREATE TABLE IF NOT EXISTS sync_push_watermarks(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,\n through_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, client_id, outbox_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n last_round_at_ms BIGINT, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n schema_version INTEGER, units JSONB, registration_digest BYTEA,\n catching_up JSONB NOT NULL DEFAULT '[]',\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS schema_version INTEGER;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS units JSONB;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS registration_digest BYTEA;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS catching_up JSONB NOT NULL DEFAULT '[]';\n-- \u00A79: a database from before wire version 4 (its client records still hold\n-- subscriptions) starts the horizon at its last commit.\nUPDATE sync_partitions SET horizon_seq = max_commit_seq\n WHERE EXISTS (SELECT 1 FROM information_schema.columns\n WHERE table_schema = current_schema()\n AND table_name = 'sync_clients'\n AND column_name = 'subscriptions');\nALTER TABLE sync_clients DROP COLUMN IF EXISTS subscriptions;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,\n PRIMARY KEY(partition, tbl, row_id)\n);\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;\nALTER TABLE sync_partitions\n ADD COLUMN IF NOT EXISTS delete_floor BIGINT NOT NULL DEFAULT 0;\nUPDATE sync_partitions SET delete_floor = t.version\n FROM (SELECT partition, max(server_version) AS version\n FROM sync_tombstones GROUP BY partition) t\n WHERE t.partition = sync_partitions.partition\n AND t.version > sync_partitions.delete_floor;\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
77
|
+
export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0,\n delete_floor BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_table_horizons(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl)\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_change_scopes_by_key\n ON sync_change_scopes(partition, var, value, commit_seq);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n outbox_id TEXT, client_seq BIGINT,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS outbox_id TEXT;\nALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS client_seq BIGINT;\nCREATE TABLE IF NOT EXISTS sync_push_watermarks(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,\n through_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, client_id, outbox_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n last_round_at_ms BIGINT, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n schema_version INTEGER, units JSONB, registration_digest BYTEA,\n catching_up JSONB NOT NULL DEFAULT '[]',\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS schema_version INTEGER;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS units JSONB;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS registration_digest BYTEA;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS catching_up JSONB NOT NULL DEFAULT '[]';\n-- \u00A79: a database from before wire version 4 (its client records still hold\n-- subscriptions) starts the horizon at its last commit.\nUPDATE sync_partitions SET horizon_seq = max_commit_seq\n WHERE EXISTS (SELECT 1 FROM information_schema.columns\n WHERE table_schema = current_schema()\n AND table_name = 'sync_clients'\n AND column_name = 'subscriptions');\nALTER TABLE sync_clients DROP COLUMN IF EXISTS subscriptions;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,\n PRIMARY KEY(partition, tbl, row_id)\n);\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;\nALTER TABLE sync_partitions\n ADD COLUMN IF NOT EXISTS delete_floor BIGINT NOT NULL DEFAULT 0;\nUPDATE sync_partitions SET delete_floor = t.version\n FROM (SELECT partition, max(server_version) AS version\n FROM sync_tombstones GROUP BY partition) t\n WHERE t.partition = sync_partitions.partition\n AND t.version > sync_partitions.delete_floor;\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
78
78
|
export declare class PostgresServerStorage implements ServerStorage {
|
|
79
79
|
#private;
|
|
80
80
|
constructor(exec: PgExecutor);
|
|
@@ -105,6 +105,7 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
105
105
|
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
|
|
106
106
|
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
107
107
|
getHorizonSeq(partition: string): Promise<number>;
|
|
108
|
+
getHorizons(partition: string): Promise<PartitionHorizons>;
|
|
108
109
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
109
110
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
110
111
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|