@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/README.md
CHANGED
|
@@ -408,7 +408,7 @@ must enumerate every row in one Workspace. Syncular does not currently expose
|
|
|
408
408
|
trusted prefix or range scans.
|
|
409
409
|
|
|
410
410
|
This is a trusted `@syncular/server` storage capability, not SSP2: it creates no
|
|
411
|
-
scope variable, named-query obligation,
|
|
411
|
+
scope variable, named-query obligation, unit, or client
|
|
412
412
|
authority. Never expose table/index/value selection through a client-controlled
|
|
413
413
|
route. Custom adapters may omit the additive method; authoritative commands
|
|
414
414
|
must check for it and fail closed. See the public
|
|
@@ -571,7 +571,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
|
|
|
571
571
|
|
|
572
572
|
| Event | When | Key fields |
|
|
573
573
|
| --- | --- | --- |
|
|
574
|
-
| `request.handled` | Once per `POST /sync`, after the response bytes are fully produced (or the request was rejected up front) | `kind` (`sync`), `partition`, `actorId`, `durationMs`, `bytesIn`, `bytesOut`, `outcome` (`ok` \| `schema_floor` \| `epoch_acquired` \| `reset` \| `rejected` \| `error`; `epoch_acquired` answers a first round without `logEpoch`, `reset` a `logEpoch` the log no longer matches), `errorCode?`, `clientId?` (absent when the request did not decode), `pushCommits`, `pulled`, `
|
|
574
|
+
| `request.handled` | Once per `POST /sync`, after the response bytes are fully produced (or the request was rejected up front) | `kind` (`sync`), `partition`, `actorId`, `durationMs`, `bytesIn`, `bytesOut`, `outcome` (`ok` \| `schema_floor` \| `epoch_acquired` \| `reset` \| `rejected` \| `error`; `epoch_acquired` answers a first round without `logEpoch`, `reset` a `logEpoch` the log no longer matches), `errorCode?`, `clientId?` (absent when the request did not decode), `pushCommits`, `pulled`, `units` (units of the request's effective list, 0 without one) |
|
|
575
575
|
| `push.applied` | A `PUSH_COMMIT` applied, or replayed from the idempotency cache (§2.3) | `clientId`, `clientCommitId`, `operations`, `commitSeq?`, `replay` |
|
|
576
576
|
| `push.rejected` | A commit rejected (§6.3) | `clientId`, `clientCommitId`, `operations`, `code` (§10.2), `opIndex`, `table?` and `rowId?` of the terminating operation |
|
|
577
577
|
| `push.conflicted` | A commit terminated by a version conflict (§6.2) | `clientId`, `clientCommitId`, `operations`, `opIndex`, `table?` and `rowId?` of the conflicting operation |
|
|
@@ -581,7 +581,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
|
|
|
581
581
|
| `reaction.completed` | A handler finished and its owner acknowledged | started fields |
|
|
582
582
|
| `reaction.dead_lettered` | A permanent or exhausted failure was recorded | started fields plus `errorCode` |
|
|
583
583
|
| `reaction.prune_completed` | One bounded terminal-reaction retention pass finished | `completedBeforeMs`, `deadLetterBeforeMs`, `limit`, `removedCompleted`, `removedDeadLetter`, `mayHaveMore` |
|
|
584
|
-
| `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `
|
|
584
|
+
| `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `acceptedThrough`, `storageMaxCommitSeq`, `registrationTraveled` (the request carried its registration list), `units[]`: `{id, scopes, status` (`active` \| `revoked` \| `reset`)`, mode` (`live` \| `catching-up` \| `bootstrap` \| `none`)`, section` (the response carried a unit section)`, fenced, fromCursor, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}`; `live`: `{status` (`active` \| `reset` \| `unregistered`)`, fromCursor` (the replica cursor)`, throughSeq, commits, changes, firstCommitSeq?, lastCommitSeq?}` |
|
|
585
585
|
| `segment.downloaded` | Every direct segment download (§5.5), success or failure | `segmentId`, `outcome` (`ok` \| `error`), `errorCode?`, `mediaType?`, `bytes?`, `durationMs` |
|
|
586
586
|
| `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (old candidates still referenced), `graceMs` |
|
|
587
587
|
| `realtime.opened` | A socket registered with the hub and got `hello` (§8.1) | `sessionId`, `clientId`, `registrations`, `cursor`, `latestSeq` |
|
|
@@ -645,14 +645,14 @@ design. Row reads need the server `schema` (`fromConfig` passes it).
|
|
|
645
645
|
| `tableCounts(partition)` | Row count per schema table. |
|
|
646
646
|
| `listRows(partition, table, {afterRowId?, limit?, scope?})` | One page of decoded rows ordered by rowId, optionally only rows whose stored `scope` (`{variable, value}`) matches, read through the scope index; `nextAfterRowId` continues. |
|
|
647
647
|
| `rowDelivery(partition, table, rowId)` | The trace verdict for every known client on one row, sorted by verdict then last seen: who has the row and who does not. |
|
|
648
|
-
| `findWrite(partition, clientCommitId)` | Every stored idempotency outcome for that clientCommitId (applied with `commitSeq`, or rejected with per-operation results; conflict bytes as base64)
|
|
649
|
-
| `listClients(partition, {limit?, sort?})` | `{clients, total}`. Each client: `clientId`, `actorId`, `wireVersion`, `schemaVersion?` (
|
|
648
|
+
| `findWrite(partition, clientCommitId)` | Every stored idempotency outcome for that clientCommitId (applied with `commitSeq`, or rejected with per-operation results; conflict bytes as base64), the ring events that carry it, and, when no outcome is stored, the §6.3.2 `watermarks[]` (`clientId`, `outboxId`, `throughSeq`) of the clients whose retained log carries the commit: a commit below `throughSeq` lost its result to the watermark ("expired below sequence N"). |
|
|
649
|
+
| `listClients(partition, {limit?, sort?})` | `{clients, total}`. Each client: `clientId`, `actorId`, `wireVersion`, `schemaVersion?` (of its latest completed round), `cursor`, `lag` (`maxCommitSeq − max(cursor, 0)`), `updatedAtMs`, `units[]` (the stored registration list: `id`, `scopes`, `catchingUpCursor?` for a unit still catching up), `active` (touched within the §4.6 active window), `connected?` and `connectedSinceMs?` (open realtime sockets on this process; present when a hub is wired, which `fromConfig` does when `config.realtime` is a `RealtimeHub`). `sort`: `seen` (default) or `lag`; `limit` default 200. |
|
|
650
650
|
| `clientDetail(partition, clientId, {eventLimit?})` | `{exists, client?, lease?, events}`: the record, its §7.3 lease when a lease store is wired, and its slice of the event tail. |
|
|
651
651
|
| `listCommits(partition, {afterSeq?, beforeSeq?, limit?, table?, changeSample?})` | Commit metadata, newest first: `commitSeq`, `clientId`, `clientCommitId`, `actorId`, `createdAtMs`, `changeCount`, `tables[]`, and with `changeSample` the first N changes as `sample[]` (`table`, `rowId`, `op`). Pass the oldest `commitSeq` of a page as `beforeSeq` for the next older page. |
|
|
652
652
|
| `commitDetail(partition, commitSeq)` | The commit's metadata plus `changes[]` (`table`, `rowId`, `op`, `rowVersion?`, `scopes`, `values?` rendered per SPEC §11, `undecodable?` for a payload written under an older column set). Throws `sync.not_found` for a missing or pruned commit. |
|
|
653
653
|
| `listReactions(partition, {statuses?, types?, limit?})` | Durable reaction lifecycle rows, newest first, including source commit, attempts, lease, completion, and bounded failure information. |
|
|
654
654
|
| `inspectRow(partition, table, rowId, {historyLimit?})` | `{exists, serverVersion?, scopes?, values?, history}`: the row decoded per SPEC §11 and its retained changes, newest first (`commitSeq`, `op`, `rowVersion?`, `actorId`, `clientId`, `createdAtMs`). Throws `sync.unknown_table` for a table outside the schema. |
|
|
655
|
-
| `traceRow(partition, {table, rowId, clientId})` | Whether the client receives the row: a `verdict` (`client-unknown`, `row-missing`, `no-
|
|
655
|
+
| `traceRow(partition, {table, rowId, clientId})` | Whether the client receives the row: a `verdict` (`client-unknown`, `row-missing`, `no-unit`, `row-deleted`, `scope-mismatch`, `scope-denied`, `pending-pull`, `delivered`, evaluated in that order) with the row, its newest change, the horizon, the client, the actor's `allowed` scopes from `resolveScopes` (`outage` for the §7.3.3 sentinel), each of the client's units that covers the table with `requestedMatch`, `mismatches[]` and `effectiveMatch` (requested ∩ allowed, §3.2), and the client's retained push rejections. |
|
|
656
656
|
| `scopeActivity(partition, {variable, value}, {limit?})` | Recent commits touching one scope key, via the §3.1 change-scope index (never a log scan). |
|
|
657
657
|
| `horizonStatus(partition)` | `{maxCommitSeq, horizonSeq, retainedCommits, activeCursorFloor, recommendedHorizonSeq, recommendation}`: the horizon a prune pass would reach now (§4.6) and `up-to-date` / `prune-recommended`. |
|
|
658
658
|
| `stats(partition)` | `{segments?, blobs?}` counts/bytes where the stores expose them (segments split rows/sqlite). |
|
|
@@ -962,8 +962,8 @@ bytes straight to the object store. **Upload authz is host-authentication only**
|
|
|
962
962
|
bytes is not a scope-bearing act, because the content address discloses
|
|
963
963
|
nothing. The grant signs `content-length` (the declared `byteLength`) and
|
|
964
964
|
`x-amz-checksum-sha256` (the `blobId` digest as base64), and the client sends
|
|
965
|
-
that checksum header,
|
|
966
|
-
other bytes. The **size cap is enforced up front** against the declared
|
|
965
|
+
that checksum header, derived from the `blobId` without hashing the body again,
|
|
966
|
+
so the object store refuses a PUT of another size or other bytes. The **size cap is enforced up front** against the declared
|
|
967
967
|
`byteLength`, before any URL is minted. An already-present blob returns `{ present: true }` (skip
|
|
968
968
|
the PUT, idempotent §5.9.3). A **single PUT only** — never a multipart or chunk
|
|
969
969
|
protocol; resumable upload, when it lands, is provider multipart behind this
|
|
@@ -1151,14 +1151,16 @@ regardless, and at least the newest `minRetainedCommits` (default 1000)
|
|
|
1151
1151
|
commits are always kept. Raise the defaults freely, lower them with care.
|
|
1152
1152
|
|
|
1153
1153
|
**What `sync.cursor_expired` means operationally.** A client whose
|
|
1154
|
-
cursor fell behind the horizon gets `
|
|
1155
|
-
|
|
1154
|
+
replica cursor fell behind the horizon gets `LIVE_START.status = reset`
|
|
1155
|
+
(a post-bootstrap unit behind it gets `SUB_START.status = reset`) and
|
|
1156
|
+
re-bootstraps the affected units from scratch (§4.6, §4.7). That is correct behavior, not an
|
|
1156
1157
|
error — but its *rate* is your pruning health signal. A steady trickle
|
|
1157
1158
|
means devices returning from >30-day absences (expected). A spike means
|
|
1158
1159
|
you pruned faster than your fleet syncs: `ageForceMs` or
|
|
1159
1160
|
`activeWindowMs` is too tight for real usage, and you are paying for it
|
|
1160
1161
|
in bootstrap load (full re-scans + segment builds), not just in resets.
|
|
1161
|
-
Observe it via `pull.served`
|
|
1162
|
+
Observe it via `pull.served` events whose `live.status` or unit `status`
|
|
1163
|
+
is `"reset"`.
|
|
1162
1164
|
|
|
1163
1165
|
**Segment TTL interplay.** Segments are cache entries, not durable state
|
|
1164
1166
|
(§5.1; default TTL 24 h). A bootstrap that resumes past segment expiry
|
|
@@ -1178,7 +1180,7 @@ same table+scope during a storm, your TTL is shorter than the storm.
|
|
|
1178
1180
|
(`push.conflicted` is normal offline-first traffic; alert only on
|
|
1179
1181
|
gross shifts.)
|
|
1180
1182
|
- `scopes.resolve_failed` — any nonzero rate. This is the fail-loud
|
|
1181
|
-
path: every occurrence revokes
|
|
1183
|
+
path: every occurrence revokes units or rejects writes for a
|
|
1182
1184
|
real request, and it is almost always a host bug or a dead dependency
|
|
1183
1185
|
of the resolver.
|
|
1184
1186
|
- `request.handled` with `outcome: "error"` and `errorCode: "internal"`
|
|
@@ -1399,10 +1401,12 @@ rows. The image database and serialized output still consume memory.
|
|
|
1399
1401
|
Realtime acknowledgements call `advanceClientCursor(partition, clientId,
|
|
1400
1402
|
actorId, logEpoch, cursor, updatedAtMs)`. Custom storage adapters must implement
|
|
1401
1403
|
this atomic update: advance the cursor and activity timestamp to their respective
|
|
1402
|
-
maxima,
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1404
|
+
maxima, cap the cursor at the lowest non-negative cursor of the record's
|
|
1405
|
+
catching-up units, preserve registration fields, and require a matching actor and
|
|
1406
|
+
current partition log epoch. Leave missing records unchanged. SQLite, Postgres,
|
|
1407
|
+
and D1 perform one update without reading or serializing the registration list.
|
|
1408
|
+
A completed sync round writes the record through `putClientRecord`, which keeps
|
|
1409
|
+
the stored registration list unless the round's list traveled.
|
|
1406
1410
|
|
|
1407
1411
|
## Reviewed schema windows
|
|
1408
1412
|
|
package/dist/admin.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* thrown `Error`) when a backend lacks one, so a host wiring an unsupported
|
|
12
12
|
* store learns immediately rather than getting a silently-empty console.
|
|
13
13
|
*/
|
|
14
|
-
import { type ColumnType, type JsonValue, type PushOperationResult, type ScopeMap } from '@syncular/core';
|
|
14
|
+
import { type ColumnType, type JsonValue, type PushOperationResult, type ScopeMap, type UnitScopes } from '@syncular/core';
|
|
15
15
|
import type { BlobStore, BlobStoreStats } from './blob-store.js';
|
|
16
16
|
import { type ResolveScopes, type SyncServerConfig } from './context.js';
|
|
17
17
|
import { type SyncularServerEvent, type SyncularServerEvents } from './events.js';
|
|
@@ -21,14 +21,22 @@ import { RealtimeHub } from './realtime.js';
|
|
|
21
21
|
import { type RetentionPolicy } from './prune.js';
|
|
22
22
|
import { type ServerSchema } from './schema.js';
|
|
23
23
|
import type { SegmentStore, SegmentStoreStats } from './segment-store.js';
|
|
24
|
-
import type { CheckpointDeclaration, CommitMetadata, ReactionStatus, StoredPushResult, RowChangeRecord, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
|
|
24
|
+
import type { CheckpointDeclaration, CommitMetadata, PushWatermark, ReactionStatus, StoredPushResult, RowChangeRecord, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
|
|
25
|
+
/** One registered unit of a client record. */
|
|
26
|
+
export interface AdminClientUnit {
|
|
27
|
+
/** The unit id (§4.1). */
|
|
28
|
+
readonly id: string;
|
|
29
|
+
readonly scopes: UnitScopes;
|
|
30
|
+
/** The unit's cursor while it catches up; absent for a live unit. */
|
|
31
|
+
readonly catchingUpCursor?: number;
|
|
32
|
+
}
|
|
25
33
|
/** A connected/known client as the console sees it (§4.5, §8.1). */
|
|
26
34
|
export interface AdminClient {
|
|
27
35
|
readonly clientId: string;
|
|
28
36
|
readonly actorId: string;
|
|
29
37
|
/** SSP2 wire version last accepted from this client. */
|
|
30
38
|
readonly wireVersion: number;
|
|
31
|
-
/**
|
|
39
|
+
/** Schema version of its latest completed round, when recorded. */
|
|
32
40
|
readonly schemaVersion?: number;
|
|
33
41
|
readonly cursor: number;
|
|
34
42
|
/**
|
|
@@ -37,11 +45,8 @@ export interface AdminClient {
|
|
|
37
45
|
*/
|
|
38
46
|
readonly lag: number;
|
|
39
47
|
readonly updatedAtMs: number;
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
readonly table: string;
|
|
43
|
-
readonly scopes: Record<string, readonly string[]>;
|
|
44
|
-
}[];
|
|
48
|
+
/** The stored registration list (§4.1); empty when none is stored. */
|
|
49
|
+
readonly units: readonly AdminClientUnit[];
|
|
45
50
|
/** True when the cursor record was touched within the active window. */
|
|
46
51
|
readonly active: boolean;
|
|
47
52
|
/**
|
|
@@ -211,27 +216,27 @@ export interface AdminCommitDetail extends CommitMetadata {
|
|
|
211
216
|
* The trace verdict, evaluated in this order:
|
|
212
217
|
* `client-unknown`: no client record for the id.
|
|
213
218
|
* `row-missing`: no row and no retained change to it.
|
|
214
|
-
* `no-
|
|
219
|
+
* `no-unit`: no registered unit of the client covers the table.
|
|
215
220
|
* `row-deleted`: the row's newest retained change is a delete the client
|
|
216
221
|
* has pulled.
|
|
217
|
-
* `scope-mismatch`: no
|
|
218
|
-
* `scope-denied`: a
|
|
222
|
+
* `scope-mismatch`: no covering unit requests the row's stored scopes.
|
|
223
|
+
* `scope-denied`: a covering unit requests them, but the actor's allowed
|
|
219
224
|
* scopes (the host `resolveScopes`) exclude the row.
|
|
220
225
|
* `pending-pull`: the client's cursor is below the row's newest change.
|
|
221
226
|
* `delivered`: the client has pulled the row's newest change.
|
|
222
227
|
*/
|
|
223
|
-
export type AdminTraceVerdict = 'client-unknown' | 'row-missing' | 'no-
|
|
224
|
-
/** One of the client's
|
|
225
|
-
export interface
|
|
228
|
+
export type AdminTraceVerdict = 'client-unknown' | 'row-missing' | 'no-unit' | 'row-deleted' | 'scope-mismatch' | 'scope-denied' | 'pending-pull' | 'delivered';
|
|
229
|
+
/** One of the client's units covering the traced table. */
|
|
230
|
+
export interface AdminTraceUnit {
|
|
226
231
|
readonly id: string;
|
|
227
|
-
readonly scopes:
|
|
228
|
-
/** The row's stored scopes
|
|
232
|
+
readonly scopes: UnitScopes;
|
|
233
|
+
/** The row's stored scopes carry the unit's values. */
|
|
229
234
|
readonly requestedMatch: boolean;
|
|
230
|
-
/** Each
|
|
235
|
+
/** Each unit variable the row's stored value fails. */
|
|
231
236
|
readonly mismatches: readonly {
|
|
232
237
|
readonly variable: string;
|
|
233
238
|
readonly rowValue: string | null;
|
|
234
|
-
readonly requested:
|
|
239
|
+
readonly requested: string;
|
|
235
240
|
}[];
|
|
236
241
|
/**
|
|
237
242
|
* The row also satisfies requested ∩ allowed (§3.2). Absent when the
|
|
@@ -256,7 +261,7 @@ export interface AdminRowTrace {
|
|
|
256
261
|
* when it returned the §7.3.3 outage sentinel. Absent without a resolver.
|
|
257
262
|
*/
|
|
258
263
|
readonly allowed?: ScopeMap | 'outage';
|
|
259
|
-
readonly
|
|
264
|
+
readonly units: readonly AdminTraceUnit[];
|
|
260
265
|
/** The client's retained push rejections and conflicts, newest first. */
|
|
261
266
|
readonly rejections: readonly SyncularServerEvent[];
|
|
262
267
|
}
|
|
@@ -323,6 +328,13 @@ export interface AdminWriteLookup {
|
|
|
323
328
|
readonly clientCommitId: string;
|
|
324
329
|
/** One outcome per client that sent this clientCommitId. */
|
|
325
330
|
readonly outcomes: readonly AdminWriteOutcome[];
|
|
331
|
+
/**
|
|
332
|
+
* With no stored outcome: the §6.3.2 watermarks of the clients whose
|
|
333
|
+
* retained commit log carries this clientCommitId. A commit below an
|
|
334
|
+
* outbox's `throughSeq` lost its result to the watermark ("expired below
|
|
335
|
+
* sequence N").
|
|
336
|
+
*/
|
|
337
|
+
readonly watermarks: readonly PushWatermark[];
|
|
326
338
|
/** Retained ring events carrying this clientCommitId, newest first. */
|
|
327
339
|
readonly events: readonly SyncularServerEvent[];
|
|
328
340
|
}
|
|
@@ -464,7 +476,7 @@ export declare class SyncularAdmin {
|
|
|
464
476
|
}): Promise<AdminRowInspection>;
|
|
465
477
|
/**
|
|
466
478
|
* Trace whether one client receives one row, combining the row's stored
|
|
467
|
-
* scopes and newest change with the client's
|
|
479
|
+
* scopes and newest change with the client's units, cursor, the
|
|
468
480
|
* actor's allowed scopes, and its retained push failures. See
|
|
469
481
|
* `AdminTraceVerdict` for the verdict order.
|
|
470
482
|
*/
|
package/dist/admin.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* thrown `Error`) when a backend lacks one, so a host wiring an unsupported
|
|
12
12
|
* store learns immediately rather than getting a silently-empty console.
|
|
13
13
|
*/
|
|
14
|
-
import { bytesToBase64, decodeRow, renderRowValues, } from '@syncular/core';
|
|
14
|
+
import { bytesToBase64, decodeRow, renderRowValues, unitId, } from '@syncular/core';
|
|
15
15
|
import { clockOf, RESOLVER_OUTAGE, } from './context.js';
|
|
16
16
|
import { SyncError } from './errors.js';
|
|
17
17
|
import { emitEvent, } from './events.js';
|
|
@@ -47,41 +47,39 @@ function required(value, what) {
|
|
|
47
47
|
const VERDICT_ORDER = [
|
|
48
48
|
'client-unknown',
|
|
49
49
|
'row-missing',
|
|
50
|
-
'no-
|
|
50
|
+
'no-unit',
|
|
51
51
|
'row-deleted',
|
|
52
52
|
'scope-mismatch',
|
|
53
53
|
'scope-denied',
|
|
54
54
|
'pending-pull',
|
|
55
55
|
'delivered',
|
|
56
56
|
];
|
|
57
|
-
/** One known client's
|
|
57
|
+
/** One known client's units covering the row's table and its verdict. */
|
|
58
58
|
function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
|
|
59
59
|
const rowScopes = row.scopes ?? {};
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
})
|
|
68
|
-
.map(([variable, values]) => ({
|
|
60
|
+
// §4.1: a unit covers every table that declares all of its variables.
|
|
61
|
+
const units = (record.registration?.units ?? [])
|
|
62
|
+
.filter((scopes) => Object.keys(scopes).every((variable) => declared.has(variable)))
|
|
63
|
+
.map((scopes) => {
|
|
64
|
+
const mismatches = Object.entries(scopes)
|
|
65
|
+
.filter(([variable, value]) => rowScopes[variable] !== value)
|
|
66
|
+
.map(([variable, value]) => ({
|
|
69
67
|
variable,
|
|
70
68
|
rowValue: rowScopes[variable] ?? null,
|
|
71
|
-
requested:
|
|
69
|
+
requested: value,
|
|
72
70
|
}));
|
|
73
71
|
const traced = {
|
|
74
|
-
id:
|
|
75
|
-
scopes
|
|
72
|
+
id: unitId(scopes),
|
|
73
|
+
scopes,
|
|
76
74
|
requestedMatch: mismatches.length === 0,
|
|
77
75
|
mismatches,
|
|
78
76
|
};
|
|
79
77
|
if (allowed === undefined || allowed === 'outage')
|
|
80
78
|
return traced;
|
|
81
|
-
const effective = computeEffective(declared,
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
});
|
|
79
|
+
const effective = computeEffective(declared, Object.fromEntries(Object.entries(scopes).map(([variable, value]) => [
|
|
80
|
+
variable,
|
|
81
|
+
[value],
|
|
82
|
+
])), { ok: true, allowed });
|
|
85
83
|
return {
|
|
86
84
|
...traced,
|
|
87
85
|
effectiveMatch: effective.status === 'active' &&
|
|
@@ -93,20 +91,20 @@ function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
|
|
|
93
91
|
const pulled = record.cursor >= (lastChange?.commitSeq ?? horizonSeq);
|
|
94
92
|
const verdict = !row.exists && lastChange === undefined
|
|
95
93
|
? 'row-missing'
|
|
96
|
-
:
|
|
97
|
-
? 'no-
|
|
94
|
+
: units.length === 0
|
|
95
|
+
? 'no-unit'
|
|
98
96
|
: !row.exists
|
|
99
97
|
? pulled
|
|
100
98
|
? 'row-deleted'
|
|
101
99
|
: 'pending-pull'
|
|
102
|
-
: !
|
|
100
|
+
: !units.some((unit) => unit.requestedMatch)
|
|
103
101
|
? 'scope-mismatch'
|
|
104
|
-
:
|
|
102
|
+
: units.every((unit) => unit.effectiveMatch === false)
|
|
105
103
|
? 'scope-denied'
|
|
106
104
|
: pulled
|
|
107
105
|
? 'delivered'
|
|
108
106
|
: 'pending-pull';
|
|
109
|
-
return {
|
|
107
|
+
return { units, verdict };
|
|
110
108
|
}
|
|
111
109
|
/** The console query surface plus its operator actions. One per process. */
|
|
112
110
|
export class SyncularAdmin {
|
|
@@ -322,7 +320,7 @@ export class SyncularAdmin {
|
|
|
322
320
|
}
|
|
323
321
|
/**
|
|
324
322
|
* Trace whether one client receives one row, combining the row's stored
|
|
325
|
-
* scopes and newest change with the client's
|
|
323
|
+
* scopes and newest change with the client's units, cursor, the
|
|
326
324
|
* actor's allowed scopes, and its retained push failures. See
|
|
327
325
|
* `AdminTraceVerdict` for the verdict order.
|
|
328
326
|
*/
|
|
@@ -345,7 +343,7 @@ export class SyncularAdmin {
|
|
|
345
343
|
...(lastChange !== undefined ? { lastChange } : {}),
|
|
346
344
|
};
|
|
347
345
|
if (record === undefined) {
|
|
348
|
-
return { ...base, verdict: 'client-unknown',
|
|
346
|
+
return { ...base, verdict: 'client-unknown', units: [] };
|
|
349
347
|
}
|
|
350
348
|
const allowed = await this.#allowedScopes(partition, record);
|
|
351
349
|
return {
|
|
@@ -503,7 +501,10 @@ export class SyncularAdmin {
|
|
|
503
501
|
: entry),
|
|
504
502
|
}));
|
|
505
503
|
const events = this.events({ partition }).filter((event) => 'clientCommitId' in event && event.clientCommitId === clientCommitId);
|
|
506
|
-
|
|
504
|
+
const watermarks = outcomes.length > 0
|
|
505
|
+
? []
|
|
506
|
+
: await required(this.#storage.listPushWatermarks?.bind(this.#storage), 'storage')(partition, clientCommitId);
|
|
507
|
+
return { clientCommitId, outcomes, watermarks, events };
|
|
507
508
|
}
|
|
508
509
|
/**
|
|
509
510
|
* Resolve an id pasted from a bug report: clients and actors containing
|
|
@@ -903,28 +904,29 @@ export class SyncularAdmin {
|
|
|
903
904
|
}
|
|
904
905
|
}
|
|
905
906
|
return (record) => {
|
|
906
|
-
let schemaVersion;
|
|
907
|
-
for (const sub of record.subscriptions) {
|
|
908
|
-
if (sub.schemaVersion !== undefined &&
|
|
909
|
-
(schemaVersion === undefined || sub.schemaVersion > schemaVersion)) {
|
|
910
|
-
schemaVersion = sub.schemaVersion;
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
907
|
const connectedSinceMs = connectedSince.get(record.clientId);
|
|
914
908
|
return {
|
|
915
909
|
clientId: record.clientId,
|
|
916
910
|
actorId: record.actorId,
|
|
917
911
|
wireVersion: record.wireVersion,
|
|
918
|
-
...(schemaVersion !== undefined
|
|
912
|
+
...(record.schemaVersion !== undefined
|
|
913
|
+
? { schemaVersion: record.schemaVersion }
|
|
914
|
+
: {}),
|
|
919
915
|
cursor: record.cursor,
|
|
920
916
|
// A never-pulled cursor (-1) has seen nothing: it lags the whole log.
|
|
921
917
|
lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
|
|
922
918
|
updatedAtMs: record.updatedAtMs,
|
|
923
|
-
|
|
924
|
-
id
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
919
|
+
units: (record.registration?.units ?? []).map((scopes) => {
|
|
920
|
+
const id = unitId(scopes);
|
|
921
|
+
const catchingUp = record.catchingUp.find((unit) => unit.id === id);
|
|
922
|
+
return {
|
|
923
|
+
id,
|
|
924
|
+
scopes,
|
|
925
|
+
...(catchingUp !== undefined
|
|
926
|
+
? { catchingUpCursor: catchingUp.cursor }
|
|
927
|
+
: {}),
|
|
928
|
+
};
|
|
929
|
+
}),
|
|
928
930
|
active: record.updatedAtMs >= activeFloorMs,
|
|
929
931
|
...(sockets !== undefined
|
|
930
932
|
? { connected: connectedSinceMs !== undefined }
|
package/dist/context.d.ts
CHANGED
|
@@ -44,7 +44,7 @@ export type ResolverOutage = typeof RESOLVER_OUTAGE;
|
|
|
44
44
|
/**
|
|
45
45
|
* Host callback: actor → allowed scopes (§3.2 step 3). Resolved at most
|
|
46
46
|
* once per request and memoized (§3.4 step 1). `'*'` means "any value for
|
|
47
|
-
* this variable". A throwing resolver revokes
|
|
47
|
+
* this variable". A throwing resolver revokes units and rejects
|
|
48
48
|
* writes with `sync.forbidden` — fail loud, never leak. Returning
|
|
49
49
|
* `RESOLVER_OUTAGE` instead opts the request into lease authorization
|
|
50
50
|
* (§7.3.3) — a deliberate outage signal, not a failure.
|
|
@@ -69,10 +69,8 @@ export interface ServerLimits {
|
|
|
69
69
|
* decoding, and a socket round's assembled request (§8.7).
|
|
70
70
|
*/
|
|
71
71
|
readonly maxRequestBytes: number;
|
|
72
|
-
/** §1.7 cap on `
|
|
72
|
+
/** §1.7 cap on `UNIT` frames per request: the units of a traveling list. */
|
|
73
73
|
readonly maxSubscriptionsPerRequest: number;
|
|
74
|
-
/** §1.7 cap on requested scope values, summed over a subscription's keys. */
|
|
75
|
-
readonly maxScopeValuesPerSubscription: number;
|
|
76
74
|
/** Cap on concurrent registered-query watches per watch session. */
|
|
77
75
|
readonly maxWatchesPerSession: number;
|
|
78
76
|
}
|
|
@@ -81,10 +79,10 @@ export declare const DEFAULT_LIMITS: ServerLimits;
|
|
|
81
79
|
export interface RealtimeNotifier {
|
|
82
80
|
notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
|
|
83
81
|
/**
|
|
84
|
-
* A pull changed the client's
|
|
85
|
-
* re-registers that client's live sockets, so a client that runs its
|
|
86
|
-
* rounds over HTTP while a socket is open receives deltas for the
|
|
87
|
-
*
|
|
82
|
+
* A pull changed the client's registration list or live set (§8.1). A
|
|
83
|
+
* hub re-registers that client's live sockets, so a client that runs its
|
|
84
|
+
* rounds over HTTP while a socket is open receives deltas for the units
|
|
85
|
+
* that joined.
|
|
88
86
|
*/
|
|
89
87
|
notifySubscriptions?(partition: string, clientId: string): void | Promise<void>;
|
|
90
88
|
}
|
package/dist/context.js
CHANGED
|
@@ -17,10 +17,8 @@ export const DEFAULT_LIMITS = {
|
|
|
17
17
|
maxCascadeOperationsPerCommit: 1_000,
|
|
18
18
|
// The size above which adapters already refuse to buffer a segment body.
|
|
19
19
|
maxRequestBytes: 16 * 1024 * 1024,
|
|
20
|
-
// One
|
|
20
|
+
// One unit per pushable operation: the existing per-request cap.
|
|
21
21
|
maxSubscriptionsPerRequest: 500,
|
|
22
|
-
// One default snapshot page (§4.2): a longer list wants a coarser scope.
|
|
23
|
-
maxScopeValuesPerSubscription: 1_000,
|
|
24
22
|
// Every commit reruns a session's affected watches one after another.
|
|
25
23
|
maxWatchesPerSession: 100,
|
|
26
24
|
};
|
package/dist/d1-storage.js
CHANGED
|
@@ -42,7 +42,8 @@ var _a;
|
|
|
42
42
|
import { decodeRow } from '@syncular/core';
|
|
43
43
|
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
44
44
|
import { matchesEffective } from './scopes.js';
|
|
45
|
-
import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction,
|
|
45
|
+
import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
|
|
46
|
+
import { commitDeleteFloor } from './storage.js';
|
|
46
47
|
import { isD1ConstraintError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
|
|
47
48
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
48
49
|
/** The rows of one `db.batch` result entry (`{ results }` per statement). */
|
|
@@ -316,18 +317,37 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
316
317
|
}
|
|
317
318
|
return queryAuthoritativeOn(this.#db, this.partition, this.#prepareQuery(query));
|
|
318
319
|
}
|
|
319
|
-
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = []) {
|
|
320
|
+
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = [], outbox) {
|
|
320
321
|
this.assertOpen();
|
|
321
322
|
if (!this.#pushApplySerialized) {
|
|
322
323
|
throw new Error('D1 push apply requires externally serialized partition writes');
|
|
323
324
|
}
|
|
324
325
|
// D1 has no interactive lock. The caller explicitly asserted that every
|
|
325
326
|
// write for this partition is already serialized (normally by its DO).
|
|
327
|
+
// The settlement's writes sit below the checkpoint, so a rejection keeps
|
|
328
|
+
// them.
|
|
329
|
+
const settled = outbox === undefined
|
|
330
|
+
? undefined
|
|
331
|
+
: await this.settleOutbox(clientId, clientCommitId, outbox);
|
|
326
332
|
this.#pushApplyCheckpoint = this.#buffer.length;
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
333
|
+
// The serve gate and the delete floor (§2.2) in one round trip.
|
|
334
|
+
const read = await this.#db
|
|
335
|
+
.prepare('SELECT (SELECT schema_version FROM sync_schema_meta WHERE id=1) AS schema_version, (SELECT delete_floor FROM sync_partitions WHERE partition=?) AS delete_floor')
|
|
336
|
+
.bind(this.partition)
|
|
337
|
+
.first();
|
|
338
|
+
const gate = {
|
|
339
|
+
storedSchemaVersion: read?.schema_version ?? 0,
|
|
340
|
+
logEpoch: undefined,
|
|
341
|
+
pending: [],
|
|
330
342
|
};
|
|
343
|
+
const deleteFloor = read?.delete_floor ?? 0;
|
|
344
|
+
return settled === undefined
|
|
345
|
+
? {
|
|
346
|
+
gate,
|
|
347
|
+
deleteFloor,
|
|
348
|
+
persisted: await this.getPushResult(clientId, clientCommitId),
|
|
349
|
+
}
|
|
350
|
+
: { gate, deleteFloor, ...settled };
|
|
331
351
|
}
|
|
332
352
|
advanceCheckpoint(_name, _ownerEpoch, _watermark, _observedRows, _nowMs) {
|
|
333
353
|
// D1 installs no fence and declares no checkpoints in this release.
|
|
@@ -344,7 +364,7 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
344
364
|
pending: [],
|
|
345
365
|
};
|
|
346
366
|
}
|
|
347
|
-
async commitRejectedPushResult(clientId, clientCommitId, result) {
|
|
367
|
+
async commitRejectedPushResult(clientId, clientCommitId, result, sequence) {
|
|
348
368
|
this.assertOpen();
|
|
349
369
|
const checkpoint = this.#pushApplyCheckpoint;
|
|
350
370
|
if (checkpoint === undefined) {
|
|
@@ -355,7 +375,7 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
355
375
|
this.#clearedTombstones.clear();
|
|
356
376
|
this.#maxCommitSeq = undefined;
|
|
357
377
|
this.#applicationOpIndexes.clear();
|
|
358
|
-
await this.putPushResult(clientId, clientCommitId, result);
|
|
378
|
+
await this.putPushResult(clientId, clientCommitId, result, sequence);
|
|
359
379
|
await this.commit();
|
|
360
380
|
// The idempotency re-check found no record under the serialized apply, so
|
|
361
381
|
// the insert above is the canonical outcome.
|
|
@@ -447,7 +467,7 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
447
467
|
}
|
|
448
468
|
const commitSeq = this.#maxCommitSeq + 1;
|
|
449
469
|
this.#maxCommitSeq = commitSeq;
|
|
450
|
-
this.run('INSERT INTO sync_partitions(partition, max_commit_seq) VALUES (
|
|
470
|
+
this.run('INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES (?,?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq, delete_floor=max(delete_floor, excluded.delete_floor)', [p, commitSeq, commitDeleteFloor(commit.changes)]);
|
|
451
471
|
this.run('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)', [
|
|
452
472
|
p,
|
|
453
473
|
commitSeq,
|
|
@@ -505,6 +525,7 @@ const D1_CORE_TABLES = [
|
|
|
505
525
|
'sync_changes',
|
|
506
526
|
'sync_change_scopes',
|
|
507
527
|
'sync_push_results',
|
|
528
|
+
'sync_push_watermarks',
|
|
508
529
|
'sync_reactions',
|
|
509
530
|
'sync_clients',
|
|
510
531
|
'sync_blob_refs',
|
|
@@ -520,9 +541,9 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
520
541
|
maxBindParams = 100;
|
|
521
542
|
/**
|
|
522
543
|
* Page reads waiting for the end of the current microtask turn. A pull
|
|
523
|
-
* starts the
|
|
544
|
+
* starts the first snapshot page and scope pin of every bootstrapping unit
|
|
524
545
|
* before awaiting any of them, so they leave as ONE `db.batch` round trip
|
|
525
|
-
* instead of one per
|
|
546
|
+
* instead of one per unit (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
526
547
|
*/
|
|
527
548
|
#pendingReads;
|
|
528
549
|
constructor(db, options = {}) {
|
|
@@ -584,22 +605,17 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
584
605
|
for (const statement of sqliteDdlStatements()) {
|
|
585
606
|
await this.#db.exec(`${statement.replace(/\s+/g, ' ')};`);
|
|
586
607
|
}
|
|
587
|
-
for (const sql of await this.#
|
|
608
|
+
for (const sql of await this.#upgradeDdl()) {
|
|
588
609
|
await this.#db.exec(`${sql};`);
|
|
589
610
|
}
|
|
590
611
|
}
|
|
591
|
-
/**
|
|
592
|
-
|
|
593
|
-
* lacks, read in one statement.
|
|
594
|
-
*/
|
|
595
|
-
async #addedColumnDdl() {
|
|
612
|
+
/** `sqliteUpgradeDdl` for this database, read in one statement. */
|
|
613
|
+
async #upgradeDdl() {
|
|
596
614
|
const { results } = await this.#db
|
|
597
|
-
.prepare(
|
|
598
|
-
.bind(
|
|
599
|
-
...new Set(SQLITE_ADDED_COLUMNS.map(([table]) => table)),
|
|
600
|
-
]))
|
|
615
|
+
.prepare(SQLITE_UPGRADE_PROBE.sql)
|
|
616
|
+
.bind(...SQLITE_UPGRADE_PROBE.params)
|
|
601
617
|
.all();
|
|
602
|
-
return
|
|
618
|
+
return sqliteUpgradeDdl(results);
|
|
603
619
|
}
|
|
604
620
|
async ensureSchema(schema, checkpoints) {
|
|
605
621
|
if (checkpoints !== undefined && checkpoints.length > 0) {
|
|
@@ -688,9 +704,11 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
688
704
|
primaryKeyPosition: column.pk,
|
|
689
705
|
})), 'sqlite');
|
|
690
706
|
}
|
|
691
|
-
// A database from
|
|
707
|
+
// A database from an earlier release gains the core-table columns and
|
|
708
|
+
// indexes it lacks in place, and loses the columns wire version 4
|
|
709
|
+
// deleted.
|
|
692
710
|
statementsExecuted++;
|
|
693
|
-
const added = await this.#
|
|
711
|
+
const added = await this.#upgradeDdl();
|
|
694
712
|
if (added.length > 0) {
|
|
695
713
|
await batch(added.map((sql) => this.#db.prepare(sql)));
|
|
696
714
|
}
|
|
@@ -810,7 +828,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
810
828
|
if (remaining < 4)
|
|
811
829
|
break;
|
|
812
830
|
statementsExecuted++;
|
|
813
|
-
const added = await this.#
|
|
831
|
+
const added = await this.#upgradeDdl();
|
|
814
832
|
if (remaining < 3 + added.length)
|
|
815
833
|
break;
|
|
816
834
|
if (!(await save({ ...state, phase: 'prepare', offset: 0 }, added.map((sql) => this.#db.prepare(sql)))))
|
package/dist/errors.js
CHANGED
|
@@ -168,6 +168,14 @@ export const ERROR_CATALOG = {
|
|
|
168
168
|
recommendedAction: 'retryLater',
|
|
169
169
|
httpStatus: 500,
|
|
170
170
|
},
|
|
171
|
+
// §6.3.2: below the outbox watermark with no retained result. A push
|
|
172
|
+
// operation-result error record only, never persisted.
|
|
173
|
+
'sync.commit_expired': {
|
|
174
|
+
category: 'invalid-request',
|
|
175
|
+
retryable: false,
|
|
176
|
+
recommendedAction: 'resetClientId',
|
|
177
|
+
httpStatus: 400,
|
|
178
|
+
},
|
|
171
179
|
// §2.4 on a non-D1 backend: the server refuses while a declared backfill
|
|
172
180
|
// checkpoint is incomplete or the database is newer than the running build.
|
|
173
181
|
// Formerly host-only; promoted to the §10.2 catalogue so an in-flight
|