@syncular/server 0.15.45 → 0.15.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -7
- package/dist/admin.d.ts +10 -4
- package/dist/admin.js +10 -0
- package/dist/authoritative-query.d.ts +20 -0
- package/dist/authoritative-query.js +184 -0
- package/dist/context.d.ts +11 -1
- package/dist/context.js +2 -0
- package/dist/d1-storage.d.ts +10 -1
- package/dist/d1-storage.js +216 -0
- package/dist/errors.d.ts +1 -1
- package/dist/errors.js +43 -1
- package/dist/events.d.ts +52 -3
- package/dist/handler.js +4 -1
- package/dist/index-bun.d.ts +2 -0
- package/dist/index-bun.js +2 -0
- package/dist/index-node.d.ts +2 -0
- package/dist/index-node.js +2 -0
- package/dist/index.d.ts +7 -2
- package/dist/index.js +7 -6
- package/dist/operations-realtime.d.ts +16 -0
- package/dist/operations-realtime.js +196 -0
- package/dist/operations.d.ts +97 -0
- package/dist/operations.js +392 -0
- package/dist/postgres-storage.d.ts +11 -2
- package/dist/postgres-storage.js +220 -0
- package/dist/pull.js +1 -1
- package/dist/push.d.ts +8 -2
- package/dist/push.js +75 -21
- package/dist/reactions.d.ts +167 -0
- package/dist/reactions.js +442 -0
- package/dist/realtime.js +4 -1
- package/dist/sqlite-blob-store.d.ts +4 -9
- package/dist/sqlite-blob-store.js +5 -10
- package/dist/sqlite-bun-driver.d.ts +11 -0
- package/dist/sqlite-bun-driver.js +27 -0
- package/dist/sqlite-bun.d.ts +24 -0
- package/dist/sqlite-bun.js +40 -0
- package/dist/sqlite-dialect.d.ts +8 -8
- package/dist/sqlite-dialect.js +22 -2
- package/dist/sqlite-driver.d.ts +26 -0
- package/dist/sqlite-driver.js +8 -0
- package/dist/sqlite-image.d.ts +7 -9
- package/dist/sqlite-image.js +26 -28
- package/dist/sqlite-lease-store.d.ts +4 -9
- package/dist/sqlite-lease-store.js +5 -10
- package/dist/sqlite-node-driver.d.ts +10 -0
- package/dist/sqlite-node-driver.js +30 -0
- package/dist/sqlite-node.d.ts +24 -0
- package/dist/sqlite-node.js +50 -0
- package/dist/sqlite-segment-store.d.ts +4 -10
- package/dist/sqlite-segment-store.js +6 -9
- package/dist/sqlite-storage.d.ts +13 -12
- package/dist/sqlite-storage.js +223 -5
- package/dist/storage-errors.js +4 -1
- package/dist/storage.d.ts +109 -0
- package/dist/validate.js +1 -0
- package/package.json +18 -3
- package/src/admin.ts +27 -3
- package/src/authoritative-query.ts +218 -0
- package/src/context.ts +12 -1
- package/src/d1-storage.ts +352 -0
- package/src/errors.ts +43 -1
- package/src/events.ts +64 -2
- package/src/handler.ts +13 -1
- package/src/index-bun.ts +9 -0
- package/src/index-node.ts +9 -0
- package/src/index.ts +40 -6
- package/src/operations-realtime.ts +272 -0
- package/src/operations.ts +720 -0
- package/src/postgres-storage.ts +351 -0
- package/src/pull.ts +1 -1
- package/src/push.ts +97 -29
- package/src/reactions.ts +741 -0
- package/src/realtime.ts +7 -1
- package/src/sqlite-blob-store.ts +11 -10
- package/src/sqlite-bun-driver.ts +42 -0
- package/src/sqlite-bun.ts +53 -0
- package/src/sqlite-dialect.ts +27 -7
- package/src/sqlite-driver.ts +44 -0
- package/src/sqlite-image.ts +44 -49
- package/src/sqlite-lease-store.ts +11 -10
- package/src/sqlite-node-driver.ts +46 -0
- package/src/sqlite-node.ts +62 -0
- package/src/sqlite-segment-store.ts +11 -11
- package/src/sqlite-storage.ts +378 -7
- package/src/storage-errors.ts +4 -1
- package/src/storage.ts +165 -0
- package/src/validate.ts +1 -0
package/README.md
CHANGED
|
@@ -8,6 +8,17 @@ pruning (§4.6), and signed-URL token issuance (§5.4). `SPEC.md` is
|
|
|
8
8
|
normative for everything on the wire; this README covers the **host
|
|
9
9
|
surface** — in particular the ops seam and the pruning runbook.
|
|
10
10
|
|
|
11
|
+
Application processes can expose generated named queries and transactional
|
|
12
|
+
commands through `RemoteOperationRegistry`. Queries stay in the server
|
|
13
|
+
registry, command mutations use the ordinary serialized push path, and
|
|
14
|
+
`RemoteOperationWatchHub` provides live replacement snapshots. The protocol is
|
|
15
|
+
specified in [`docs/REMOTE.md`](../../docs/REMOTE.md) and the practical setup is
|
|
16
|
+
in the [remote operations guide](https://syncular.dev/guide-remote-operations/).
|
|
17
|
+
|
|
18
|
+
Application intent belongs in immutable domain event rows written in the same
|
|
19
|
+
commit as the state change. `SyncularServerEvents` below remains operational
|
|
20
|
+
telemetry. See the [domain event guide](https://syncular.dev/guide-domain-events/).
|
|
21
|
+
|
|
11
22
|
## Deployment matrix (runtime adapters)
|
|
12
23
|
|
|
13
24
|
The server core is **runtime-neutral TypeScript** — `handleSyncRequest` and
|
|
@@ -18,7 +29,7 @@ The supported set, and what deliberately does **not** get an adapter:
|
|
|
18
29
|
|
|
19
30
|
| Runtime | Adapter | Transport | Storage | Status |
|
|
20
31
|
| --- | --- | --- | --- | --- |
|
|
21
|
-
| **Bun / Node
|
|
32
|
+
| **Bun / Node 22.13+** | `@syncular/server-hono` | HTTP (`POST /sync`, segments, blobs) **+ WS realtime** (§8, host-driven upgrade) | `SqliteServerStorage` through `@syncular/server/sqlite`, Postgres, memory | **Supported now** — the reference deployment; runs the full conformance catalog on both bindings. |
|
|
22
33
|
| **Cloudflare Workers** | `@syncular/server-workers` | HTTP binding via Hono (Workers-native) **+ optional WS realtime** (§8) | `D1ServerStorage` behind one per-partition Durable Object queue; R2-as-S3 for segments/blobs | **Supported now** — D1 sync writes always traverse the DO; WebSocket upgrades remain optional. |
|
|
23
34
|
| Raw Deno / edge-misc | — | — | — | **Not adapted** (policy below). |
|
|
24
35
|
|
|
@@ -65,9 +76,9 @@ or the storage projection cannot migrate:
|
|
|
65
76
|
```ts
|
|
66
77
|
import {
|
|
67
78
|
ensureSyncServerReady,
|
|
68
|
-
SqliteServerStorage,
|
|
69
79
|
type SyncServerConfig,
|
|
70
80
|
} from '@syncular/server';
|
|
81
|
+
import { SqliteServerStorage } from '@syncular/server/sqlite';
|
|
71
82
|
|
|
72
83
|
const config: SyncServerConfig = {
|
|
73
84
|
schema,
|
|
@@ -80,6 +91,12 @@ await ensureSyncServerReady(config);
|
|
|
80
91
|
Bun.serve({ fetch: app.fetch });
|
|
81
92
|
```
|
|
82
93
|
|
|
94
|
+
`@syncular/server/sqlite` selects `bun:sqlite` on Bun and the built-in
|
|
95
|
+
`node:sqlite` module on Node. It covers server storage, segment storage, blob
|
|
96
|
+
storage, leases, and SQLite-image generation without an external SQLite
|
|
97
|
+
package. The runtime-specific database wrappers are `BunSqliteDatabase` and
|
|
98
|
+
`NodeSqliteDatabase` when a host needs direct access to the native handle.
|
|
99
|
+
|
|
83
100
|
The helper accepts the generated `ServerSchema`, compiles it, and calls the
|
|
84
101
|
storage backend's low-level `ensureSchema(CompiledSchema)`. A failure is a
|
|
85
102
|
`SyncServerReadinessError` with stable code `sync.schema_not_ready`, a `phase`
|
|
@@ -263,6 +280,118 @@ must check for it and fail closed. See the public
|
|
|
263
280
|
for a user-scoped key-grant table revoked through a Workspace index and for the
|
|
264
281
|
atomic reverse-index/queue fallback required by ordered or derived lookups.
|
|
265
282
|
|
|
283
|
+
## Durable server reactions
|
|
284
|
+
|
|
285
|
+
`reactionPlanner` turns an accepted candidate commit into bounded work records.
|
|
286
|
+
It runs after operation and whole-commit validation, inside the authoritative
|
|
287
|
+
push transaction. It may use the candidate-state reader and must perform no
|
|
288
|
+
external side effects. A rejected or replayed commit does not run it.
|
|
289
|
+
|
|
290
|
+
```ts
|
|
291
|
+
import {
|
|
292
|
+
ReactionRunner,
|
|
293
|
+
type ReactionPlanner,
|
|
294
|
+
type SyncServerConfig,
|
|
295
|
+
} from '@syncular/server';
|
|
296
|
+
|
|
297
|
+
type AppReactions = {
|
|
298
|
+
'invoice.email': { invoiceId: string };
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
const reactionPlanner: ReactionPlanner<AppReactions> = ({ operations }) =>
|
|
302
|
+
operations.flatMap((operation) =>
|
|
303
|
+
operation.table === 'invoice_events' &&
|
|
304
|
+
operation.row?.kind === 'invoice_finalized' &&
|
|
305
|
+
typeof operation.row.invoice_id === 'string'
|
|
306
|
+
? [{
|
|
307
|
+
key: `invoice:${operation.row.invoice_id}`,
|
|
308
|
+
type: 'invoice.email',
|
|
309
|
+
version: 1,
|
|
310
|
+
payload: { invoiceId: operation.row.invoice_id },
|
|
311
|
+
maxAttempts: 8,
|
|
312
|
+
}]
|
|
313
|
+
: [],
|
|
314
|
+
);
|
|
315
|
+
|
|
316
|
+
const config: SyncServerConfig = {
|
|
317
|
+
schema, storage, segments, resolveScopes, reactionPlanner,
|
|
318
|
+
};
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
App rows, commit metadata, reaction rows, and the push idempotency result land
|
|
322
|
+
in one transaction. The handler idempotency key is derived from the source
|
|
323
|
+
`partition`, `clientId`, `clientCommitId`, and planner `key`. Reaction rows use
|
|
324
|
+
their own partition-scoped table and survive `pruneCommitLog`.
|
|
325
|
+
|
|
326
|
+
Drive delivery from a host scheduler or queue wake:
|
|
327
|
+
|
|
328
|
+
```ts
|
|
329
|
+
const runner = new ReactionRunner<AppReactions>({
|
|
330
|
+
storage,
|
|
331
|
+
partition: 'main',
|
|
332
|
+
workerId: 'invoice-worker-1',
|
|
333
|
+
handlers: {
|
|
334
|
+
'invoice.email': async ({ payload, idempotencyKey, extendLease }) => {
|
|
335
|
+
await extendLease();
|
|
336
|
+
await emailProvider.send({
|
|
337
|
+
invoiceId: payload.invoiceId,
|
|
338
|
+
idempotencyKey,
|
|
339
|
+
});
|
|
340
|
+
},
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
await runner.runOnce();
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
Claims and acknowledgements compare a lease owner. Expired leases can be
|
|
348
|
+
claimed by another worker, and long handlers can call `extendLease()`. Ordinary
|
|
349
|
+
throws and `RetryableReactionError` retry with bounded exponential backoff.
|
|
350
|
+
`PermanentReactionError` and exhausted retry limits enter `dead-letter`.
|
|
351
|
+
`retryDeadLetterReaction` resets one row for an explicit operator retry.
|
|
352
|
+
Each `runOnce()` uses a fresh lease token and rechecks ownership before
|
|
353
|
+
starting every handler in a claimed batch.
|
|
354
|
+
|
|
355
|
+
Schedule terminal retention separately from commit-log pruning:
|
|
356
|
+
|
|
357
|
+
```ts
|
|
358
|
+
import { pruneReactions } from '@syncular/server';
|
|
359
|
+
|
|
360
|
+
let result;
|
|
361
|
+
do {
|
|
362
|
+
result = await pruneReactions({
|
|
363
|
+
storage,
|
|
364
|
+
partition: 'main',
|
|
365
|
+
nowMs: Date.now(),
|
|
366
|
+
events,
|
|
367
|
+
});
|
|
368
|
+
} while (result.mayHaveMore);
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
Defaults retain completed rows for 30 days, dead-lettered rows for 90 days,
|
|
372
|
+
and remove at most 1,000 rows per pass. Override them with
|
|
373
|
+
`retention: { completedRetentionMs, deadLetterRetentionMs, batchSize }`.
|
|
374
|
+
Only terminal rows strictly older than their cutoff are eligible. Pending and
|
|
375
|
+
leased work is preserved, including expired leases. Cleanup and manual retry
|
|
376
|
+
serialize at storage, so one transition wins a race. Each pass emits
|
|
377
|
+
`reaction.prune_completed` with both cutoffs, both removal counts, the limit,
|
|
378
|
+
and `mayHaveMore`.
|
|
379
|
+
|
|
380
|
+
Delivery is at least once. A crash after the handler's external call and before
|
|
381
|
+
acknowledgement can run that call again. Handlers receive the same stable
|
|
382
|
+
`idempotencyKey` on every attempt and should pass it to external providers.
|
|
383
|
+
Syncular does not claim exactly-once external effects.
|
|
384
|
+
|
|
385
|
+
Planned payloads are plain JSON, versioned, limited to 64 KiB and 16 levels,
|
|
386
|
+
with at most 100 reactions per commit. Failure details are plain JSON limited
|
|
387
|
+
to 8 KiB. The planner API cannot enforce purity in JavaScript; running an
|
|
388
|
+
external effect from the planner violates the transaction contract.
|
|
389
|
+
|
|
390
|
+
SQLite and PostgreSQL use their existing push transactions. D1 appends the
|
|
391
|
+
reaction writes to the same atomic batch as the source commit and retains its
|
|
392
|
+
mandatory per-partition Durable Object coordination for pushes. D1 claims use
|
|
393
|
+
one `UPDATE ... RETURNING` statement, so there is no claim read/write gap.
|
|
394
|
+
|
|
266
395
|
## Structured events (the ops seam)
|
|
267
396
|
|
|
268
397
|
One optional interface, `SyncularServerEvents`, carries every
|
|
@@ -310,6 +439,12 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
|
|
|
310
439
|
| `push.applied` | A `PUSH_COMMIT` applied, or replayed from the idempotency cache (§2.3) | `clientId`, `clientCommitId`, `operations`, `commitSeq?`, `replay` |
|
|
311
440
|
| `push.rejected` | A commit rejected (§6.3) | `clientId`, `clientCommitId`, `operations`, `code` (§10.2), `opIndex` |
|
|
312
441
|
| `push.conflicted` | A commit terminated by a version conflict (§6.2) | `clientId`, `clientCommitId`, `operations`, `opIndex` |
|
|
442
|
+
| `reaction.queued` | A planned reaction committed with its source push | `clientId`, `clientCommitId`, `commitSeq`, `idempotencyKey`, `reactionType`, `version` |
|
|
443
|
+
| `reaction.started` | A worker claimed and began one attempt | `workerId`, `idempotencyKey`, `reactionType`, `version`, `attempt` |
|
|
444
|
+
| `reaction.retried` | A retryable attempt failed and was rescheduled | started fields plus `nextAttemptAtMs`, `errorCode` |
|
|
445
|
+
| `reaction.completed` | A handler finished and its owner acknowledged | started fields |
|
|
446
|
+
| `reaction.dead_lettered` | A permanent or exhausted failure was recorded | started fields plus `errorCode` |
|
|
447
|
+
| `reaction.prune_completed` | One bounded terminal-reaction retention pass finished | `completedBeforeMs`, `deadLetterBeforeMs`, `limit`, `removedCompleted`, `removedDeadLetter`, `mayHaveMore` |
|
|
313
448
|
| `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `subscriptions[]`: `{id, table, status, mode` (`bootstrap` \| `incremental` \| `none`)`, fromCursor, nextCursor, commits, changes, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}` |
|
|
314
449
|
| `segment.downloaded` | Every direct segment download (§5.5), success or failure | `segmentId`, `outcome` (`ok` \| `error`), `errorCode?`, `mediaType?`, `bytes?`, `durationMs` |
|
|
315
450
|
| `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (keep-set size), `graceMs` |
|
|
@@ -325,10 +460,9 @@ exists) `partition` / `actorId`.
|
|
|
325
460
|
|
|
326
461
|
## Admin / console surface (`SyncularAdmin`)
|
|
327
462
|
|
|
328
|
-
The operator-facing read surface over the server core
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
reads is entirely the host's. It delivers the 80% operator value (who's
|
|
463
|
+
The operator-facing read surface over the server core lives in this package
|
|
464
|
+
and adds **zero** wire protocol. Authorization for these reads is entirely the
|
|
465
|
+
host's. It delivers the 80% operator value (who's
|
|
332
466
|
connected, what's flowing,
|
|
333
467
|
horizon health, the event tail) as a handful of read-only, partition-scoped,
|
|
334
468
|
JSON-able queries.
|
|
@@ -367,6 +501,7 @@ by design:
|
|
|
367
501
|
| `listClients(partition)` | Known clients: `clientId`, `actorId`, `cursor`, `lag` (commits not yet pulled: `maxCommitSeq − max(cursor, 0)`), `updatedAtMs`, `subscriptions[]`, and an `active` flag (cursor touched within the §4.6 active window). |
|
|
368
502
|
| `clientDetail(partition, clientId, {eventLimit?})` | One client's drill-down: `{exists, client?, lease?, events}` — the record (with lag), its §7.3 lease when a lease store is wired, and its slice of the event tail. Answers "why is this client stale" in one read. |
|
|
369
503
|
| `listCommits(partition, {afterSeq?, limit?, table?})` | Commit-log **metadata** (never payloads), newest first: `commitSeq`, `clientId`, `clientCommitId`, `actorId`, `createdAtMs`, `changeCount`, `tables[]`. |
|
|
504
|
+
| `listReactions(partition, {statuses?, types?, limit?})` | Durable reaction lifecycle rows, newest first, including source commit, attempts, lease, completion, and bounded failure information. |
|
|
370
505
|
| `inspectRow(partition, table, rowId)` | `{exists, serverVersion?, scopes?}` — current row version + stored scopes, payload **not** decoded. |
|
|
371
506
|
| `scopeActivity(partition, {variable, value}, {limit?})` | Recent commits touching one scope key, via the §3.1 change-scope index (never a log scan). |
|
|
372
507
|
| `horizonStatus(partition)` | `{maxCommitSeq, horizonSeq, retainedCommits, activeCursorFloor, recommendedHorizonSeq, recommendation}` — the horizon a prune pass would reach now (§4.6) + a coarse `up-to-date` / `prune-recommended`. |
|
|
@@ -413,6 +548,7 @@ app.route('/admin', routes);
|
|
|
413
548
|
| `GET /clients` | `listClients` |
|
|
414
549
|
| `GET /clients/:clientId?eventLimit` | `clientDetail` |
|
|
415
550
|
| `GET /commits?afterSeq&limit&table` | `listCommits` |
|
|
551
|
+
| `GET /reactions?status&type&limit` | `listReactions` |
|
|
416
552
|
| `GET /rows/:table/:rowId` | `inspectRow` |
|
|
417
553
|
| `GET /scope-activity?variable&value&limit` | `scopeActivity` |
|
|
418
554
|
| `GET /horizon` | `horizonStatus` |
|
|
@@ -844,7 +980,8 @@ same table+scope during a storm, your TTL is shorter than the storm.
|
|
|
844
980
|
|
|
845
981
|
## Postgres storage (the production database path)
|
|
846
982
|
|
|
847
|
-
`SqliteServerStorage`
|
|
983
|
+
`SqliteServerStorage` from `@syncular/server/sqlite` uses `bun:sqlite` or
|
|
984
|
+
Node's built-in `node:sqlite`. For
|
|
848
985
|
production, `PostgresServerStorage` implements the same `ServerStorage`
|
|
849
986
|
contract against Postgres, with the inverted scope index carried through
|
|
850
987
|
as **covering indexes** so scope fanout is an index range scan, never a
|
package/dist/admin.d.ts
CHANGED
|
@@ -4,9 +4,8 @@
|
|
|
4
4
|
* `ServerStorage`, the optional segment/blob store stats, and an in-memory
|
|
5
5
|
* event ring. It delivers the 80% operator value (who's connected, what's
|
|
6
6
|
* flowing, horizon health, the event tail) as a handful of queries in the
|
|
7
|
-
* server package — no separate
|
|
8
|
-
*
|
|
9
|
-
* this is host surface, mirrored in the server README).
|
|
7
|
+
* server package — no separate UI package, no framework, and no wire-protocol
|
|
8
|
+
* surface. This host surface is mirrored in the server README.
|
|
10
9
|
*
|
|
11
10
|
* Nothing here is on the sync hot path. Every method is a plain read; the
|
|
12
11
|
* additive optional storage/store methods it depends on are documented as
|
|
@@ -22,7 +21,7 @@ import type { LeaseRecord, LeaseStore } from './lease-store.js';
|
|
|
22
21
|
import { type RetentionPolicy } from './prune.js';
|
|
23
22
|
import { type ServerSchema } from './schema.js';
|
|
24
23
|
import type { SegmentStore, SegmentStoreStats } from './segment-store.js';
|
|
25
|
-
import type { CommitMetadata, ScopeCommitActivity, ServerStorage } from './storage.js';
|
|
24
|
+
import type { CommitMetadata, ReactionStatus, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
|
|
26
25
|
/** A connected/known client as the console sees it (§4.5, §8.1). */
|
|
27
26
|
export interface AdminClient {
|
|
28
27
|
readonly clientId: string;
|
|
@@ -123,6 +122,11 @@ export interface AdminListCommitsOptions {
|
|
|
123
122
|
export interface AdminScopeActivityOptions {
|
|
124
123
|
readonly limit?: number;
|
|
125
124
|
}
|
|
125
|
+
export interface AdminListReactionsOptions {
|
|
126
|
+
readonly statuses?: readonly ReactionStatus[];
|
|
127
|
+
readonly types?: readonly string[];
|
|
128
|
+
readonly limit?: number;
|
|
129
|
+
}
|
|
126
130
|
export interface AdminStats {
|
|
127
131
|
readonly segments?: SegmentStoreStats;
|
|
128
132
|
readonly blobs?: BlobStoreStats;
|
|
@@ -171,6 +175,8 @@ export declare class SyncularAdmin {
|
|
|
171
175
|
}): Promise<AdminClientDetail>;
|
|
172
176
|
/** Commit-log metadata (no payloads), newest first. */
|
|
173
177
|
listCommits(partition: string, options?: AdminListCommitsOptions): Promise<CommitMetadata[]>;
|
|
178
|
+
/** Pending, leased, completed, and dead-lettered durable reactions. */
|
|
179
|
+
listReactions(partition: string, options?: AdminListReactionsOptions): Promise<StoredReaction[]>;
|
|
174
180
|
/**
|
|
175
181
|
* Inspect a single row: current server_version, stored scopes, and the
|
|
176
182
|
* blobIds it references (when the store tracks references). Payload bytes
|
package/dist/admin.js
CHANGED
|
@@ -3,6 +3,7 @@ import { DEFAULT_RETENTION } from './prune.js';
|
|
|
3
3
|
import { compileSchema } from './schema.js';
|
|
4
4
|
const DEFAULT_COMMIT_LIMIT = 50;
|
|
5
5
|
const DEFAULT_SCOPE_LIMIT = 50;
|
|
6
|
+
const DEFAULT_REACTION_LIMIT = 100;
|
|
6
7
|
const DEFAULT_CLIENT_EVENT_LIMIT = 100;
|
|
7
8
|
const DEFAULT_METRICS_WINDOW_MS = 5 * 60 * 1000;
|
|
8
9
|
const DEFAULT_METRICS_BUCKETS = 30;
|
|
@@ -120,6 +121,15 @@ export class SyncularAdmin {
|
|
|
120
121
|
...(options.table !== undefined ? { table: options.table } : {}),
|
|
121
122
|
});
|
|
122
123
|
}
|
|
124
|
+
/** Pending, leased, completed, and dead-lettered durable reactions. */
|
|
125
|
+
async listReactions(partition, options = {}) {
|
|
126
|
+
const read = required(this.#storage.listReactions?.bind(this.#storage), 'storage');
|
|
127
|
+
return read(partition, {
|
|
128
|
+
limit: options.limit ?? DEFAULT_REACTION_LIMIT,
|
|
129
|
+
...(options.statuses !== undefined ? { statuses: options.statuses } : {}),
|
|
130
|
+
...(options.types !== undefined ? { types: options.types } : {}),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
123
133
|
/**
|
|
124
134
|
* Inspect a single row: current server_version, stored scopes, and the
|
|
125
135
|
* blobIds it references (when the store tracks references). Payload bytes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { CompiledTable } from './schema.js';
|
|
2
|
+
import type { AuthoritativeQueryValue } from './storage.js';
|
|
3
|
+
export interface PreparedAuthoritativeQuery {
|
|
4
|
+
readonly sql: string;
|
|
5
|
+
readonly params: readonly (AuthoritativeQueryValue | typeof PARTITION_BIND)[];
|
|
6
|
+
}
|
|
7
|
+
export interface BoundAuthoritativeQuery {
|
|
8
|
+
readonly sql: string;
|
|
9
|
+
readonly params: readonly AuthoritativeQueryValue[];
|
|
10
|
+
}
|
|
11
|
+
declare const PARTITION_BIND: unique symbol;
|
|
12
|
+
/**
|
|
13
|
+
* Turn generated local SQL into a partition-local authoritative statement.
|
|
14
|
+
* Only relations declared by the generated descriptor are rewritten. Values
|
|
15
|
+
* remain parameters; request data is never interpolated into SQL.
|
|
16
|
+
*/
|
|
17
|
+
export declare function prepareAuthoritativeQuery(sql: string, params: readonly AuthoritativeQueryValue[], declaredTables: readonly string[], tables: ReadonlyMap<string, CompiledTable>): PreparedAuthoritativeQuery;
|
|
18
|
+
export declare function bindAuthoritativePartition(prepared: PreparedAuthoritativeQuery, partition: string): BoundAuthoritativeQuery;
|
|
19
|
+
export declare function postgresPlaceholders(sql: string): string;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { quoteIdent, SYNC_PARTITION_COLUMN } from './relational-rows.js';
|
|
2
|
+
const PARTITION_BIND = Symbol('syncular.authoritative_partition');
|
|
3
|
+
const RESERVED_ALIAS = new Set([
|
|
4
|
+
'on',
|
|
5
|
+
'where',
|
|
6
|
+
'group',
|
|
7
|
+
'order',
|
|
8
|
+
'inner',
|
|
9
|
+
'left',
|
|
10
|
+
'right',
|
|
11
|
+
'full',
|
|
12
|
+
'outer',
|
|
13
|
+
'natural',
|
|
14
|
+
'join',
|
|
15
|
+
'cross',
|
|
16
|
+
'using',
|
|
17
|
+
'limit',
|
|
18
|
+
'having',
|
|
19
|
+
]);
|
|
20
|
+
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
|
|
21
|
+
const TABLE_REF_RE = new RegExp(`\\b(FROM|(?:NATURAL\\s+)?(?:(?:LEFT|RIGHT|FULL)(?:\\s+OUTER)?|INNER|CROSS)?\\s*JOIN)\\s+((?:\\(\\s*)*)(${IDENT})(?:\\s+(?:AS\\s+)?((?!(?:${[...RESERVED_ALIAS].join('|')})\\b)${IDENT}))?`, 'gi');
|
|
22
|
+
function protectedSqlEnd(sql, index) {
|
|
23
|
+
const char = sql[index];
|
|
24
|
+
const next = sql[index + 1];
|
|
25
|
+
if (char === "'" || char === '"' || char === '`') {
|
|
26
|
+
let end = index + 1;
|
|
27
|
+
while (end < sql.length) {
|
|
28
|
+
if (sql[end] === char && sql[end + 1] === char)
|
|
29
|
+
end += 2;
|
|
30
|
+
else if (sql[end] === char)
|
|
31
|
+
return end + 1;
|
|
32
|
+
else
|
|
33
|
+
end += 1;
|
|
34
|
+
}
|
|
35
|
+
return sql.length;
|
|
36
|
+
}
|
|
37
|
+
if (char === '[') {
|
|
38
|
+
const end = sql.indexOf(']', index + 1);
|
|
39
|
+
return end < 0 ? sql.length : end + 1;
|
|
40
|
+
}
|
|
41
|
+
if (char === '-' && next === '-') {
|
|
42
|
+
const end = sql.indexOf('\n', index);
|
|
43
|
+
return end < 0 ? sql.length : end;
|
|
44
|
+
}
|
|
45
|
+
if (char === '/' && next === '*') {
|
|
46
|
+
const end = sql.indexOf('*/', index + 2);
|
|
47
|
+
return end < 0 ? sql.length : end + 2;
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
function maskedSql(sql) {
|
|
52
|
+
let out = '';
|
|
53
|
+
let index = 0;
|
|
54
|
+
while (index < sql.length) {
|
|
55
|
+
const end = protectedSqlEnd(sql, index);
|
|
56
|
+
if (end === undefined)
|
|
57
|
+
out += sql[index];
|
|
58
|
+
else
|
|
59
|
+
out += sql.slice(index, end).replace(/[^\n]/g, ' ');
|
|
60
|
+
index = end ?? index + 1;
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Turn generated local SQL into a partition-local authoritative statement.
|
|
66
|
+
* Only relations declared by the generated descriptor are rewritten. Values
|
|
67
|
+
* remain parameters; request data is never interpolated into SQL.
|
|
68
|
+
*/
|
|
69
|
+
export function prepareAuthoritativeQuery(sql, params, declaredTables, tables) {
|
|
70
|
+
if (maskedSql(sql).includes(';'))
|
|
71
|
+
throw new Error('registered query must be one SELECT');
|
|
72
|
+
const declared = new Set(declaredTables);
|
|
73
|
+
const masked = maskedSql(sql);
|
|
74
|
+
const replacements = [];
|
|
75
|
+
const found = new Set();
|
|
76
|
+
for (const match of masked.matchAll(TABLE_REF_RE)) {
|
|
77
|
+
const rawTable = match[3];
|
|
78
|
+
const table = tables.get(rawTable);
|
|
79
|
+
if (table === undefined)
|
|
80
|
+
continue;
|
|
81
|
+
if (!declared.has(table.name)) {
|
|
82
|
+
throw new Error('registered query table metadata does not match its SQL');
|
|
83
|
+
}
|
|
84
|
+
if (!table.materialize) {
|
|
85
|
+
throw new Error('registered query targets a non-materialized table');
|
|
86
|
+
}
|
|
87
|
+
let alias = match[4];
|
|
88
|
+
if (alias !== undefined && RESERVED_ALIAS.has(alias.toLowerCase())) {
|
|
89
|
+
alias = undefined;
|
|
90
|
+
}
|
|
91
|
+
const matchStart = match.index ?? 0;
|
|
92
|
+
const afterOperator = match[1].length;
|
|
93
|
+
const relative = match[0]
|
|
94
|
+
.toLowerCase()
|
|
95
|
+
.indexOf(rawTable.toLowerCase(), afterOperator);
|
|
96
|
+
const start = matchStart + relative;
|
|
97
|
+
const projection = table.columns.map((column) => quoteIdent(column.name));
|
|
98
|
+
replacements.push({
|
|
99
|
+
start,
|
|
100
|
+
end: start + rawTable.length,
|
|
101
|
+
text: `(SELECT ${projection.join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=/*syncular_partition*/?)${alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
|
|
102
|
+
});
|
|
103
|
+
found.add(table.name);
|
|
104
|
+
}
|
|
105
|
+
if (found.size !== declared.size ||
|
|
106
|
+
[...declared].some((table) => !found.has(table))) {
|
|
107
|
+
throw new Error('registered query table metadata does not match its SQL');
|
|
108
|
+
}
|
|
109
|
+
let rewritten = sql;
|
|
110
|
+
for (const replacement of replacements.sort((left, right) => right.start - left.start)) {
|
|
111
|
+
rewritten =
|
|
112
|
+
rewritten.slice(0, replacement.start) +
|
|
113
|
+
replacement.text +
|
|
114
|
+
rewritten.slice(replacement.end);
|
|
115
|
+
}
|
|
116
|
+
const bound = [];
|
|
117
|
+
let anonymousIndex = 0;
|
|
118
|
+
let rendered = '';
|
|
119
|
+
for (let index = 0; index < rewritten.length; index += 1) {
|
|
120
|
+
if (rewritten.startsWith('/*syncular_partition*/?', index)) {
|
|
121
|
+
rendered += '?';
|
|
122
|
+
bound.push(PARTITION_BIND);
|
|
123
|
+
index += '/*syncular_partition*/?'.length - 1;
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const protectedEnd = protectedSqlEnd(rewritten, index);
|
|
127
|
+
if (protectedEnd !== undefined) {
|
|
128
|
+
rendered += rewritten.slice(index, protectedEnd);
|
|
129
|
+
index = protectedEnd - 1;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const char = rewritten[index];
|
|
133
|
+
if (char !== '?') {
|
|
134
|
+
rendered += char;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
let end = index + 1;
|
|
138
|
+
while (end < rewritten.length && /[0-9]/.test(rewritten[end])) {
|
|
139
|
+
end += 1;
|
|
140
|
+
}
|
|
141
|
+
const numbered = rewritten.slice(index + 1, end);
|
|
142
|
+
const parameterIndex = numbered.length > 0
|
|
143
|
+
? Number.parseInt(numbered, 10) - 1
|
|
144
|
+
: anonymousIndex++;
|
|
145
|
+
if (numbered.length > 0) {
|
|
146
|
+
anonymousIndex = Math.max(anonymousIndex, parameterIndex + 1);
|
|
147
|
+
}
|
|
148
|
+
if (parameterIndex < 0 || parameterIndex >= params.length) {
|
|
149
|
+
throw new Error('registered query bind metadata does not match its SQL');
|
|
150
|
+
}
|
|
151
|
+
const value = params[parameterIndex];
|
|
152
|
+
if (value === undefined) {
|
|
153
|
+
throw new Error('registered query bind metadata does not match its SQL');
|
|
154
|
+
}
|
|
155
|
+
rendered += '?';
|
|
156
|
+
bound.push(value);
|
|
157
|
+
index = end - 1;
|
|
158
|
+
}
|
|
159
|
+
return { sql: rendered, params: bound };
|
|
160
|
+
}
|
|
161
|
+
export function bindAuthoritativePartition(prepared, partition) {
|
|
162
|
+
return {
|
|
163
|
+
sql: prepared.sql,
|
|
164
|
+
params: prepared.params.map((value) => value === PARTITION_BIND ? partition : value),
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
export function postgresPlaceholders(sql) {
|
|
168
|
+
let bind = 0;
|
|
169
|
+
let rendered = '';
|
|
170
|
+
for (let index = 0; index < sql.length; index += 1) {
|
|
171
|
+
const protectedEnd = protectedSqlEnd(sql, index);
|
|
172
|
+
if (protectedEnd !== undefined) {
|
|
173
|
+
rendered += sql.slice(index, protectedEnd);
|
|
174
|
+
index = protectedEnd - 1;
|
|
175
|
+
}
|
|
176
|
+
else if (sql[index] === '?') {
|
|
177
|
+
rendered += `$${++bind}`;
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
rendered += sql[index];
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return rendered;
|
|
184
|
+
}
|
package/dist/context.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ import type { BlobStore } from './blob-store.js';
|
|
|
9
9
|
import type { CrdtMergerRegistry } from './crdt-merger.js';
|
|
10
10
|
import type { SyncularServerEvents } from './events.js';
|
|
11
11
|
import type { LeaseStore } from './lease-store.js';
|
|
12
|
+
import type { AnyReactionPlanner } from './reactions.js';
|
|
12
13
|
import type { ServerSchema } from './schema.js';
|
|
13
14
|
import type { SegmentStore } from './segment-store.js';
|
|
14
15
|
import type { BlobPresignConfig, BlobUploadPresignConfig, SegmentUrlConfig } from './signed-url.js';
|
|
@@ -17,6 +18,8 @@ import type { ServerStorage, StoredCommit } from './storage.js';
|
|
|
17
18
|
import type { CommitValidator, ValidatorRegistry } from './validate.js';
|
|
18
19
|
/** SSP2 body content type (§1.1). */
|
|
19
20
|
export declare const SSP2_CONTENT_TYPE = "application/vnd.syncular.sync.v2";
|
|
21
|
+
/** Internal idempotency namespace. Ordinary SSP2 client IDs cannot use it. */
|
|
22
|
+
export declare const REMOTE_COMMAND_CLIENT_ID_PREFIX = "[\"remote-command\",";
|
|
20
23
|
export interface ResolveScopesArgs {
|
|
21
24
|
readonly partition: string;
|
|
22
25
|
readonly actorId: string;
|
|
@@ -101,6 +104,12 @@ export interface SyncServerConfig {
|
|
|
101
104
|
* commit-log/idempotency append. A throw rolls back the complete commit.
|
|
102
105
|
*/
|
|
103
106
|
readonly commitValidator?: CommitValidator;
|
|
107
|
+
/**
|
|
108
|
+
* Pure durable-reaction planner. Runs once after candidate validation and
|
|
109
|
+
* before commit-log/idempotency append. Its bounded records are enqueued in
|
|
110
|
+
* the same transaction; handlers run later through `ReactionRunner`.
|
|
111
|
+
*/
|
|
112
|
+
readonly reactionPlanner?: AnyReactionPlanner;
|
|
104
113
|
readonly resolveScopes: ResolveScopes;
|
|
105
114
|
/**
|
|
106
115
|
* §7.3 auth leases. Absent ⇒ the feature is off: no `LEASE` frame is
|
|
@@ -139,7 +148,8 @@ export interface SyncServerConfig {
|
|
|
139
148
|
* §5.3 sqlite-image builder, injected so the pull path never
|
|
140
149
|
* statically imports `bun:sqlite`. Absent ⇒ the sqlite-image lane is off
|
|
141
150
|
* (bit-2 clients are served the rows lane) — the Workers/edge posture. A
|
|
142
|
-
* Bun
|
|
151
|
+
* Bun or Node host wires `buildSqliteImage` from
|
|
152
|
+
* `@syncular/server/sqlite`.
|
|
143
153
|
*/
|
|
144
154
|
readonly sqliteImageBuilder?: SqliteImageBuilder;
|
|
145
155
|
readonly realtime?: RealtimeNotifier;
|
package/dist/context.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** SSP2 body content type (§1.1). */
|
|
2
2
|
export const SSP2_CONTENT_TYPE = 'application/vnd.syncular.sync.v2';
|
|
3
|
+
/** Internal idempotency namespace. Ordinary SSP2 client IDs cannot use it. */
|
|
4
|
+
export const REMOTE_COMMAND_CLIENT_ID_PREFIX = '["remote-command",';
|
|
3
5
|
/**
|
|
4
6
|
* Sentinel a resolver returns to signal a **live-authorization outage**
|
|
5
7
|
* (§7.3.3): the live authority is unreachable, so the server SHOULD
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
2
|
-
import type { ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredRow } from './storage.js';
|
|
2
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
3
3
|
export interface D1PreparedStatement {
|
|
4
4
|
bind(...values: unknown[]): D1PreparedStatement;
|
|
5
5
|
first<T = Record<string, unknown>>(): Promise<T | null>;
|
|
@@ -38,12 +38,21 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
38
38
|
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
39
39
|
begin(partition: string): Promise<StorageTransaction>;
|
|
40
40
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
41
|
+
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
41
42
|
getHorizonSeq(partition: string): Promise<number>;
|
|
42
43
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
43
44
|
pruneCommitsThrough(partition: string, seq: number): Promise<number>;
|
|
44
45
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
45
46
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
46
47
|
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
48
|
+
claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
|
|
49
|
+
completeReaction(partition: string, idempotencyKey: string, leaseOwner: string, completedAtMs: number): Promise<boolean>;
|
|
50
|
+
extendReactionLease(partition: string, idempotencyKey: string, leaseOwner: string, leaseExpiresAtMs: number): Promise<boolean>;
|
|
51
|
+
failReaction(partition: string, idempotencyKey: string, update: ReactionFailureUpdate): Promise<boolean>;
|
|
52
|
+
retryReaction(partition: string, idempotencyKey: string, nowMs: number): Promise<boolean>;
|
|
53
|
+
getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
|
|
54
|
+
listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
|
|
55
|
+
pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
47
56
|
readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
|
|
48
57
|
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
49
58
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|