@syncular/server 0.31.0 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -74
- package/dist/admin.d.ts +394 -50
- package/dist/admin.js +714 -82
- package/dist/d1-storage.d.ts +6 -5
- package/dist/d1-storage.js +136 -21
- package/dist/events-ring.d.ts +12 -2
- package/dist/events-ring.js +27 -2
- package/dist/events.d.ts +21 -3
- package/dist/handler.js +36 -13
- package/dist/operations.js +4 -4
- package/dist/postgres-storage.d.ts +6 -5
- package/dist/postgres-storage.js +102 -18
- package/dist/push.js +28 -7
- package/dist/realtime.d.ts +15 -0
- package/dist/realtime.js +46 -7
- package/dist/seed.js +4 -3
- package/dist/sqlite-storage.d.ts +6 -5
- package/dist/sqlite-storage.js +115 -18
- package/dist/storage.d.ts +58 -7
- package/package.json +2 -2
- package/src/admin.ts +1215 -120
- package/src/d1-storage.ts +199 -25
- package/src/events-ring.ts +31 -2
- package/src/events.ts +27 -4
- package/src/handler.ts +47 -16
- package/src/operations.ts +8 -4
- package/src/postgres-storage.ts +173 -28
- package/src/push.ts +40 -9
- package/src/realtime.ts +60 -7
- package/src/seed.ts +4 -3
- package/src/sqlite-storage.ts +181 -26
- package/src/storage.ts +72 -9
package/src/d1-storage.ts
CHANGED
|
@@ -68,6 +68,12 @@ import {
|
|
|
68
68
|
rewriteRowSql,
|
|
69
69
|
rewriteValues,
|
|
70
70
|
SCHEMA_META_DDL_SQLITE,
|
|
71
|
+
SYNC_COLUMN_VERSIONS_COLUMN,
|
|
72
|
+
SYNC_PARTITION_COLUMN,
|
|
73
|
+
SYNC_PAYLOAD_COLUMN,
|
|
74
|
+
SYNC_ROW_ID_COLUMN,
|
|
75
|
+
SYNC_SCOPES_COLUMN,
|
|
76
|
+
SYNC_VERSION_COLUMN,
|
|
71
77
|
type StoredColumnLayout,
|
|
72
78
|
scanRowPageSql,
|
|
73
79
|
schemaDdl,
|
|
@@ -86,10 +92,12 @@ import {
|
|
|
86
92
|
collectCommitWindowPage,
|
|
87
93
|
deserializePushResult,
|
|
88
94
|
placeholders,
|
|
95
|
+
type SqliteChangeRecord,
|
|
89
96
|
type SqliteCommitWindowRecord,
|
|
90
97
|
type SqliteRowRecord,
|
|
91
98
|
serializePushResult,
|
|
92
99
|
sqliteDdlStatements,
|
|
100
|
+
toStoredChange,
|
|
93
101
|
toStoredRow,
|
|
94
102
|
} from './sqlite-dialect';
|
|
95
103
|
import type {
|
|
@@ -97,8 +105,11 @@ import type {
|
|
|
97
105
|
AuthoritativeQueryResult,
|
|
98
106
|
CheckpointDeclaration,
|
|
99
107
|
ClientCursorInfo,
|
|
108
|
+
ClientPushResult,
|
|
100
109
|
ClientRecord,
|
|
101
110
|
ClientSubscription,
|
|
111
|
+
CommitChangeSummary,
|
|
112
|
+
CommitDetail,
|
|
102
113
|
CommitMetadata,
|
|
103
114
|
CommitMetadataQuery,
|
|
104
115
|
CommitWindowQuery,
|
|
@@ -113,6 +124,9 @@ import type {
|
|
|
113
124
|
ReactionFailureUpdate,
|
|
114
125
|
ReactionListQuery,
|
|
115
126
|
ReactionPruneQuery,
|
|
127
|
+
RowChangeRecord,
|
|
128
|
+
RowChangesQuery,
|
|
129
|
+
RowListQuery,
|
|
116
130
|
RowScanQuery,
|
|
117
131
|
ScopeActivityQuery,
|
|
118
132
|
ScopeCommitActivity,
|
|
@@ -288,6 +302,7 @@ class D1Transaction implements StorageTransaction {
|
|
|
288
302
|
* `tbl\u0000rowId` → the pending state, consulted by `getRow` before D1.
|
|
289
303
|
*/
|
|
290
304
|
readonly #pending = new Map<string, PendingRow>();
|
|
305
|
+
readonly #clearedTombstones = new Set<string>();
|
|
291
306
|
|
|
292
307
|
constructor(
|
|
293
308
|
db: Pick<D1Database, 'prepare' | 'batch'>,
|
|
@@ -333,6 +348,9 @@ class D1Transaction implements StorageTransaction {
|
|
|
333
348
|
rowId: string,
|
|
334
349
|
): Promise<number | undefined> {
|
|
335
350
|
this.#assertOpen();
|
|
351
|
+
if (this.#clearedTombstones.has(D1Transaction.#key(table, rowId))) {
|
|
352
|
+
return undefined;
|
|
353
|
+
}
|
|
336
354
|
// Persisted read, no #pending overlay: a delete buffered by this commit
|
|
337
355
|
// records its tombstone at appendCommit, matching the SQLite/Postgres
|
|
338
356
|
// adapters where the tombstone insert also runs at commit append.
|
|
@@ -347,6 +365,7 @@ class D1Transaction implements StorageTransaction {
|
|
|
347
365
|
|
|
348
366
|
async clearTombstone(table: string, rowId: string): Promise<void> {
|
|
349
367
|
this.#assertOpen();
|
|
368
|
+
this.#clearedTombstones.add(D1Transaction.#key(table, rowId));
|
|
350
369
|
this.#buffer_(
|
|
351
370
|
'DELETE FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?',
|
|
352
371
|
[this.#partition, table, rowId],
|
|
@@ -434,7 +453,9 @@ class D1Transaction implements StorageTransaction {
|
|
|
434
453
|
}
|
|
435
454
|
}
|
|
436
455
|
return [...rows.values()]
|
|
437
|
-
.sort((left, right) =>
|
|
456
|
+
.sort((left, right) =>
|
|
457
|
+
left.rowId < right.rowId ? -1 : left.rowId > right.rowId ? 1 : 0,
|
|
458
|
+
)
|
|
438
459
|
.slice(0, query.limit);
|
|
439
460
|
}
|
|
440
461
|
|
|
@@ -488,7 +509,9 @@ class D1Transaction implements StorageTransaction {
|
|
|
488
509
|
if (matches) rows.set(rowId, pending.row);
|
|
489
510
|
}
|
|
490
511
|
return [...rows.values()]
|
|
491
|
-
.sort((left, right) =>
|
|
512
|
+
.sort((left, right) =>
|
|
513
|
+
left.rowId < right.rowId ? -1 : left.rowId > right.rowId ? 1 : 0,
|
|
514
|
+
)
|
|
492
515
|
.slice(0, query.limit);
|
|
493
516
|
}
|
|
494
517
|
|
|
@@ -560,6 +583,7 @@ class D1Transaction implements StorageTransaction {
|
|
|
560
583
|
}
|
|
561
584
|
this.#buffer.length = checkpoint;
|
|
562
585
|
this.#pending.clear();
|
|
586
|
+
this.#clearedTombstones.clear();
|
|
563
587
|
this.#maxCommitSeq = undefined;
|
|
564
588
|
this.#applicationOpIndexes.clear();
|
|
565
589
|
await this.putPushResult(clientId, clientCommitId, result);
|
|
@@ -2479,17 +2503,22 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2479
2503
|
partition: string,
|
|
2480
2504
|
query: CommitMetadataQuery,
|
|
2481
2505
|
): Promise<CommitMetadata[]> {
|
|
2506
|
+
// Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
|
|
2507
|
+
// ended by `beforeSeq` (exclusive upper bound) when paging older. An
|
|
2508
|
+
// omitted bound is the largest safe commitSeq, so one statement serves
|
|
2509
|
+
// both the bounded and unbounded read.
|
|
2510
|
+
const beforeSeq = query.beforeSeq ?? Number.MAX_SAFE_INTEGER;
|
|
2482
2511
|
const { results: rows } = query.table
|
|
2483
2512
|
? await this.#db
|
|
2484
2513
|
.prepare(
|
|
2485
2514
|
`SELECT c.commit_seq, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms
|
|
2486
2515
|
FROM sync_commits c
|
|
2487
|
-
WHERE c.partition=? AND c.commit_seq>?
|
|
2516
|
+
WHERE c.partition=? AND c.commit_seq>? AND c.commit_seq<?
|
|
2488
2517
|
AND EXISTS (SELECT 1 FROM sync_changes ch
|
|
2489
2518
|
WHERE ch.partition=c.partition AND ch.commit_seq=c.commit_seq AND ch.tbl=?)
|
|
2490
2519
|
ORDER BY c.commit_seq DESC LIMIT ?`,
|
|
2491
2520
|
)
|
|
2492
|
-
.bind(partition, query.afterSeq, query.table, query.limit)
|
|
2521
|
+
.bind(partition, query.afterSeq, beforeSeq, query.table, query.limit)
|
|
2493
2522
|
.all<{
|
|
2494
2523
|
commit_seq: number;
|
|
2495
2524
|
client_id: string;
|
|
@@ -2501,10 +2530,10 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2501
2530
|
.prepare(
|
|
2502
2531
|
`SELECT commit_seq, client_id, client_commit_id, actor_id, created_at_ms
|
|
2503
2532
|
FROM sync_commits
|
|
2504
|
-
WHERE partition=? AND commit_seq>?
|
|
2533
|
+
WHERE partition=? AND commit_seq>? AND commit_seq<?
|
|
2505
2534
|
ORDER BY commit_seq DESC LIMIT ?`,
|
|
2506
2535
|
)
|
|
2507
|
-
.bind(partition, query.afterSeq, query.limit)
|
|
2536
|
+
.bind(partition, query.afterSeq, beforeSeq, query.limit)
|
|
2508
2537
|
.all<{
|
|
2509
2538
|
commit_seq: number;
|
|
2510
2539
|
client_id: string;
|
|
@@ -2513,6 +2542,46 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2513
2542
|
created_at_ms: number;
|
|
2514
2543
|
}>();
|
|
2515
2544
|
const out: CommitMetadata[] = [];
|
|
2545
|
+
// `changeSample`: extra statement(s) for the whole page — the first
|
|
2546
|
+
// `sample` changes (change-index order) of every listed commit.
|
|
2547
|
+
const sampleCount =
|
|
2548
|
+
query.changeSample !== undefined &&
|
|
2549
|
+
Number.isInteger(query.changeSample) &&
|
|
2550
|
+
query.changeSample > 0
|
|
2551
|
+
? query.changeSample
|
|
2552
|
+
: 0;
|
|
2553
|
+
const samples = new Map<number, CommitChangeSummary[]>();
|
|
2554
|
+
if (sampleCount > 0 && rows.length > 0) {
|
|
2555
|
+
const seqs = rows.map((row) => row.commit_seq);
|
|
2556
|
+
// D1 caps the bound parameters of one statement, so a page larger than
|
|
2557
|
+
// that cap splits into a few chunk statements, never one per commit.
|
|
2558
|
+
const chunkSize = D1_MAX_BIND_PARAMS - 2;
|
|
2559
|
+
for (let offset = 0; offset < seqs.length; offset += chunkSize) {
|
|
2560
|
+
const chunk = seqs.slice(offset, offset + chunkSize);
|
|
2561
|
+
const { results: changes } = await this.#db
|
|
2562
|
+
.prepare(
|
|
2563
|
+
`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
2564
|
+
WHERE partition=? AND commit_seq IN (${placeholders(chunk.length)}) AND idx<?
|
|
2565
|
+
ORDER BY commit_seq, idx`,
|
|
2566
|
+
)
|
|
2567
|
+
.bind(partition, ...chunk, sampleCount)
|
|
2568
|
+
.all<{
|
|
2569
|
+
commit_seq: number;
|
|
2570
|
+
tbl: string;
|
|
2571
|
+
row_id: string;
|
|
2572
|
+
op: number;
|
|
2573
|
+
}>();
|
|
2574
|
+
for (const change of changes) {
|
|
2575
|
+
const sample = samples.get(change.commit_seq) ?? [];
|
|
2576
|
+
sample.push({
|
|
2577
|
+
table: change.tbl,
|
|
2578
|
+
rowId: change.row_id,
|
|
2579
|
+
op: change.op === 1 ? 'upsert' : 'delete',
|
|
2580
|
+
});
|
|
2581
|
+
samples.set(change.commit_seq, sample);
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2516
2585
|
for (const row of rows) {
|
|
2517
2586
|
const { results: changes } = await this.#db
|
|
2518
2587
|
.prepare(
|
|
@@ -2528,11 +2597,135 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2528
2597
|
createdAtMs: row.created_at_ms,
|
|
2529
2598
|
changeCount: changes.reduce((sum, c) => sum + c.n, 0),
|
|
2530
2599
|
tables: changes.map((c) => c.tbl),
|
|
2600
|
+
...(sampleCount > 0
|
|
2601
|
+
? { sample: samples.get(row.commit_seq) ?? [] }
|
|
2602
|
+
: {}),
|
|
2531
2603
|
});
|
|
2532
2604
|
}
|
|
2533
2605
|
return out;
|
|
2534
2606
|
}
|
|
2535
2607
|
|
|
2608
|
+
async getCommit(
|
|
2609
|
+
partition: string,
|
|
2610
|
+
commitSeq: number,
|
|
2611
|
+
): Promise<CommitDetail | undefined> {
|
|
2612
|
+
const meta = await this.#db
|
|
2613
|
+
.prepare(
|
|
2614
|
+
'SELECT client_id, client_commit_id, actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?',
|
|
2615
|
+
)
|
|
2616
|
+
.bind(partition, commitSeq)
|
|
2617
|
+
.first<{
|
|
2618
|
+
client_id: string;
|
|
2619
|
+
client_commit_id: string;
|
|
2620
|
+
actor_id: string;
|
|
2621
|
+
created_at_ms: number;
|
|
2622
|
+
}>();
|
|
2623
|
+
if (meta === null) return undefined;
|
|
2624
|
+
const { results: changeRows } = await this.#db
|
|
2625
|
+
.prepare(
|
|
2626
|
+
'SELECT tbl, row_id, op, row_version, scopes, payload FROM sync_changes WHERE partition=? AND commit_seq=? ORDER BY idx',
|
|
2627
|
+
)
|
|
2628
|
+
.bind(partition, commitSeq)
|
|
2629
|
+
.all<SqliteChangeRecord>();
|
|
2630
|
+
const changes = changeRows.map(toStoredChange);
|
|
2631
|
+
return {
|
|
2632
|
+
commitSeq,
|
|
2633
|
+
clientId: meta.client_id,
|
|
2634
|
+
clientCommitId: meta.client_commit_id,
|
|
2635
|
+
actorId: meta.actor_id,
|
|
2636
|
+
createdAtMs: meta.created_at_ms,
|
|
2637
|
+
changeCount: changes.length,
|
|
2638
|
+
tables: [...new Set(changes.map((change) => change.table))].sort(),
|
|
2639
|
+
changes,
|
|
2640
|
+
};
|
|
2641
|
+
}
|
|
2642
|
+
|
|
2643
|
+
async listRowChanges(
|
|
2644
|
+
partition: string,
|
|
2645
|
+
query: RowChangesQuery,
|
|
2646
|
+
): Promise<RowChangeRecord[]> {
|
|
2647
|
+
// Retained-log read for one row: no dedicated index (the write path pays
|
|
2648
|
+
// nothing for it), bounded by retention (§4.6).
|
|
2649
|
+
const { results: rows } = await this.#db
|
|
2650
|
+
.prepare(
|
|
2651
|
+
`SELECT ch.commit_seq, ch.op, ch.row_version, c.actor_id, c.client_id, c.created_at_ms
|
|
2652
|
+
FROM sync_changes ch
|
|
2653
|
+
JOIN sync_commits c
|
|
2654
|
+
ON c.partition=ch.partition AND c.commit_seq=ch.commit_seq
|
|
2655
|
+
WHERE ch.partition=? AND ch.tbl=? AND ch.row_id=?
|
|
2656
|
+
ORDER BY ch.commit_seq DESC, ch.idx DESC LIMIT ?`,
|
|
2657
|
+
)
|
|
2658
|
+
.bind(partition, query.table, query.rowId, query.limit)
|
|
2659
|
+
.all<{
|
|
2660
|
+
commit_seq: number;
|
|
2661
|
+
op: number;
|
|
2662
|
+
row_version: number | null;
|
|
2663
|
+
actor_id: string;
|
|
2664
|
+
client_id: string;
|
|
2665
|
+
created_at_ms: number;
|
|
2666
|
+
}>();
|
|
2667
|
+
return rows.map((row) => ({
|
|
2668
|
+
commitSeq: row.commit_seq,
|
|
2669
|
+
op: row.op === 1 ? 'upsert' : 'delete',
|
|
2670
|
+
...(row.row_version !== null ? { rowVersion: row.row_version } : {}),
|
|
2671
|
+
actorId: row.actor_id,
|
|
2672
|
+
clientId: row.client_id,
|
|
2673
|
+
createdAtMs: row.created_at_ms,
|
|
2674
|
+
}));
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
async listRows(partition: string, query: RowListQuery): Promise<StoredRow[]> {
|
|
2678
|
+
// Keyset page over the row table's (partition, rowId) primary key: an
|
|
2679
|
+
// index range read, no scope filter, no per-row lookup.
|
|
2680
|
+
const { results } = await this.#schemaDatabase()
|
|
2681
|
+
.prepare(
|
|
2682
|
+
`SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
2683
|
+
FROM ${quoteIdent(this.table(query.table).name)}
|
|
2684
|
+
WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>?
|
|
2685
|
+
ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ?`,
|
|
2686
|
+
)
|
|
2687
|
+
.bind(partition, query.afterRowId ?? '', query.limit)
|
|
2688
|
+
.all<SqliteRowRecord>();
|
|
2689
|
+
return results.map(toStoredRow);
|
|
2690
|
+
}
|
|
2691
|
+
|
|
2692
|
+
async countRows(partition: string, table: string): Promise<number> {
|
|
2693
|
+
const record = await this.#schemaDatabase()
|
|
2694
|
+
.prepare(
|
|
2695
|
+
`SELECT count(*) AS n FROM ${quoteIdent(this.table(table).name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?`,
|
|
2696
|
+
)
|
|
2697
|
+
.bind(partition)
|
|
2698
|
+
.first<{ n: number }>();
|
|
2699
|
+
return record?.n ?? 0;
|
|
2700
|
+
}
|
|
2701
|
+
|
|
2702
|
+
async listPushResults(
|
|
2703
|
+
partition: string,
|
|
2704
|
+
clientCommitId: string,
|
|
2705
|
+
): Promise<ClientPushResult[]> {
|
|
2706
|
+
// Retained outcomes for one clientCommitId: no dedicated index, ordered
|
|
2707
|
+
// by client so the read is deterministic.
|
|
2708
|
+
const { results } = await this.#schemaDatabase()
|
|
2709
|
+
.prepare(
|
|
2710
|
+
'SELECT client_id, result FROM sync_push_results WHERE partition=? AND client_commit_id=? ORDER BY client_id',
|
|
2711
|
+
)
|
|
2712
|
+
.bind(partition, clientCommitId)
|
|
2713
|
+
.all<{ client_id: string; result: string }>();
|
|
2714
|
+
return results.map((record) => {
|
|
2715
|
+
try {
|
|
2716
|
+
return {
|
|
2717
|
+
clientId: record.client_id,
|
|
2718
|
+
result: deserializePushResult(record.result),
|
|
2719
|
+
};
|
|
2720
|
+
} catch {
|
|
2721
|
+
throw syncError(
|
|
2722
|
+
'sync.idempotency_cache_miss',
|
|
2723
|
+
'persisted push result unreadable (§6.3)',
|
|
2724
|
+
);
|
|
2725
|
+
}
|
|
2726
|
+
});
|
|
2727
|
+
}
|
|
2728
|
+
|
|
2536
2729
|
async scopeActivity(
|
|
2537
2730
|
partition: string,
|
|
2538
2731
|
query: ScopeActivityQuery,
|
|
@@ -2571,25 +2764,6 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
2571
2764
|
return out;
|
|
2572
2765
|
}
|
|
2573
2766
|
|
|
2574
|
-
async getRowScopes(
|
|
2575
|
-
partition: string,
|
|
2576
|
-
table: string,
|
|
2577
|
-
rowId: string,
|
|
2578
|
-
): Promise<
|
|
2579
|
-
{ serverVersion: number; scopes: Record<string, string> } | undefined
|
|
2580
|
-
> {
|
|
2581
|
-
const db = this.#schemaDatabase();
|
|
2582
|
-
const record = await db
|
|
2583
|
-
.prepare(selectRowScopesSql(this.table(table), 'sqlite'))
|
|
2584
|
-
.bind(partition, rowId)
|
|
2585
|
-
.first<{ server_version: number; scopes: string }>();
|
|
2586
|
-
if (record === null) return undefined;
|
|
2587
|
-
return {
|
|
2588
|
-
serverVersion: record.server_version,
|
|
2589
|
-
scopes: JSON.parse(record.scopes) as Record<string, string>,
|
|
2590
|
-
};
|
|
2591
|
-
}
|
|
2592
|
-
|
|
2593
2767
|
async listPartitions(): Promise<string[]> {
|
|
2594
2768
|
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
2595
2769
|
}
|
package/src/events-ring.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* without any infrastructure dependency.
|
|
4
4
|
*
|
|
5
5
|
* `RingBufferEvents` is a `SyncularServerEvents` sink that retains the last
|
|
6
|
-
* N events in memory and exposes a `query({type?,
|
|
7
|
-
* actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
|
|
6
|
+
* N events in memory and exposes a `query({type?, partition?, sinceMs?,
|
|
7
|
+
* clientId?, actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
|
|
8
8
|
* composes with any other sink through `composeEvents(...sinks)`, so a host
|
|
9
9
|
* can keep `consoleJsonEvents()` (or a Sentry adapter) AND feed the console
|
|
10
10
|
* event tail from the same emissions. Fire-and-forget discipline is
|
|
@@ -20,6 +20,10 @@ import {
|
|
|
20
20
|
export interface RingEventQuery {
|
|
21
21
|
/** Restrict to one event `type` (e.g. `push.applied`). */
|
|
22
22
|
readonly type?: SyncularServerEvent['type'];
|
|
23
|
+
/** Restrict to one partition (every event carries one). */
|
|
24
|
+
readonly partition?: string;
|
|
25
|
+
/** Restrict to the events of one sync request. */
|
|
26
|
+
readonly requestId?: string;
|
|
23
27
|
/** Only events with `atMs >= sinceMs`. */
|
|
24
28
|
readonly sinceMs?: number;
|
|
25
29
|
/** Restrict to events carrying this `clientId` (not every type does). */
|
|
@@ -41,6 +45,12 @@ export function matchesRingQuery(
|
|
|
41
45
|
query: Omit<RingEventQuery, 'limit'>,
|
|
42
46
|
): boolean {
|
|
43
47
|
if (query.type !== undefined && event.type !== query.type) return false;
|
|
48
|
+
if (query.partition !== undefined && event.partition !== query.partition) {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
if (query.requestId !== undefined && event.requestId !== query.requestId) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
44
54
|
if (query.sinceMs !== undefined && event.atMs < query.sinceMs) return false;
|
|
45
55
|
const carrier = event as { clientId?: string; actorId?: string };
|
|
46
56
|
if (query.clientId !== undefined && carrier.clientId !== query.clientId) {
|
|
@@ -67,6 +77,7 @@ export class RingBufferEvents implements SyncularServerEvents {
|
|
|
67
77
|
/** Index of the oldest element in the circular buffer. */
|
|
68
78
|
#head = 0;
|
|
69
79
|
#size = 0;
|
|
80
|
+
#evictedThroughMs: number | undefined;
|
|
70
81
|
/** Live subscribers (the SSE tail). Each notify is guarded. */
|
|
71
82
|
readonly #listeners = new Set<(event: SyncularServerEvent) => void>();
|
|
72
83
|
|
|
@@ -84,6 +95,8 @@ export class RingBufferEvents implements SyncularServerEvents {
|
|
|
84
95
|
this.#size += 1;
|
|
85
96
|
} else {
|
|
86
97
|
// Full: overwrite the oldest and advance the head.
|
|
98
|
+
const evicted = this.#buffer[this.#head];
|
|
99
|
+
if (evicted !== undefined) this.#markEvicted(evicted.atMs);
|
|
87
100
|
this.#buffer[this.#head] = event;
|
|
88
101
|
this.#head = (this.#head + 1) % this.#capacity;
|
|
89
102
|
}
|
|
@@ -119,6 +132,21 @@ export class RingBufferEvents implements SyncularServerEvents {
|
|
|
119
132
|
return this.#size;
|
|
120
133
|
}
|
|
121
134
|
|
|
135
|
+
/**
|
|
136
|
+
* The newest `atMs` among events the ring dropped (evicted when full, or
|
|
137
|
+
* cleared); `undefined` while nothing was dropped. A window that starts
|
|
138
|
+
* at or before it is incomplete.
|
|
139
|
+
*/
|
|
140
|
+
get evictedThroughMs(): number | undefined {
|
|
141
|
+
return this.#evictedThroughMs;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
#markEvicted(atMs: number): void {
|
|
145
|
+
if (this.#evictedThroughMs === undefined || atMs > this.#evictedThroughMs) {
|
|
146
|
+
this.#evictedThroughMs = atMs;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
122
150
|
/**
|
|
123
151
|
* Newest-first slice of the retained events. Filters by `type` and
|
|
124
152
|
* `sinceMs`, then caps to `limit`. Returns a fresh array — the caller may
|
|
@@ -139,6 +167,7 @@ export class RingBufferEvents implements SyncularServerEvents {
|
|
|
139
167
|
|
|
140
168
|
/** Drop all retained events. */
|
|
141
169
|
clear(): void {
|
|
170
|
+
for (const event of this.#buffer) this.#markEvicted(event.atMs);
|
|
142
171
|
this.#buffer = [];
|
|
143
172
|
this.#head = 0;
|
|
144
173
|
this.#size = 0;
|
package/src/events.ts
CHANGED
|
@@ -29,11 +29,21 @@ export interface RequestHandledEvent {
|
|
|
29
29
|
/**
|
|
30
30
|
* `ok` — response streamed to END;
|
|
31
31
|
* `schema_floor` — §2.4 required-schema answer;
|
|
32
|
-
* `
|
|
32
|
+
* `epoch_acquired` — §2.1 answer to a request without `logEpoch` (a new
|
|
33
|
+
* or wiped replica's first round);
|
|
34
|
+
* `reset` — §2.1 answer to a `logEpoch` that no longer matches the log;
|
|
33
35
|
* `rejected` — request validation failed before any bytes (§1.7);
|
|
34
36
|
* `error` — in-band ERROR frame (§1.6) or a thrown host failure.
|
|
35
37
|
*/
|
|
36
|
-
readonly outcome:
|
|
38
|
+
readonly outcome:
|
|
39
|
+
| 'ok'
|
|
40
|
+
| 'schema_floor'
|
|
41
|
+
| 'epoch_acquired'
|
|
42
|
+
| 'reset'
|
|
43
|
+
| 'rejected'
|
|
44
|
+
| 'error';
|
|
45
|
+
/** The REQ_HEADER clientId; absent when the request failed to decode. */
|
|
46
|
+
readonly clientId?: string;
|
|
37
47
|
/** §10.2 code for `rejected`/`error`; `"internal"` for non-SyncErrors. */
|
|
38
48
|
readonly errorCode?: string;
|
|
39
49
|
readonly pushCommits: number;
|
|
@@ -64,6 +74,9 @@ export interface PushRejectedEvent extends PushEventBase {
|
|
|
64
74
|
readonly type: 'push.rejected';
|
|
65
75
|
readonly code: string;
|
|
66
76
|
readonly opIndex: number;
|
|
77
|
+
/** The terminating operation's table and rowId. */
|
|
78
|
+
readonly table?: string;
|
|
79
|
+
readonly rowId?: string;
|
|
67
80
|
/** True when the rejection was replayed from the idempotency cache. */
|
|
68
81
|
readonly replay: boolean;
|
|
69
82
|
/** Original host time for outcomes recorded by a metadata-aware server. */
|
|
@@ -76,6 +89,9 @@ export interface PushRejectedEvent extends PushEventBase {
|
|
|
76
89
|
export interface PushConflictedEvent extends PushEventBase {
|
|
77
90
|
readonly type: 'push.conflicted';
|
|
78
91
|
readonly opIndex: number;
|
|
92
|
+
/** The conflicting operation's table and rowId. */
|
|
93
|
+
readonly table?: string;
|
|
94
|
+
readonly rowId?: string;
|
|
79
95
|
/** True when the conflict was replayed from the idempotency cache. */
|
|
80
96
|
readonly replay: boolean;
|
|
81
97
|
readonly recordedAtMs?: number;
|
|
@@ -306,7 +322,13 @@ export interface LeaseRevokedEvent {
|
|
|
306
322
|
readonly leaseId: string;
|
|
307
323
|
}
|
|
308
324
|
|
|
309
|
-
|
|
325
|
+
/**
|
|
326
|
+
* Every event. `requestId` is present on the events emitted while serving
|
|
327
|
+
* one sync request (`request.handled`, `push.*`, `pull.served`,
|
|
328
|
+
* `lease.issued`, `reaction.queued`, `scopes.resolve_failed`) and shared by
|
|
329
|
+
* all of them, so a sink can group one request's events.
|
|
330
|
+
*/
|
|
331
|
+
export type SyncularServerEvent = (
|
|
310
332
|
| RequestHandledEvent
|
|
311
333
|
| PushAppliedEvent
|
|
312
334
|
| PushRejectedEvent
|
|
@@ -329,7 +351,8 @@ export type SyncularServerEvent =
|
|
|
329
351
|
| PruneCompletedEvent
|
|
330
352
|
| ScopesResolveFailedEvent
|
|
331
353
|
| LeaseIssuedEvent
|
|
332
|
-
| LeaseRevokedEvent
|
|
354
|
+
| LeaseRevokedEvent
|
|
355
|
+
) & { readonly requestId?: string };
|
|
333
356
|
|
|
334
357
|
/**
|
|
335
358
|
* The seam. Optional on the server config — when absent, no event object
|
package/src/handler.ts
CHANGED
|
@@ -373,7 +373,7 @@ async function planRequest(
|
|
|
373
373
|
|
|
374
374
|
/** Mutable outcome box shared with the instrumented stream wrapper. */
|
|
375
375
|
interface RequestReport {
|
|
376
|
-
outcome: 'ok' | 'schema_floor' | 'reset' | 'error';
|
|
376
|
+
outcome: 'ok' | 'schema_floor' | 'epoch_acquired' | 'reset' | 'error';
|
|
377
377
|
errorCode?: string;
|
|
378
378
|
}
|
|
379
379
|
|
|
@@ -421,11 +421,14 @@ function emitPushEvent(
|
|
|
421
421
|
}
|
|
422
422
|
// Rejected (§6.3): exactly one terminating operation record.
|
|
423
423
|
const record = frame.results[0];
|
|
424
|
+
const op = record !== undefined ? push.operations[record.opIndex] : undefined;
|
|
425
|
+
const target = op !== undefined ? { table: op.table, rowId: op.rowId } : {};
|
|
424
426
|
if (record?.status === 'conflict') {
|
|
425
427
|
emitEvent(events, {
|
|
426
428
|
type: 'push.conflicted',
|
|
427
429
|
...base,
|
|
428
430
|
opIndex: record.opIndex,
|
|
431
|
+
...target,
|
|
429
432
|
replay: processed.replayed,
|
|
430
433
|
...(processed.recordedAtMs !== undefined
|
|
431
434
|
? { recordedAtMs: processed.recordedAtMs }
|
|
@@ -444,6 +447,7 @@ function emitPushEvent(
|
|
|
444
447
|
? record.code
|
|
445
448
|
: 'sync.invalid_request',
|
|
446
449
|
opIndex: record?.opIndex ?? 0,
|
|
450
|
+
...target,
|
|
447
451
|
replay: processed.replayed,
|
|
448
452
|
...(processed.recordedAtMs !== undefined
|
|
449
453
|
? { recordedAtMs: processed.recordedAtMs }
|
|
@@ -491,7 +495,10 @@ async function* streamResponse(
|
|
|
491
495
|
plan.wireVersion,
|
|
492
496
|
);
|
|
493
497
|
if (plan.epochReset) {
|
|
494
|
-
if (report !== undefined)
|
|
498
|
+
if (report !== undefined) {
|
|
499
|
+
report.outcome =
|
|
500
|
+
plan.header.logEpoch === undefined ? 'epoch_acquired' : 'reset';
|
|
501
|
+
}
|
|
495
502
|
yield END_FRAME_BYTES;
|
|
496
503
|
return;
|
|
497
504
|
}
|
|
@@ -519,11 +526,12 @@ async function* streamResponse(
|
|
|
519
526
|
}
|
|
520
527
|
try {
|
|
521
528
|
// Push half (§6): one PUSH_RESULT per PUSH_COMMIT, in request order.
|
|
529
|
+
const currentSchema = compileSchema(ctx.schema);
|
|
522
530
|
let acceptedThrough = 0;
|
|
523
531
|
for (const push of plan.pushes) {
|
|
524
532
|
const processed = await processPushCommitWithTrace(
|
|
525
533
|
ctx,
|
|
526
|
-
|
|
534
|
+
currentSchema,
|
|
527
535
|
plan.resolved,
|
|
528
536
|
plan.header.clientId,
|
|
529
537
|
push,
|
|
@@ -536,20 +544,36 @@ async function* streamResponse(
|
|
|
536
544
|
const frame = {
|
|
537
545
|
...processed.frame,
|
|
538
546
|
results: processed.frame.results.map((result) => {
|
|
539
|
-
if (result.status !== 'conflict')
|
|
540
|
-
|
|
547
|
+
if (result.status !== 'conflict' || schema === currentSchema) {
|
|
548
|
+
return result;
|
|
549
|
+
}
|
|
550
|
+
const operation = push.operations[result.opIndex];
|
|
551
|
+
// §2.3: a replay returns the persisted result without rebuilding
|
|
552
|
+
// operations, so a reused clientCommitId with changed operations
|
|
553
|
+
// leaves no table to project the stored conflict row through.
|
|
554
|
+
if (operation === undefined) {
|
|
555
|
+
throw syncError(
|
|
556
|
+
'sync.invalid_request',
|
|
557
|
+
'replayed clientCommitId carries changed operations',
|
|
558
|
+
JSON.stringify({
|
|
559
|
+
clientCommitId: push.clientCommitId,
|
|
560
|
+
opIndex: result.opIndex,
|
|
561
|
+
}),
|
|
562
|
+
);
|
|
563
|
+
}
|
|
541
564
|
const target = schema.tables.get(operation.table)!;
|
|
542
|
-
const
|
|
543
|
-
operation.table,
|
|
544
|
-
)!;
|
|
565
|
+
const bitmapBytes = Math.ceil(target.columns.length / 8);
|
|
545
566
|
return {
|
|
546
567
|
...result,
|
|
547
|
-
serverRow: projectRowPayload(
|
|
568
|
+
serverRow: projectRowPayload(
|
|
569
|
+
currentSchema.tables.get(operation.table)!,
|
|
570
|
+
target,
|
|
571
|
+
result.serverRow,
|
|
572
|
+
),
|
|
548
573
|
conflictColumns: result.conflictColumns
|
|
549
|
-
.slice(0,
|
|
574
|
+
.slice(0, bitmapBytes)
|
|
550
575
|
.map((byte, i) =>
|
|
551
|
-
i ===
|
|
552
|
-
target.columns.length % 8 !== 0
|
|
576
|
+
i === bitmapBytes - 1 && target.columns.length % 8 !== 0
|
|
553
577
|
? byte & ((1 << (target.columns.length % 8)) - 1)
|
|
554
578
|
: byte,
|
|
555
579
|
),
|
|
@@ -729,14 +753,20 @@ export async function createSyncResponseStream(
|
|
|
729
753
|
bytes: Uint8Array,
|
|
730
754
|
ctx: SyncRequestContext,
|
|
731
755
|
): Promise<AsyncIterable<Uint8Array>> {
|
|
732
|
-
const
|
|
733
|
-
if (
|
|
756
|
+
const sink = ctx.events;
|
|
757
|
+
if (sink === undefined) return createStreamCore(bytes, ctx, undefined);
|
|
734
758
|
// Instrumented path: measure with the ctx clock, count bytes at the
|
|
735
|
-
// seam, emit exactly one `request.handled` per request.
|
|
759
|
+
// seam, emit exactly one `request.handled` per request. Every event of
|
|
760
|
+
// this request carries one `requestId`.
|
|
761
|
+
const requestId = crypto.randomUUID();
|
|
762
|
+
const events: SyncularServerEvents = {
|
|
763
|
+
emit: (event) => sink.emit({ ...event, requestId }),
|
|
764
|
+
};
|
|
765
|
+
const requestCtx = { ...ctx, events };
|
|
736
766
|
const clock = clockOf(ctx);
|
|
737
767
|
const startedAtMs = clock();
|
|
738
768
|
try {
|
|
739
|
-
return await createStreamCore(bytes,
|
|
769
|
+
return await createStreamCore(bytes, requestCtx, events, startedAtMs);
|
|
740
770
|
} catch (error) {
|
|
741
771
|
emitEvent(events, {
|
|
742
772
|
type: 'request.handled',
|
|
@@ -878,6 +908,7 @@ async function* instrumentedStream(
|
|
|
878
908
|
bytesIn,
|
|
879
909
|
bytesOut,
|
|
880
910
|
outcome: report.outcome,
|
|
911
|
+
clientId: plan.header.clientId,
|
|
881
912
|
...(report.errorCode !== undefined
|
|
882
913
|
? { errorCode: report.errorCode }
|
|
883
914
|
: {}),
|
package/src/operations.ts
CHANGED
|
@@ -21,7 +21,11 @@ import {
|
|
|
21
21
|
import { reportError, SyncError, syncError } from './errors';
|
|
22
22
|
import { processPushOperationsWithTrace } from './push';
|
|
23
23
|
import { compileSchema } from './schema';
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
authorizeWrite,
|
|
26
|
+
renderScopeValue,
|
|
27
|
+
type ResolvedScopes,
|
|
28
|
+
} from './scopes';
|
|
25
29
|
import type {
|
|
26
30
|
AuthoritativeQueryRequest,
|
|
27
31
|
AuthoritativeQueryResult,
|
|
@@ -534,11 +538,11 @@ function commandOperations(
|
|
|
534
538
|
}
|
|
535
539
|
return mutation.values[column.name] ?? null;
|
|
536
540
|
});
|
|
537
|
-
const rowId = values[table.primaryKeyIndex];
|
|
538
|
-
if (
|
|
541
|
+
const rowId = renderScopeValue(values[table.primaryKeyIndex]);
|
|
542
|
+
if (rowId === undefined) {
|
|
539
543
|
throw syncError(
|
|
540
544
|
'operation.invalid_request',
|
|
541
|
-
'command upsert requires a non-empty
|
|
545
|
+
'command upsert requires a non-empty primary key',
|
|
542
546
|
);
|
|
543
547
|
}
|
|
544
548
|
return {
|