@syncular/server 0.31.0 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -74
- package/dist/admin.d.ts +394 -50
- package/dist/admin.js +714 -82
- package/dist/d1-storage.d.ts +6 -5
- package/dist/d1-storage.js +136 -21
- package/dist/events-ring.d.ts +12 -2
- package/dist/events-ring.js +27 -2
- package/dist/events.d.ts +21 -3
- package/dist/handler.js +36 -13
- package/dist/operations.js +4 -4
- package/dist/postgres-storage.d.ts +6 -5
- package/dist/postgres-storage.js +102 -18
- package/dist/push.js +28 -7
- package/dist/realtime.d.ts +15 -0
- package/dist/realtime.js +46 -7
- package/dist/seed.js +4 -3
- package/dist/sqlite-storage.d.ts +6 -5
- package/dist/sqlite-storage.js +115 -18
- package/dist/storage.d.ts +58 -7
- package/package.json +2 -2
- package/src/admin.ts +1215 -120
- package/src/d1-storage.ts +199 -25
- package/src/events-ring.ts +31 -2
- package/src/events.ts +27 -4
- package/src/handler.ts +47 -16
- package/src/operations.ts +8 -4
- package/src/postgres-storage.ts +173 -28
- package/src/push.ts +40 -9
- package/src/realtime.ts +60 -7
- package/src/seed.ts +4 -3
- package/src/sqlite-storage.ts +181 -26
- package/src/storage.ts +72 -9
package/dist/d1-storage.d.ts
CHANGED
|
@@ -42,7 +42,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
42
42
|
*/
|
|
43
43
|
import { type ScopeMap } from '@syncular/core';
|
|
44
44
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
45
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
45
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
46
46
|
export interface D1PreparedStatement {
|
|
47
47
|
bind(...values: unknown[]): D1PreparedStatement;
|
|
48
48
|
first<T = Record<string, unknown>>(): Promise<T | null>;
|
|
@@ -136,10 +136,11 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
136
136
|
listReferencedBlobIds(partition: string): Promise<string[]>;
|
|
137
137
|
listClientRecords(partition: string): Promise<ClientRecord[]>;
|
|
138
138
|
listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
139
|
+
getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
|
|
140
|
+
listRowChanges(partition: string, query: RowChangesQuery): Promise<RowChangeRecord[]>;
|
|
141
|
+
listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
|
|
142
|
+
countRows(partition: string, table: string): Promise<number>;
|
|
143
|
+
listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
139
144
|
scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
|
|
140
|
-
getRowScopes(partition: string, table: string, rowId: string): Promise<{
|
|
141
|
-
serverVersion: number;
|
|
142
|
-
scopes: Record<string, string>;
|
|
143
|
-
} | undefined>;
|
|
144
145
|
listPartitions(): Promise<string[]>;
|
|
145
146
|
}
|
package/dist/d1-storage.js
CHANGED
|
@@ -45,9 +45,9 @@ import { StorageQueryError } from './storage-errors.js';
|
|
|
45
45
|
import { decodeRow } from '@syncular/core';
|
|
46
46
|
import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
47
47
|
import { syncError } from './errors.js';
|
|
48
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, latestScopedChangeSeqSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
|
|
48
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, latestScopedChangeSeqSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
|
|
49
49
|
import { matchesEffective } from './scopes.js';
|
|
50
|
-
import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, serializePushResult, sqliteDdlStatements, toStoredRow, } from './sqlite-dialect.js';
|
|
50
|
+
import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, serializePushResult, sqliteDdlStatements, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
|
|
51
51
|
import { isD1ConstraintError, StorageConstraintError } from './storage-errors.js';
|
|
52
52
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
53
53
|
function toStoredReaction(record) {
|
|
@@ -144,6 +144,7 @@ class D1Transaction {
|
|
|
144
144
|
* `tbl\u0000rowId` → the pending state, consulted by `getRow` before D1.
|
|
145
145
|
*/
|
|
146
146
|
#pending = new Map();
|
|
147
|
+
#clearedTombstones = new Set();
|
|
147
148
|
constructor(db, partition, resolveTable, prepareQuery, pushApplySerialized) {
|
|
148
149
|
this.#db = db;
|
|
149
150
|
this.#partition = partition;
|
|
@@ -175,6 +176,9 @@ class D1Transaction {
|
|
|
175
176
|
}
|
|
176
177
|
async getTombstoneSeq(table, rowId) {
|
|
177
178
|
this.#assertOpen();
|
|
179
|
+
if (this.#clearedTombstones.has(_a.#key(table, rowId))) {
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
178
182
|
// Persisted read, no #pending overlay: a delete buffered by this commit
|
|
179
183
|
// records its tombstone at appendCommit, matching the SQLite/Postgres
|
|
180
184
|
// adapters where the tombstone insert also runs at commit append.
|
|
@@ -186,6 +190,7 @@ class D1Transaction {
|
|
|
186
190
|
}
|
|
187
191
|
async clearTombstone(table, rowId) {
|
|
188
192
|
this.#assertOpen();
|
|
193
|
+
this.#clearedTombstones.add(_a.#key(table, rowId));
|
|
189
194
|
this.#buffer_('DELETE FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?', [this.#partition, table, rowId]);
|
|
190
195
|
}
|
|
191
196
|
async getPushResult(clientId, clientCommitId) {
|
|
@@ -250,7 +255,7 @@ class D1Transaction {
|
|
|
250
255
|
}
|
|
251
256
|
}
|
|
252
257
|
return [...rows.values()]
|
|
253
|
-
.sort((left, right) => left.rowId.
|
|
258
|
+
.sort((left, right) => left.rowId < right.rowId ? -1 : left.rowId > right.rowId ? 1 : 0)
|
|
254
259
|
.slice(0, query.limit);
|
|
255
260
|
}
|
|
256
261
|
async scanRowsByIndex(query) {
|
|
@@ -287,7 +292,7 @@ class D1Transaction {
|
|
|
287
292
|
rows.set(rowId, pending.row);
|
|
288
293
|
}
|
|
289
294
|
return [...rows.values()]
|
|
290
|
-
.sort((left, right) => left.rowId.
|
|
295
|
+
.sort((left, right) => left.rowId < right.rowId ? -1 : left.rowId > right.rowId ? 1 : 0)
|
|
291
296
|
.slice(0, query.limit);
|
|
292
297
|
}
|
|
293
298
|
async queryAuthoritative(query) {
|
|
@@ -334,6 +339,7 @@ class D1Transaction {
|
|
|
334
339
|
}
|
|
335
340
|
this.#buffer.length = checkpoint;
|
|
336
341
|
this.#pending.clear();
|
|
342
|
+
this.#clearedTombstones.clear();
|
|
337
343
|
this.#maxCommitSeq = undefined;
|
|
338
344
|
this.#applicationOpIndexes.clear();
|
|
339
345
|
await this.putPushResult(clientId, clientCommitId, result);
|
|
@@ -1664,24 +1670,61 @@ export class D1ServerStorage {
|
|
|
1664
1670
|
}));
|
|
1665
1671
|
}
|
|
1666
1672
|
async listCommitMetadata(partition, query) {
|
|
1673
|
+
// Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
|
|
1674
|
+
// ended by `beforeSeq` (exclusive upper bound) when paging older. An
|
|
1675
|
+
// omitted bound is the largest safe commitSeq, so one statement serves
|
|
1676
|
+
// both the bounded and unbounded read.
|
|
1677
|
+
const beforeSeq = query.beforeSeq ?? Number.MAX_SAFE_INTEGER;
|
|
1667
1678
|
const { results: rows } = query.table
|
|
1668
1679
|
? await this.#db
|
|
1669
1680
|
.prepare(`SELECT c.commit_seq, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms
|
|
1670
1681
|
FROM sync_commits c
|
|
1671
|
-
WHERE c.partition=? AND c.commit_seq>?
|
|
1682
|
+
WHERE c.partition=? AND c.commit_seq>? AND c.commit_seq<?
|
|
1672
1683
|
AND EXISTS (SELECT 1 FROM sync_changes ch
|
|
1673
1684
|
WHERE ch.partition=c.partition AND ch.commit_seq=c.commit_seq AND ch.tbl=?)
|
|
1674
1685
|
ORDER BY c.commit_seq DESC LIMIT ?`)
|
|
1675
|
-
.bind(partition, query.afterSeq, query.table, query.limit)
|
|
1686
|
+
.bind(partition, query.afterSeq, beforeSeq, query.table, query.limit)
|
|
1676
1687
|
.all()
|
|
1677
1688
|
: await this.#db
|
|
1678
1689
|
.prepare(`SELECT commit_seq, client_id, client_commit_id, actor_id, created_at_ms
|
|
1679
1690
|
FROM sync_commits
|
|
1680
|
-
WHERE partition=? AND commit_seq>?
|
|
1691
|
+
WHERE partition=? AND commit_seq>? AND commit_seq<?
|
|
1681
1692
|
ORDER BY commit_seq DESC LIMIT ?`)
|
|
1682
|
-
.bind(partition, query.afterSeq, query.limit)
|
|
1693
|
+
.bind(partition, query.afterSeq, beforeSeq, query.limit)
|
|
1683
1694
|
.all();
|
|
1684
1695
|
const out = [];
|
|
1696
|
+
// `changeSample`: extra statement(s) for the whole page — the first
|
|
1697
|
+
// `sample` changes (change-index order) of every listed commit.
|
|
1698
|
+
const sampleCount = query.changeSample !== undefined &&
|
|
1699
|
+
Number.isInteger(query.changeSample) &&
|
|
1700
|
+
query.changeSample > 0
|
|
1701
|
+
? query.changeSample
|
|
1702
|
+
: 0;
|
|
1703
|
+
const samples = new Map();
|
|
1704
|
+
if (sampleCount > 0 && rows.length > 0) {
|
|
1705
|
+
const seqs = rows.map((row) => row.commit_seq);
|
|
1706
|
+
// D1 caps the bound parameters of one statement, so a page larger than
|
|
1707
|
+
// that cap splits into a few chunk statements, never one per commit.
|
|
1708
|
+
const chunkSize = D1_MAX_BIND_PARAMS - 2;
|
|
1709
|
+
for (let offset = 0; offset < seqs.length; offset += chunkSize) {
|
|
1710
|
+
const chunk = seqs.slice(offset, offset + chunkSize);
|
|
1711
|
+
const { results: changes } = await this.#db
|
|
1712
|
+
.prepare(`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
1713
|
+
WHERE partition=? AND commit_seq IN (${placeholders(chunk.length)}) AND idx<?
|
|
1714
|
+
ORDER BY commit_seq, idx`)
|
|
1715
|
+
.bind(partition, ...chunk, sampleCount)
|
|
1716
|
+
.all();
|
|
1717
|
+
for (const change of changes) {
|
|
1718
|
+
const sample = samples.get(change.commit_seq) ?? [];
|
|
1719
|
+
sample.push({
|
|
1720
|
+
table: change.tbl,
|
|
1721
|
+
rowId: change.row_id,
|
|
1722
|
+
op: change.op === 1 ? 'upsert' : 'delete',
|
|
1723
|
+
});
|
|
1724
|
+
samples.set(change.commit_seq, sample);
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1685
1728
|
for (const row of rows) {
|
|
1686
1729
|
const { results: changes } = await this.#db
|
|
1687
1730
|
.prepare('SELECT tbl, count(*) AS n FROM sync_changes WHERE partition=? AND commit_seq=? GROUP BY tbl')
|
|
@@ -1695,10 +1738,95 @@ export class D1ServerStorage {
|
|
|
1695
1738
|
createdAtMs: row.created_at_ms,
|
|
1696
1739
|
changeCount: changes.reduce((sum, c) => sum + c.n, 0),
|
|
1697
1740
|
tables: changes.map((c) => c.tbl),
|
|
1741
|
+
...(sampleCount > 0
|
|
1742
|
+
? { sample: samples.get(row.commit_seq) ?? [] }
|
|
1743
|
+
: {}),
|
|
1698
1744
|
});
|
|
1699
1745
|
}
|
|
1700
1746
|
return out;
|
|
1701
1747
|
}
|
|
1748
|
+
async getCommit(partition, commitSeq) {
|
|
1749
|
+
const meta = await this.#db
|
|
1750
|
+
.prepare('SELECT client_id, client_commit_id, actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?')
|
|
1751
|
+
.bind(partition, commitSeq)
|
|
1752
|
+
.first();
|
|
1753
|
+
if (meta === null)
|
|
1754
|
+
return undefined;
|
|
1755
|
+
const { results: changeRows } = await this.#db
|
|
1756
|
+
.prepare('SELECT tbl, row_id, op, row_version, scopes, payload FROM sync_changes WHERE partition=? AND commit_seq=? ORDER BY idx')
|
|
1757
|
+
.bind(partition, commitSeq)
|
|
1758
|
+
.all();
|
|
1759
|
+
const changes = changeRows.map(toStoredChange);
|
|
1760
|
+
return {
|
|
1761
|
+
commitSeq,
|
|
1762
|
+
clientId: meta.client_id,
|
|
1763
|
+
clientCommitId: meta.client_commit_id,
|
|
1764
|
+
actorId: meta.actor_id,
|
|
1765
|
+
createdAtMs: meta.created_at_ms,
|
|
1766
|
+
changeCount: changes.length,
|
|
1767
|
+
tables: [...new Set(changes.map((change) => change.table))].sort(),
|
|
1768
|
+
changes,
|
|
1769
|
+
};
|
|
1770
|
+
}
|
|
1771
|
+
async listRowChanges(partition, query) {
|
|
1772
|
+
// Retained-log read for one row: no dedicated index (the write path pays
|
|
1773
|
+
// nothing for it), bounded by retention (§4.6).
|
|
1774
|
+
const { results: rows } = await this.#db
|
|
1775
|
+
.prepare(`SELECT ch.commit_seq, ch.op, ch.row_version, c.actor_id, c.client_id, c.created_at_ms
|
|
1776
|
+
FROM sync_changes ch
|
|
1777
|
+
JOIN sync_commits c
|
|
1778
|
+
ON c.partition=ch.partition AND c.commit_seq=ch.commit_seq
|
|
1779
|
+
WHERE ch.partition=? AND ch.tbl=? AND ch.row_id=?
|
|
1780
|
+
ORDER BY ch.commit_seq DESC, ch.idx DESC LIMIT ?`)
|
|
1781
|
+
.bind(partition, query.table, query.rowId, query.limit)
|
|
1782
|
+
.all();
|
|
1783
|
+
return rows.map((row) => ({
|
|
1784
|
+
commitSeq: row.commit_seq,
|
|
1785
|
+
op: row.op === 1 ? 'upsert' : 'delete',
|
|
1786
|
+
...(row.row_version !== null ? { rowVersion: row.row_version } : {}),
|
|
1787
|
+
actorId: row.actor_id,
|
|
1788
|
+
clientId: row.client_id,
|
|
1789
|
+
createdAtMs: row.created_at_ms,
|
|
1790
|
+
}));
|
|
1791
|
+
}
|
|
1792
|
+
async listRows(partition, query) {
|
|
1793
|
+
// Keyset page over the row table's (partition, rowId) primary key: an
|
|
1794
|
+
// index range read, no scope filter, no per-row lookup.
|
|
1795
|
+
const { results } = await this.#schemaDatabase()
|
|
1796
|
+
.prepare(`SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
1797
|
+
FROM ${quoteIdent(this.table(query.table).name)}
|
|
1798
|
+
WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>?
|
|
1799
|
+
ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ?`)
|
|
1800
|
+
.bind(partition, query.afterRowId ?? '', query.limit)
|
|
1801
|
+
.all();
|
|
1802
|
+
return results.map(toStoredRow);
|
|
1803
|
+
}
|
|
1804
|
+
async countRows(partition, table) {
|
|
1805
|
+
const record = await this.#schemaDatabase()
|
|
1806
|
+
.prepare(`SELECT count(*) AS n FROM ${quoteIdent(this.table(table).name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?`)
|
|
1807
|
+
.bind(partition)
|
|
1808
|
+
.first();
|
|
1809
|
+
return record?.n ?? 0;
|
|
1810
|
+
}
|
|
1811
|
+
async listPushResults(partition, clientCommitId) {
|
|
1812
|
+
// Retained outcomes for one clientCommitId: no dedicated index, ordered
|
|
1813
|
+
// by client so the read is deterministic.
|
|
1814
|
+
const { results } = await this.#schemaDatabase()
|
|
1815
|
+
.prepare('SELECT client_id, result FROM sync_push_results WHERE partition=? AND client_commit_id=? ORDER BY client_id')
|
|
1816
|
+
.bind(partition, clientCommitId)
|
|
1817
|
+
.all();
|
|
1818
|
+
return results.map((record) => {
|
|
1819
|
+
try {
|
|
1820
|
+
return {
|
|
1821
|
+
clientId: record.client_id,
|
|
1822
|
+
result: deserializePushResult(record.result),
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
catch {
|
|
1826
|
+
throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
|
|
1827
|
+
}
|
|
1828
|
+
});
|
|
1829
|
+
}
|
|
1702
1830
|
async scopeActivity(partition, query) {
|
|
1703
1831
|
const { results: rows } = await this.#db
|
|
1704
1832
|
.prepare(`SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
|
|
@@ -1728,19 +1856,6 @@ export class D1ServerStorage {
|
|
|
1728
1856
|
}
|
|
1729
1857
|
return out;
|
|
1730
1858
|
}
|
|
1731
|
-
async getRowScopes(partition, table, rowId) {
|
|
1732
|
-
const db = this.#schemaDatabase();
|
|
1733
|
-
const record = await db
|
|
1734
|
-
.prepare(selectRowScopesSql(this.table(table), 'sqlite'))
|
|
1735
|
-
.bind(partition, rowId)
|
|
1736
|
-
.first();
|
|
1737
|
-
if (record === null)
|
|
1738
|
-
return undefined;
|
|
1739
|
-
return {
|
|
1740
|
-
serverVersion: record.server_version,
|
|
1741
|
-
scopes: JSON.parse(record.scopes),
|
|
1742
|
-
};
|
|
1743
|
-
}
|
|
1744
1859
|
async listPartitions() {
|
|
1745
1860
|
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1746
1861
|
}
|
package/dist/events-ring.d.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* without any infrastructure dependency.
|
|
4
4
|
*
|
|
5
5
|
* `RingBufferEvents` is a `SyncularServerEvents` sink that retains the last
|
|
6
|
-
* N events in memory and exposes a `query({type?,
|
|
7
|
-
* actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
|
|
6
|
+
* N events in memory and exposes a `query({type?, partition?, sinceMs?,
|
|
7
|
+
* clientId?, actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
|
|
8
8
|
* composes with any other sink through `composeEvents(...sinks)`, so a host
|
|
9
9
|
* can keep `consoleJsonEvents()` (or a Sentry adapter) AND feed the console
|
|
10
10
|
* event tail from the same emissions. Fire-and-forget discipline is
|
|
@@ -15,6 +15,10 @@ import { type SyncularServerEvent, type SyncularServerEvents } from './events.js
|
|
|
15
15
|
export interface RingEventQuery {
|
|
16
16
|
/** Restrict to one event `type` (e.g. `push.applied`). */
|
|
17
17
|
readonly type?: SyncularServerEvent['type'];
|
|
18
|
+
/** Restrict to one partition (every event carries one). */
|
|
19
|
+
readonly partition?: string;
|
|
20
|
+
/** Restrict to the events of one sync request. */
|
|
21
|
+
readonly requestId?: string;
|
|
18
22
|
/** Only events with `atMs >= sinceMs`. */
|
|
19
23
|
readonly sinceMs?: number;
|
|
20
24
|
/** Restrict to events carrying this `clientId` (not every type does). */
|
|
@@ -56,6 +60,12 @@ export declare class RingBufferEvents implements SyncularServerEvents {
|
|
|
56
60
|
get capacity(): number;
|
|
57
61
|
/** The number of events currently retained. */
|
|
58
62
|
get size(): number;
|
|
63
|
+
/**
|
|
64
|
+
* The newest `atMs` among events the ring dropped (evicted when full, or
|
|
65
|
+
* cleared); `undefined` while nothing was dropped. A window that starts
|
|
66
|
+
* at or before it is incomplete.
|
|
67
|
+
*/
|
|
68
|
+
get evictedThroughMs(): number | undefined;
|
|
59
69
|
/**
|
|
60
70
|
* Newest-first slice of the retained events. Filters by `type` and
|
|
61
71
|
* `sinceMs`, then caps to `limit`. Returns a fresh array — the caller may
|
package/dist/events-ring.js
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* without any infrastructure dependency.
|
|
4
4
|
*
|
|
5
5
|
* `RingBufferEvents` is a `SyncularServerEvents` sink that retains the last
|
|
6
|
-
* N events in memory and exposes a `query({type?,
|
|
7
|
-
* actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
|
|
6
|
+
* N events in memory and exposes a `query({type?, partition?, sinceMs?,
|
|
7
|
+
* clientId?, actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
|
|
8
8
|
* composes with any other sink through `composeEvents(...sinks)`, so a host
|
|
9
9
|
* can keep `consoleJsonEvents()` (or a Sentry adapter) AND feed the console
|
|
10
10
|
* event tail from the same emissions. Fire-and-forget discipline is
|
|
@@ -21,6 +21,12 @@ import { emitEvent, } from './events.js';
|
|
|
21
21
|
export function matchesRingQuery(event, query) {
|
|
22
22
|
if (query.type !== undefined && event.type !== query.type)
|
|
23
23
|
return false;
|
|
24
|
+
if (query.partition !== undefined && event.partition !== query.partition) {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
if (query.requestId !== undefined && event.requestId !== query.requestId) {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
24
30
|
if (query.sinceMs !== undefined && event.atMs < query.sinceMs)
|
|
25
31
|
return false;
|
|
26
32
|
const carrier = event;
|
|
@@ -46,6 +52,7 @@ export class RingBufferEvents {
|
|
|
46
52
|
/** Index of the oldest element in the circular buffer. */
|
|
47
53
|
#head = 0;
|
|
48
54
|
#size = 0;
|
|
55
|
+
#evictedThroughMs;
|
|
49
56
|
/** Live subscribers (the SSE tail). Each notify is guarded. */
|
|
50
57
|
#listeners = new Set();
|
|
51
58
|
constructor(options) {
|
|
@@ -62,6 +69,9 @@ export class RingBufferEvents {
|
|
|
62
69
|
}
|
|
63
70
|
else {
|
|
64
71
|
// Full: overwrite the oldest and advance the head.
|
|
72
|
+
const evicted = this.#buffer[this.#head];
|
|
73
|
+
if (evicted !== undefined)
|
|
74
|
+
this.#markEvicted(evicted.atMs);
|
|
65
75
|
this.#buffer[this.#head] = event;
|
|
66
76
|
this.#head = (this.#head + 1) % this.#capacity;
|
|
67
77
|
}
|
|
@@ -94,6 +104,19 @@ export class RingBufferEvents {
|
|
|
94
104
|
get size() {
|
|
95
105
|
return this.#size;
|
|
96
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* The newest `atMs` among events the ring dropped (evicted when full, or
|
|
109
|
+
* cleared); `undefined` while nothing was dropped. A window that starts
|
|
110
|
+
* at or before it is incomplete.
|
|
111
|
+
*/
|
|
112
|
+
get evictedThroughMs() {
|
|
113
|
+
return this.#evictedThroughMs;
|
|
114
|
+
}
|
|
115
|
+
#markEvicted(atMs) {
|
|
116
|
+
if (this.#evictedThroughMs === undefined || atMs > this.#evictedThroughMs) {
|
|
117
|
+
this.#evictedThroughMs = atMs;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
97
120
|
/**
|
|
98
121
|
* Newest-first slice of the retained events. Filters by `type` and
|
|
99
122
|
* `sinceMs`, then caps to `limit`. Returns a fresh array — the caller may
|
|
@@ -115,6 +138,8 @@ export class RingBufferEvents {
|
|
|
115
138
|
}
|
|
116
139
|
/** Drop all retained events. */
|
|
117
140
|
clear() {
|
|
141
|
+
for (const event of this.#buffer)
|
|
142
|
+
this.#markEvicted(event.atMs);
|
|
118
143
|
this.#buffer = [];
|
|
119
144
|
this.#head = 0;
|
|
120
145
|
this.#size = 0;
|
package/dist/events.d.ts
CHANGED
|
@@ -28,11 +28,15 @@ export interface RequestHandledEvent {
|
|
|
28
28
|
/**
|
|
29
29
|
* `ok` — response streamed to END;
|
|
30
30
|
* `schema_floor` — §2.4 required-schema answer;
|
|
31
|
-
* `
|
|
31
|
+
* `epoch_acquired` — §2.1 answer to a request without `logEpoch` (a new
|
|
32
|
+
* or wiped replica's first round);
|
|
33
|
+
* `reset` — §2.1 answer to a `logEpoch` that no longer matches the log;
|
|
32
34
|
* `rejected` — request validation failed before any bytes (§1.7);
|
|
33
35
|
* `error` — in-band ERROR frame (§1.6) or a thrown host failure.
|
|
34
36
|
*/
|
|
35
|
-
readonly outcome: 'ok' | 'schema_floor' | 'reset' | 'rejected' | 'error';
|
|
37
|
+
readonly outcome: 'ok' | 'schema_floor' | 'epoch_acquired' | 'reset' | 'rejected' | 'error';
|
|
38
|
+
/** The REQ_HEADER clientId; absent when the request failed to decode. */
|
|
39
|
+
readonly clientId?: string;
|
|
36
40
|
/** §10.2 code for `rejected`/`error`; `"internal"` for non-SyncErrors. */
|
|
37
41
|
readonly errorCode?: string;
|
|
38
42
|
readonly pushCommits: number;
|
|
@@ -60,6 +64,9 @@ export interface PushRejectedEvent extends PushEventBase {
|
|
|
60
64
|
readonly type: 'push.rejected';
|
|
61
65
|
readonly code: string;
|
|
62
66
|
readonly opIndex: number;
|
|
67
|
+
/** The terminating operation's table and rowId. */
|
|
68
|
+
readonly table?: string;
|
|
69
|
+
readonly rowId?: string;
|
|
63
70
|
/** True when the rejection was replayed from the idempotency cache. */
|
|
64
71
|
readonly replay: boolean;
|
|
65
72
|
/** Original host time for outcomes recorded by a metadata-aware server. */
|
|
@@ -71,6 +78,9 @@ export interface PushRejectedEvent extends PushEventBase {
|
|
|
71
78
|
export interface PushConflictedEvent extends PushEventBase {
|
|
72
79
|
readonly type: 'push.conflicted';
|
|
73
80
|
readonly opIndex: number;
|
|
81
|
+
/** The conflicting operation's table and rowId. */
|
|
82
|
+
readonly table?: string;
|
|
83
|
+
readonly rowId?: string;
|
|
74
84
|
/** True when the conflict was replayed from the idempotency cache. */
|
|
75
85
|
readonly replay: boolean;
|
|
76
86
|
readonly recordedAtMs?: number;
|
|
@@ -277,7 +287,15 @@ export interface LeaseRevokedEvent {
|
|
|
277
287
|
readonly partition: string;
|
|
278
288
|
readonly leaseId: string;
|
|
279
289
|
}
|
|
280
|
-
|
|
290
|
+
/**
|
|
291
|
+
* Every event. `requestId` is present on the events emitted while serving
|
|
292
|
+
* one sync request (`request.handled`, `push.*`, `pull.served`,
|
|
293
|
+
* `lease.issued`, `reaction.queued`, `scopes.resolve_failed`) and shared by
|
|
294
|
+
* all of them, so a sink can group one request's events.
|
|
295
|
+
*/
|
|
296
|
+
export type SyncularServerEvent = (RequestHandledEvent | PushAppliedEvent | PushRejectedEvent | PushConflictedEvent | ReactionQueuedEvent | ReactionStartedEvent | ReactionRetriedEvent | ReactionCompletedEvent | ReactionDeadLetteredEvent | ReactionPruneCompletedEvent | PullServedEvent | SegmentDownloadedEvent | BlobUploadedEvent | BlobDownloadedEvent | BlobSweptEvent | RealtimeOpenedEvent | RealtimeClosedEvent | RealtimeDeltaEvent | RealtimeWakeEvent | PruneCompletedEvent | ScopesResolveFailedEvent | LeaseIssuedEvent | LeaseRevokedEvent) & {
|
|
297
|
+
readonly requestId?: string;
|
|
298
|
+
};
|
|
281
299
|
/**
|
|
282
300
|
* The seam. Optional on the server config — when absent, no event object
|
|
283
301
|
* is ever built. Implementations receive every event synchronously and
|
package/dist/handler.js
CHANGED
|
@@ -253,11 +253,14 @@ function emitPushEvent(events, ctx, clientId, push, processed) {
|
|
|
253
253
|
}
|
|
254
254
|
// Rejected (§6.3): exactly one terminating operation record.
|
|
255
255
|
const record = frame.results[0];
|
|
256
|
+
const op = record !== undefined ? push.operations[record.opIndex] : undefined;
|
|
257
|
+
const target = op !== undefined ? { table: op.table, rowId: op.rowId } : {};
|
|
256
258
|
if (record?.status === 'conflict') {
|
|
257
259
|
emitEvent(events, {
|
|
258
260
|
type: 'push.conflicted',
|
|
259
261
|
...base,
|
|
260
262
|
opIndex: record.opIndex,
|
|
263
|
+
...target,
|
|
261
264
|
replay: processed.replayed,
|
|
262
265
|
...(processed.recordedAtMs !== undefined
|
|
263
266
|
? { recordedAtMs: processed.recordedAtMs }
|
|
@@ -275,6 +278,7 @@ function emitPushEvent(events, ctx, clientId, push, processed) {
|
|
|
275
278
|
? record.code
|
|
276
279
|
: 'sync.invalid_request',
|
|
277
280
|
opIndex: record?.opIndex ?? 0,
|
|
281
|
+
...target,
|
|
278
282
|
replay: processed.replayed,
|
|
279
283
|
...(processed.recordedAtMs !== undefined
|
|
280
284
|
? { recordedAtMs: processed.recordedAtMs }
|
|
@@ -309,8 +313,10 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
309
313
|
: {}),
|
|
310
314
|
}, plan.wireVersion);
|
|
311
315
|
if (plan.epochReset) {
|
|
312
|
-
if (report !== undefined)
|
|
313
|
-
report.outcome =
|
|
316
|
+
if (report !== undefined) {
|
|
317
|
+
report.outcome =
|
|
318
|
+
plan.header.logEpoch === undefined ? 'epoch_acquired' : 'reset';
|
|
319
|
+
}
|
|
314
320
|
yield END_FRAME_BYTES;
|
|
315
321
|
return;
|
|
316
322
|
}
|
|
@@ -335,25 +341,35 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
335
341
|
}
|
|
336
342
|
try {
|
|
337
343
|
// Push half (§6): one PUSH_RESULT per PUSH_COMMIT, in request order.
|
|
344
|
+
const currentSchema = compileSchema(ctx.schema);
|
|
338
345
|
let acceptedThrough = 0;
|
|
339
346
|
for (const push of plan.pushes) {
|
|
340
|
-
const processed = await processPushCommitWithTrace(ctx,
|
|
347
|
+
const processed = await processPushCommitWithTrace(ctx, currentSchema, plan.resolved, plan.header.clientId, push, schema);
|
|
341
348
|
acceptedThrough = Math.max(acceptedThrough, processed.frame.commitSeq ?? 0);
|
|
342
349
|
const frame = {
|
|
343
350
|
...processed.frame,
|
|
344
351
|
results: processed.frame.results.map((result) => {
|
|
345
|
-
if (result.status !== 'conflict')
|
|
352
|
+
if (result.status !== 'conflict' || schema === currentSchema) {
|
|
346
353
|
return result;
|
|
354
|
+
}
|
|
347
355
|
const operation = push.operations[result.opIndex];
|
|
356
|
+
// §2.3: a replay returns the persisted result without rebuilding
|
|
357
|
+
// operations, so a reused clientCommitId with changed operations
|
|
358
|
+
// leaves no table to project the stored conflict row through.
|
|
359
|
+
if (operation === undefined) {
|
|
360
|
+
throw syncError('sync.invalid_request', 'replayed clientCommitId carries changed operations', JSON.stringify({
|
|
361
|
+
clientCommitId: push.clientCommitId,
|
|
362
|
+
opIndex: result.opIndex,
|
|
363
|
+
}));
|
|
364
|
+
}
|
|
348
365
|
const target = schema.tables.get(operation.table);
|
|
349
|
-
const
|
|
366
|
+
const bitmapBytes = Math.ceil(target.columns.length / 8);
|
|
350
367
|
return {
|
|
351
368
|
...result,
|
|
352
|
-
serverRow: projectRowPayload(
|
|
369
|
+
serverRow: projectRowPayload(currentSchema.tables.get(operation.table), target, result.serverRow),
|
|
353
370
|
conflictColumns: result.conflictColumns
|
|
354
|
-
.slice(0,
|
|
355
|
-
.map((byte, i) => i ===
|
|
356
|
-
target.columns.length % 8 !== 0
|
|
371
|
+
.slice(0, bitmapBytes)
|
|
372
|
+
.map((byte, i) => i === bitmapBytes - 1 && target.columns.length % 8 !== 0
|
|
357
373
|
? byte & ((1 << (target.columns.length % 8)) - 1)
|
|
358
374
|
: byte),
|
|
359
375
|
};
|
|
@@ -498,15 +514,21 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
498
514
|
* bytes are produced; adapters map them to HTTP JSON errors (§1.1).
|
|
499
515
|
*/
|
|
500
516
|
export async function createSyncResponseStream(bytes, ctx) {
|
|
501
|
-
const
|
|
502
|
-
if (
|
|
517
|
+
const sink = ctx.events;
|
|
518
|
+
if (sink === undefined)
|
|
503
519
|
return createStreamCore(bytes, ctx, undefined);
|
|
504
520
|
// Instrumented path: measure with the ctx clock, count bytes at the
|
|
505
|
-
// seam, emit exactly one `request.handled` per request.
|
|
521
|
+
// seam, emit exactly one `request.handled` per request. Every event of
|
|
522
|
+
// this request carries one `requestId`.
|
|
523
|
+
const requestId = crypto.randomUUID();
|
|
524
|
+
const events = {
|
|
525
|
+
emit: (event) => sink.emit({ ...event, requestId }),
|
|
526
|
+
};
|
|
527
|
+
const requestCtx = { ...ctx, events };
|
|
506
528
|
const clock = clockOf(ctx);
|
|
507
529
|
const startedAtMs = clock();
|
|
508
530
|
try {
|
|
509
|
-
return await createStreamCore(bytes,
|
|
531
|
+
return await createStreamCore(bytes, requestCtx, events, startedAtMs);
|
|
510
532
|
}
|
|
511
533
|
catch (error) {
|
|
512
534
|
emitEvent(events, {
|
|
@@ -617,6 +639,7 @@ async function* instrumentedStream(stream, ctx, events, plan, report, bytesIn, s
|
|
|
617
639
|
bytesIn,
|
|
618
640
|
bytesOut,
|
|
619
641
|
outcome: report.outcome,
|
|
642
|
+
clientId: plan.header.clientId,
|
|
620
643
|
...(report.errorCode !== undefined
|
|
621
644
|
? { errorCode: report.errorCode }
|
|
622
645
|
: {}),
|
package/dist/operations.js
CHANGED
|
@@ -4,7 +4,7 @@ import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPar
|
|
|
4
4
|
import { reportError, SyncError, syncError } from './errors.js';
|
|
5
5
|
import { processPushOperationsWithTrace } from './push.js';
|
|
6
6
|
import { compileSchema } from './schema.js';
|
|
7
|
-
import { authorizeWrite } from './scopes.js';
|
|
7
|
+
import { authorizeWrite, renderScopeValue, } from './scopes.js';
|
|
8
8
|
import { transactionQuery } from './storage-query.js';
|
|
9
9
|
import { toValidateRow } from './validate.js';
|
|
10
10
|
function scopeAllowed(allowed, variable, value) {
|
|
@@ -265,9 +265,9 @@ function commandOperations(mutations, schema) {
|
|
|
265
265
|
}
|
|
266
266
|
return mutation.values[column.name] ?? null;
|
|
267
267
|
});
|
|
268
|
-
const rowId = values[table.primaryKeyIndex];
|
|
269
|
-
if (
|
|
270
|
-
throw syncError('operation.invalid_request', 'command upsert requires a non-empty
|
|
268
|
+
const rowId = renderScopeValue(values[table.primaryKeyIndex]);
|
|
269
|
+
if (rowId === undefined) {
|
|
270
|
+
throw syncError('operation.invalid_request', 'command upsert requires a non-empty primary key');
|
|
271
271
|
}
|
|
272
272
|
return {
|
|
273
273
|
table: table.name,
|
|
@@ -39,7 +39,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
39
39
|
import type { ScopeMap } from '@syncular/core';
|
|
40
40
|
import { type PgExecutor } from './pg-executor.js';
|
|
41
41
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
42
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
42
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
43
43
|
/**
|
|
44
44
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
45
45
|
*
|
|
@@ -136,10 +136,11 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
136
136
|
listReferencedBlobIds(partition: string): Promise<string[]>;
|
|
137
137
|
listClientRecords(partition: string): Promise<ClientRecord[]>;
|
|
138
138
|
listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
139
|
+
getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
|
|
140
|
+
listRowChanges(partition: string, query: RowChangesQuery): Promise<RowChangeRecord[]>;
|
|
141
|
+
listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
|
|
142
|
+
countRows(partition: string, table: string): Promise<number>;
|
|
143
|
+
listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
139
144
|
scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
|
|
140
|
-
getRowScopes(partition: string, table: string, rowId: string): Promise<{
|
|
141
|
-
serverVersion: number;
|
|
142
|
-
scopes: Record<string, string>;
|
|
143
|
-
} | undefined>;
|
|
144
145
|
listPartitions(): Promise<string[]>;
|
|
145
146
|
}
|