@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/dist/sqlite-dialect.d.ts
CHANGED
|
@@ -14,11 +14,11 @@
|
|
|
14
14
|
* promises; the synchronous SQLite driver resolves them with rows it read at
|
|
15
15
|
* call time. Both classes run the identical `test/storage-contract.ts`.
|
|
16
16
|
*/
|
|
17
|
-
import type {
|
|
17
|
+
import type { UnitScopes } from '@syncular/core';
|
|
18
18
|
import { type BoundAuthoritativeQuery } from './authoritative-query.js';
|
|
19
19
|
import type { CompiledTable } from './schema.js';
|
|
20
20
|
import type { SqliteValue } from './sqlite-driver.js';
|
|
21
|
-
import type { AuthoritativeQueryRequest, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitPruneQuery, CommitPruneResult,
|
|
21
|
+
import type { AuthoritativeQueryRequest, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitPruneQuery, CommitPruneResult, IndexRowScanQuery, LiveWindow, LiveWindowQuery, NewCommit, OutboxSequence, OutboxSettlement, NewReaction, PartitionRegistryEntry, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, StoredChange, StoredPushResult, StoredReaction, StoredRow, Tombstone } from './storage.js';
|
|
22
22
|
/**
|
|
23
23
|
* Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
|
|
24
24
|
* the whole string via `db.exec(SQLITE_DDL)`; D1 applies each statement
|
|
@@ -32,19 +32,27 @@ import type { AuthoritativeQueryRequest, ClientCursorInfo, ClientPushResult, Cli
|
|
|
32
32
|
* the Postgres storage documents (§3.1, performance-by-
|
|
33
33
|
* construction).
|
|
34
34
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
|
|
38
|
-
* per value. The per-row scope replacement (`deleteSqliteRowScopesSql`)
|
|
39
|
-
* binds `(partition, tbl, var, value, row_id)` and needs no secondary index.
|
|
40
|
-
* `DROP INDEX IF EXISTS` stays in the DDL so existing databases lose the
|
|
41
|
-
* index the next time the DDL runs: on every `SqliteServerStorage`
|
|
42
|
-
* construction, and on D1 in `migrate()` and the core phase of
|
|
43
|
-
* `migrateSchema`.
|
|
35
|
+
* Changes to the core tables of an existing database (added and dropped
|
|
36
|
+
* columns and indexes) live in `sqliteUpgradeDdl`, which runs after this DDL.
|
|
44
37
|
*/
|
|
45
|
-
export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq INTEGER NOT NULL DEFAULT 0,\n horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required INTEGER NOT NULL DEFAULT 0,\n last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\
|
|
46
|
-
/**
|
|
47
|
-
|
|
38
|
+
export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq INTEGER NOT NULL DEFAULT 0,\n horizon_seq INTEGER NOT NULL DEFAULT 0,\n delete_floor INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required INTEGER NOT NULL DEFAULT 0,\n last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n writer_version INTEGER,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n outbox_id TEXT, client_seq INTEGER,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_push_watermarks(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,\n through_seq INTEGER NOT NULL,\n PRIMARY KEY(partition, client_id, outbox_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n last_failure TEXT,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor INTEGER NOT NULL,\n updated_at_ms INTEGER NOT NULL,\n last_round_at_ms INTEGER, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n schema_version INTEGER, units TEXT, registration_digest BLOB,\n catching_up TEXT NOT NULL DEFAULT '[]',\n PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq INTEGER NOT NULL, server_version INTEGER, scopes TEXT,\n PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark INTEGER NOT NULL DEFAULT 0,\n owner_epoch INTEGER NOT NULL DEFAULT 0,\n observed_rows INTEGER NOT NULL DEFAULT 0,\n updated_at_ms INTEGER NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
39
|
+
/**
|
|
40
|
+
* One statement that reads what `sqliteUpgradeDdl` needs: the columns of
|
|
41
|
+
* every table with added or dropped columns, and which added indexes exist.
|
|
42
|
+
*/
|
|
43
|
+
export declare const SQLITE_UPGRADE_PROBE: SqliteStatementSpec;
|
|
44
|
+
/**
|
|
45
|
+
* The statements that bring the core tables of a database from an earlier
|
|
46
|
+
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
47
|
+
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
48
|
+
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
49
|
+
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
50
|
+
* and the first open of a database at the current schema version.
|
|
51
|
+
*/
|
|
52
|
+
export declare function sqliteUpgradeDdl(present: readonly {
|
|
53
|
+
readonly tbl: string;
|
|
54
|
+
readonly name: string;
|
|
55
|
+
}[]): string[];
|
|
48
56
|
/** Split the DDL into individual statements (D1 applies them one by one). */
|
|
49
57
|
export declare function sqliteDdlStatements(): string[];
|
|
50
58
|
/** `?,?,…` for an `IN (…)` clause of `count` positional parameters. */
|
|
@@ -79,47 +87,6 @@ export interface SqliteTombstoneRecord {
|
|
|
79
87
|
}
|
|
80
88
|
export declare function toTombstone(record: SqliteTombstoneRecord): Tombstone;
|
|
81
89
|
export declare function toStoredChange(record: SqliteChangeRecord): StoredChange;
|
|
82
|
-
/**
|
|
83
|
-
* One result row of `commitWindowPageSql` (candidate LEFT JOIN commit meta
|
|
84
|
-
* LEFT JOIN changes): meta/change columns are NULL when the joined row
|
|
85
|
-
* vanished (see the builder's LEFT JOIN contract). `payload` is a BLOB —
|
|
86
|
-
* Native SQLite hands back `Uint8Array`, D1 `ArrayBuffer`; `toStoredChange`
|
|
87
|
-
* normalizes via `asUint8Array`.
|
|
88
|
-
*/
|
|
89
|
-
export interface SqliteCommitWindowRecord {
|
|
90
|
-
commit_seq: number;
|
|
91
|
-
actor_id: string | null;
|
|
92
|
-
created_at_ms: number | null;
|
|
93
|
-
tbl: string | null;
|
|
94
|
-
row_id: string | null;
|
|
95
|
-
op: number | null;
|
|
96
|
-
row_version: number | null;
|
|
97
|
-
scopes: string | null;
|
|
98
|
-
payload: Uint8Array | null;
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Fold one `commitWindowPageSql` page into commits, preserving the exact
|
|
102
|
-
* semantics of the old per-candidate loop:
|
|
103
|
-
*
|
|
104
|
-
* - rows arrive ordered (commit_seq, idx); consecutive rows with the same
|
|
105
|
-
* `commit_seq` are one candidate commit;
|
|
106
|
-
* - every candidate advances the cursor (`lastSeq`) — including vanished
|
|
107
|
-
* commits (NULL meta) and commits whose changes all fail the exact
|
|
108
|
-
* multi-variable scope verification (`matchesEffective`);
|
|
109
|
-
* - a commit is emitted only with its matching changes, in `idx` order;
|
|
110
|
-
* - stops after the commit that accumulates at least `remaining` matching
|
|
111
|
-
* changes (never splitting a commit).
|
|
112
|
-
*
|
|
113
|
-
* `candidateCount` counts the processed candidates so the caller can detect
|
|
114
|
-
* a short page (window exhausted) exactly as it did with the old candidate
|
|
115
|
-
* query.
|
|
116
|
-
*/
|
|
117
|
-
export declare function collectCommitWindowPage(records: readonly SqliteCommitWindowRecord[], scopeFilter: ScopeMap, remaining: number): {
|
|
118
|
-
commits: StoredCommit[];
|
|
119
|
-
delivered: number;
|
|
120
|
-
lastSeq: number;
|
|
121
|
-
candidateCount: number;
|
|
122
|
-
};
|
|
123
90
|
/** One statement of an atomic unit: its SQL text and bound parameters. */
|
|
124
91
|
export interface SqliteStatementSpec {
|
|
125
92
|
readonly sql: string;
|
|
@@ -172,16 +139,15 @@ export declare abstract class SqliteFamilyStorage {
|
|
|
172
139
|
pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
173
140
|
latestScopedChangeSeq(partition: string, query: {
|
|
174
141
|
readonly table: string;
|
|
175
|
-
readonly
|
|
142
|
+
readonly scopes: UnitScopes;
|
|
176
143
|
}): Promise<number>;
|
|
177
|
-
|
|
144
|
+
readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
|
|
178
145
|
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
179
146
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
180
147
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
181
148
|
putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
|
|
182
149
|
recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
|
|
183
150
|
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
184
|
-
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
185
151
|
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
186
152
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
187
153
|
listRowsReferencingBlob(partition: string, blobId: string): Promise<{
|
|
@@ -199,6 +165,7 @@ export declare abstract class SqliteFamilyStorage {
|
|
|
199
165
|
listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
|
|
200
166
|
countRows(partition: string, table: string): Promise<number>;
|
|
201
167
|
listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
168
|
+
listPushWatermarks(partition: string, clientCommitId: string): Promise<PushWatermark[]>;
|
|
202
169
|
scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
|
|
203
170
|
listPartitions(): Promise<string[]>;
|
|
204
171
|
}
|
|
@@ -233,6 +200,16 @@ export declare abstract class SqliteFamilyTransaction {
|
|
|
233
200
|
deleteRow(table: string, rowId: string): Promise<void>;
|
|
234
201
|
setBlobRefs(table: string, rowId: string, blobIds: readonly string[]): Promise<void>;
|
|
235
202
|
appendCommit(commit: NewCommit): Promise<number>;
|
|
236
|
-
putPushResult(clientId: string, clientCommitId: string, result: StoredPushResult): Promise<void>;
|
|
203
|
+
putPushResult(clientId: string, clientCommitId: string, result: StoredPushResult, sequence?: OutboxSequence): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* The §9 settlement and §6.3.2 steps 1 and 2 for one request, then the
|
|
206
|
+
* idempotency read of its first commit. The reads run before the writes,
|
|
207
|
+
* so D1, which buffers the writes until commit, answers exactly as SQLite:
|
|
208
|
+
* the first commit's result counts as deleted when the writes delete it.
|
|
209
|
+
*/
|
|
210
|
+
protected settleOutbox(clientId: string, clientCommitId: string, outbox: OutboxSettlement): Promise<{
|
|
211
|
+
readonly watermark: number;
|
|
212
|
+
readonly persisted: StoredPushResult | undefined;
|
|
213
|
+
}>;
|
|
237
214
|
enqueueReactions(reactions: readonly NewReaction[]): Promise<void>;
|
|
238
215
|
}
|