@syncular/server 0.94.0 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/context.d.ts +4 -10
- package/dist/errors.js +8 -0
- package/dist/frame-bytes.js +1 -0
- package/dist/handler.d.ts +20 -2
- package/dist/handler.js +230 -57
- package/dist/postgres-storage.d.ts +3 -2
- package/dist/postgres-storage.js +18 -2
- package/dist/pull.d.ts +11 -3
- package/dist/pull.js +28 -21
- package/dist/push.js +41 -77
- package/dist/realtime.d.ts +5 -11
- package/dist/realtime.js +10 -23
- package/dist/relational-rows.d.ts +4 -2
- package/dist/relational-rows.js +8 -3
- package/dist/schema-window.js +2 -2
- package/dist/schema.d.ts +2 -2
- package/dist/schema.js +2 -2
- package/dist/sqlite-dialect.d.ts +3 -1
- package/dist/sqlite-dialect.js +37 -5
- package/dist/storage.d.ts +12 -0
- package/dist/validate.d.ts +21 -39
- package/dist/validate.js +4 -3
- package/package.json +2 -2
- package/src/context.ts +4 -14
- package/src/errors.ts +8 -0
- package/src/frame-bytes.ts +1 -0
- package/src/handler.ts +286 -73
- package/src/postgres-storage.ts +32 -6
- package/src/pull.ts +62 -37
- package/src/push.ts +51 -129
- package/src/realtime.ts +10 -22
- package/src/relational-rows.ts +10 -3
- package/src/schema-window.ts +2 -2
- package/src/schema.ts +4 -4
- package/src/sqlite-dialect.ts +47 -6
- package/src/storage.ts +13 -0
- package/src/validate.ts +21 -48
package/src/schema.ts
CHANGED
|
@@ -154,8 +154,8 @@ export interface CompiledTable {
|
|
|
154
154
|
* image copies ciphertext wholesale with no per-row decrypt pass, so an
|
|
155
155
|
* encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
|
|
156
156
|
readonly encryptedColumnIndices: readonly number[];
|
|
157
|
-
/** §6.
|
|
158
|
-
*
|
|
157
|
+
/** §6.7: column indices marked `serverOwned`. Empty ⇒ no server column
|
|
158
|
+
* values are accepted from the table's validator. */
|
|
159
159
|
readonly serverOwnedColumnIndices: readonly number[];
|
|
160
160
|
}
|
|
161
161
|
|
|
@@ -404,7 +404,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
404
404
|
if (column.type === 'blob_ref') blobRefColumnIndices.push(index);
|
|
405
405
|
// §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
|
|
406
406
|
if (column.encrypted === true) encryptedColumnIndices.push(index);
|
|
407
|
-
// §6.
|
|
407
|
+
// §6.7: a validator never changes who may write the row (owner set), and
|
|
408
408
|
// a client's optimistic row holds NULL until the server row arrives.
|
|
409
409
|
if (column.serverOwned === true) {
|
|
410
410
|
if (
|
|
@@ -419,7 +419,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
419
419
|
)
|
|
420
420
|
) {
|
|
421
421
|
throw new Error(
|
|
422
|
-
`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.
|
|
422
|
+
`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.7)`,
|
|
423
423
|
);
|
|
424
424
|
}
|
|
425
425
|
serverOwnedColumnIndices.push(index);
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -46,8 +46,10 @@ import type { CompiledTable } from './schema';
|
|
|
46
46
|
import { matchesEffective } from './scopes';
|
|
47
47
|
import type { SqliteValue } from './sqlite-driver';
|
|
48
48
|
import {
|
|
49
|
+
asJson,
|
|
49
50
|
type ClientRowRecord,
|
|
50
51
|
CLIENT_RECORD_COLUMNS,
|
|
52
|
+
type PartitionHorizons,
|
|
51
53
|
PARTITION_TOUCH_INTERVAL_MS,
|
|
52
54
|
pushResultFromJson,
|
|
53
55
|
pushResultToJson,
|
|
@@ -257,6 +259,20 @@ const SQLITE_ADDED_COLUMNS = [
|
|
|
257
259
|
['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
|
|
258
260
|
] as const;
|
|
259
261
|
|
|
262
|
+
/**
|
|
263
|
+
* Core tables that `sqliteUpgradeDdl` creates, keyed by one of their columns,
|
|
264
|
+
* so a D1 database at an unchanged schema version gains them at its first
|
|
265
|
+
* open. `sync_table_horizons` holds the §4.6 table horizons.
|
|
266
|
+
*/
|
|
267
|
+
const SQLITE_ADDED_TABLES = [
|
|
268
|
+
[
|
|
269
|
+
'sync_table_horizons',
|
|
270
|
+
'commit_seq',
|
|
271
|
+
// One line: D1's `exec` runs each line as a statement.
|
|
272
|
+
'CREATE TABLE IF NOT EXISTS sync_table_horizons(partition TEXT NOT NULL, tbl TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(partition, tbl))',
|
|
273
|
+
],
|
|
274
|
+
] as const;
|
|
275
|
+
|
|
260
276
|
/** Core-table columns wire version 4 deleted: the version 3 subscription list. */
|
|
261
277
|
const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']] as const;
|
|
262
278
|
|
|
@@ -294,9 +310,11 @@ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
|
|
|
294
310
|
params: [
|
|
295
311
|
JSON.stringify([
|
|
296
312
|
...new Set(
|
|
297
|
-
[
|
|
298
|
-
|
|
299
|
-
|
|
313
|
+
[
|
|
314
|
+
...SQLITE_ADDED_COLUMNS,
|
|
315
|
+
...SQLITE_DROPPED_COLUMNS,
|
|
316
|
+
...SQLITE_ADDED_TABLES,
|
|
317
|
+
].map(([table]) => table),
|
|
300
318
|
),
|
|
301
319
|
]),
|
|
302
320
|
JSON.stringify([
|
|
@@ -309,7 +327,7 @@ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
|
|
|
309
327
|
/**
|
|
310
328
|
* The statements that bring the core tables of a database from an earlier
|
|
311
329
|
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
312
|
-
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
330
|
+
* tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
313
331
|
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
314
332
|
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
315
333
|
* and the first open of a database at the current schema version.
|
|
@@ -320,6 +338,9 @@ export function sqliteUpgradeDdl(
|
|
|
320
338
|
const has = (table: string, name: string) =>
|
|
321
339
|
present.some((row) => row.tbl === table && row.name === name);
|
|
322
340
|
return [
|
|
341
|
+
...SQLITE_ADDED_TABLES.filter(([table, name]) => !has(table, name)).map(
|
|
342
|
+
([, , ddl]) => ddl,
|
|
343
|
+
),
|
|
323
344
|
...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(
|
|
324
345
|
([table, name, definition]) =>
|
|
325
346
|
`ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
|
|
@@ -605,6 +626,20 @@ export abstract class SqliteFamilyStorage {
|
|
|
605
626
|
return row?.horizon_seq ?? 0;
|
|
606
627
|
}
|
|
607
628
|
|
|
629
|
+
async getHorizons(partition: string): Promise<PartitionHorizons> {
|
|
630
|
+
const [row] = await this.all<{ seq: number; tables: string }>(
|
|
631
|
+
`SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=?),0) AS seq,
|
|
632
|
+
(SELECT json_group_object(tbl, commit_seq) FROM sync_table_horizons WHERE partition=?) AS tables`,
|
|
633
|
+
[partition, partition],
|
|
634
|
+
);
|
|
635
|
+
return {
|
|
636
|
+
seq: row?.seq ?? 0,
|
|
637
|
+
tables: new Map(
|
|
638
|
+
Object.entries(asJson<Record<string, number>>(row?.tables ?? '{}')),
|
|
639
|
+
),
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
|
|
608
643
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
609
644
|
await this.all(
|
|
610
645
|
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
@@ -649,8 +684,14 @@ export abstract class SqliteFamilyStorage {
|
|
|
649
684
|
AND ${epochGuard}`,
|
|
650
685
|
params: [partition, partition, partition, query.logEpoch],
|
|
651
686
|
},
|
|
652
|
-
// §4.6: tombstones prune in the same pass as the
|
|
653
|
-
|
|
687
|
+
// §4.6: tombstones and table horizons prune in the same pass as the
|
|
688
|
+
// commit log.
|
|
689
|
+
...[
|
|
690
|
+
'sync_commits',
|
|
691
|
+
'sync_changes',
|
|
692
|
+
'sync_tombstones',
|
|
693
|
+
'sync_table_horizons',
|
|
694
|
+
].map((table) => ({
|
|
654
695
|
sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
|
|
655
696
|
params: [partition, partition, partition, query.logEpoch],
|
|
656
697
|
})),
|
package/src/storage.ts
CHANGED
|
@@ -792,6 +792,17 @@ export function toClientRecord(record: ClientRowRecord): ClientRecord {
|
|
|
792
792
|
};
|
|
793
793
|
}
|
|
794
794
|
|
|
795
|
+
/** A partition's pruning horizon and its table horizons (§4.6). */
|
|
796
|
+
export interface PartitionHorizons {
|
|
797
|
+
/** `horizonSeq`. */
|
|
798
|
+
readonly seq: number;
|
|
799
|
+
/**
|
|
800
|
+
* Re-keyed table → the commit sequence number its re-key took. The changes
|
|
801
|
+
* of a table at or below it are never served.
|
|
802
|
+
*/
|
|
803
|
+
readonly tables: ReadonlyMap<string, number>;
|
|
804
|
+
}
|
|
805
|
+
|
|
795
806
|
/** One unit of the §4.5 scan set. */
|
|
796
807
|
export interface LiveScanUnit {
|
|
797
808
|
/** The unit's scope map; its first variable (keys sorted) is its index key. */
|
|
@@ -1313,6 +1324,8 @@ export interface ServerStorage {
|
|
|
1313
1324
|
|
|
1314
1325
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
1315
1326
|
getHorizonSeq(partition: string): Promise<number>;
|
|
1327
|
+
/** `horizonSeq` and the table horizons (§4.6) in one read. */
|
|
1328
|
+
getHorizons(partition: string): Promise<PartitionHorizons>;
|
|
1316
1329
|
/** Monotonic within the current epoch; use atomic pruning for maintenance. */
|
|
1317
1330
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
1318
1331
|
/**
|
package/src/validate.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
* row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
|
|
6
6
|
* commit transaction, once per operation. It is the seam for business
|
|
7
7
|
* rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
|
|
8
|
-
* "status ∈ {…}")
|
|
9
|
-
* atomically (§6.4) with a
|
|
10
|
-
* in its rejection record
|
|
8
|
+
* "status ∈ {…}") and for the values of server-owned columns. A throw (or
|
|
9
|
+
* rejected promise) rejects the whole commit atomically (§6.4) with a
|
|
10
|
+
* host-defined code the client surfaces unchanged in its rejection record
|
|
11
|
+
* (§6.3).
|
|
11
12
|
*
|
|
12
13
|
* The feature is OFF by default (no `validators` on the config): the push
|
|
13
14
|
* path pays only an `undefined` check per operation and builds no context
|
|
@@ -55,8 +56,9 @@ export type ValidateOpKind = 'upsert' | 'delete';
|
|
|
55
56
|
|
|
56
57
|
/**
|
|
57
58
|
* One push operation as the validator sees it (§6.7). For an `upsert`,
|
|
58
|
-
* `row` is the post-merge, post-scope-strip row that will be written
|
|
59
|
-
*
|
|
59
|
+
* `row` is the post-merge, post-scope-strip row that will be written, with
|
|
60
|
+
* each server-owned column as stored (NULL on an insert), and `stored` is
|
|
61
|
+
* the prior stored row (or `undefined` on insert). For a
|
|
60
62
|
* `delete`, `row` is `undefined` and `stored` is the row about to be
|
|
61
63
|
* removed (a delete only reaches a validator when a stored row exists —
|
|
62
64
|
* an absent-row delete is an idempotent no-op that never validates, §6.2).
|
|
@@ -188,7 +190,7 @@ export type CommitValidator = (
|
|
|
188
190
|
* inside and the push never completes, and on a pool-backed executor (`pg`,
|
|
189
191
|
* `Bun.sql`) it reads committed state without the staged writes, or waits
|
|
190
192
|
* forever once push transactions hold every pool connection. An authority
|
|
191
|
-
* read
|
|
193
|
+
* read goes through `read` instead.
|
|
192
194
|
*/
|
|
193
195
|
export interface ValidateContext {
|
|
194
196
|
/** Host-authenticated actor (§1.1) performing the write. */
|
|
@@ -198,17 +200,13 @@ export interface ValidateContext {
|
|
|
198
200
|
/** The context `authenticate` returned for this request (§1.1); absent without one. */
|
|
199
201
|
readonly context?: string;
|
|
200
202
|
/**
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
* PostgreSQL return the transaction's snapshot, including the operations
|
|
205
|
-
* staged before this one; D1 fails with
|
|
203
|
+
* The §6.8 candidate-state reader of the push transaction: it sees every
|
|
204
|
+
* operation staged before this one. `read.queryAuthoritative` runs a
|
|
205
|
+
* generated registered query on that transaction; D1 fails with
|
|
206
206
|
* `sync.storage.query_over_staged_writes` when the query reads a table the
|
|
207
207
|
* commit has written.
|
|
208
208
|
*/
|
|
209
|
-
readonly
|
|
210
|
-
query: AuthoritativeQueryRequest,
|
|
211
|
-
) => Promise<AuthoritativeQueryResult>;
|
|
209
|
+
readonly read: CommitValidationReader;
|
|
212
210
|
}
|
|
213
211
|
|
|
214
212
|
/**
|
|
@@ -220,8 +218,15 @@ export interface ValidateContext {
|
|
|
220
218
|
* `sync.constraint_violation` code (a validator SHOULD throw
|
|
221
219
|
* `ValidationRejection` to control the code). MUST NOT mutate the row.
|
|
222
220
|
*
|
|
221
|
+
* On an insert or update of a pushed commit or a registered command, the
|
|
222
|
+
* validator may return values for the table's server-owned columns; a column
|
|
223
|
+
* it omits keeps its value in `row`. A key that names no server-owned column,
|
|
224
|
+
* a value the column's type refuses, or a changed scope column on an update
|
|
225
|
+
* rejects the commit with `sync.constraint_violation`. The server ignores
|
|
226
|
+
* returned values on a delete, a rescope (§2.2) and a §6.11 cascade.
|
|
227
|
+
*
|
|
223
228
|
* The hook runs inside the push transaction. An authority read uses
|
|
224
|
-
* `ctx.queryAuthoritative`, which executes on that transaction; calling
|
|
229
|
+
* `ctx.read.queryAuthoritative`, which executes on that transaction; calling
|
|
225
230
|
* `ServerStorage.queryAuthoritative` from here deadlocks on a
|
|
226
231
|
* single-connection executor and reads committed state on a pool-backed one.
|
|
227
232
|
* A rule over sibling operations belongs in the whole-commit
|
|
@@ -230,7 +235,7 @@ export interface ValidateContext {
|
|
|
230
235
|
export type Validator = (
|
|
231
236
|
op: ValidateOperation,
|
|
232
237
|
ctx: ValidateContext,
|
|
233
|
-
) => void | Promise<void>;
|
|
238
|
+
) => ValidateRow | void | Promise<ValidateRow | void>;
|
|
234
239
|
|
|
235
240
|
/** Table name → validator (§6.7). Absent tables are unvalidated. */
|
|
236
241
|
export type ValidatorRegistry = Readonly<Record<string, Validator>>;
|
|
@@ -315,35 +320,3 @@ export function toValidateRow(
|
|
|
315
320
|
});
|
|
316
321
|
return row;
|
|
317
322
|
}
|
|
318
|
-
|
|
319
|
-
/** §6.12: the accepted upsert a server column hook completes. `row` is the
|
|
320
|
-
* merged row (present columns over stored values, crdt columns merged) with
|
|
321
|
-
* every server-owned column as stored, or NULL on an insert. */
|
|
322
|
-
export interface ServerColumnsInput {
|
|
323
|
-
readonly table: string;
|
|
324
|
-
readonly rowId: string;
|
|
325
|
-
readonly row: ValidateRow;
|
|
326
|
-
readonly stored: ValidateRow | undefined;
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
export interface ServerColumnsContext {
|
|
330
|
-
readonly actorId: string;
|
|
331
|
-
readonly partition: string;
|
|
332
|
-
readonly context?: string;
|
|
333
|
-
/** The §6.8 candidate-state reader of the push transaction. */
|
|
334
|
-
readonly read: CommitValidationReader;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
/**
|
|
338
|
-
* §6.12: returns values for the table's server-owned columns; a column it
|
|
339
|
-
* omits keeps its value in `row`. A key that names no server-owned column, a
|
|
340
|
-
* value the column's type refuses, or a changed scope column on an update
|
|
341
|
-
* rejects the commit with `sync.constraint_violation`, like any other throw
|
|
342
|
-
* that is not a {@link ValidationRejection}.
|
|
343
|
-
*/
|
|
344
|
-
export type ServerColumnsHook = (
|
|
345
|
-
input: ServerColumnsInput,
|
|
346
|
-
ctx: ServerColumnsContext,
|
|
347
|
-
) => ValidateRow | Promise<ValidateRow>;
|
|
348
|
-
|
|
349
|
-
export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;
|