@syncular/server 0.95.0 → 0.97.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/context.d.ts +4 -16
- package/dist/context.js +0 -3
- package/dist/d1-storage.js +1 -1
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +36 -0
- package/dist/frame-bytes.js +1 -0
- package/dist/handler.d.ts +20 -2
- package/dist/handler.js +199 -59
- package/dist/postgres-storage.d.ts +3 -2
- package/dist/postgres-storage.js +18 -2
- package/dist/pull.d.ts +2 -2
- package/dist/pull.js +22 -16
- package/dist/push.js +41 -77
- package/dist/realtime.d.ts +5 -11
- package/dist/realtime.js +10 -23
- package/dist/relational-rows.d.ts +23 -13
- package/dist/relational-rows.js +49 -30
- package/dist/schema-window.js +2 -2
- package/dist/schema.d.ts +2 -2
- package/dist/schema.js +2 -2
- package/dist/segment-download.js +11 -4
- package/dist/sqlite-dialect.d.ts +3 -1
- package/dist/sqlite-dialect.js +37 -6
- package/dist/storage-query.d.ts +5 -2
- package/dist/storage-query.js +7 -4
- package/dist/storage.d.ts +12 -0
- package/dist/validate.d.ts +21 -39
- package/dist/validate.js +4 -3
- package/package.json +2 -2
- package/src/context.ts +4 -23
- package/src/d1-storage.ts +0 -1
- package/src/errors.ts +46 -0
- package/src/frame-bytes.ts +1 -0
- package/src/handler.ts +257 -73
- package/src/postgres-storage.ts +32 -6
- package/src/pull.ts +50 -35
- package/src/push.ts +51 -129
- package/src/realtime.ts +10 -22
- package/src/relational-rows.ts +51 -30
- package/src/schema-window.ts +2 -2
- package/src/schema.ts +4 -4
- package/src/segment-download.ts +19 -12
- package/src/sqlite-dialect.ts +47 -7
- package/src/storage-query.ts +7 -4
- package/src/storage.ts +13 -0
- package/src/validate.ts +21 -48
package/src/relational-rows.ts
CHANGED
|
@@ -675,12 +675,22 @@ export function indexRowPageStatement(
|
|
|
675
675
|
|
|
676
676
|
/**
|
|
677
677
|
* One-round-trip page scan for `scanRows`: candidates from the inverted
|
|
678
|
-
* scope index
|
|
679
|
-
*
|
|
680
|
-
*
|
|
681
|
-
*
|
|
682
|
-
*
|
|
683
|
-
*
|
|
678
|
+
* scope index LEFT JOINed to the row table, so a whole page arrives in ONE
|
|
679
|
+
* statement instead of one lookup per candidate. That per-candidate lookup
|
|
680
|
+
* was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
|
|
681
|
+
* ~100k network round-trips (~10 s at 0.1 ms each) before this join.
|
|
682
|
+
*
|
|
683
|
+
* The candidates read at most `limit` entries of the covering
|
|
684
|
+
* `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
|
|
685
|
+
* `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
|
|
686
|
+
* cannot prove a single value, and then every entry after the cursor is read
|
|
687
|
+
* and sorted before the LIMIT: a cost that grows with the entries left, so a
|
|
688
|
+
* bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
|
|
689
|
+
* value through a LATERAL join and merges them. SQLite has no LATERAL; it
|
|
690
|
+
* bounds the ranges from above instead: the smallest of the values' `limit`th
|
|
691
|
+
* entries after the cursor. That value alone has `limit` entries up to its
|
|
692
|
+
* bound, so the page ends there; a value with fewer entries has no bound, and
|
|
693
|
+
* when no value has one the empty blob `x''` sorts after every TEXT row id.
|
|
684
694
|
*
|
|
685
695
|
* LEFT JOIN (not INNER): a candidate whose row vanished (index entry
|
|
686
696
|
* without a row) must still reach the caller — its `row_id` advances the
|
|
@@ -688,11 +698,9 @@ export function indexRowPageStatement(
|
|
|
688
698
|
* disappearing from the page.
|
|
689
699
|
*
|
|
690
700
|
* Bind order:
|
|
691
|
-
* - postgres: [partition, tbl, var, ...values, afterRowId, limit]
|
|
692
|
-
* (the join reuses `$1` for the partition);
|
|
701
|
+
* - postgres: [partition, tbl, var, ...values, afterRowId, limit];
|
|
693
702
|
* - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
|
|
694
|
-
* limit
|
|
695
|
-
* join). The values expand through `json_each`, so the statement text and
|
|
703
|
+
* limit]. The values expand through `json_each`, so the statement text and
|
|
696
704
|
* parameter count do not depend on `valueCount` and D1's 100-parameter
|
|
697
705
|
* cap is unreachable through scope values.
|
|
698
706
|
*/
|
|
@@ -701,21 +709,26 @@ export function scanRowPageSql(
|
|
|
701
709
|
valueCount: number,
|
|
702
710
|
dialect: RelationalDialect,
|
|
703
711
|
): string {
|
|
704
|
-
const
|
|
705
|
-
const
|
|
712
|
+
const after = dialect === 'sqlite' ? '?5' : `$${4 + valueCount}`;
|
|
713
|
+
const limit = dialect === 'sqlite' ? '?6' : `$${5 + valueCount}`;
|
|
714
|
+
const candidates =
|
|
706
715
|
dialect === 'sqlite'
|
|
707
|
-
?
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
716
|
+
? `SELECT DISTINCT row_id FROM sync_row_scopes
|
|
717
|
+
WHERE partition=?1 AND tbl=?2 AND var=?3 AND value IN (SELECT value FROM json_each(?4))
|
|
718
|
+
AND row_id>?5
|
|
719
|
+
AND row_id<=coalesce((SELECT min((SELECT row_id FROM sync_row_scopes
|
|
720
|
+
WHERE partition=?1 AND tbl=?2 AND var=?3 AND value=v.value AND row_id>?5
|
|
721
|
+
ORDER BY row_id LIMIT 1 OFFSET ?6-1)) FROM json_each(?4) v), x'')
|
|
722
|
+
ORDER BY row_id LIMIT ?6`
|
|
723
|
+
: `SELECT DISTINCT x.row_id FROM (VALUES ${Array.from({ length: valueCount }, (_, i) => `($${4 + i})`).join(',')}) v(value)
|
|
724
|
+
CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
|
|
725
|
+
WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=v.value AND row_id>${after}
|
|
726
|
+
ORDER BY row_id LIMIT ${limit}) x
|
|
727
|
+
ORDER BY x.row_id LIMIT ${limit}`;
|
|
712
728
|
return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
713
|
-
FROM (
|
|
714
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
715
|
-
AND row_id>${after}
|
|
716
|
-
ORDER BY row_id LIMIT ${limit}) c
|
|
729
|
+
FROM (${candidates}) c
|
|
717
730
|
LEFT JOIN ${quoteIdent(table.name)} r
|
|
718
|
-
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${
|
|
731
|
+
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?1' : '$1'} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
|
|
719
732
|
ORDER BY c.row_id`;
|
|
720
733
|
}
|
|
721
734
|
|
|
@@ -774,7 +787,7 @@ export function scanProbeSql(dialect: RelationalDialect): string {
|
|
|
774
787
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
775
788
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
|
776
789
|
* array of `{i, tbl, var, vals, after, lim}` records; each record runs the
|
|
777
|
-
* same
|
|
790
|
+
* same per-value candidate ranges through a LATERAL join, so every
|
|
778
791
|
* subscription keeps its own keyset cursor and LIMIT. The candidate side is
|
|
779
792
|
* table-agnostic (a spec names its table), so the statement joins the row
|
|
780
793
|
* table once per table present in the batch through a UNION ALL branch that
|
|
@@ -800,10 +813,11 @@ export function scanRowBatchSql(tables: readonly CompiledTable[]): string {
|
|
|
800
813
|
return `WITH cand AS (
|
|
801
814
|
SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
|
|
802
815
|
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
|
|
803
|
-
CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM
|
|
804
|
-
|
|
805
|
-
AND row_id>s.after
|
|
806
|
-
|
|
816
|
+
CROSS JOIN LATERAL (SELECT DISTINCT x.row_id FROM unnest(s.vals) v(value)
|
|
817
|
+
CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
|
|
818
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value AND row_id>s.after
|
|
819
|
+
ORDER BY row_id LIMIT s.lim) x
|
|
820
|
+
ORDER BY x.row_id LIMIT s.lim) c)
|
|
807
821
|
${projection}
|
|
808
822
|
ORDER BY sub, row_id`;
|
|
809
823
|
}
|
|
@@ -1010,8 +1024,10 @@ export function assertStoredScopePatterns(
|
|
|
1010
1024
|
|
|
1011
1025
|
/**
|
|
1012
1026
|
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
1013
|
-
* entries and tombstones
|
|
1014
|
-
*
|
|
1027
|
+
* entries and tombstones. The re-key takes a commit sequence number of its
|
|
1028
|
+
* own in every partition and records it as each re-keyed table's horizon
|
|
1029
|
+
* (§4.6), past the changes logged under the previous patterns. Empty when
|
|
1030
|
+
* nothing re-keys.
|
|
1015
1031
|
*/
|
|
1016
1032
|
export function rekeyCleanupSql(tables: readonly string[]): string[] {
|
|
1017
1033
|
if (tables.length === 0) return [];
|
|
@@ -1019,7 +1035,12 @@ export function rekeyCleanupSql(tables: readonly string[]): string[] {
|
|
|
1019
1035
|
return [
|
|
1020
1036
|
`DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
|
|
1021
1037
|
`DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
|
|
1022
|
-
'UPDATE sync_partitions SET
|
|
1038
|
+
'UPDATE sync_partitions SET max_commit_seq=max_commit_seq+1',
|
|
1039
|
+
...names.map(
|
|
1040
|
+
(name) => `INSERT INTO sync_table_horizons(partition, tbl, commit_seq)
|
|
1041
|
+
SELECT partition, ${name}, max_commit_seq FROM sync_partitions WHERE true
|
|
1042
|
+
ON CONFLICT(partition, tbl) DO UPDATE SET commit_seq=excluded.commit_seq`,
|
|
1043
|
+
),
|
|
1023
1044
|
];
|
|
1024
1045
|
}
|
|
1025
1046
|
|
package/src/schema-window.ts
CHANGED
|
@@ -31,7 +31,7 @@ export function schemaWindowOf(
|
|
|
31
31
|
if (
|
|
32
32
|
next === undefined ||
|
|
33
33
|
table.columns.length > next.columns.length ||
|
|
34
|
-
// §6.
|
|
34
|
+
// §6.7: a version may mark an existing column server-owned; the flag
|
|
35
35
|
// never reaches the wire.
|
|
36
36
|
table.columns.some(
|
|
37
37
|
(column, i) =>
|
|
@@ -42,7 +42,7 @@ export function schemaWindowOf(
|
|
|
42
42
|
.slice(table.columns.length)
|
|
43
43
|
.some((column) => !column.nullable) ||
|
|
44
44
|
table.primaryKeyIndex !== next.primaryKeyIndex ||
|
|
45
|
-
// §6.
|
|
45
|
+
// §6.7: a pattern over an appended server-owned column is new to the
|
|
46
46
|
// current version; an older client neither holds nor writes it.
|
|
47
47
|
JSON.stringify(table.scopePatterns) !==
|
|
48
48
|
JSON.stringify(
|
package/src/schema.ts
CHANGED
|
@@ -154,8 +154,8 @@ export interface CompiledTable {
|
|
|
154
154
|
* image copies ciphertext wholesale with no per-row decrypt pass, so an
|
|
155
155
|
* encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
|
|
156
156
|
readonly encryptedColumnIndices: readonly number[];
|
|
157
|
-
/** §6.
|
|
158
|
-
*
|
|
157
|
+
/** §6.7: column indices marked `serverOwned`. Empty ⇒ no server column
|
|
158
|
+
* values are accepted from the table's validator. */
|
|
159
159
|
readonly serverOwnedColumnIndices: readonly number[];
|
|
160
160
|
}
|
|
161
161
|
|
|
@@ -404,7 +404,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
404
404
|
if (column.type === 'blob_ref') blobRefColumnIndices.push(index);
|
|
405
405
|
// §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
|
|
406
406
|
if (column.encrypted === true) encryptedColumnIndices.push(index);
|
|
407
|
-
// §6.
|
|
407
|
+
// §6.7: a validator never changes who may write the row (owner set), and
|
|
408
408
|
// a client's optimistic row holds NULL until the server row arrives.
|
|
409
409
|
if (column.serverOwned === true) {
|
|
410
410
|
if (
|
|
@@ -419,7 +419,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
419
419
|
)
|
|
420
420
|
) {
|
|
421
421
|
throw new Error(
|
|
422
|
-
`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.
|
|
422
|
+
`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.7)`,
|
|
423
423
|
);
|
|
424
424
|
}
|
|
425
425
|
serverOwnedColumnIndices.push(index);
|
package/src/segment-download.ts
CHANGED
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
RESOLVER_OUTAGE,
|
|
16
16
|
touchAuthenticatedPartition,
|
|
17
17
|
} from './context';
|
|
18
|
-
import { SyncError, syncError } from './errors';
|
|
18
|
+
import { SyncError, segmentStoreFailure, syncError } from './errors';
|
|
19
19
|
import { emitEvent } from './events';
|
|
20
20
|
import { compileSchema } from './schema';
|
|
21
21
|
import { schemaWindowOf } from './schema-window';
|
|
@@ -158,7 +158,9 @@ async function downloadSegment(
|
|
|
158
158
|
request: SegmentDownloadRequest,
|
|
159
159
|
): Promise<SegmentDownloadResult> {
|
|
160
160
|
const registry = await touchAuthenticatedPartition(ctx);
|
|
161
|
-
const entry = await ctx.segments
|
|
161
|
+
const entry = await ctx.segments
|
|
162
|
+
.get(request.segmentId)
|
|
163
|
+
.catch(segmentStoreFailure(ctx.onError, 'segments', 'get'));
|
|
162
164
|
if (entry === undefined) {
|
|
163
165
|
throw syncError('sync.not_found', 'unknown segment (§5.5)');
|
|
164
166
|
}
|
|
@@ -178,16 +180,21 @@ async function openSegment(
|
|
|
178
180
|
const registry = await touchAuthenticatedPartition(ctx);
|
|
179
181
|
const opened =
|
|
180
182
|
ctx.segments.open !== undefined
|
|
181
|
-
? await ctx.segments
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
183
|
+
? await ctx.segments
|
|
184
|
+
.open(request.segmentId)
|
|
185
|
+
.catch(segmentStoreFailure(ctx.onError, 'segments', 'open'))
|
|
186
|
+
: await ctx.segments
|
|
187
|
+
.get(request.segmentId)
|
|
188
|
+
.catch(segmentStoreFailure(ctx.onError, 'segments', 'get'))
|
|
189
|
+
.then((entry) =>
|
|
190
|
+
entry === undefined
|
|
191
|
+
? undefined
|
|
192
|
+
: {
|
|
193
|
+
record: entry.record,
|
|
194
|
+
body: bytesStream(entry.bytes),
|
|
195
|
+
byteLength: entry.bytes.length,
|
|
196
|
+
},
|
|
197
|
+
);
|
|
191
198
|
if (opened === undefined) {
|
|
192
199
|
throw syncError('sync.not_found', 'unknown segment (§5.5)');
|
|
193
200
|
}
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -46,8 +46,10 @@ import type { CompiledTable } from './schema';
|
|
|
46
46
|
import { matchesEffective } from './scopes';
|
|
47
47
|
import type { SqliteValue } from './sqlite-driver';
|
|
48
48
|
import {
|
|
49
|
+
asJson,
|
|
49
50
|
type ClientRowRecord,
|
|
50
51
|
CLIENT_RECORD_COLUMNS,
|
|
52
|
+
type PartitionHorizons,
|
|
51
53
|
PARTITION_TOUCH_INTERVAL_MS,
|
|
52
54
|
pushResultFromJson,
|
|
53
55
|
pushResultToJson,
|
|
@@ -257,6 +259,20 @@ const SQLITE_ADDED_COLUMNS = [
|
|
|
257
259
|
['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
|
|
258
260
|
] as const;
|
|
259
261
|
|
|
262
|
+
/**
|
|
263
|
+
* Core tables that `sqliteUpgradeDdl` creates, keyed by one of their columns,
|
|
264
|
+
* so a D1 database at an unchanged schema version gains them at its first
|
|
265
|
+
* open. `sync_table_horizons` holds the §4.6 table horizons.
|
|
266
|
+
*/
|
|
267
|
+
const SQLITE_ADDED_TABLES = [
|
|
268
|
+
[
|
|
269
|
+
'sync_table_horizons',
|
|
270
|
+
'commit_seq',
|
|
271
|
+
// One line: D1's `exec` runs each line as a statement.
|
|
272
|
+
'CREATE TABLE IF NOT EXISTS sync_table_horizons(partition TEXT NOT NULL, tbl TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(partition, tbl))',
|
|
273
|
+
],
|
|
274
|
+
] as const;
|
|
275
|
+
|
|
260
276
|
/** Core-table columns wire version 4 deleted: the version 3 subscription list. */
|
|
261
277
|
const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']] as const;
|
|
262
278
|
|
|
@@ -294,9 +310,11 @@ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
|
|
|
294
310
|
params: [
|
|
295
311
|
JSON.stringify([
|
|
296
312
|
...new Set(
|
|
297
|
-
[
|
|
298
|
-
|
|
299
|
-
|
|
313
|
+
[
|
|
314
|
+
...SQLITE_ADDED_COLUMNS,
|
|
315
|
+
...SQLITE_DROPPED_COLUMNS,
|
|
316
|
+
...SQLITE_ADDED_TABLES,
|
|
317
|
+
].map(([table]) => table),
|
|
300
318
|
),
|
|
301
319
|
]),
|
|
302
320
|
JSON.stringify([
|
|
@@ -309,7 +327,7 @@ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
|
|
|
309
327
|
/**
|
|
310
328
|
* The statements that bring the core tables of a database from an earlier
|
|
311
329
|
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
312
|
-
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
330
|
+
* tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
313
331
|
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
314
332
|
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
315
333
|
* and the first open of a database at the current schema version.
|
|
@@ -320,6 +338,9 @@ export function sqliteUpgradeDdl(
|
|
|
320
338
|
const has = (table: string, name: string) =>
|
|
321
339
|
present.some((row) => row.tbl === table && row.name === name);
|
|
322
340
|
return [
|
|
341
|
+
...SQLITE_ADDED_TABLES.filter(([table, name]) => !has(table, name)).map(
|
|
342
|
+
([, , ddl]) => ddl,
|
|
343
|
+
),
|
|
323
344
|
...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(
|
|
324
345
|
([table, name, definition]) =>
|
|
325
346
|
`ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
|
|
@@ -605,6 +626,20 @@ export abstract class SqliteFamilyStorage {
|
|
|
605
626
|
return row?.horizon_seq ?? 0;
|
|
606
627
|
}
|
|
607
628
|
|
|
629
|
+
async getHorizons(partition: string): Promise<PartitionHorizons> {
|
|
630
|
+
const [row] = await this.all<{ seq: number; tables: string }>(
|
|
631
|
+
`SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=?),0) AS seq,
|
|
632
|
+
(SELECT json_group_object(tbl, commit_seq) FROM sync_table_horizons WHERE partition=?) AS tables`,
|
|
633
|
+
[partition, partition],
|
|
634
|
+
);
|
|
635
|
+
return {
|
|
636
|
+
seq: row?.seq ?? 0,
|
|
637
|
+
tables: new Map(
|
|
638
|
+
Object.entries(asJson<Record<string, number>>(row?.tables ?? '{}')),
|
|
639
|
+
),
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
|
|
608
643
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
609
644
|
await this.all(
|
|
610
645
|
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
@@ -649,8 +684,14 @@ export abstract class SqliteFamilyStorage {
|
|
|
649
684
|
AND ${epochGuard}`,
|
|
650
685
|
params: [partition, partition, partition, query.logEpoch],
|
|
651
686
|
},
|
|
652
|
-
// §4.6: tombstones prune in the same pass as the
|
|
653
|
-
|
|
687
|
+
// §4.6: tombstones and table horizons prune in the same pass as the
|
|
688
|
+
// commit log.
|
|
689
|
+
...[
|
|
690
|
+
'sync_commits',
|
|
691
|
+
'sync_changes',
|
|
692
|
+
'sync_tombstones',
|
|
693
|
+
'sync_table_horizons',
|
|
694
|
+
].map((table) => ({
|
|
654
695
|
sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
|
|
655
696
|
params: [partition, partition, partition, query.logEpoch],
|
|
656
697
|
})),
|
|
@@ -1120,7 +1161,6 @@ export abstract class SqliteFamilyStorage {
|
|
|
1120
1161
|
JSON.stringify(values),
|
|
1121
1162
|
afterRowId,
|
|
1122
1163
|
batchSize,
|
|
1123
|
-
partition,
|
|
1124
1164
|
]);
|
|
1125
1165
|
if (records.length === 0) break;
|
|
1126
1166
|
for (const record of records) {
|
package/src/storage-query.ts
CHANGED
|
@@ -35,8 +35,11 @@ export interface ScanProbe {
|
|
|
35
35
|
*
|
|
36
36
|
* A single-variable filter needs no probe. Otherwise `count` counts each
|
|
37
37
|
* variable's entries in one statement, up to `cap`. The cap starts at
|
|
38
|
-
* max(64, limit) and grows fourfold
|
|
39
|
-
*
|
|
38
|
+
* max(64, limit) and grows fourfold while the fewest count is shared by
|
|
39
|
+
* several variables at the cap. A round runs only while every tied variable
|
|
40
|
+
* has at least `cap` entries, so the probe ends at the first cap above the
|
|
41
|
+
* smallest count `n`: at most 4n entries per variable in the last round, and
|
|
42
|
+
* the pick is the variable with the fewest entries whatever the starting cap.
|
|
40
43
|
*
|
|
41
44
|
* `chosen` memoizes the pick per (partition, table, filter), at most 1,024
|
|
42
45
|
* entries, so the page loop of a bootstrap probes once. A stale pick only
|
|
@@ -78,7 +81,7 @@ export async function chooseScanVariable(
|
|
|
78
81
|
const cached = chosen.get(key);
|
|
79
82
|
if (cached !== undefined) return cached;
|
|
80
83
|
let tied = variables;
|
|
81
|
-
for (let cap = Math.max(64, query.limit); ; cap
|
|
84
|
+
for (let cap = Math.max(64, query.limit); ; cap *= 4) {
|
|
82
85
|
const counts = tied.map(() => 0);
|
|
83
86
|
const probes = tied.map((variable, i) => ({
|
|
84
87
|
i,
|
|
@@ -92,7 +95,7 @@ export async function chooseScanVariable(
|
|
|
92
95
|
const fewest = Math.min(...counts);
|
|
93
96
|
if (fewest === 0) return undefined;
|
|
94
97
|
tied = tied.filter((_, i) => counts[i] === fewest);
|
|
95
|
-
if (fewest < cap || tied.length === 1
|
|
98
|
+
if (fewest < cap || tied.length === 1) break;
|
|
96
99
|
}
|
|
97
100
|
if (chosen.size >= 1_024) chosen.delete(chosen.keys().next().value!);
|
|
98
101
|
chosen.set(key, tied[0]!);
|
package/src/storage.ts
CHANGED
|
@@ -792,6 +792,17 @@ export function toClientRecord(record: ClientRowRecord): ClientRecord {
|
|
|
792
792
|
};
|
|
793
793
|
}
|
|
794
794
|
|
|
795
|
+
/** A partition's pruning horizon and its table horizons (§4.6). */
|
|
796
|
+
export interface PartitionHorizons {
|
|
797
|
+
/** `horizonSeq`. */
|
|
798
|
+
readonly seq: number;
|
|
799
|
+
/**
|
|
800
|
+
* Re-keyed table → the commit sequence number its re-key took. The changes
|
|
801
|
+
* of a table at or below it are never served.
|
|
802
|
+
*/
|
|
803
|
+
readonly tables: ReadonlyMap<string, number>;
|
|
804
|
+
}
|
|
805
|
+
|
|
795
806
|
/** One unit of the §4.5 scan set. */
|
|
796
807
|
export interface LiveScanUnit {
|
|
797
808
|
/** The unit's scope map; its first variable (keys sorted) is its index key. */
|
|
@@ -1313,6 +1324,8 @@ export interface ServerStorage {
|
|
|
1313
1324
|
|
|
1314
1325
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
1315
1326
|
getHorizonSeq(partition: string): Promise<number>;
|
|
1327
|
+
/** `horizonSeq` and the table horizons (§4.6) in one read. */
|
|
1328
|
+
getHorizons(partition: string): Promise<PartitionHorizons>;
|
|
1316
1329
|
/** Monotonic within the current epoch; use atomic pruning for maintenance. */
|
|
1317
1330
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
1318
1331
|
/**
|
package/src/validate.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
* row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
|
|
6
6
|
* commit transaction, once per operation. It is the seam for business
|
|
7
7
|
* rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
|
|
8
|
-
* "status ∈ {…}")
|
|
9
|
-
* atomically (§6.4) with a
|
|
10
|
-
* in its rejection record
|
|
8
|
+
* "status ∈ {…}") and for the values of server-owned columns. A throw (or
|
|
9
|
+
* rejected promise) rejects the whole commit atomically (§6.4) with a
|
|
10
|
+
* host-defined code the client surfaces unchanged in its rejection record
|
|
11
|
+
* (§6.3).
|
|
11
12
|
*
|
|
12
13
|
* The feature is OFF by default (no `validators` on the config): the push
|
|
13
14
|
* path pays only an `undefined` check per operation and builds no context
|
|
@@ -55,8 +56,9 @@ export type ValidateOpKind = 'upsert' | 'delete';
|
|
|
55
56
|
|
|
56
57
|
/**
|
|
57
58
|
* One push operation as the validator sees it (§6.7). For an `upsert`,
|
|
58
|
-
* `row` is the post-merge, post-scope-strip row that will be written
|
|
59
|
-
*
|
|
59
|
+
* `row` is the post-merge, post-scope-strip row that will be written, with
|
|
60
|
+
* each server-owned column as stored (NULL on an insert), and `stored` is
|
|
61
|
+
* the prior stored row (or `undefined` on insert). For a
|
|
60
62
|
* `delete`, `row` is `undefined` and `stored` is the row about to be
|
|
61
63
|
* removed (a delete only reaches a validator when a stored row exists —
|
|
62
64
|
* an absent-row delete is an idempotent no-op that never validates, §6.2).
|
|
@@ -188,7 +190,7 @@ export type CommitValidator = (
|
|
|
188
190
|
* inside and the push never completes, and on a pool-backed executor (`pg`,
|
|
189
191
|
* `Bun.sql`) it reads committed state without the staged writes, or waits
|
|
190
192
|
* forever once push transactions hold every pool connection. An authority
|
|
191
|
-
* read
|
|
193
|
+
* read goes through `read` instead.
|
|
192
194
|
*/
|
|
193
195
|
export interface ValidateContext {
|
|
194
196
|
/** Host-authenticated actor (§1.1) performing the write. */
|
|
@@ -198,17 +200,13 @@ export interface ValidateContext {
|
|
|
198
200
|
/** The context `authenticate` returned for this request (§1.1); absent without one. */
|
|
199
201
|
readonly context?: string;
|
|
200
202
|
/**
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
* PostgreSQL return the transaction's snapshot, including the operations
|
|
205
|
-
* staged before this one; D1 fails with
|
|
203
|
+
* The §6.8 candidate-state reader of the push transaction: it sees every
|
|
204
|
+
* operation staged before this one. `read.queryAuthoritative` runs a
|
|
205
|
+
* generated registered query on that transaction; D1 fails with
|
|
206
206
|
* `sync.storage.query_over_staged_writes` when the query reads a table the
|
|
207
207
|
* commit has written.
|
|
208
208
|
*/
|
|
209
|
-
readonly
|
|
210
|
-
query: AuthoritativeQueryRequest,
|
|
211
|
-
) => Promise<AuthoritativeQueryResult>;
|
|
209
|
+
readonly read: CommitValidationReader;
|
|
212
210
|
}
|
|
213
211
|
|
|
214
212
|
/**
|
|
@@ -220,8 +218,15 @@ export interface ValidateContext {
|
|
|
220
218
|
* `sync.constraint_violation` code (a validator SHOULD throw
|
|
221
219
|
* `ValidationRejection` to control the code). MUST NOT mutate the row.
|
|
222
220
|
*
|
|
221
|
+
* On an insert or update of a pushed commit or a registered command, the
|
|
222
|
+
* validator may return values for the table's server-owned columns; a column
|
|
223
|
+
* it omits keeps its value in `row`. A key that names no server-owned column,
|
|
224
|
+
* a value the column's type refuses, or a changed scope column on an update
|
|
225
|
+
* rejects the commit with `sync.constraint_violation`. The server ignores
|
|
226
|
+
* returned values on a delete, a rescope (§2.2) and a §6.11 cascade.
|
|
227
|
+
*
|
|
223
228
|
* The hook runs inside the push transaction. An authority read uses
|
|
224
|
-
* `ctx.queryAuthoritative`, which executes on that transaction; calling
|
|
229
|
+
* `ctx.read.queryAuthoritative`, which executes on that transaction; calling
|
|
225
230
|
* `ServerStorage.queryAuthoritative` from here deadlocks on a
|
|
226
231
|
* single-connection executor and reads committed state on a pool-backed one.
|
|
227
232
|
* A rule over sibling operations belongs in the whole-commit
|
|
@@ -230,7 +235,7 @@ export interface ValidateContext {
|
|
|
230
235
|
export type Validator = (
|
|
231
236
|
op: ValidateOperation,
|
|
232
237
|
ctx: ValidateContext,
|
|
233
|
-
) => void | Promise<void>;
|
|
238
|
+
) => ValidateRow | void | Promise<ValidateRow | void>;
|
|
234
239
|
|
|
235
240
|
/** Table name → validator (§6.7). Absent tables are unvalidated. */
|
|
236
241
|
export type ValidatorRegistry = Readonly<Record<string, Validator>>;
|
|
@@ -315,35 +320,3 @@ export function toValidateRow(
|
|
|
315
320
|
});
|
|
316
321
|
return row;
|
|
317
322
|
}
|
|
318
|
-
|
|
319
|
-
/** §6.12: the accepted upsert a server column hook completes. `row` is the
|
|
320
|
-
* merged row (present columns over stored values, crdt columns merged) with
|
|
321
|
-
* every server-owned column as stored, or NULL on an insert. */
|
|
322
|
-
export interface ServerColumnsInput {
|
|
323
|
-
readonly table: string;
|
|
324
|
-
readonly rowId: string;
|
|
325
|
-
readonly row: ValidateRow;
|
|
326
|
-
readonly stored: ValidateRow | undefined;
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
export interface ServerColumnsContext {
|
|
330
|
-
readonly actorId: string;
|
|
331
|
-
readonly partition: string;
|
|
332
|
-
readonly context?: string;
|
|
333
|
-
/** The §6.8 candidate-state reader of the push transaction. */
|
|
334
|
-
readonly read: CommitValidationReader;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
/**
|
|
338
|
-
* §6.12: returns values for the table's server-owned columns; a column it
|
|
339
|
-
* omits keeps its value in `row`. A key that names no server-owned column, a
|
|
340
|
-
* value the column's type refuses, or a changed scope column on an update
|
|
341
|
-
* rejects the commit with `sync.constraint_violation`, like any other throw
|
|
342
|
-
* that is not a {@link ValidationRejection}.
|
|
343
|
-
*/
|
|
344
|
-
export type ServerColumnsHook = (
|
|
345
|
-
input: ServerColumnsInput,
|
|
346
|
-
ctx: ServerColumnsContext,
|
|
347
|
-
) => ValidateRow | Promise<ValidateRow>;
|
|
348
|
-
|
|
349
|
-
export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;
|