@syncular/server 0.95.0 → 0.97.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/context.d.ts +4 -16
- package/dist/context.js +0 -3
- package/dist/d1-storage.js +1 -1
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +36 -0
- package/dist/frame-bytes.js +1 -0
- package/dist/handler.d.ts +20 -2
- package/dist/handler.js +199 -59
- package/dist/postgres-storage.d.ts +3 -2
- package/dist/postgres-storage.js +18 -2
- package/dist/pull.d.ts +2 -2
- package/dist/pull.js +22 -16
- package/dist/push.js +41 -77
- package/dist/realtime.d.ts +5 -11
- package/dist/realtime.js +10 -23
- package/dist/relational-rows.d.ts +23 -13
- package/dist/relational-rows.js +49 -30
- package/dist/schema-window.js +2 -2
- package/dist/schema.d.ts +2 -2
- package/dist/schema.js +2 -2
- package/dist/segment-download.js +11 -4
- package/dist/sqlite-dialect.d.ts +3 -1
- package/dist/sqlite-dialect.js +37 -6
- package/dist/storage-query.d.ts +5 -2
- package/dist/storage-query.js +7 -4
- package/dist/storage.d.ts +12 -0
- package/dist/validate.d.ts +21 -39
- package/dist/validate.js +4 -3
- package/package.json +2 -2
- package/src/context.ts +4 -23
- package/src/d1-storage.ts +0 -1
- package/src/errors.ts +46 -0
- package/src/frame-bytes.ts +1 -0
- package/src/handler.ts +257 -73
- package/src/postgres-storage.ts +32 -6
- package/src/pull.ts +50 -35
- package/src/push.ts +51 -129
- package/src/realtime.ts +10 -22
- package/src/relational-rows.ts +51 -30
- package/src/schema-window.ts +2 -2
- package/src/schema.ts +4 -4
- package/src/segment-download.ts +19 -12
- package/src/sqlite-dialect.ts +47 -7
- package/src/storage-query.ts +7 -4
- package/src/storage.ts +13 -0
- package/src/validate.ts +21 -48
package/dist/relational-rows.js
CHANGED
|
@@ -538,12 +538,22 @@ export function indexRowPageStatement(table, index, values, partition, afterRowI
|
|
|
538
538
|
}
|
|
539
539
|
/**
|
|
540
540
|
* One-round-trip page scan for `scanRows`: candidates from the inverted
|
|
541
|
-
* scope index
|
|
542
|
-
*
|
|
543
|
-
*
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
*
|
|
541
|
+
* scope index LEFT JOINed to the row table, so a whole page arrives in ONE
|
|
542
|
+
* statement instead of one lookup per candidate. That per-candidate lookup
|
|
543
|
+
* was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
|
|
544
|
+
* ~100k network round-trips (~10 s at 0.1 ms each) before this join.
|
|
545
|
+
*
|
|
546
|
+
* The candidates read at most `limit` entries of the covering
|
|
547
|
+
* `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
|
|
548
|
+
* `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
|
|
549
|
+
* cannot prove a single value, and then every entry after the cursor is read
|
|
550
|
+
* and sorted before the LIMIT: a cost that grows with the entries left, so a
|
|
551
|
+
* bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
|
|
552
|
+
* value through a LATERAL join and merges them. SQLite has no LATERAL; it
|
|
553
|
+
* bounds the ranges from above instead: the smallest of the values' `limit`th
|
|
554
|
+
* entries after the cursor. That value alone has `limit` entries up to its
|
|
555
|
+
* bound, so the page ends there; a value with fewer entries has no bound, and
|
|
556
|
+
* when no value has one the empty blob `x''` sorts after every TEXT row id.
|
|
547
557
|
*
|
|
548
558
|
* LEFT JOIN (not INNER): a candidate whose row vanished (index entry
|
|
549
559
|
* without a row) must still reach the caller — its `row_id` advances the
|
|
@@ -551,29 +561,32 @@ export function indexRowPageStatement(table, index, values, partition, afterRowI
|
|
|
551
561
|
* disappearing from the page.
|
|
552
562
|
*
|
|
553
563
|
* Bind order:
|
|
554
|
-
* - postgres: [partition, tbl, var, ...values, afterRowId, limit]
|
|
555
|
-
* (the join reuses `$1` for the partition);
|
|
564
|
+
* - postgres: [partition, tbl, var, ...values, afterRowId, limit];
|
|
556
565
|
* - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
|
|
557
|
-
* limit
|
|
558
|
-
* join). The values expand through `json_each`, so the statement text and
|
|
566
|
+
* limit]. The values expand through `json_each`, so the statement text and
|
|
559
567
|
* parameter count do not depend on `valueCount` and D1's 100-parameter
|
|
560
568
|
* cap is unreachable through scope values.
|
|
561
569
|
*/
|
|
562
570
|
export function scanRowPageSql(table, valueCount, dialect) {
|
|
563
|
-
const
|
|
564
|
-
const
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
571
|
+
const after = dialect === 'sqlite' ? '?5' : `$${4 + valueCount}`;
|
|
572
|
+
const limit = dialect === 'sqlite' ? '?6' : `$${5 + valueCount}`;
|
|
573
|
+
const candidates = dialect === 'sqlite'
|
|
574
|
+
? `SELECT DISTINCT row_id FROM sync_row_scopes
|
|
575
|
+
WHERE partition=?1 AND tbl=?2 AND var=?3 AND value IN (SELECT value FROM json_each(?4))
|
|
576
|
+
AND row_id>?5
|
|
577
|
+
AND row_id<=coalesce((SELECT min((SELECT row_id FROM sync_row_scopes
|
|
578
|
+
WHERE partition=?1 AND tbl=?2 AND var=?3 AND value=v.value AND row_id>?5
|
|
579
|
+
ORDER BY row_id LIMIT 1 OFFSET ?6-1)) FROM json_each(?4) v), x'')
|
|
580
|
+
ORDER BY row_id LIMIT ?6`
|
|
581
|
+
: `SELECT DISTINCT x.row_id FROM (VALUES ${Array.from({ length: valueCount }, (_, i) => `($${4 + i})`).join(',')}) v(value)
|
|
582
|
+
CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
|
|
583
|
+
WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=v.value AND row_id>${after}
|
|
584
|
+
ORDER BY row_id LIMIT ${limit}) x
|
|
585
|
+
ORDER BY x.row_id LIMIT ${limit}`;
|
|
570
586
|
return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
571
|
-
FROM (
|
|
572
|
-
WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
|
|
573
|
-
AND row_id>${after}
|
|
574
|
-
ORDER BY row_id LIMIT ${limit}) c
|
|
587
|
+
FROM (${candidates}) c
|
|
575
588
|
LEFT JOIN ${quoteIdent(table.name)} r
|
|
576
|
-
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${
|
|
589
|
+
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?1' : '$1'} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
|
|
577
590
|
ORDER BY c.row_id`;
|
|
578
591
|
}
|
|
579
592
|
/**
|
|
@@ -629,7 +642,7 @@ export function scanProbeSql(dialect) {
|
|
|
629
642
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
630
643
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
|
631
644
|
* array of `{i, tbl, var, vals, after, lim}` records; each record runs the
|
|
632
|
-
* same
|
|
645
|
+
* same per-value candidate ranges through a LATERAL join, so every
|
|
633
646
|
* subscription keeps its own keyset cursor and LIMIT. The candidate side is
|
|
634
647
|
* table-agnostic (a spec names its table), so the statement joins the row
|
|
635
648
|
* table once per table present in the batch through a UNION ALL branch that
|
|
@@ -655,10 +668,11 @@ export function scanRowBatchSql(tables) {
|
|
|
655
668
|
return `WITH cand AS (
|
|
656
669
|
SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
|
|
657
670
|
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
|
|
658
|
-
CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM
|
|
659
|
-
|
|
660
|
-
AND row_id>s.after
|
|
661
|
-
|
|
671
|
+
CROSS JOIN LATERAL (SELECT DISTINCT x.row_id FROM unnest(s.vals) v(value)
|
|
672
|
+
CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
|
|
673
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value AND row_id>s.after
|
|
674
|
+
ORDER BY row_id LIMIT s.lim) x
|
|
675
|
+
ORDER BY x.row_id LIMIT s.lim) c)
|
|
662
676
|
${projection}
|
|
663
677
|
ORDER BY sub, row_id`;
|
|
664
678
|
}
|
|
@@ -816,8 +830,10 @@ export function assertStoredScopePatterns(schema, storedJson) {
|
|
|
816
830
|
}
|
|
817
831
|
/**
|
|
818
832
|
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
819
|
-
* entries and tombstones
|
|
820
|
-
*
|
|
833
|
+
* entries and tombstones. The re-key takes a commit sequence number of its
|
|
834
|
+
* own in every partition and records it as each re-keyed table's horizon
|
|
835
|
+
* (§4.6), past the changes logged under the previous patterns. Empty when
|
|
836
|
+
* nothing re-keys.
|
|
821
837
|
*/
|
|
822
838
|
export function rekeyCleanupSql(tables) {
|
|
823
839
|
if (tables.length === 0)
|
|
@@ -826,7 +842,10 @@ export function rekeyCleanupSql(tables) {
|
|
|
826
842
|
return [
|
|
827
843
|
`DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
|
|
828
844
|
`DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
|
|
829
|
-
'UPDATE sync_partitions SET
|
|
845
|
+
'UPDATE sync_partitions SET max_commit_seq=max_commit_seq+1',
|
|
846
|
+
...names.map((name) => `INSERT INTO sync_table_horizons(partition, tbl, commit_seq)
|
|
847
|
+
SELECT partition, ${name}, max_commit_seq FROM sync_partitions WHERE true
|
|
848
|
+
ON CONFLICT(partition, tbl) DO UPDATE SET commit_seq=excluded.commit_seq`),
|
|
830
849
|
];
|
|
831
850
|
}
|
|
832
851
|
/**
|
package/dist/schema-window.js
CHANGED
|
@@ -19,7 +19,7 @@ export function schemaWindowOf(current, configured) {
|
|
|
19
19
|
const next = head.tables.get(table.name);
|
|
20
20
|
if (next === undefined ||
|
|
21
21
|
table.columns.length > next.columns.length ||
|
|
22
|
-
// §6.
|
|
22
|
+
// §6.7: a version may mark an existing column server-owned; the flag
|
|
23
23
|
// never reaches the wire.
|
|
24
24
|
table.columns.some((column, i) => JSON.stringify({ ...column, serverOwned: undefined }) !==
|
|
25
25
|
JSON.stringify({ ...next.columns[i], serverOwned: undefined })) ||
|
|
@@ -27,7 +27,7 @@ export function schemaWindowOf(current, configured) {
|
|
|
27
27
|
.slice(table.columns.length)
|
|
28
28
|
.some((column) => !column.nullable) ||
|
|
29
29
|
table.primaryKeyIndex !== next.primaryKeyIndex ||
|
|
30
|
-
// §6.
|
|
30
|
+
// §6.7: a pattern over an appended server-owned column is new to the
|
|
31
31
|
// current version; an older client neither holds nor writes it.
|
|
32
32
|
JSON.stringify(table.scopePatterns) !==
|
|
33
33
|
JSON.stringify(next.scopePatterns.filter((pattern) => pattern.columnIndex < table.columns.length ||
|
package/dist/schema.d.ts
CHANGED
|
@@ -141,8 +141,8 @@ export interface CompiledTable {
|
|
|
141
141
|
* image copies ciphertext wholesale with no per-row decrypt pass, so an
|
|
142
142
|
* encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
|
|
143
143
|
readonly encryptedColumnIndices: readonly number[];
|
|
144
|
-
/** §6.
|
|
145
|
-
*
|
|
144
|
+
/** §6.7: column indices marked `serverOwned`. Empty ⇒ no server column
|
|
145
|
+
* values are accepted from the table's validator. */
|
|
146
146
|
readonly serverOwnedColumnIndices: readonly number[];
|
|
147
147
|
}
|
|
148
148
|
export interface CompiledSchema {
|
package/dist/schema.js
CHANGED
|
@@ -174,7 +174,7 @@ export function compileSchema(schema) {
|
|
|
174
174
|
// §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
|
|
175
175
|
if (column.encrypted === true)
|
|
176
176
|
encryptedColumnIndices.push(index);
|
|
177
|
-
// §6.
|
|
177
|
+
// §6.7: a validator never changes who may write the row (owner set), and
|
|
178
178
|
// a client's optimistic row holds NULL until the server row arrives.
|
|
179
179
|
if (column.serverOwned === true) {
|
|
180
180
|
if (index === primaryKeyIndex ||
|
|
@@ -183,7 +183,7 @@ export function compileSchema(schema) {
|
|
|
183
183
|
column.encrypted === true ||
|
|
184
184
|
scopePatterns.some((pattern) => pattern.columnIndex === index &&
|
|
185
185
|
writeScopeSets[0].includes(pattern.variable))) {
|
|
186
|
-
throw new Error(`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.
|
|
186
|
+
throw new Error(`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.7)`);
|
|
187
187
|
}
|
|
188
188
|
serverOwnedColumnIndices.push(index);
|
|
189
189
|
}
|
package/dist/segment-download.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
2
|
-
import { SyncError, syncError } from './errors.js';
|
|
2
|
+
import { SyncError, segmentStoreFailure, syncError } from './errors.js';
|
|
3
3
|
import { emitEvent } from './events.js';
|
|
4
4
|
import { compileSchema } from './schema.js';
|
|
5
5
|
import { schemaWindowOf } from './schema-window.js';
|
|
@@ -79,7 +79,9 @@ async function withDownloadEvent(ctx, request, run, bytesOf) {
|
|
|
79
79
|
}
|
|
80
80
|
async function downloadSegment(ctx, request) {
|
|
81
81
|
const registry = await touchAuthenticatedPartition(ctx);
|
|
82
|
-
const entry = await ctx.segments
|
|
82
|
+
const entry = await ctx.segments
|
|
83
|
+
.get(request.segmentId)
|
|
84
|
+
.catch(segmentStoreFailure(ctx.onError, 'segments', 'get'));
|
|
83
85
|
if (entry === undefined) {
|
|
84
86
|
throw syncError('sync.not_found', 'unknown segment (§5.5)');
|
|
85
87
|
}
|
|
@@ -89,8 +91,13 @@ async function downloadSegment(ctx, request) {
|
|
|
89
91
|
async function openSegment(ctx, request) {
|
|
90
92
|
const registry = await touchAuthenticatedPartition(ctx);
|
|
91
93
|
const opened = ctx.segments.open !== undefined
|
|
92
|
-
? await ctx.segments
|
|
93
|
-
|
|
94
|
+
? await ctx.segments
|
|
95
|
+
.open(request.segmentId)
|
|
96
|
+
.catch(segmentStoreFailure(ctx.onError, 'segments', 'open'))
|
|
97
|
+
: await ctx.segments
|
|
98
|
+
.get(request.segmentId)
|
|
99
|
+
.catch(segmentStoreFailure(ctx.onError, 'segments', 'get'))
|
|
100
|
+
.then((entry) => entry === undefined
|
|
94
101
|
? undefined
|
|
95
102
|
: {
|
|
96
103
|
record: entry.record,
|
package/dist/sqlite-dialect.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ import type { UnitScopes } from '@syncular/core';
|
|
|
18
18
|
import { type BoundAuthoritativeQuery } from './authoritative-query.js';
|
|
19
19
|
import type { CompiledTable } from './schema.js';
|
|
20
20
|
import type { SqliteValue } from './sqlite-driver.js';
|
|
21
|
+
import { type PartitionHorizons } from './storage.js';
|
|
21
22
|
import type { AuthoritativeQueryRequest, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitPruneQuery, CommitPruneResult, IndexRowScanQuery, LiveWindow, LiveWindowQuery, NewCommit, OutboxSequence, OutboxSettlement, NewReaction, PartitionRegistryEntry, PartitionTouch, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, StoredPushResult, StoredReaction, StoredRow, Tombstone } from './storage.js';
|
|
22
23
|
/**
|
|
23
24
|
* Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
|
|
@@ -44,7 +45,7 @@ export declare const SQLITE_UPGRADE_PROBE: SqliteStatementSpec;
|
|
|
44
45
|
/**
|
|
45
46
|
* The statements that bring the core tables of a database from an earlier
|
|
46
47
|
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
47
|
-
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
48
|
+
* tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
48
49
|
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
49
50
|
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
50
51
|
* and the first open of a database at the current schema version.
|
|
@@ -125,6 +126,7 @@ export declare abstract class SqliteFamilyStorage {
|
|
|
125
126
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
126
127
|
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
127
128
|
getHorizonSeq(partition: string): Promise<number>;
|
|
129
|
+
getHorizons(partition: string): Promise<PartitionHorizons>;
|
|
128
130
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
129
131
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
130
132
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -4,7 +4,7 @@ import { validateCommitPruneQuery } from './prune.js';
|
|
|
4
4
|
import { StorageQueryError } from './storage-errors.js';
|
|
5
5
|
import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, countRowsSql, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
|
|
6
6
|
import { matchesEffective } from './scopes.js';
|
|
7
|
-
import { CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
|
|
7
|
+
import { asJson, CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
|
|
8
8
|
import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
9
9
|
/**
|
|
10
10
|
* Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
|
|
@@ -156,6 +156,19 @@ const SQLITE_ADDED_COLUMNS = [
|
|
|
156
156
|
['sync_push_results', 'client_seq', 'INTEGER'],
|
|
157
157
|
['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
|
|
158
158
|
];
|
|
159
|
+
/**
|
|
160
|
+
* Core tables that `sqliteUpgradeDdl` creates, keyed by one of their columns,
|
|
161
|
+
* so a D1 database at an unchanged schema version gains them at its first
|
|
162
|
+
* open. `sync_table_horizons` holds the §4.6 table horizons.
|
|
163
|
+
*/
|
|
164
|
+
const SQLITE_ADDED_TABLES = [
|
|
165
|
+
[
|
|
166
|
+
'sync_table_horizons',
|
|
167
|
+
'commit_seq',
|
|
168
|
+
// One line: D1's `exec` runs each line as a statement.
|
|
169
|
+
'CREATE TABLE IF NOT EXISTS sync_table_horizons(partition TEXT NOT NULL, tbl TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(partition, tbl))',
|
|
170
|
+
],
|
|
171
|
+
];
|
|
159
172
|
/** Core-table columns wire version 4 deleted: the version 3 subscription list. */
|
|
160
173
|
const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']];
|
|
161
174
|
/**
|
|
@@ -189,7 +202,11 @@ export const SQLITE_UPGRADE_PROBE = {
|
|
|
189
202
|
WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
|
|
190
203
|
params: [
|
|
191
204
|
JSON.stringify([
|
|
192
|
-
...new Set([
|
|
205
|
+
...new Set([
|
|
206
|
+
...SQLITE_ADDED_COLUMNS,
|
|
207
|
+
...SQLITE_DROPPED_COLUMNS,
|
|
208
|
+
...SQLITE_ADDED_TABLES,
|
|
209
|
+
].map(([table]) => table)),
|
|
193
210
|
]),
|
|
194
211
|
JSON.stringify([
|
|
195
212
|
...SQLITE_ADDED_INDEXES.map(([name]) => name),
|
|
@@ -200,7 +217,7 @@ export const SQLITE_UPGRADE_PROBE = {
|
|
|
200
217
|
/**
|
|
201
218
|
* The statements that bring the core tables of a database from an earlier
|
|
202
219
|
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
203
|
-
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
220
|
+
* tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
204
221
|
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
205
222
|
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
206
223
|
* and the first open of a database at the current schema version.
|
|
@@ -208,6 +225,7 @@ export const SQLITE_UPGRADE_PROBE = {
|
|
|
208
225
|
export function sqliteUpgradeDdl(present) {
|
|
209
226
|
const has = (table, name) => present.some((row) => row.tbl === table && row.name === name);
|
|
210
227
|
return [
|
|
228
|
+
...SQLITE_ADDED_TABLES.filter(([table, name]) => !has(table, name)).map(([, , ddl]) => ddl),
|
|
211
229
|
...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(([table, name, definition]) => `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`),
|
|
212
230
|
// A database from before wire version 4 (and the delete floor) starts
|
|
213
231
|
// the delete floor at the highest version its tombstones recorded
|
|
@@ -358,6 +376,14 @@ export class SqliteFamilyStorage {
|
|
|
358
376
|
const [row] = await this.all('SELECT horizon_seq FROM sync_partitions WHERE partition=?', [partition]);
|
|
359
377
|
return row?.horizon_seq ?? 0;
|
|
360
378
|
}
|
|
379
|
+
async getHorizons(partition) {
|
|
380
|
+
const [row] = await this.all(`SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=?),0) AS seq,
|
|
381
|
+
(SELECT json_group_object(tbl, commit_seq) FROM sync_table_horizons WHERE partition=?) AS tables`, [partition, partition]);
|
|
382
|
+
return {
|
|
383
|
+
seq: row?.seq ?? 0,
|
|
384
|
+
tables: new Map(Object.entries(asJson(row?.tables ?? '{}'))),
|
|
385
|
+
};
|
|
386
|
+
}
|
|
361
387
|
async setHorizonSeq(partition, seq) {
|
|
362
388
|
await this.all(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
363
389
|
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`, [partition, seq]);
|
|
@@ -393,8 +419,14 @@ export class SqliteFamilyStorage {
|
|
|
393
419
|
AND ${epochGuard}`,
|
|
394
420
|
params: [partition, partition, partition, query.logEpoch],
|
|
395
421
|
},
|
|
396
|
-
// §4.6: tombstones prune in the same pass as the
|
|
397
|
-
|
|
422
|
+
// §4.6: tombstones and table horizons prune in the same pass as the
|
|
423
|
+
// commit log.
|
|
424
|
+
...[
|
|
425
|
+
'sync_commits',
|
|
426
|
+
'sync_changes',
|
|
427
|
+
'sync_tombstones',
|
|
428
|
+
'sync_table_horizons',
|
|
429
|
+
].map((table) => ({
|
|
398
430
|
sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
|
|
399
431
|
params: [partition, partition, partition, query.logEpoch],
|
|
400
432
|
})),
|
|
@@ -706,7 +738,6 @@ export class SqliteFamilyStorage {
|
|
|
706
738
|
JSON.stringify(values),
|
|
707
739
|
afterRowId,
|
|
708
740
|
batchSize,
|
|
709
|
-
partition,
|
|
710
741
|
]);
|
|
711
742
|
if (records.length === 0)
|
|
712
743
|
break;
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -20,8 +20,11 @@ export interface ScanProbe {
|
|
|
20
20
|
*
|
|
21
21
|
* A single-variable filter needs no probe. Otherwise `count` counts each
|
|
22
22
|
* variable's entries in one statement, up to `cap`. The cap starts at
|
|
23
|
-
* max(64, limit) and grows fourfold
|
|
24
|
-
*
|
|
23
|
+
* max(64, limit) and grows fourfold while the fewest count is shared by
|
|
24
|
+
* several variables at the cap. A round runs only while every tied variable
|
|
25
|
+
* has at least `cap` entries, so the probe ends at the first cap above the
|
|
26
|
+
* smallest count `n`: at most 4n entries per variable in the last round, and
|
|
27
|
+
* the pick is the variable with the fewest entries whatever the starting cap.
|
|
25
28
|
*
|
|
26
29
|
* `chosen` memoizes the pick per (partition, table, filter), at most 1,024
|
|
27
30
|
* entries, so the page loop of a bootstrap probes once. A stale pick only
|
package/dist/storage-query.js
CHANGED
|
@@ -10,8 +10,11 @@ import { StorageQueryError } from './storage-errors.js';
|
|
|
10
10
|
*
|
|
11
11
|
* A single-variable filter needs no probe. Otherwise `count` counts each
|
|
12
12
|
* variable's entries in one statement, up to `cap`. The cap starts at
|
|
13
|
-
* max(64, limit) and grows fourfold
|
|
14
|
-
*
|
|
13
|
+
* max(64, limit) and grows fourfold while the fewest count is shared by
|
|
14
|
+
* several variables at the cap. A round runs only while every tied variable
|
|
15
|
+
* has at least `cap` entries, so the probe ends at the first cap above the
|
|
16
|
+
* smallest count `n`: at most 4n entries per variable in the last round, and
|
|
17
|
+
* the pick is the variable with the fewest entries whatever the starting cap.
|
|
15
18
|
*
|
|
16
19
|
* `chosen` memoizes the pick per (partition, table, filter), at most 1,024
|
|
17
20
|
* entries, so the page loop of a bootstrap probes once. A stale pick only
|
|
@@ -42,7 +45,7 @@ export async function chooseScanVariable(partition, query, chosen, count) {
|
|
|
42
45
|
if (cached !== undefined)
|
|
43
46
|
return cached;
|
|
44
47
|
let tied = variables;
|
|
45
|
-
for (let cap = Math.max(64, query.limit);; cap
|
|
48
|
+
for (let cap = Math.max(64, query.limit);; cap *= 4) {
|
|
46
49
|
const counts = tied.map(() => 0);
|
|
47
50
|
const probes = tied.map((variable, i) => ({
|
|
48
51
|
i,
|
|
@@ -57,7 +60,7 @@ export async function chooseScanVariable(partition, query, chosen, count) {
|
|
|
57
60
|
if (fewest === 0)
|
|
58
61
|
return undefined;
|
|
59
62
|
tied = tied.filter((_, i) => counts[i] === fewest);
|
|
60
|
-
if (fewest < cap || tied.length === 1
|
|
63
|
+
if (fewest < cap || tied.length === 1)
|
|
61
64
|
break;
|
|
62
65
|
}
|
|
63
66
|
if (chosen.size >= 1_024)
|
package/dist/storage.d.ts
CHANGED
|
@@ -440,6 +440,16 @@ export declare function toStoredChange(record: StoredChangeRecord): StoredChange
|
|
|
440
440
|
export declare function toStoredReaction(record: StoredReactionRecord): StoredReaction;
|
|
441
441
|
export declare const CLIENT_RECORD_COLUMNS = "client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures";
|
|
442
442
|
export declare function toClientRecord(record: ClientRowRecord): ClientRecord;
|
|
443
|
+
/** A partition's pruning horizon and its table horizons (§4.6). */
|
|
444
|
+
export interface PartitionHorizons {
|
|
445
|
+
/** `horizonSeq`. */
|
|
446
|
+
readonly seq: number;
|
|
447
|
+
/**
|
|
448
|
+
* Re-keyed table → the commit sequence number its re-key took. The changes
|
|
449
|
+
* of a table at or below it are never served.
|
|
450
|
+
*/
|
|
451
|
+
readonly tables: ReadonlyMap<string, number>;
|
|
452
|
+
}
|
|
443
453
|
/** One unit of the §4.5 scan set. */
|
|
444
454
|
export interface LiveScanUnit {
|
|
445
455
|
/** The unit's scope map; its first variable (keys sorted) is its index key. */
|
|
@@ -866,6 +876,8 @@ export interface ServerStorage {
|
|
|
866
876
|
begin(partition: string): Promise<StorageTransaction>;
|
|
867
877
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
868
878
|
getHorizonSeq(partition: string): Promise<number>;
|
|
879
|
+
/** `horizonSeq` and the table horizons (§4.6) in one read. */
|
|
880
|
+
getHorizons(partition: string): Promise<PartitionHorizons>;
|
|
869
881
|
/** Monotonic within the current epoch; use atomic pruning for maintenance. */
|
|
870
882
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
871
883
|
/**
|
package/dist/validate.d.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
* row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
|
|
6
6
|
* commit transaction, once per operation. It is the seam for business
|
|
7
7
|
* rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
|
|
8
|
-
* "status ∈ {…}")
|
|
9
|
-
* atomically (§6.4) with a
|
|
10
|
-
* in its rejection record
|
|
8
|
+
* "status ∈ {…}") and for the values of server-owned columns. A throw (or
|
|
9
|
+
* rejected promise) rejects the whole commit atomically (§6.4) with a
|
|
10
|
+
* host-defined code the client surfaces unchanged in its rejection record
|
|
11
|
+
* (§6.3).
|
|
11
12
|
*
|
|
12
13
|
* The feature is OFF by default (no `validators` on the config): the push
|
|
13
14
|
* path pays only an `undefined` check per operation and builds no context
|
|
@@ -36,8 +37,9 @@ export type ValidateRow = Readonly<Record<string, RowValue>>;
|
|
|
36
37
|
export type ValidateOpKind = 'upsert' | 'delete';
|
|
37
38
|
/**
|
|
38
39
|
* One push operation as the validator sees it (§6.7). For an `upsert`,
|
|
39
|
-
* `row` is the post-merge, post-scope-strip row that will be written
|
|
40
|
-
*
|
|
40
|
+
* `row` is the post-merge, post-scope-strip row that will be written, with
|
|
41
|
+
* each server-owned column as stored (NULL on an insert), and `stored` is
|
|
42
|
+
* the prior stored row (or `undefined` on insert). For a
|
|
41
43
|
* `delete`, `row` is `undefined` and `stored` is the row about to be
|
|
42
44
|
* removed (a delete only reaches a validator when a stored row exists —
|
|
43
45
|
* an absent-row delete is an idempotent no-op that never validates, §6.2).
|
|
@@ -151,7 +153,7 @@ export type CommitValidator = (input: ValidateCommitInput) => void | Promise<voi
|
|
|
151
153
|
* inside and the push never completes, and on a pool-backed executor (`pg`,
|
|
152
154
|
* `Bun.sql`) it reads committed state without the staged writes, or waits
|
|
153
155
|
* forever once push transactions hold every pool connection. An authority
|
|
154
|
-
* read
|
|
156
|
+
* read goes through `read` instead.
|
|
155
157
|
*/
|
|
156
158
|
export interface ValidateContext {
|
|
157
159
|
/** Host-authenticated actor (§1.1) performing the write. */
|
|
@@ -161,15 +163,13 @@ export interface ValidateContext {
|
|
|
161
163
|
/** The context `authenticate` returned for this request (§1.1); absent without one. */
|
|
162
164
|
readonly context?: string;
|
|
163
165
|
/**
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
* PostgreSQL return the transaction's snapshot, including the operations
|
|
168
|
-
* staged before this one; D1 fails with
|
|
166
|
+
* The §6.8 candidate-state reader of the push transaction: it sees every
|
|
167
|
+
* operation staged before this one. `read.queryAuthoritative` runs a
|
|
168
|
+
* generated registered query on that transaction; D1 fails with
|
|
169
169
|
* `sync.storage.query_over_staged_writes` when the query reads a table the
|
|
170
170
|
* commit has written.
|
|
171
171
|
*/
|
|
172
|
-
readonly
|
|
172
|
+
readonly read: CommitValidationReader;
|
|
173
173
|
}
|
|
174
174
|
/**
|
|
175
175
|
* A per-table validation hook (§6.7). Called once per push operation on
|
|
@@ -180,14 +180,21 @@ export interface ValidateContext {
|
|
|
180
180
|
* `sync.constraint_violation` code (a validator SHOULD throw
|
|
181
181
|
* `ValidationRejection` to control the code). MUST NOT mutate the row.
|
|
182
182
|
*
|
|
183
|
+
* On an insert or update of a pushed commit or a registered command, the
|
|
184
|
+
* validator may return values for the table's server-owned columns; a column
|
|
185
|
+
* it omits keeps its value in `row`. A key that names no server-owned column,
|
|
186
|
+
* a value the column's type refuses, or a changed scope column on an update
|
|
187
|
+
* rejects the commit with `sync.constraint_violation`. The server ignores
|
|
188
|
+
* returned values on a delete, a rescope (§2.2) and a §6.11 cascade.
|
|
189
|
+
*
|
|
183
190
|
* The hook runs inside the push transaction. An authority read uses
|
|
184
|
-
* `ctx.queryAuthoritative`, which executes on that transaction; calling
|
|
191
|
+
* `ctx.read.queryAuthoritative`, which executes on that transaction; calling
|
|
185
192
|
* `ServerStorage.queryAuthoritative` from here deadlocks on a
|
|
186
193
|
* single-connection executor and reads committed state on a pool-backed one.
|
|
187
194
|
* A rule over sibling operations belongs in the whole-commit
|
|
188
195
|
* `CommitValidator` (§6.8).
|
|
189
196
|
*/
|
|
190
|
-
export type Validator = (op: ValidateOperation, ctx: ValidateContext) => void | Promise<void>;
|
|
197
|
+
export type Validator = (op: ValidateOperation, ctx: ValidateContext) => ValidateRow | void | Promise<ValidateRow | void>;
|
|
191
198
|
/** Table name → validator (§6.7). Absent tables are unvalidated. */
|
|
192
199
|
export type ValidatorRegistry = Readonly<Record<string, Validator>>;
|
|
193
200
|
/**
|
|
@@ -222,28 +229,3 @@ export declare class CommitAuthorizationRejection extends SyncError {
|
|
|
222
229
|
}
|
|
223
230
|
/** Build the column-keyed row object a validator inspects (§6.7). */
|
|
224
231
|
export declare function toValidateRow(columns: readonly RowColumn[], values: readonly RowValue[]): ValidateRow;
|
|
225
|
-
/** §6.12: the accepted upsert a server column hook completes. `row` is the
|
|
226
|
-
* merged row (present columns over stored values, crdt columns merged) with
|
|
227
|
-
* every server-owned column as stored, or NULL on an insert. */
|
|
228
|
-
export interface ServerColumnsInput {
|
|
229
|
-
readonly table: string;
|
|
230
|
-
readonly rowId: string;
|
|
231
|
-
readonly row: ValidateRow;
|
|
232
|
-
readonly stored: ValidateRow | undefined;
|
|
233
|
-
}
|
|
234
|
-
export interface ServerColumnsContext {
|
|
235
|
-
readonly actorId: string;
|
|
236
|
-
readonly partition: string;
|
|
237
|
-
readonly context?: string;
|
|
238
|
-
/** The §6.8 candidate-state reader of the push transaction. */
|
|
239
|
-
readonly read: CommitValidationReader;
|
|
240
|
-
}
|
|
241
|
-
/**
|
|
242
|
-
* §6.12: returns values for the table's server-owned columns; a column it
|
|
243
|
-
* omits keeps its value in `row`. A key that names no server-owned column, a
|
|
244
|
-
* value the column's type refuses, or a changed scope column on an update
|
|
245
|
-
* rejects the commit with `sync.constraint_violation`, like any other throw
|
|
246
|
-
* that is not a {@link ValidationRejection}.
|
|
247
|
-
*/
|
|
248
|
-
export type ServerColumnsHook = (input: ServerColumnsInput, ctx: ServerColumnsContext) => ValidateRow | Promise<ValidateRow>;
|
|
249
|
-
export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;
|
package/dist/validate.js
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
* row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
|
|
6
6
|
* commit transaction, once per operation. It is the seam for business
|
|
7
7
|
* rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
|
|
8
|
-
* "status ∈ {…}")
|
|
9
|
-
* atomically (§6.4) with a
|
|
10
|
-
* in its rejection record
|
|
8
|
+
* "status ∈ {…}") and for the values of server-owned columns. A throw (or
|
|
9
|
+
* rejected promise) rejects the whole commit atomically (§6.4) with a
|
|
10
|
+
* host-defined code the client surfaces unchanged in its rejection record
|
|
11
|
+
* (§6.3).
|
|
11
12
|
*
|
|
12
13
|
* The feature is OFF by default (no `validators` on the config): the push
|
|
13
14
|
* path pays only an `undefined` check per operation and builds no context
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.97.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.97.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/context.ts
CHANGED
|
@@ -25,11 +25,7 @@ import type {
|
|
|
25
25
|
ServerStorage,
|
|
26
26
|
StoredCommit,
|
|
27
27
|
} from './storage';
|
|
28
|
-
import type {
|
|
29
|
-
CommitValidator,
|
|
30
|
-
ServerColumnsRegistry,
|
|
31
|
-
ValidatorRegistry,
|
|
32
|
-
} from './validate';
|
|
28
|
+
import type { CommitValidator, ValidatorRegistry } from './validate';
|
|
33
29
|
|
|
34
30
|
/** SSP2 body content type (§1.1). */
|
|
35
31
|
export const SSP2_CONTENT_TYPE = 'application/vnd.syncular.sync.v2';
|
|
@@ -124,12 +120,6 @@ export interface ServerLimits {
|
|
|
124
120
|
* bound gets an image of its own.
|
|
125
121
|
*/
|
|
126
122
|
readonly maxImageSegmentBytes: number;
|
|
127
|
-
/**
|
|
128
|
-
* Segment puts one pull keeps in flight. The pull scans and encodes its
|
|
129
|
-
* pages in order and stores up to this many at once, so it holds this
|
|
130
|
-
* many encoded segments in memory.
|
|
131
|
-
*/
|
|
132
|
-
readonly maxConcurrentSegmentPuts: number;
|
|
133
123
|
}
|
|
134
124
|
|
|
135
125
|
export const DEFAULT_LIMITS: ServerLimits = {
|
|
@@ -146,9 +136,6 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
146
136
|
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
147
137
|
// §5.3 DECISION (2026-11-17): a client holds about three images at once.
|
|
148
138
|
maxImageSegmentBytes: 32 * 1024 * 1024,
|
|
149
|
-
// A Workers invocation holds six connections open at once, and an R2 put
|
|
150
|
-
// uses one; past six the runtime queues the put.
|
|
151
|
-
maxConcurrentSegmentPuts: 6,
|
|
152
139
|
};
|
|
153
140
|
|
|
154
141
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
@@ -200,19 +187,13 @@ export interface SyncServerConfig {
|
|
|
200
187
|
* builds no validator context — zero cost. Present ⇒ each push operation
|
|
201
188
|
* on a listed table runs its validator after decode + §3.4 scope
|
|
202
189
|
* authorization, inside the commit transaction; a throw rejects the
|
|
203
|
-
* whole commit atomically (§6.4) with the host code (§6.3).
|
|
190
|
+
* whole commit atomically (§6.4) with the host code (§6.3). On an insert
|
|
191
|
+
* or update the validator may return values of the table's server-owned
|
|
192
|
+
* columns, which are stored and fanned out in the same commit. Kept a host
|
|
204
193
|
* callback (like `resolveScopes`) — business rules live in the host
|
|
205
194
|
* process, never on the wire.
|
|
206
195
|
*/
|
|
207
196
|
readonly validators?: ValidatorRegistry;
|
|
208
|
-
/**
|
|
209
|
-
* §6.12 per-table server column hooks (`table` → hook). Each accepted
|
|
210
|
-
* upsert on a listed table runs its hook inside the push transaction,
|
|
211
|
-
* after §3.4 authorization and before §6.7 validation; the returned values
|
|
212
|
-
* of the table's server-owned columns are stored and fanned out in the
|
|
213
|
-
* same commit.
|
|
214
|
-
*/
|
|
215
|
-
readonly serverColumns?: ServerColumnsRegistry;
|
|
216
197
|
/**
|
|
217
198
|
* §6.8 whole-commit validator. When present, the storage serializes this
|
|
218
199
|
* partition before operation reads/writes, then invokes the callback once
|