@syncular/server 0.45.1 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/d1-storage.js +12 -0
- package/dist/errors.js +9 -0
- package/dist/operations.d.ts +8 -0
- package/dist/operations.js +18 -0
- package/dist/postgres-storage.js +22 -1
- package/dist/push.js +43 -5
- package/dist/relational-rows.d.ts +7 -0
- package/dist/relational-rows.js +9 -0
- package/dist/sqlite-storage.js +11 -0
- package/dist/storage.d.ts +16 -0
- package/dist/storage.js +19 -0
- package/dist/validate.d.ts +9 -0
- package/package.json +2 -2
- package/src/d1-storage.ts +15 -0
- package/src/errors.ts +9 -0
- package/src/operations.ts +40 -0
- package/src/postgres-storage.ts +25 -0
- package/src/push.ts +47 -3
- package/src/relational-rows.ts +10 -0
- package/src/sqlite-storage.ts +14 -0
- package/src/storage.ts +34 -0
- package/src/validate.ts +12 -0
package/README.md
CHANGED
|
@@ -255,7 +255,8 @@ const config: SyncServerConfig = {
|
|
|
255
255
|
};
|
|
256
256
|
```
|
|
257
257
|
|
|
258
|
-
The callback also receives `read.getRow()
|
|
258
|
+
The callback also receives `read.getRow()` and its batch form
|
|
259
|
+
`read.getRows(table, rowIds)`, bounded `read.scanRows()`, and
|
|
259
260
|
`read.queryAuthoritative()` APIs. Those reads see the final candidate state, including staged sibling upserts and
|
|
260
261
|
deletes. Syncular serializes the partition before any operation read so two
|
|
261
262
|
aggregate validators cannot both accept mutually invalid candidates. It also
|
package/dist/d1-storage.js
CHANGED
|
@@ -175,6 +175,18 @@ class D1Transaction {
|
|
|
175
175
|
.first();
|
|
176
176
|
return record === null ? undefined : toStoredRow(record);
|
|
177
177
|
}
|
|
178
|
+
async getRows(table, rowIds) {
|
|
179
|
+
this.#assertOpen();
|
|
180
|
+
// One lookup per id: `getRow` owns the buffered-write overlay, and a
|
|
181
|
+
// mutable id list cannot share one bound statement across D1's limit.
|
|
182
|
+
const rows = new Map();
|
|
183
|
+
for (const rowId of new Set(rowIds)) {
|
|
184
|
+
const row = await this.getRow(table, rowId);
|
|
185
|
+
if (row !== undefined)
|
|
186
|
+
rows.set(rowId, row);
|
|
187
|
+
}
|
|
188
|
+
return rows;
|
|
189
|
+
}
|
|
178
190
|
async getTombstoneSeq(table, rowId) {
|
|
179
191
|
this.#assertOpen();
|
|
180
192
|
if (this.#clearedTombstones.has(_a.#key(table, rowId))) {
|
package/dist/errors.js
CHANGED
|
@@ -139,6 +139,15 @@ export const ERROR_CATALOG = {
|
|
|
139
139
|
recommendedAction: 'fixRequest',
|
|
140
140
|
httpStatus: 400,
|
|
141
141
|
},
|
|
142
|
+
// §6.8: the whole-commit validator threw an exception that is not a
|
|
143
|
+
// deliberate rejection. A host fault; the thrown message rides the §6.3.1
|
|
144
|
+
// details and the original reaches the host's `onError`.
|
|
145
|
+
'sync.validator_error': {
|
|
146
|
+
category: 'internal',
|
|
147
|
+
retryable: false,
|
|
148
|
+
recommendedAction: 'inspectServer',
|
|
149
|
+
httpStatus: 500,
|
|
150
|
+
},
|
|
142
151
|
'sync.missing_scopes': {
|
|
143
152
|
category: 'internal',
|
|
144
153
|
retryable: false,
|
package/dist/operations.d.ts
CHANGED
|
@@ -68,6 +68,14 @@ export interface RemoteCommandContext {
|
|
|
68
68
|
readonly operationId: string;
|
|
69
69
|
readonly requestId: string;
|
|
70
70
|
getRow(table: string, rowId: string): Promise<ValidateRow | undefined>;
|
|
71
|
+
/**
|
|
72
|
+
* Batch form of `getRow` with the same scope authorization: a row outside
|
|
73
|
+
* the actor's resolved scopes is missing from the map, exactly as `getRow`
|
|
74
|
+
* returns `undefined` for it. Reads observe this commit's staged writes and
|
|
75
|
+
* deletes. Postgres answers any number of ids in one statement; SQLite and
|
|
76
|
+
* D1 issue one lookup per id.
|
|
77
|
+
*/
|
|
78
|
+
getRows(table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, ValidateRow>>;
|
|
71
79
|
/**
|
|
72
80
|
* Run a generated registered query on the command's push transaction
|
|
73
81
|
* (§6.7). The storage binds every relation to the partition; unlike
|
package/dist/operations.js
CHANGED
|
@@ -5,6 +5,7 @@ import { reportError, SyncError, syncError } from './errors.js';
|
|
|
5
5
|
import { processPushOperationsWithTrace } from './push.js';
|
|
6
6
|
import { compileSchema } from './schema.js';
|
|
7
7
|
import { authorizeWrite, renderScopeValue, } from './scopes.js';
|
|
8
|
+
import { readCandidateRows } from './storage.js';
|
|
8
9
|
import { transactionQuery } from './storage-query.js';
|
|
9
10
|
import { toValidateRow, ValidationRejection, } from './validate.js';
|
|
10
11
|
function scopeAllowed(allowed, variable, value) {
|
|
@@ -296,6 +297,23 @@ function commandContext(ctx, tx, resolved, schema, clientId, operationId, reques
|
|
|
296
297
|
}
|
|
297
298
|
return toValidateRow(table.columns, decodeRow(table.columns, stored.payload));
|
|
298
299
|
},
|
|
300
|
+
getRows: async (tableName, rowIds) => {
|
|
301
|
+
const table = schema.tables.get(tableName);
|
|
302
|
+
if (table === undefined) {
|
|
303
|
+
throw syncError('operation.invalid_request', 'command read targets an unknown table');
|
|
304
|
+
}
|
|
305
|
+
const storedRows = await readCandidateRows(tx, tableName, rowIds);
|
|
306
|
+
const rows = new Map();
|
|
307
|
+
for (const rowId of rowIds) {
|
|
308
|
+
const stored = storedRows.get(rowId);
|
|
309
|
+
if (stored === undefined ||
|
|
310
|
+
!authorizeWrite(table, stored.scopes, resolved)) {
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
rows.set(rowId, toValidateRow(table.columns, decodeRow(table.columns, stored.payload)));
|
|
314
|
+
}
|
|
315
|
+
return rows;
|
|
316
|
+
},
|
|
299
317
|
queryAuthoritative: (query) => transactionQuery(tx, query),
|
|
300
318
|
};
|
|
301
319
|
}
|
package/dist/postgres-storage.js
CHANGED
|
@@ -5,7 +5,7 @@ import { serveNotReadyError } from './readiness.js';
|
|
|
5
5
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, selectRowScopesSql, selectRowSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
8
|
+
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, selectRowScopesSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
9
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
11
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
@@ -829,6 +829,27 @@ class PostgresTransaction {
|
|
|
829
829
|
}
|
|
830
830
|
return getRowOn(this.#client, this.#resolveTable(table), this.#partition, rowId);
|
|
831
831
|
}
|
|
832
|
+
async getRows(table, rowIds) {
|
|
833
|
+
this.#assertOpen();
|
|
834
|
+
const rows = new Map();
|
|
835
|
+
const missing = [];
|
|
836
|
+
for (const rowId of new Set(rowIds)) {
|
|
837
|
+
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
838
|
+
if (prefetched === undefined)
|
|
839
|
+
missing.push(rowId);
|
|
840
|
+
else if (prefetched !== null)
|
|
841
|
+
rows.set(rowId, prefetched);
|
|
842
|
+
}
|
|
843
|
+
if (missing.length === 0)
|
|
844
|
+
return rows;
|
|
845
|
+
// One statement for every id the admission read did not already answer.
|
|
846
|
+
// `#client` flushes this transaction's staged upserts first, so the rows
|
|
847
|
+
// are the final candidate state.
|
|
848
|
+
const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table)), [this.#partition, missing]);
|
|
849
|
+
for (const record of records)
|
|
850
|
+
rows.set(record.row_id, toStoredRow(record));
|
|
851
|
+
return rows;
|
|
852
|
+
}
|
|
832
853
|
async prefetchRows(targets) {
|
|
833
854
|
this.#assertOpen();
|
|
834
855
|
const tables = [];
|
package/dist/push.js
CHANGED
|
@@ -20,12 +20,12 @@ import { retainRollbackFailure } from './storage-errors.js';
|
|
|
20
20
|
*/
|
|
21
21
|
import { decodeRow, decodeSparseRow, encodeRow, parseBlobRef, } from '@syncular/core';
|
|
22
22
|
import { clockOf, limitsOf } from './context.js';
|
|
23
|
-
import { SyncError } from './errors.js';
|
|
23
|
+
import { reportError, SyncError } from './errors.js';
|
|
24
24
|
import { emitEvent } from './events.js';
|
|
25
25
|
import { prepareReactions, toNewReactions, } from './reactions.js';
|
|
26
26
|
import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes.js';
|
|
27
27
|
import { decodeColumnVersions, encodeColumnVersions } from './relational-rows.js';
|
|
28
|
-
import { serveGateRefusal } from './storage.js';
|
|
28
|
+
import { readCandidateRows, serveGateRefusal } from './storage.js';
|
|
29
29
|
import { StorageConstraintError } from './storage-errors.js';
|
|
30
30
|
import { transactionQuery } from './storage-query.js';
|
|
31
31
|
import { serveNotReadyError } from './readiness.js';
|
|
@@ -662,6 +662,21 @@ function commitValidationReader(tx, schema) {
|
|
|
662
662
|
serverVersion: stored.serverVersion,
|
|
663
663
|
};
|
|
664
664
|
},
|
|
665
|
+
getRows: async (tableName, rowIds) => {
|
|
666
|
+
const table = tableFor(tableName);
|
|
667
|
+
const storedRows = await readCandidateRows(tx, tableName, rowIds);
|
|
668
|
+
const rows = new Map();
|
|
669
|
+
for (const rowId of rowIds) {
|
|
670
|
+
const stored = storedRows.get(rowId);
|
|
671
|
+
if (stored === undefined)
|
|
672
|
+
continue;
|
|
673
|
+
rows.set(rowId, {
|
|
674
|
+
row: toValidateRow(table.columns, decodeRow(table.columns, stored.payload)),
|
|
675
|
+
serverVersion: stored.serverVersion,
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
return rows;
|
|
679
|
+
},
|
|
665
680
|
scanRows: async ({ table: tableName, scopeFilter, afterRowId = null, limit = 100, }) => {
|
|
666
681
|
const table = tableFor(tableName);
|
|
667
682
|
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
@@ -684,7 +699,7 @@ function commitValidationReader(tx, schema) {
|
|
|
684
699
|
queryAuthoritative: (query) => transactionQuery(tx, query),
|
|
685
700
|
};
|
|
686
701
|
}
|
|
687
|
-
async function runCommitValidator(validator, tx, schema, clientId, clientCommitId, actorId, partition, operations) {
|
|
702
|
+
async function runCommitValidator(validator, onError, tx, schema, clientId, clientCommitId, actorId, partition, operations) {
|
|
688
703
|
if (validator === undefined)
|
|
689
704
|
return undefined;
|
|
690
705
|
try {
|
|
@@ -707,7 +722,30 @@ async function runCommitValidator(validator, tx, schema, clientId, clientCommitI
|
|
|
707
722
|
if (error instanceof ValidationRejection) {
|
|
708
723
|
return errorRecord(operations[0]?.opIndex ?? 0, error.code, error.message, false, error.details);
|
|
709
724
|
}
|
|
710
|
-
|
|
725
|
+
// §6.8: a throw that is not a deliberate rejection is a host fault, not a
|
|
726
|
+
// rule refusal. The host's observation hook receives the original (its
|
|
727
|
+
// stack included); the client gets a distinct code and, in the §6.3.1
|
|
728
|
+
// details, the bounded thrown message. Stacks and thrown values stay off
|
|
729
|
+
// the wire.
|
|
730
|
+
reportError(error, onError, 'sync');
|
|
731
|
+
let text = '';
|
|
732
|
+
let bytes = 0;
|
|
733
|
+
const encoder = new TextEncoder();
|
|
734
|
+
const thrown = error instanceof Error ? error.message : String(error);
|
|
735
|
+
for (const char of thrown.replace(/\p{Cc}+/gu, ' ').trim()) {
|
|
736
|
+
const size = encoder.encode(char).length;
|
|
737
|
+
if (bytes + size > 256)
|
|
738
|
+
break;
|
|
739
|
+
bytes += size;
|
|
740
|
+
text += char;
|
|
741
|
+
}
|
|
742
|
+
text = text.trimEnd();
|
|
743
|
+
return errorRecord(operations[0]?.opIndex ?? 0, 'sync.validator_error', 'whole-commit validator failed', false, {
|
|
744
|
+
reason: 'validator_threw',
|
|
745
|
+
...(text === ''
|
|
746
|
+
? {}
|
|
747
|
+
: { references: { validator: 'commit', message: text } }),
|
|
748
|
+
});
|
|
711
749
|
}
|
|
712
750
|
return undefined;
|
|
713
751
|
}
|
|
@@ -1114,7 +1152,7 @@ export async function processPushOperationsWithTrace(ctx, schema, resolved, clie
|
|
|
1114
1152
|
}
|
|
1115
1153
|
}
|
|
1116
1154
|
if (terminated === undefined) {
|
|
1117
|
-
const commitReject = await runCommitValidator(commitValidator, tx, schema, clientId, clientCommitId, ctx.actorId, partition, validatedOperations);
|
|
1155
|
+
const commitReject = await runCommitValidator(commitValidator, ctx.onError, tx, schema, clientId, clientCommitId, ctx.actorId, partition, validatedOperations);
|
|
1118
1156
|
if (commitReject?.kind === 'terminate') {
|
|
1119
1157
|
// An appended §6.11 cascade operation carries a synthetic opIndex;
|
|
1120
1158
|
// surface the originating client operation instead.
|
|
@@ -188,6 +188,13 @@ export declare function upsertSql(table: CompiledTable, dialect: RelationalDiale
|
|
|
188
188
|
* `toStoredRow` converters keep working. Params: [partition, rowId].
|
|
189
189
|
*/
|
|
190
190
|
export declare function selectRowSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
191
|
+
/**
|
|
192
|
+
* SELECT many stored rows of one table by primary key in one statement.
|
|
193
|
+
* Postgres binds the ids as one `text[]` (`row_id = ANY($2)`), so the
|
|
194
|
+
* statement count does not grow with the id count. Params: [partition,
|
|
195
|
+
* rowIds]; absent ids contribute no row.
|
|
196
|
+
*/
|
|
197
|
+
export declare function selectRowsSql(table: CompiledTable): string;
|
|
191
198
|
export interface IndexRowPageStatement {
|
|
192
199
|
readonly sql: string;
|
|
193
200
|
readonly params: readonly unknown[];
|
package/dist/relational-rows.js
CHANGED
|
@@ -491,6 +491,15 @@ export function selectRowSql(table, dialect) {
|
|
|
491
491
|
const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
|
|
492
492
|
return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
|
|
493
493
|
}
|
|
494
|
+
/**
|
|
495
|
+
* SELECT many stored rows of one table by primary key in one statement.
|
|
496
|
+
* Postgres binds the ids as one `text[]` (`row_id = ANY($2)`), so the
|
|
497
|
+
* statement count does not grow with the id count. Params: [partition,
|
|
498
|
+
* rowIds]; absent ids contribute no row.
|
|
499
|
+
*/
|
|
500
|
+
export function selectRowsSql(table) {
|
|
501
|
+
return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=ANY($2::text[])`;
|
|
502
|
+
}
|
|
494
503
|
/**
|
|
495
504
|
* Bounded exact lookup through one declared relational index. Unlike
|
|
496
505
|
* `scanRowPageSql`, this is a trusted server-host query: it never reads or
|
package/dist/sqlite-storage.js
CHANGED
|
@@ -80,6 +80,17 @@ class SqliteTransaction {
|
|
|
80
80
|
this.#assertOpen();
|
|
81
81
|
return this.#storage.getRow(this.#partition, table, rowId);
|
|
82
82
|
}
|
|
83
|
+
async getRows(table, rowIds) {
|
|
84
|
+
this.#assertOpen();
|
|
85
|
+
// SQLite is in-process: one indexed lookup per id costs no round trip.
|
|
86
|
+
const rows = new Map();
|
|
87
|
+
for (const rowId of new Set(rowIds)) {
|
|
88
|
+
const row = await this.#storage.getRow(this.#partition, table, rowId);
|
|
89
|
+
if (row !== undefined)
|
|
90
|
+
rows.set(rowId, row);
|
|
91
|
+
}
|
|
92
|
+
return rows;
|
|
93
|
+
}
|
|
83
94
|
async getTombstoneSeq(table, rowId) {
|
|
84
95
|
this.#assertOpen();
|
|
85
96
|
const row = this.#storage.db
|
package/dist/storage.d.ts
CHANGED
|
@@ -425,6 +425,16 @@ export interface AuthoritativeQueryResult {
|
|
|
425
425
|
*/
|
|
426
426
|
export interface StorageTransaction {
|
|
427
427
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
428
|
+
/**
|
|
429
|
+
* Optional candidate-state batch read by primary key. A storage with
|
|
430
|
+
* per-statement network cost answers any number of ids in one statement
|
|
431
|
+
* (Postgres binds the ids as one array); SQLite and D1 issue one lookup per
|
|
432
|
+
* id. Absent ids are missing from the map and repeated ids read once. The
|
|
433
|
+
* result observes this transaction's staged writes and deletes, exactly as
|
|
434
|
+
* `getRow` does. Both commit readers fall back to one `getRow` per id when
|
|
435
|
+
* a custom transaction omits the capability.
|
|
436
|
+
*/
|
|
437
|
+
getRows?(table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, StoredRow>>;
|
|
428
438
|
/**
|
|
429
439
|
* Optional read-ahead: the push layer names every row a commit's
|
|
430
440
|
* operations target, across every table, before it applies them. A storage
|
|
@@ -542,6 +552,12 @@ export interface StorageTransaction {
|
|
|
542
552
|
commit(): Promise<void>;
|
|
543
553
|
rollback(): Promise<void>;
|
|
544
554
|
}
|
|
555
|
+
/**
|
|
556
|
+
* The commit readers' batch read: delegate to the transaction's `getRows`
|
|
557
|
+
* when it implements the capability, otherwise issue one `getRow` per id.
|
|
558
|
+
* Both paths read the same candidate state; only the statement count differs.
|
|
559
|
+
*/
|
|
560
|
+
export declare function readCandidateRows(tx: StorageTransaction, table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, StoredRow>>;
|
|
545
561
|
export interface ServerStorage {
|
|
546
562
|
/**
|
|
547
563
|
* Create/migrate the relational per-app row tables for `schema`
|
package/dist/storage.js
CHANGED
|
@@ -75,3 +75,22 @@ export function clientRoundState(lastRoundAtMs, lastErrorCode, consecutiveFailur
|
|
|
75
75
|
/** Durable partition identity refreshed after host authentication (§2.1). */
|
|
76
76
|
/** Longest age of `lastAuthenticatedAtMs` that `touchPartition` leaves unwritten. */
|
|
77
77
|
export const PARTITION_TOUCH_INTERVAL_MS = 60_000;
|
|
78
|
+
/**
|
|
79
|
+
* The commit readers' batch read: delegate to the transaction's `getRows`
|
|
80
|
+
* when it implements the capability, otherwise issue one `getRow` per id.
|
|
81
|
+
* Both paths read the same candidate state; only the statement count differs.
|
|
82
|
+
*/
|
|
83
|
+
export async function readCandidateRows(tx, table, rowIds) {
|
|
84
|
+
const ids = [...new Set(rowIds)];
|
|
85
|
+
if (ids.length === 0)
|
|
86
|
+
return new Map();
|
|
87
|
+
if (tx.getRows !== undefined)
|
|
88
|
+
return tx.getRows(table, ids);
|
|
89
|
+
const rows = new Map();
|
|
90
|
+
for (const rowId of ids) {
|
|
91
|
+
const row = await tx.getRow(table, rowId);
|
|
92
|
+
if (row !== undefined)
|
|
93
|
+
rows.set(rowId, row);
|
|
94
|
+
}
|
|
95
|
+
return rows;
|
|
96
|
+
}
|
package/dist/validate.d.ts
CHANGED
|
@@ -81,6 +81,15 @@ export interface CommitValidationScanInput {
|
|
|
81
81
|
*/
|
|
82
82
|
export interface CommitValidationReader {
|
|
83
83
|
getRow(table: string, rowId: string): Promise<CommitValidationRow | undefined>;
|
|
84
|
+
/**
|
|
85
|
+
* Batch candidate-state read by primary key, with the same read-your-own-
|
|
86
|
+
* writes guarantee as `getRow`: staged sibling upserts are present and
|
|
87
|
+
* staged deletes are absent. An absent id is missing from the map and a
|
|
88
|
+
* repeated id reads once. There is no scope declaration and no paging: the
|
|
89
|
+
* caller already holds partition authority. Postgres answers any number of
|
|
90
|
+
* ids in one statement; SQLite and D1 issue one lookup per id.
|
|
91
|
+
*/
|
|
92
|
+
getRows(table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, CommitValidationRow>>;
|
|
84
93
|
scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
|
|
85
94
|
/**
|
|
86
95
|
* Run a generated registered query on the commit transaction (§6.8). The
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.47.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.47.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/d1-storage.ts
CHANGED
|
@@ -360,6 +360,21 @@ class D1Transaction implements StorageTransaction {
|
|
|
360
360
|
return record === null ? undefined : toStoredRow(record);
|
|
361
361
|
}
|
|
362
362
|
|
|
363
|
+
async getRows(
|
|
364
|
+
table: string,
|
|
365
|
+
rowIds: readonly string[],
|
|
366
|
+
): Promise<ReadonlyMap<string, StoredRow>> {
|
|
367
|
+
this.#assertOpen();
|
|
368
|
+
// One lookup per id: `getRow` owns the buffered-write overlay, and a
|
|
369
|
+
// mutable id list cannot share one bound statement across D1's limit.
|
|
370
|
+
const rows = new Map<string, StoredRow>();
|
|
371
|
+
for (const rowId of new Set(rowIds)) {
|
|
372
|
+
const row = await this.getRow(table, rowId);
|
|
373
|
+
if (row !== undefined) rows.set(rowId, row);
|
|
374
|
+
}
|
|
375
|
+
return rows;
|
|
376
|
+
}
|
|
377
|
+
|
|
363
378
|
async getTombstoneSeq(
|
|
364
379
|
table: string,
|
|
365
380
|
rowId: string,
|
package/src/errors.ts
CHANGED
|
@@ -147,6 +147,15 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
|
|
|
147
147
|
recommendedAction: 'fixRequest',
|
|
148
148
|
httpStatus: 400,
|
|
149
149
|
},
|
|
150
|
+
// §6.8: the whole-commit validator threw an exception that is not a
|
|
151
|
+
// deliberate rejection. A host fault; the thrown message rides the §6.3.1
|
|
152
|
+
// details and the original reaches the host's `onError`.
|
|
153
|
+
'sync.validator_error': {
|
|
154
|
+
category: 'internal',
|
|
155
|
+
retryable: false,
|
|
156
|
+
recommendedAction: 'inspectServer',
|
|
157
|
+
httpStatus: 500,
|
|
158
|
+
},
|
|
150
159
|
'sync.missing_scopes': {
|
|
151
160
|
category: 'internal',
|
|
152
161
|
retryable: false,
|
package/src/operations.ts
CHANGED
|
@@ -32,6 +32,7 @@ import type {
|
|
|
32
32
|
AuthoritativeQueryValue,
|
|
33
33
|
} from './storage';
|
|
34
34
|
import type { StorageTransaction } from './storage';
|
|
35
|
+
import { readCandidateRows } from './storage';
|
|
35
36
|
import { transactionQuery } from './storage-query';
|
|
36
37
|
import {
|
|
37
38
|
toValidateRow,
|
|
@@ -126,6 +127,17 @@ export interface RemoteCommandContext {
|
|
|
126
127
|
readonly operationId: string;
|
|
127
128
|
readonly requestId: string;
|
|
128
129
|
getRow(table: string, rowId: string): Promise<ValidateRow | undefined>;
|
|
130
|
+
/**
|
|
131
|
+
* Batch form of `getRow` with the same scope authorization: a row outside
|
|
132
|
+
* the actor's resolved scopes is missing from the map, exactly as `getRow`
|
|
133
|
+
* returns `undefined` for it. Reads observe this commit's staged writes and
|
|
134
|
+
* deletes. Postgres answers any number of ids in one statement; SQLite and
|
|
135
|
+
* D1 issue one lookup per id.
|
|
136
|
+
*/
|
|
137
|
+
getRows(
|
|
138
|
+
table: string,
|
|
139
|
+
rowIds: readonly string[],
|
|
140
|
+
): Promise<ReadonlyMap<string, ValidateRow>>;
|
|
129
141
|
/**
|
|
130
142
|
* Run a generated registered query on the command's push transaction
|
|
131
143
|
* (§6.7). The storage binds every relation to the partition; unlike
|
|
@@ -599,6 +611,34 @@ function commandContext(
|
|
|
599
611
|
decodeRow(table.columns, stored.payload),
|
|
600
612
|
);
|
|
601
613
|
},
|
|
614
|
+
getRows: async (tableName, rowIds) => {
|
|
615
|
+
const table = schema.tables.get(tableName);
|
|
616
|
+
if (table === undefined) {
|
|
617
|
+
throw syncError(
|
|
618
|
+
'operation.invalid_request',
|
|
619
|
+
'command read targets an unknown table',
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
const storedRows = await readCandidateRows(tx, tableName, rowIds);
|
|
623
|
+
const rows = new Map<string, ValidateRow>();
|
|
624
|
+
for (const rowId of rowIds) {
|
|
625
|
+
const stored = storedRows.get(rowId);
|
|
626
|
+
if (
|
|
627
|
+
stored === undefined ||
|
|
628
|
+
!authorizeWrite(table, stored.scopes, resolved)
|
|
629
|
+
) {
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
rows.set(
|
|
633
|
+
rowId,
|
|
634
|
+
toValidateRow(
|
|
635
|
+
table.columns,
|
|
636
|
+
decodeRow(table.columns, stored.payload),
|
|
637
|
+
),
|
|
638
|
+
);
|
|
639
|
+
}
|
|
640
|
+
return rows;
|
|
641
|
+
},
|
|
602
642
|
queryAuthoritative: (query) => transactionQuery(tx, query),
|
|
603
643
|
};
|
|
604
644
|
}
|
package/src/postgres-storage.ts
CHANGED
|
@@ -92,6 +92,7 @@ import {
|
|
|
92
92
|
serverIndexDdl,
|
|
93
93
|
selectRowScopesSql,
|
|
94
94
|
selectRowSql,
|
|
95
|
+
selectRowsSql,
|
|
95
96
|
prefetchRowsBatchSql,
|
|
96
97
|
selectRowsForRewriteSql,
|
|
97
98
|
upsertSql,
|
|
@@ -1300,6 +1301,30 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1300
1301
|
);
|
|
1301
1302
|
}
|
|
1302
1303
|
|
|
1304
|
+
async getRows(
|
|
1305
|
+
table: string,
|
|
1306
|
+
rowIds: readonly string[],
|
|
1307
|
+
): Promise<ReadonlyMap<string, StoredRow>> {
|
|
1308
|
+
this.#assertOpen();
|
|
1309
|
+
const rows = new Map<string, StoredRow>();
|
|
1310
|
+
const missing: string[] = [];
|
|
1311
|
+
for (const rowId of new Set(rowIds)) {
|
|
1312
|
+
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
1313
|
+
if (prefetched === undefined) missing.push(rowId);
|
|
1314
|
+
else if (prefetched !== null) rows.set(rowId, prefetched);
|
|
1315
|
+
}
|
|
1316
|
+
if (missing.length === 0) return rows;
|
|
1317
|
+
// One statement for every id the admission read did not already answer.
|
|
1318
|
+
// `#client` flushes this transaction's staged upserts first, so the rows
|
|
1319
|
+
// are the final candidate state.
|
|
1320
|
+
const { rows: records } = await this.#client.query<RowRecord>(
|
|
1321
|
+
selectRowsSql(this.#resolveTable(table)),
|
|
1322
|
+
[this.#partition, missing],
|
|
1323
|
+
);
|
|
1324
|
+
for (const record of records) rows.set(record.row_id, toStoredRow(record));
|
|
1325
|
+
return rows;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1303
1328
|
async prefetchRows(
|
|
1304
1329
|
targets: readonly {
|
|
1305
1330
|
readonly table: string;
|
package/src/push.ts
CHANGED
|
@@ -35,7 +35,7 @@ import type { BlobStore } from './blob-store';
|
|
|
35
35
|
import type { SyncRequestContext } from './context';
|
|
36
36
|
import { clockOf, limitsOf } from './context';
|
|
37
37
|
import type { CrdtMergerRegistry } from './crdt-merger';
|
|
38
|
-
import { SyncError } from './errors';
|
|
38
|
+
import { reportError, SyncError, type SyncularErrorHandler } from './errors';
|
|
39
39
|
import { emitEvent } from './events';
|
|
40
40
|
import {
|
|
41
41
|
prepareReactions,
|
|
@@ -54,12 +54,13 @@ import type {
|
|
|
54
54
|
StoredPushResult,
|
|
55
55
|
StoredRow,
|
|
56
56
|
} from './storage';
|
|
57
|
-
import { serveGateRefusal } from './storage';
|
|
57
|
+
import { readCandidateRows, serveGateRefusal } from './storage';
|
|
58
58
|
import { StorageConstraintError } from './storage-errors';
|
|
59
59
|
import { transactionQuery } from './storage-query';
|
|
60
60
|
import { serveNotReadyError } from './readiness';
|
|
61
61
|
import type {
|
|
62
62
|
CommitValidationReader,
|
|
63
|
+
CommitValidationRow,
|
|
63
64
|
CommitValidator,
|
|
64
65
|
ValidateCommitOperation,
|
|
65
66
|
ValidateOpKind,
|
|
@@ -1061,6 +1062,23 @@ function commitValidationReader(
|
|
|
1061
1062
|
serverVersion: stored.serverVersion,
|
|
1062
1063
|
};
|
|
1063
1064
|
},
|
|
1065
|
+
getRows: async (tableName, rowIds) => {
|
|
1066
|
+
const table = tableFor(tableName);
|
|
1067
|
+
const storedRows = await readCandidateRows(tx, tableName, rowIds);
|
|
1068
|
+
const rows = new Map<string, CommitValidationRow>();
|
|
1069
|
+
for (const rowId of rowIds) {
|
|
1070
|
+
const stored = storedRows.get(rowId);
|
|
1071
|
+
if (stored === undefined) continue;
|
|
1072
|
+
rows.set(rowId, {
|
|
1073
|
+
row: toValidateRow(
|
|
1074
|
+
table.columns,
|
|
1075
|
+
decodeRow(table.columns, stored.payload),
|
|
1076
|
+
),
|
|
1077
|
+
serverVersion: stored.serverVersion,
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
return rows;
|
|
1081
|
+
},
|
|
1064
1082
|
scanRows: async ({
|
|
1065
1083
|
table: tableName,
|
|
1066
1084
|
scopeFilter,
|
|
@@ -1098,6 +1116,7 @@ function commitValidationReader(
|
|
|
1098
1116
|
|
|
1099
1117
|
async function runCommitValidator(
|
|
1100
1118
|
validator: CommitValidator | undefined,
|
|
1119
|
+
onError: SyncularErrorHandler | undefined,
|
|
1101
1120
|
tx: StorageTransaction,
|
|
1102
1121
|
schema: CompiledSchema,
|
|
1103
1122
|
clientId: string,
|
|
@@ -1142,10 +1161,34 @@ async function runCommitValidator(
|
|
|
1142
1161
|
error.details,
|
|
1143
1162
|
);
|
|
1144
1163
|
}
|
|
1164
|
+
// §6.8: a throw that is not a deliberate rejection is a host fault, not a
|
|
1165
|
+
// rule refusal. The host's observation hook receives the original (its
|
|
1166
|
+
// stack included); the client gets a distinct code and, in the §6.3.1
|
|
1167
|
+
// details, the bounded thrown message. Stacks and thrown values stay off
|
|
1168
|
+
// the wire.
|
|
1169
|
+
reportError(error, onError, 'sync');
|
|
1170
|
+
let text = '';
|
|
1171
|
+
let bytes = 0;
|
|
1172
|
+
const encoder = new TextEncoder();
|
|
1173
|
+
const thrown = error instanceof Error ? error.message : String(error);
|
|
1174
|
+
for (const char of thrown.replace(/\p{Cc}+/gu, ' ').trim()) {
|
|
1175
|
+
const size = encoder.encode(char).length;
|
|
1176
|
+
if (bytes + size > 256) break;
|
|
1177
|
+
bytes += size;
|
|
1178
|
+
text += char;
|
|
1179
|
+
}
|
|
1180
|
+
text = text.trimEnd();
|
|
1145
1181
|
return errorRecord(
|
|
1146
1182
|
operations[0]?.opIndex ?? 0,
|
|
1147
|
-
'sync.
|
|
1183
|
+
'sync.validator_error',
|
|
1148
1184
|
'whole-commit validator failed',
|
|
1185
|
+
false,
|
|
1186
|
+
{
|
|
1187
|
+
reason: 'validator_threw',
|
|
1188
|
+
...(text === ''
|
|
1189
|
+
? {}
|
|
1190
|
+
: { references: { validator: 'commit', message: text } }),
|
|
1191
|
+
},
|
|
1149
1192
|
);
|
|
1150
1193
|
}
|
|
1151
1194
|
return undefined;
|
|
@@ -1751,6 +1794,7 @@ export async function processPushOperationsWithTrace(
|
|
|
1751
1794
|
if (terminated === undefined) {
|
|
1752
1795
|
const commitReject = await runCommitValidator(
|
|
1753
1796
|
commitValidator,
|
|
1797
|
+
ctx.onError,
|
|
1754
1798
|
tx,
|
|
1755
1799
|
schema,
|
|
1756
1800
|
clientId,
|
package/src/relational-rows.ts
CHANGED
|
@@ -607,6 +607,16 @@ export function selectRowSql(
|
|
|
607
607
|
return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
|
|
608
608
|
}
|
|
609
609
|
|
|
610
|
+
/**
|
|
611
|
+
* SELECT many stored rows of one table by primary key in one statement.
|
|
612
|
+
* Postgres binds the ids as one `text[]` (`row_id = ANY($2)`), so the
|
|
613
|
+
* statement count does not grow with the id count. Params: [partition,
|
|
614
|
+
* rowIds]; absent ids contribute no row.
|
|
615
|
+
*/
|
|
616
|
+
export function selectRowsSql(table: CompiledTable): string {
|
|
617
|
+
return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=ANY($2::text[])`;
|
|
618
|
+
}
|
|
619
|
+
|
|
610
620
|
export interface IndexRowPageStatement {
|
|
611
621
|
readonly sql: string;
|
|
612
622
|
readonly params: readonly unknown[];
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -234,6 +234,20 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
234
234
|
return this.#storage.getRow(this.#partition, table, rowId);
|
|
235
235
|
}
|
|
236
236
|
|
|
237
|
+
async getRows(
|
|
238
|
+
table: string,
|
|
239
|
+
rowIds: readonly string[],
|
|
240
|
+
): Promise<ReadonlyMap<string, StoredRow>> {
|
|
241
|
+
this.#assertOpen();
|
|
242
|
+
// SQLite is in-process: one indexed lookup per id costs no round trip.
|
|
243
|
+
const rows = new Map<string, StoredRow>();
|
|
244
|
+
for (const rowId of new Set(rowIds)) {
|
|
245
|
+
const row = await this.#storage.getRow(this.#partition, table, rowId);
|
|
246
|
+
if (row !== undefined) rows.set(rowId, row);
|
|
247
|
+
}
|
|
248
|
+
return rows;
|
|
249
|
+
}
|
|
250
|
+
|
|
237
251
|
async getTombstoneSeq(
|
|
238
252
|
table: string,
|
|
239
253
|
rowId: string,
|
package/src/storage.ts
CHANGED
|
@@ -557,6 +557,19 @@ export interface AuthoritativeQueryResult {
|
|
|
557
557
|
*/
|
|
558
558
|
export interface StorageTransaction {
|
|
559
559
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
560
|
+
/**
|
|
561
|
+
* Optional candidate-state batch read by primary key. A storage with
|
|
562
|
+
* per-statement network cost answers any number of ids in one statement
|
|
563
|
+
* (Postgres binds the ids as one array); SQLite and D1 issue one lookup per
|
|
564
|
+
* id. Absent ids are missing from the map and repeated ids read once. The
|
|
565
|
+
* result observes this transaction's staged writes and deletes, exactly as
|
|
566
|
+
* `getRow` does. Both commit readers fall back to one `getRow` per id when
|
|
567
|
+
* a custom transaction omits the capability.
|
|
568
|
+
*/
|
|
569
|
+
getRows?(
|
|
570
|
+
table: string,
|
|
571
|
+
rowIds: readonly string[],
|
|
572
|
+
): Promise<ReadonlyMap<string, StoredRow>>;
|
|
560
573
|
/**
|
|
561
574
|
* Optional read-ahead: the push layer names every row a commit's
|
|
562
575
|
* operations target, across every table, before it applies them. A storage
|
|
@@ -704,6 +717,27 @@ export interface StorageTransaction {
|
|
|
704
717
|
rollback(): Promise<void>;
|
|
705
718
|
}
|
|
706
719
|
|
|
720
|
+
/**
|
|
721
|
+
* The commit readers' batch read: delegate to the transaction's `getRows`
|
|
722
|
+
* when it implements the capability, otherwise issue one `getRow` per id.
|
|
723
|
+
* Both paths read the same candidate state; only the statement count differs.
|
|
724
|
+
*/
|
|
725
|
+
export async function readCandidateRows(
|
|
726
|
+
tx: StorageTransaction,
|
|
727
|
+
table: string,
|
|
728
|
+
rowIds: readonly string[],
|
|
729
|
+
): Promise<ReadonlyMap<string, StoredRow>> {
|
|
730
|
+
const ids = [...new Set(rowIds)];
|
|
731
|
+
if (ids.length === 0) return new Map();
|
|
732
|
+
if (tx.getRows !== undefined) return tx.getRows(table, ids);
|
|
733
|
+
const rows = new Map<string, StoredRow>();
|
|
734
|
+
for (const rowId of ids) {
|
|
735
|
+
const row = await tx.getRow(table, rowId);
|
|
736
|
+
if (row !== undefined) rows.set(rowId, row);
|
|
737
|
+
}
|
|
738
|
+
return rows;
|
|
739
|
+
}
|
|
740
|
+
|
|
707
741
|
export interface ServerStorage {
|
|
708
742
|
/**
|
|
709
743
|
* Create/migrate the relational per-app row tables for `schema`
|
package/src/validate.ts
CHANGED
|
@@ -107,6 +107,18 @@ export interface CommitValidationReader {
|
|
|
107
107
|
table: string,
|
|
108
108
|
rowId: string,
|
|
109
109
|
): Promise<CommitValidationRow | undefined>;
|
|
110
|
+
/**
|
|
111
|
+
* Batch candidate-state read by primary key, with the same read-your-own-
|
|
112
|
+
* writes guarantee as `getRow`: staged sibling upserts are present and
|
|
113
|
+
* staged deletes are absent. An absent id is missing from the map and a
|
|
114
|
+
* repeated id reads once. There is no scope declaration and no paging: the
|
|
115
|
+
* caller already holds partition authority. Postgres answers any number of
|
|
116
|
+
* ids in one statement; SQLite and D1 issue one lookup per id.
|
|
117
|
+
*/
|
|
118
|
+
getRows(
|
|
119
|
+
table: string,
|
|
120
|
+
rowIds: readonly string[],
|
|
121
|
+
): Promise<ReadonlyMap<string, CommitValidationRow>>;
|
|
110
122
|
scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
|
|
111
123
|
/**
|
|
112
124
|
* Run a generated registered query on the commit transaction (§6.8). The
|