@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/dist/pg-executor.d.ts
CHANGED
|
@@ -47,11 +47,14 @@ export type PgRow = Record<string, unknown>;
|
|
|
47
47
|
* a time. An adapter does not need to queue calls itself, and must not
|
|
48
48
|
* accept two in flight: node-postgres warns about that today and rejects it
|
|
49
49
|
* in node-postgres 9.
|
|
50
|
+
*
|
|
51
|
+
* The result carries returned rows only. Drivers disagree on the affected-row
|
|
52
|
+
* count of a statement without result rows (Bun.sql reports none), so every
|
|
53
|
+
* write whose outcome matters ends in `RETURNING` and counts `rows`.
|
|
50
54
|
*/
|
|
51
55
|
export interface PgQueryable {
|
|
52
56
|
query<Row = PgRow>(text: string, params?: readonly unknown[]): Promise<{
|
|
53
57
|
rows: Row[];
|
|
54
|
-
rowCount: number;
|
|
55
58
|
}>;
|
|
56
59
|
}
|
|
57
60
|
/**
|
|
@@ -69,7 +69,7 @@ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDec
|
|
|
69
69
|
* set, §5.9.5) as an index range, never a scan. `postgres-explain
|
|
70
70
|
* .test.ts` asserts an `Index` node here too.
|
|
71
71
|
*/
|
|
72
|
-
export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n last_round_at_ms BIGINT, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
72
|
+
export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n last_round_at_ms BIGINT, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,\n PRIMARY KEY(partition, tbl, row_id)\n);\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;\nALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
73
73
|
export declare class PostgresServerStorage implements ServerStorage {
|
|
74
74
|
#private;
|
|
75
75
|
constructor(exec: PgExecutor);
|
package/dist/postgres-storage.js
CHANGED
|
@@ -5,7 +5,7 @@ import { serveNotReadyError } from './readiness.js';
|
|
|
5
5
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl,
|
|
8
|
+
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
9
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
11
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
@@ -128,9 +128,11 @@ CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
|
|
|
128
128
|
ON sync_blob_refs(partition, blob_id);
|
|
129
129
|
CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
130
130
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
131
|
-
commit_seq BIGINT NOT NULL,
|
|
131
|
+
commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,
|
|
132
132
|
PRIMARY KEY(partition, tbl, row_id)
|
|
133
133
|
);
|
|
134
|
+
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;
|
|
135
|
+
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;
|
|
134
136
|
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
135
137
|
partition TEXT NOT NULL, name TEXT NOT NULL,
|
|
136
138
|
schema_version INTEGER NOT NULL,
|
|
@@ -278,11 +280,13 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
|
|
|
278
280
|
NULL::text AS tbl, NULL::text AS lookup_id, NULL::text AS row_id,
|
|
279
281
|
NULL::bigint AS server_version, NULL::jsonb AS scopes,
|
|
280
282
|
NULL::bytea AS payload, NULL::bytea AS column_versions,
|
|
281
|
-
NULL::bigint AS tombstone_seq
|
|
283
|
+
NULL::bigint AS tombstone_seq, NULL::bigint AS tombstone_version,
|
|
284
|
+
NULL::jsonb AS tombstone_scopes
|
|
282
285
|
UNION ALL
|
|
283
286
|
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb,
|
|
284
287
|
p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
|
|
285
|
-
p.payload, p.column_versions, p.tombstone_seq
|
|
288
|
+
p.payload, p.column_versions, p.tombstone_seq,
|
|
289
|
+
p.tombstone_version, p.tombstone_scopes
|
|
286
290
|
FROM (${prefetchRowsBatchSql(prefetch.tables, {
|
|
287
291
|
partition: '$1',
|
|
288
292
|
specs: `$${params.length}`,
|
|
@@ -466,6 +470,19 @@ function toStoredRow(record) {
|
|
|
466
470
|
: { columnVersions: asBytes(record.column_versions) }),
|
|
467
471
|
};
|
|
468
472
|
}
|
|
473
|
+
function toTombstone(seq, version, scopes) {
|
|
474
|
+
if (seq === null || seq === undefined)
|
|
475
|
+
return null;
|
|
476
|
+
return {
|
|
477
|
+
commitSeq: asNumber(seq),
|
|
478
|
+
...(version === null || version === undefined
|
|
479
|
+
? {}
|
|
480
|
+
: { serverVersion: asNumber(version) }),
|
|
481
|
+
...(scopes === null || scopes === undefined
|
|
482
|
+
? {}
|
|
483
|
+
: { scopes: asJson(scopes) }),
|
|
484
|
+
};
|
|
485
|
+
}
|
|
469
486
|
function toStoredChange(record) {
|
|
470
487
|
return {
|
|
471
488
|
table: record.tbl,
|
|
@@ -851,7 +868,7 @@ class PostgresTransaction {
|
|
|
851
868
|
// One statement for every id the admission read did not already answer.
|
|
852
869
|
// `#client` flushes this transaction's staged upserts first, so the rows
|
|
853
870
|
// are the final candidate state.
|
|
854
|
-
const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table)), [this.#partition, missing]);
|
|
871
|
+
const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table), 'postgres'), [this.#partition, missing]);
|
|
855
872
|
for (const record of records)
|
|
856
873
|
rows.set(record.row_id, toStoredRow(record));
|
|
857
874
|
return rows;
|
|
@@ -872,26 +889,25 @@ class PostgresTransaction {
|
|
|
872
889
|
const { rows } = await this.#client.query(prefetchRowsBatchSql(tables), [this.#partition, JSON.stringify(specs)]);
|
|
873
890
|
this.#recordPrefetched(rows);
|
|
874
891
|
}
|
|
875
|
-
/** Answer the following `getRow`/`
|
|
892
|
+
/** Answer the following `getRow`/`getTombstone` calls from a read snapshot. */
|
|
876
893
|
#recordPrefetched(records) {
|
|
877
894
|
for (const record of records) {
|
|
878
895
|
const key = `${record.tbl}\u0000${record.lookup_id}`;
|
|
879
896
|
this.#prefetchedRows.set(key, record.row_id === null || record.row_id === undefined
|
|
880
897
|
? null
|
|
881
898
|
: toStoredRow({ ...record, row_id: record.row_id }));
|
|
882
|
-
this.#prefetchedTombstones.set(key, record.tombstone_seq
|
|
883
|
-
? null
|
|
884
|
-
: asNumber(record.tombstone_seq));
|
|
899
|
+
this.#prefetchedTombstones.set(key, toTombstone(record.tombstone_seq, record.tombstone_version, record.tombstone_scopes));
|
|
885
900
|
}
|
|
886
901
|
}
|
|
887
|
-
async
|
|
902
|
+
async getTombstone(table, rowId) {
|
|
888
903
|
this.#assertOpen();
|
|
889
904
|
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
890
905
|
if (prefetched !== undefined)
|
|
891
906
|
return prefetched ?? undefined;
|
|
892
|
-
const { rows } = await this.#client.query('SELECT commit_seq FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
893
|
-
const
|
|
894
|
-
return
|
|
907
|
+
const { rows } = await this.#client.query('SELECT commit_seq, server_version, scopes FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
908
|
+
const row = rows[0];
|
|
909
|
+
return (toTombstone(row?.commit_seq, row?.server_version, row?.scopes) ??
|
|
910
|
+
undefined);
|
|
895
911
|
}
|
|
896
912
|
async clearTombstone(table, rowId) {
|
|
897
913
|
this.#assertOpen();
|
|
@@ -987,10 +1003,8 @@ class PostgresTransaction {
|
|
|
987
1003
|
async advanceCheckpoint(name, ownerEpoch, watermark, observedRows, nowMs) {
|
|
988
1004
|
this.#assertOpen();
|
|
989
1005
|
await this.#ensurePushSavepoint();
|
|
990
|
-
// `RETURNING`:
|
|
991
|
-
//
|
|
992
|
-
// adapter derives `rowCount` from the returned rows, so a bare UPDATE
|
|
993
|
-
// reports 0 and the CAS would never advance on a real Bun.sql host.
|
|
1006
|
+
// `RETURNING`: `PgQueryable` reports returned rows only (no affected-row
|
|
1007
|
+
// count), so every affected-row check counts returned rows.
|
|
994
1008
|
const result = await this.#client.query(`UPDATE sync_backfill_checkpoints
|
|
995
1009
|
SET watermark=$1, observed_rows=$2, updated_at_ms=$3
|
|
996
1010
|
WHERE partition=$4 AND name=$5 AND owner_epoch=$6
|
|
@@ -1152,11 +1166,13 @@ class PostgresTransaction {
|
|
|
1152
1166
|
// chunks run under the same commitSeq with their global idx. Serialized
|
|
1153
1167
|
// scopes bind as text before parsing JSONB; drivers that encode JSONB
|
|
1154
1168
|
// parameters would otherwise store the string as a scalar. A commit can
|
|
1155
|
-
// delete one row twice, so a chunk's tombstone insert keeps
|
|
1156
|
-
// key and a later chunk
|
|
1169
|
+
// delete one row twice, so a chunk's tombstone insert keeps the latest
|
|
1170
|
+
// delete per key and a later chunk overwrites it with its own. `version`
|
|
1171
|
+
// is an upsert's row version or a delete's deleted version (§2.2); only
|
|
1172
|
+
// the tombstone keeps the latter.
|
|
1157
1173
|
const valuesOf = (params, slice, firstIdx) => slice.map((change, offset) => {
|
|
1158
1174
|
const base = params.length;
|
|
1159
|
-
params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
|
|
1175
|
+
params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? change.deletedVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
|
|
1160
1176
|
return `($${base + 1}::int,$${base + 2}::text,$${base + 3}::text,$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
|
|
1161
1177
|
});
|
|
1162
1178
|
const scopesCte = `scoped AS (
|
|
@@ -1165,16 +1181,21 @@ class PostgresTransaction {
|
|
|
1165
1181
|
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
1166
1182
|
ON CONFLICT DO NOTHING
|
|
1167
1183
|
)`;
|
|
1168
|
-
const tombstones = (movedParam) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1169
|
-
SELECT DISTINCT ON (tbl, row_id) partition,
|
|
1170
|
-
|
|
1171
|
-
|
|
1184
|
+
const tombstones = (movedParam) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
|
|
1185
|
+
SELECT DISTINCT ON (inserted.tbl, inserted.row_id) inserted.partition,
|
|
1186
|
+
inserted.tbl, inserted.row_id, inserted.commit_seq,
|
|
1187
|
+
v.version, inserted.scopes
|
|
1188
|
+
FROM inserted JOIN v ON v.idx = inserted.idx
|
|
1189
|
+
WHERE inserted.op = 2
|
|
1172
1190
|
AND NOT EXISTS (
|
|
1173
1191
|
SELECT 1
|
|
1174
1192
|
FROM jsonb_to_recordset(${movedParam}::text::jsonb) AS moved(tbl text, row_id text)
|
|
1175
1193
|
WHERE moved.tbl = inserted.tbl AND moved.row_id = inserted.row_id)
|
|
1194
|
+
ORDER BY inserted.tbl, inserted.row_id, inserted.idx DESC
|
|
1176
1195
|
ON CONFLICT (partition, tbl, row_id)
|
|
1177
|
-
DO UPDATE SET commit_seq = excluded.commit_seq
|
|
1196
|
+
DO UPDATE SET commit_seq = excluded.commit_seq,
|
|
1197
|
+
server_version = excluded.server_version,
|
|
1198
|
+
scopes = excluded.scopes`;
|
|
1178
1199
|
const firstSlice = commit.changes.slice(0, PG_CHANGES_PER_STATEMENT);
|
|
1179
1200
|
const params = [...head];
|
|
1180
1201
|
const values = valuesOf(params, firstSlice, 0);
|
|
@@ -1182,12 +1203,14 @@ class PostgresTransaction {
|
|
|
1182
1203
|
const movedParam = `$${params.length}`;
|
|
1183
1204
|
const { rows } = await q.query(`WITH allocated AS (${allocate}),
|
|
1184
1205
|
allocated_commit AS (${commitRow}),
|
|
1206
|
+
v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1207
|
+
VALUES ${values.join(',')}
|
|
1208
|
+
),
|
|
1185
1209
|
inserted AS (
|
|
1186
1210
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1187
|
-
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.
|
|
1188
|
-
FROM allocated
|
|
1189
|
-
|
|
1190
|
-
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1211
|
+
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, CASE WHEN v.op = 1 THEN v.version END, v.scopes, v.payload
|
|
1212
|
+
FROM allocated CROSS JOIN v
|
|
1213
|
+
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1191
1214
|
),
|
|
1192
1215
|
${scopesCte},
|
|
1193
1216
|
tombstoned AS (${tombstones(movedParam)})
|
|
@@ -1198,11 +1221,14 @@ class PostgresTransaction {
|
|
|
1198
1221
|
const chunkValues = valuesOf(chunkParams, commit.changes.slice(first, first + PG_CHANGES_PER_STATEMENT), first);
|
|
1199
1222
|
chunkParams.push(moved);
|
|
1200
1223
|
const chunkMovedParam = `$${chunkParams.length}`;
|
|
1201
|
-
await q.query(`WITH
|
|
1224
|
+
await q.query(`WITH v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1225
|
+
VALUES ${chunkValues.join(',')}
|
|
1226
|
+
),
|
|
1227
|
+
inserted AS (
|
|
1202
1228
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1203
|
-
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.
|
|
1204
|
-
FROM
|
|
1205
|
-
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1229
|
+
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, CASE WHEN v.op = 1 THEN v.version END, v.scopes, v.payload
|
|
1230
|
+
FROM v
|
|
1231
|
+
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1206
1232
|
),
|
|
1207
1233
|
${scopesCte},
|
|
1208
1234
|
tombstoned AS (${tombstones(chunkMovedParam)})
|
|
@@ -1654,9 +1680,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1654
1680
|
*/
|
|
1655
1681
|
async begin(partition) {
|
|
1656
1682
|
let resolveReady;
|
|
1657
|
-
|
|
1658
|
-
|
|
1683
|
+
let rejectReady;
|
|
1684
|
+
const ready = new Promise((res, rej) => {
|
|
1685
|
+
resolveReady = res;
|
|
1686
|
+
rejectReady = rej;
|
|
1659
1687
|
});
|
|
1688
|
+
let opened = false;
|
|
1660
1689
|
let resolveScope;
|
|
1661
1690
|
let rejectScope;
|
|
1662
1691
|
const scope = new Promise((res, rej) => {
|
|
@@ -1666,6 +1695,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1666
1695
|
const done = this.#exec
|
|
1667
1696
|
.transaction(async (client) => {
|
|
1668
1697
|
const tx = new PostgresTransaction(client, partition, this.#schemaVersion ?? 0, (name) => this.table(name), (query) => this.#prepareAuthoritativeQuery(partition, query), resolveScope, rejectScope);
|
|
1698
|
+
opened = true;
|
|
1669
1699
|
resolveReady(tx);
|
|
1670
1700
|
// Hold the transaction open until commit()/rollback() settles `scope`.
|
|
1671
1701
|
await scope;
|
|
@@ -1673,6 +1703,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1673
1703
|
.catch((error) => {
|
|
1674
1704
|
if (error instanceof RollbackSignal)
|
|
1675
1705
|
return;
|
|
1706
|
+
// The executor failed before the callback ran (no connection, BEGIN
|
|
1707
|
+
// refused): `begin()` itself rejects, and no transaction exists to
|
|
1708
|
+
// observe `done`.
|
|
1709
|
+
if (!opened) {
|
|
1710
|
+
rejectReady(error);
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1676
1713
|
throw error;
|
|
1677
1714
|
});
|
|
1678
1715
|
const tx = await ready;
|
|
@@ -1728,15 +1765,23 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1728
1765
|
const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
|
|
1729
1766
|
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
1730
1767
|
await client.query('UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1', [partition, horizonSeq]);
|
|
1731
|
-
const removed = await client.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1768
|
+
const removed = await client.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2 RETURNING commit_seq', [partition, horizonSeq]);
|
|
1769
|
+
// Scope entries go by the keys of the changes being pruned, before
|
|
1770
|
+
// those changes are deleted: a `commit_seq` predicate alone would scan
|
|
1771
|
+
// the partition's whole scope index (`commit_seq` is its last key column).
|
|
1772
|
+
await client.query(`DELETE FROM sync_change_scopes s USING (
|
|
1773
|
+
SELECT DISTINCT c.tbl, e.key, e.value, c.commit_seq
|
|
1774
|
+
FROM sync_changes c CROSS JOIN LATERAL jsonb_each_text(c.scopes) e
|
|
1775
|
+
WHERE c.partition=$1 AND c.commit_seq<=$2) k
|
|
1776
|
+
WHERE s.partition=$1 AND s.tbl=k.tbl AND s.var=k.key
|
|
1777
|
+
AND s.value=k.value AND s.commit_seq=k.commit_seq`, [partition, horizonSeq]);
|
|
1732
1778
|
await client.query('DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1733
|
-
await client.query('DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1734
1779
|
// §4.6: tombstones prune in the same pass as the commit log.
|
|
1735
1780
|
await client.query('DELETE FROM sync_tombstones WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1736
1781
|
return {
|
|
1737
1782
|
previousHorizonSeq,
|
|
1738
1783
|
horizonSeq,
|
|
1739
|
-
removedCommits: removed.
|
|
1784
|
+
removedCommits: removed.rows.length,
|
|
1740
1785
|
};
|
|
1741
1786
|
});
|
|
1742
1787
|
}
|
|
@@ -1892,21 +1937,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1892
1937
|
SET status='completed', completed_at_ms=$4,
|
|
1893
1938
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1894
1939
|
WHERE partition=$1 AND idempotency_key=$2
|
|
1895
|
-
AND status='leased' AND lease_owner=$3
|
|
1896
|
-
|
|
1940
|
+
AND status='leased' AND lease_owner=$3
|
|
1941
|
+
RETURNING idempotency_key`, [partition, idempotencyKey, leaseOwner, completedAtMs]);
|
|
1942
|
+
return result.rows.length === 1;
|
|
1897
1943
|
}
|
|
1898
1944
|
async extendReactionLease(partition, idempotencyKey, leaseOwner, leaseExpiresAtMs) {
|
|
1899
1945
|
const result = await this.#exec.query(`UPDATE sync_reactions SET lease_expires_at_ms=$4
|
|
1900
1946
|
WHERE partition=$1 AND idempotency_key=$2
|
|
1901
|
-
AND status='leased' AND lease_owner=$3
|
|
1902
|
-
|
|
1947
|
+
AND status='leased' AND lease_owner=$3
|
|
1948
|
+
RETURNING idempotency_key`, [partition, idempotencyKey, leaseOwner, leaseExpiresAtMs]);
|
|
1949
|
+
return result.rows.length === 1;
|
|
1903
1950
|
}
|
|
1904
1951
|
async failReaction(partition, idempotencyKey, update) {
|
|
1905
1952
|
const result = await this.#exec.query(`UPDATE sync_reactions
|
|
1906
1953
|
SET status=$4, available_at_ms=$5, last_failure=$6,
|
|
1907
1954
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1908
1955
|
WHERE partition=$1 AND idempotency_key=$2
|
|
1909
|
-
AND status='leased' AND lease_owner=$3
|
|
1956
|
+
AND status='leased' AND lease_owner=$3
|
|
1957
|
+
RETURNING idempotency_key`, [
|
|
1910
1958
|
partition,
|
|
1911
1959
|
idempotencyKey,
|
|
1912
1960
|
update.leaseOwner,
|
|
@@ -1914,15 +1962,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1914
1962
|
update.retryAtMs ?? update.failure.atMs,
|
|
1915
1963
|
JSON.stringify(update.failure),
|
|
1916
1964
|
]);
|
|
1917
|
-
return result.
|
|
1965
|
+
return result.rows.length === 1;
|
|
1918
1966
|
}
|
|
1919
1967
|
async retryReaction(partition, idempotencyKey, nowMs) {
|
|
1920
1968
|
const result = await this.#exec.query(`UPDATE sync_reactions
|
|
1921
1969
|
SET status='pending', attempts=0, available_at_ms=$3,
|
|
1922
1970
|
last_failure=NULL, lease_owner=NULL, lease_expires_at_ms=NULL,
|
|
1923
1971
|
completed_at_ms=NULL
|
|
1924
|
-
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'
|
|
1925
|
-
|
|
1972
|
+
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'
|
|
1973
|
+
RETURNING idempotency_key`, [partition, idempotencyKey, nowMs]);
|
|
1974
|
+
return result.rows.length === 1;
|
|
1926
1975
|
}
|
|
1927
1976
|
async getReaction(partition, idempotencyKey) {
|
|
1928
1977
|
const { rows } = await this.#exec.query('SELECT * FROM sync_reactions WHERE partition=$1 AND idempotency_key=$2', [partition, idempotencyKey]);
|
|
@@ -2179,7 +2228,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2179
2228
|
WHERE partition=$1 AND client_id=$2`, [partition, clientId, cursor, updatedAtMs]);
|
|
2180
2229
|
}
|
|
2181
2230
|
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
2182
|
-
const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2', [partition, cutoffMs]);
|
|
2231
|
+
const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2 AND cursor>=0', [partition, cutoffMs]);
|
|
2183
2232
|
return rows[0].cursor === null ? null : asNumber(rows[0].cursor);
|
|
2184
2233
|
}
|
|
2185
2234
|
async listClientCursors(partition) {
|
|
@@ -2191,25 +2240,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2191
2240
|
}));
|
|
2192
2241
|
}
|
|
2193
2242
|
async listRowsReferencingBlob(partition, blobId) {
|
|
2194
|
-
//
|
|
2195
|
-
// stored scopes come from
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
return out;
|
|
2243
|
+
// One statement: the by-blob index (§5.9.4/§5.9.5) supplies the candidate
|
|
2244
|
+
// rows and each row's stored scopes come from its row table for the §3.4
|
|
2245
|
+
// authorization test (see `rowsReferencingBlobSql`).
|
|
2246
|
+
if (this.#tables === undefined || this.#tables.size === 0)
|
|
2247
|
+
return [];
|
|
2248
|
+
const { rows } = await this.#exec.query(rowsReferencingBlobSql(this.#tables.values(), 'postgres'), [
|
|
2249
|
+
partition,
|
|
2250
|
+
blobId,
|
|
2251
|
+
]);
|
|
2252
|
+
return rows.flatMap((row) => row.scopes === null
|
|
2253
|
+
? []
|
|
2254
|
+
: [
|
|
2255
|
+
{
|
|
2256
|
+
table: row.tbl,
|
|
2257
|
+
rowId: row.row_id,
|
|
2258
|
+
scopes: asJson(row.scopes),
|
|
2259
|
+
},
|
|
2260
|
+
]);
|
|
2213
2261
|
}
|
|
2214
2262
|
async listReferencedBlobIds(partition) {
|
|
2215
2263
|
const { rows } = await this.#exec.query('SELECT DISTINCT blob_id FROM sync_blob_refs WHERE partition=$1', [partition]);
|
|
@@ -2241,7 +2289,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2241
2289
|
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
2242
2290
|
AND s.commit_seq<=$5 AND c2.op=1 AND c2.scopes->>$3=s.value
|
|
2243
2291
|
LIMIT $6) d
|
|
2244
|
-
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx
|
|
2292
|
+
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx
|
|
2293
|
+
RETURNING c.commit_seq`, params);
|
|
2245
2294
|
const scopes = await client.query(`DELETE FROM sync_change_scopes cs USING (
|
|
2246
2295
|
SELECT s.commit_seq FROM sync_change_scopes s
|
|
2247
2296
|
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
@@ -2251,8 +2300,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2251
2300
|
AND c.tbl=s.tbl AND c.scopes->>$3=s.value)
|
|
2252
2301
|
LIMIT $6) d
|
|
2253
2302
|
WHERE cs.partition=$1 AND cs.tbl=$2 AND cs.var=$3 AND cs.value=$4
|
|
2254
|
-
AND cs.commit_seq=d.commit_seq
|
|
2255
|
-
|
|
2303
|
+
AND cs.commit_seq=d.commit_seq
|
|
2304
|
+
RETURNING cs.commit_seq`, params);
|
|
2305
|
+
// §4.6.1: stored conflict results of the binding lose their row payload.
|
|
2306
|
+
const results = await client.query(`UPDATE sync_push_results p SET result=jsonb_set(p.result, '{results}', (
|
|
2307
|
+
SELECT jsonb_agg(e - 'serverRow' ORDER BY i)
|
|
2308
|
+
FROM jsonb_array_elements(p.result->'results')
|
|
2309
|
+
WITH ORDINALITY AS t(e, i)))
|
|
2310
|
+
FROM (
|
|
2311
|
+
SELECT client_id, client_commit_id FROM sync_push_results
|
|
2312
|
+
WHERE partition=$1
|
|
2313
|
+
AND result->'conflictAuthorization'->>'table'=$2
|
|
2314
|
+
AND result->'conflictAuthorization'->'scopes'->>$3=$4
|
|
2315
|
+
AND jsonb_path_exists(result, '$.results[*].serverRow')
|
|
2316
|
+
LIMIT $5) d
|
|
2317
|
+
WHERE p.partition=$1 AND p.client_id=d.client_id
|
|
2318
|
+
AND p.client_commit_id=d.client_commit_id
|
|
2319
|
+
RETURNING p.client_id`, [partition, query.table, query.variable, query.value, query.limit]);
|
|
2320
|
+
return changes.rows.length + scopes.rows.length + results.rows.length;
|
|
2256
2321
|
});
|
|
2257
2322
|
}
|
|
2258
2323
|
// -- admin/console read surface --------------------------------------------
|
|
@@ -2287,41 +2352,52 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2287
2352
|
FROM sync_commits
|
|
2288
2353
|
WHERE partition=$1 AND commit_seq>$2 AND commit_seq<$3
|
|
2289
2354
|
ORDER BY commit_seq DESC LIMIT $4`, [partition, query.afterSeq, beforeSeq, query.limit]);
|
|
2290
|
-
//
|
|
2291
|
-
// `sample` changes (change-index order)
|
|
2355
|
+
// One grouped statement for the page's per-table change counts and, with
|
|
2356
|
+
// `changeSample`, one for the first `sample` changes (change-index order)
|
|
2357
|
+
// of every listed commit.
|
|
2292
2358
|
const sampleCount = query.changeSample !== undefined &&
|
|
2293
2359
|
Number.isInteger(query.changeSample) &&
|
|
2294
2360
|
query.changeSample > 0
|
|
2295
2361
|
? query.changeSample
|
|
2296
2362
|
: 0;
|
|
2363
|
+
const seqs = rows.map((row) => asNumber(row.commit_seq));
|
|
2364
|
+
const counts = new Map();
|
|
2365
|
+
for (const count of seqs.length === 0
|
|
2366
|
+
? []
|
|
2367
|
+
: (await this.#exec.query(`SELECT commit_seq, tbl, count(*) AS n FROM sync_changes
|
|
2368
|
+
WHERE partition=$1 AND commit_seq=ANY($2::bigint[])
|
|
2369
|
+
GROUP BY commit_seq, tbl ORDER BY commit_seq, tbl`, [partition, seqs])).rows) {
|
|
2370
|
+
const seq = asNumber(count.commit_seq);
|
|
2371
|
+
counts.set(seq, [...(counts.get(seq) ?? []), count]);
|
|
2372
|
+
}
|
|
2297
2373
|
const samples = new Map();
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2374
|
+
for (const change of sampleCount === 0 || seqs.length === 0
|
|
2375
|
+
? []
|
|
2376
|
+
: (await this.#exec.query(`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
2377
|
+
WHERE partition=$1 AND commit_seq=ANY($2::bigint[]) AND idx<$3
|
|
2378
|
+
ORDER BY commit_seq, idx`, [partition, seqs, sampleCount])).rows) {
|
|
2379
|
+
const seq = asNumber(change.commit_seq);
|
|
2380
|
+
samples.set(seq, [
|
|
2381
|
+
...(samples.get(seq) ?? []),
|
|
2382
|
+
{
|
|
2306
2383
|
table: change.tbl,
|
|
2307
2384
|
rowId: change.row_id,
|
|
2308
2385
|
op: asNumber(change.op) === 1 ? 'upsert' : 'delete',
|
|
2309
|
-
}
|
|
2310
|
-
|
|
2311
|
-
}
|
|
2386
|
+
},
|
|
2387
|
+
]);
|
|
2312
2388
|
}
|
|
2313
2389
|
const out = [];
|
|
2314
2390
|
for (const row of rows) {
|
|
2315
2391
|
const commitSeq = asNumber(row.commit_seq);
|
|
2316
|
-
const changes =
|
|
2392
|
+
const changes = counts.get(commitSeq) ?? [];
|
|
2317
2393
|
out.push({
|
|
2318
2394
|
commitSeq,
|
|
2319
2395
|
clientId: row.client_id,
|
|
2320
2396
|
clientCommitId: row.client_commit_id,
|
|
2321
2397
|
actorId: row.actor_id,
|
|
2322
2398
|
createdAtMs: asNumber(row.created_at_ms),
|
|
2323
|
-
changeCount: changes.
|
|
2324
|
-
tables: changes.
|
|
2399
|
+
changeCount: changes.reduce((sum, c) => sum + asNumber(c.n), 0),
|
|
2400
|
+
tables: changes.map((c) => c.tbl),
|
|
2325
2401
|
...(sampleCount > 0 ? { sample: samples.get(commitSeq) ?? [] } : {}),
|
|
2326
2402
|
});
|
|
2327
2403
|
}
|
|
@@ -2395,26 +2471,27 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2395
2471
|
});
|
|
2396
2472
|
}
|
|
2397
2473
|
async scopeActivity(partition, query) {
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2474
|
+
// One statement for the page: candidate (commit, table) pairs from the
|
|
2475
|
+
// change-scope index, joined to the commit metadata with the table's
|
|
2476
|
+
// change count. `tbl` sits between `partition` and `var` in the index key,
|
|
2477
|
+
// so the `var`/`value` filter ranges over the partition's change-scope
|
|
2478
|
+
// entries; only the LIMIT bounds the output. A candidate whose commit was
|
|
2479
|
+
// pruned has no metadata and is skipped.
|
|
2480
|
+
const { rows } = await this.#exec.query(`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
|
|
2481
|
+
(SELECT count(*) FROM sync_changes ch
|
|
2482
|
+
WHERE ch.partition=$1 AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n
|
|
2483
|
+
FROM (SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
|
|
2484
|
+
WHERE partition=$1 AND var=$2 AND value=$3
|
|
2485
|
+
ORDER BY commit_seq DESC LIMIT $4) s
|
|
2486
|
+
JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=s.commit_seq
|
|
2487
|
+
ORDER BY s.commit_seq DESC`, [partition, query.variable, query.value, query.limit]);
|
|
2488
|
+
return rows.map((row) => ({
|
|
2489
|
+
commitSeq: asNumber(row.commit_seq),
|
|
2490
|
+
table: row.tbl,
|
|
2491
|
+
createdAtMs: asNumber(row.created_at_ms),
|
|
2492
|
+
actorId: row.actor_id,
|
|
2493
|
+
changeCount: asNumber(row.n),
|
|
2494
|
+
}));
|
|
2418
2495
|
}
|
|
2419
2496
|
async listPartitions() {
|
|
2420
2497
|
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
package/dist/pull.d.ts
CHANGED
|
@@ -40,31 +40,39 @@ export interface PullSectionTrace {
|
|
|
40
40
|
readonly segments: PullSegmentSummary[];
|
|
41
41
|
}
|
|
42
42
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
46
|
-
* subscription in one statement, and the horizon re-check runs once per pull.
|
|
43
|
+
* One subscription's section decision (§4.5–§4.7) with the storage reads it
|
|
44
|
+
* needs before its first frame.
|
|
47
45
|
*/
|
|
48
|
-
export
|
|
49
|
-
|
|
46
|
+
export type PreparedSection = {
|
|
47
|
+
readonly kind: 'revoked';
|
|
48
|
+
}
|
|
49
|
+
/** §4.6: the cursor is behind the horizon. */
|
|
50
|
+
| {
|
|
51
|
+
readonly kind: 'reset';
|
|
52
|
+
}
|
|
53
|
+
/** §4.5 window read for an in-range cursor. */
|
|
54
|
+
| {
|
|
55
|
+
readonly kind: 'incremental';
|
|
50
56
|
readonly commits: StoredCommit[];
|
|
51
|
-
|
|
52
|
-
readonly
|
|
53
|
-
/**
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
*/
|
|
57
|
-
readonly firstPage: StoredRow[] | undefined;
|
|
58
|
-
/** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
|
|
59
|
-
readonly pin: number | undefined;
|
|
57
|
+
} | {
|
|
58
|
+
readonly kind: 'bootstrap';
|
|
59
|
+
/** §4.7 pin: a usable resume token's, the sqlite lane's scope pin, or `maxCommitSeq`. */
|
|
60
|
+
readonly asOf: number;
|
|
61
|
+
readonly startCursor: string | null;
|
|
60
62
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* smaller table can never match a stored image.
|
|
63
|
+
* The first snapshot page (`limitSnapshotRows + 1` rows from
|
|
64
|
+
* `startCursor`), which is also the §5.3 image eligibility probe.
|
|
64
65
|
*/
|
|
66
|
+
readonly firstPage: StoredRow[];
|
|
67
|
+
/** §5.3 stored image under the reuse key, read only for an eligible probe. */
|
|
65
68
|
readonly image: SegmentRecord | undefined;
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Decide the section of every plan and run its pre-frame storage reads. The
|
|
72
|
+
* reads of all plans start before any is awaited, so a storage that batches
|
|
73
|
+
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
74
|
+
* plan in one statement per table, and the horizon re-check runs once.
|
|
75
|
+
*/
|
|
68
76
|
export declare function prepareSections(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plans: readonly SubscriptionPlan[], maxSeq: number, horizonSeq: number, logEpoch: string): Promise<PreparedSection[]>;
|
|
69
77
|
/**
|
|
70
78
|
* Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
|