@syncular/server 0.39.2 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/admin.d.ts +6 -0
- package/dist/admin.js +9 -0
- package/dist/d1-storage.d.ts +3 -2
- package/dist/d1-storage.js +60 -15
- package/dist/handler.js +79 -8
- package/dist/postgres-storage.d.ts +4 -3
- package/dist/postgres-storage.js +96 -46
- package/dist/relational-rows.d.ts +9 -0
- package/dist/relational-rows.js +30 -0
- package/dist/sqlite-dialect.d.ts +7 -1
- package/dist/sqlite-dialect.js +8 -0
- package/dist/sqlite-storage.d.ts +3 -2
- package/dist/sqlite-storage.js +57 -8
- package/dist/storage.d.ts +31 -1
- package/dist/storage.js +8 -0
- package/package.json +2 -2
- package/src/admin.ts +15 -0
- package/src/d1-storage.ts +109 -14
- package/src/handler.ts +126 -19
- package/src/postgres-storage.ts +154 -58
- package/src/relational-rows.ts +34 -0
- package/src/sqlite-dialect.ts +9 -0
- package/src/sqlite-storage.ts +102 -4
- package/src/storage.ts +56 -1
package/dist/postgres-storage.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune.js';
|
|
2
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
3
|
-
import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage.js';
|
|
3
|
+
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal, } from './storage.js';
|
|
4
4
|
import { serveNotReadyError } from './readiness.js';
|
|
5
5
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
8
|
+
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
9
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
11
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
@@ -109,10 +109,16 @@ CREATE TABLE IF NOT EXISTS sync_clients(
|
|
|
109
109
|
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
110
110
|
cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
|
|
111
111
|
updated_at_ms BIGINT NOT NULL,
|
|
112
|
+
last_round_at_ms BIGINT, last_error_code TEXT,
|
|
113
|
+
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
112
114
|
PRIMARY KEY(partition, client_id)
|
|
113
115
|
);
|
|
114
116
|
ALTER TABLE sync_clients
|
|
115
117
|
ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;
|
|
118
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;
|
|
119
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;
|
|
120
|
+
ALTER TABLE sync_clients
|
|
121
|
+
ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;
|
|
116
122
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
117
123
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
118
124
|
blob_id TEXT NOT NULL,
|
|
@@ -169,6 +175,56 @@ async function installCheckpointOn(client, partition, name, schemaVersion, nowMs
|
|
|
169
175
|
sync_writer_fence.required_writer_version,
|
|
170
176
|
EXCLUDED.required_writer_version)`, [partition, schemaVersion]);
|
|
171
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* A scalar subquery over the table names in the JSON array bound to
|
|
180
|
+
* `namesParam`: the free-standing indexes of each table as a JSON array of
|
|
181
|
+
* `{tbl, name, unique, partial, columns}`. Parity with the SQLite/D1
|
|
182
|
+
* `origin === 'c'` filter: constraint-owned indexes (PRIMARY KEY, UNIQUE
|
|
183
|
+
* constraints) can only be removed through their constraint, so DROP INDEX on
|
|
184
|
+
* them would abort the transaction. Tables resolve through `to_regclass`,
|
|
185
|
+
* which follows the session's `search_path` like the row queries do.
|
|
186
|
+
*/
|
|
187
|
+
function indexesSubquery(namesParam) {
|
|
188
|
+
return `(SELECT coalesce(json_agg(json_build_object(
|
|
189
|
+
'tbl', tables.name,
|
|
190
|
+
'name', index_class.relname,
|
|
191
|
+
'unique', index_meta.indisunique,
|
|
192
|
+
'partial', index_meta.indpred IS NOT NULL,
|
|
193
|
+
'columns', ARRAY(
|
|
194
|
+
SELECT attribute.attname::text
|
|
195
|
+
FROM unnest(index_meta.indkey::int2[])
|
|
196
|
+
WITH ORDINALITY AS key(attnum, ord)
|
|
197
|
+
LEFT JOIN pg_catalog.pg_attribute AS attribute
|
|
198
|
+
ON attribute.attrelid = index_meta.indrelid
|
|
199
|
+
AND attribute.attnum = key.attnum
|
|
200
|
+
WHERE key.ord <= index_meta.indnkeyatts
|
|
201
|
+
ORDER BY key.ord))), '[]'::json)::text
|
|
202
|
+
FROM jsonb_array_elements_text(${namesParam}::text::jsonb) AS tables(name)
|
|
203
|
+
JOIN pg_catalog.pg_index AS index_meta
|
|
204
|
+
ON index_meta.indrelid = to_regclass(tables.name)
|
|
205
|
+
JOIN pg_catalog.pg_class AS index_class
|
|
206
|
+
ON index_class.oid = index_meta.indexrelid
|
|
207
|
+
WHERE NOT index_meta.indisprimary
|
|
208
|
+
AND NOT EXISTS (
|
|
209
|
+
SELECT 1
|
|
210
|
+
FROM pg_catalog.pg_constraint AS owning_constraint
|
|
211
|
+
WHERE owning_constraint.conindid = index_meta.indexrelid
|
|
212
|
+
))`;
|
|
213
|
+
}
|
|
214
|
+
/** Table name to index name to {@link indexSignature}, from {@link indexesSubquery} JSON. */
|
|
215
|
+
function parseIndexSignatures(json) {
|
|
216
|
+
const byTable = new Map();
|
|
217
|
+
for (const index of JSON.parse(json)) {
|
|
218
|
+
const indexes = byTable.get(index.tbl) ?? new Map();
|
|
219
|
+
indexes.set(index.name, indexSignature(index.columns, index.unique, index.partial));
|
|
220
|
+
byTable.set(index.tbl, indexes);
|
|
221
|
+
}
|
|
222
|
+
return byTable;
|
|
223
|
+
}
|
|
224
|
+
async function readIndexSignatures(client, tableNames) {
|
|
225
|
+
const { rows } = await client.query(`SELECT ${indexesSubquery('$1')} AS indexes`, [JSON.stringify(tableNames)]);
|
|
226
|
+
return parseIndexSignatures(rows[0]?.indexes ?? '[]');
|
|
227
|
+
}
|
|
172
228
|
/** RFC 0007 serve gate on either the pool or a transaction's client. */
|
|
173
229
|
async function readServeGateOn(client, partition, runningSchemaVersion) {
|
|
174
230
|
// One statement: every request reads the gate twice (entry and verify),
|
|
@@ -1069,7 +1125,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1069
1125
|
JOIN pg_attribute attribute
|
|
1070
1126
|
ON attribute.attrelid = to_regclass(tables.name)
|
|
1071
1127
|
WHERE attribute.attnum > 0
|
|
1072
|
-
AND NOT attribute.attisdropped) AS columns
|
|
1128
|
+
AND NOT attribute.attisdropped) AS columns,
|
|
1129
|
+
${indexesSubquery('$2')} AS indexes
|
|
1073
1130
|
FROM (SELECT 1) AS one
|
|
1074
1131
|
LEFT JOIN sync_schema_meta meta ON meta.id=1`, [schema.version, JSON.stringify([...schema.tables.keys()])]);
|
|
1075
1132
|
const marker = probe[0];
|
|
@@ -1105,6 +1162,19 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1105
1162
|
primaryKeyPosition: column.pk ?? 0,
|
|
1106
1163
|
})), 'postgres');
|
|
1107
1164
|
}
|
|
1165
|
+
// Server-only indexes reconcile at every open of a database already at
|
|
1166
|
+
// this version: they never reach a client, so no version bump is needed.
|
|
1167
|
+
// The unlocked read finds the common no-op case; the diff is recomputed
|
|
1168
|
+
// under the migration barrier before any DDL runs.
|
|
1169
|
+
const reconcile = serverIndexDdl(schema, parseIndexSignatures(marker.indexes));
|
|
1170
|
+
if (reconcile.length > 0) {
|
|
1171
|
+
await this.#exec.transaction(async (client) => {
|
|
1172
|
+
await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
|
|
1173
|
+
for (const statement of serverIndexDdl(schema, await readIndexSignatures(client, [...schema.tables.keys()]))) {
|
|
1174
|
+
await client.query(statement);
|
|
1175
|
+
}
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1108
1178
|
}
|
|
1109
1179
|
if (stored === undefined || stored < schema.version) {
|
|
1110
1180
|
// Introspect existing app tables, apply the migration subset
|
|
@@ -1145,48 +1215,15 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1145
1215
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
1146
1216
|
const existing = new Map();
|
|
1147
1217
|
const existingIndexes = new Map();
|
|
1218
|
+
const tableIndexes = await readIndexSignatures(client, [
|
|
1219
|
+
...schema.tables.keys(),
|
|
1220
|
+
]);
|
|
1148
1221
|
for (const table of schema.tables.values()) {
|
|
1149
1222
|
const { rows } = await client.query(`SELECT column_name FROM information_schema.columns
|
|
1150
1223
|
WHERE table_schema = current_schema() AND table_name = $1`, [table.name]);
|
|
1151
1224
|
if (rows.length > 0) {
|
|
1152
1225
|
existing.set(table.name, new Set(rows.map((r) => r.column_name)));
|
|
1153
|
-
|
|
1154
|
-
// free-standing indexes enter the rebuild set. Constraint-owned
|
|
1155
|
-
// indexes (PRIMARY KEY, UNIQUE constraints) can only be removed
|
|
1156
|
-
// through their constraint, so DROP INDEX on them would abort
|
|
1157
|
-
// the migration transaction.
|
|
1158
|
-
const indexes = await client.query(`SELECT index_class.relname AS index_name,
|
|
1159
|
-
index_meta.indisunique AS is_unique,
|
|
1160
|
-
index_meta.indpred IS NOT NULL AS is_partial,
|
|
1161
|
-
ARRAY(
|
|
1162
|
-
SELECT attribute.attname::text
|
|
1163
|
-
FROM unnest(index_meta.indkey::int2[])
|
|
1164
|
-
WITH ORDINALITY AS key(attnum, ord)
|
|
1165
|
-
LEFT JOIN pg_catalog.pg_attribute AS attribute
|
|
1166
|
-
ON attribute.attrelid = table_class.oid
|
|
1167
|
-
AND attribute.attnum = key.attnum
|
|
1168
|
-
WHERE key.ord <= index_meta.indnkeyatts
|
|
1169
|
-
ORDER BY key.ord
|
|
1170
|
-
) AS columns
|
|
1171
|
-
FROM pg_catalog.pg_class AS table_class
|
|
1172
|
-
JOIN pg_catalog.pg_namespace AS namespace
|
|
1173
|
-
ON namespace.oid = table_class.relnamespace
|
|
1174
|
-
JOIN pg_catalog.pg_index AS index_meta
|
|
1175
|
-
ON index_meta.indrelid = table_class.oid
|
|
1176
|
-
JOIN pg_catalog.pg_class AS index_class
|
|
1177
|
-
ON index_class.oid = index_meta.indexrelid
|
|
1178
|
-
WHERE namespace.nspname = current_schema()
|
|
1179
|
-
AND table_class.relname = $1
|
|
1180
|
-
AND NOT index_meta.indisprimary
|
|
1181
|
-
AND NOT EXISTS (
|
|
1182
|
-
SELECT 1
|
|
1183
|
-
FROM pg_catalog.pg_constraint AS owning_constraint
|
|
1184
|
-
WHERE owning_constraint.conindid = index_meta.indexrelid
|
|
1185
|
-
)`, [table.name]);
|
|
1186
|
-
existingIndexes.set(table.name, new Map(indexes.rows.map((index) => [
|
|
1187
|
-
index.index_name,
|
|
1188
|
-
indexSignature(index.columns, index.is_unique, index.is_partial),
|
|
1189
|
-
])));
|
|
1226
|
+
existingIndexes.set(table.name, tableIndexes.get(table.name) ?? new Map());
|
|
1190
1227
|
}
|
|
1191
1228
|
}
|
|
1192
1229
|
if (retiredTables.length > 0) {
|
|
@@ -1774,7 +1811,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1774
1811
|
return scanRowsByIndexOn(this.#exec, this.table(query.table), partition, query);
|
|
1775
1812
|
}
|
|
1776
1813
|
async getClientRecord(partition, clientId) {
|
|
1777
|
-
const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2', [partition, clientId]);
|
|
1814
|
+
const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 AND client_id=$2', [partition, clientId]);
|
|
1778
1815
|
const record = rows[0];
|
|
1779
1816
|
if (record === undefined)
|
|
1780
1817
|
return undefined;
|
|
@@ -1785,16 +1822,22 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1785
1822
|
cursor: asNumber(record.cursor),
|
|
1786
1823
|
updatedAtMs: asNumber(record.updated_at_ms),
|
|
1787
1824
|
subscriptions: asJson(record.subscriptions),
|
|
1825
|
+
...clientRoundState(record.last_round_at_ms === null
|
|
1826
|
+
? null
|
|
1827
|
+
: asNumber(record.last_round_at_ms), record.last_error_code, asNumber(record.consecutive_failures)),
|
|
1788
1828
|
};
|
|
1789
1829
|
}
|
|
1790
|
-
async putClientRecord(partition, record) {
|
|
1791
|
-
await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms)
|
|
1792
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
1830
|
+
async putClientRecord(partition, record, options) {
|
|
1831
|
+
await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms)
|
|
1832
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$7)
|
|
1793
1833
|
ON CONFLICT (partition, client_id) DO UPDATE
|
|
1794
1834
|
SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
|
|
1795
1835
|
cursor=EXCLUDED.cursor,
|
|
1796
1836
|
subscriptions=EXCLUDED.subscriptions,
|
|
1797
|
-
updated_at_ms=EXCLUDED.updated_at_ms
|
|
1837
|
+
updated_at_ms=EXCLUDED.updated_at_ms,
|
|
1838
|
+
last_round_at_ms=EXCLUDED.last_round_at_ms${options?.keepRoundOutcome === true
|
|
1839
|
+
? ''
|
|
1840
|
+
: ',\n last_error_code=NULL, consecutive_failures=0'}`, [
|
|
1798
1841
|
partition,
|
|
1799
1842
|
record.clientId,
|
|
1800
1843
|
record.actorId,
|
|
@@ -1804,6 +1847,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1804
1847
|
record.updatedAtMs,
|
|
1805
1848
|
]);
|
|
1806
1849
|
}
|
|
1850
|
+
async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
|
|
1851
|
+
await this.#exec.query(`UPDATE sync_clients
|
|
1852
|
+
SET last_round_at_ms=$1, last_error_code=$2,
|
|
1853
|
+
consecutive_failures=consecutive_failures+1
|
|
1854
|
+
WHERE partition=$3 AND client_id=$4 AND actor_id=$5`, [atMs, errorCode, partition, clientId, actorId]);
|
|
1855
|
+
}
|
|
1807
1856
|
async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
|
|
1808
1857
|
await this.#exec.query(`UPDATE sync_clients
|
|
1809
1858
|
SET cursor=GREATEST(cursor, $1), updated_at_ms=GREATEST(updated_at_ms, $2)
|
|
@@ -1895,7 +1944,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1895
1944
|
}
|
|
1896
1945
|
// -- admin/console read surface --------------------------------------------
|
|
1897
1946
|
async listClientRecords(partition) {
|
|
1898
|
-
const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC', [partition]);
|
|
1947
|
+
const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC', [partition]);
|
|
1899
1948
|
return rows.map((r) => ({
|
|
1900
1949
|
clientId: r.client_id,
|
|
1901
1950
|
actorId: r.actor_id,
|
|
@@ -1905,6 +1954,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1905
1954
|
subscriptions: (typeof r.subscriptions === 'string'
|
|
1906
1955
|
? JSON.parse(r.subscriptions)
|
|
1907
1956
|
: r.subscriptions),
|
|
1957
|
+
...clientRoundState(r.last_round_at_ms === null ? null : asNumber(r.last_round_at_ms), r.last_error_code, asNumber(r.consecutive_failures)),
|
|
1908
1958
|
}));
|
|
1909
1959
|
}
|
|
1910
1960
|
async listCommitMetadata(partition, query) {
|
|
@@ -404,3 +404,12 @@ export declare function rewritePlan(table: CompiledTable, oldLayout: readonly St
|
|
|
404
404
|
* names and survive every bump.
|
|
405
405
|
*/
|
|
406
406
|
export declare function schemaDdl(schema: CompiledSchema, existingColumnsByTable: ReadonlyMap<string, ReadonlySet<string>>, dialect: RelationalDialect, existingIndexesByTable?: ReadonlyMap<string, ReadonlyMap<string, string>>): string[];
|
|
407
|
+
/**
|
|
408
|
+
* DDL that reconciles the server-only indexes of a database already at
|
|
409
|
+
* `schema.version`, so declaring or changing a `serverIndexes` entry needs no
|
|
410
|
+
* schema-version bump (server indexes never reach a client). It creates
|
|
411
|
+
* missing server indexes and drops every Syncular-owned index the schema no
|
|
412
|
+
* longer names. Declared `indexes` belong to the version bump and are never
|
|
413
|
+
* created or rebuilt here.
|
|
414
|
+
*/
|
|
415
|
+
export declare function serverIndexDdl(schema: CompiledSchema, existingIndexesByTable: ReadonlyMap<string, ReadonlyMap<string, string>>): string[];
|
package/dist/relational-rows.js
CHANGED
|
@@ -969,3 +969,33 @@ export function schemaDdl(schema, existingColumnsByTable, dialect, existingIndex
|
|
|
969
969
|
// receiving table re-creates it.
|
|
970
970
|
return [...drops, ...creates];
|
|
971
971
|
}
|
|
972
|
+
/**
|
|
973
|
+
* DDL that reconciles the server-only indexes of a database already at
|
|
974
|
+
* `schema.version`, so declaring or changing a `serverIndexes` entry needs no
|
|
975
|
+
* schema-version bump (server indexes never reach a client). It creates
|
|
976
|
+
* missing server indexes and drops every Syncular-owned index the schema no
|
|
977
|
+
* longer names. Declared `indexes` belong to the version bump and are never
|
|
978
|
+
* created or rebuilt here.
|
|
979
|
+
*/
|
|
980
|
+
export function serverIndexDdl(schema, existingIndexesByTable) {
|
|
981
|
+
const drops = [];
|
|
982
|
+
const creates = [];
|
|
983
|
+
for (const table of schema.tables.values()) {
|
|
984
|
+
const existing = existingIndexesByTable.get(table.name);
|
|
985
|
+
if (existing === undefined)
|
|
986
|
+
continue;
|
|
987
|
+
const desired = desiredIndexes(table);
|
|
988
|
+
for (const indexName of existing.keys()) {
|
|
989
|
+
if (indexName.startsWith(SYNC_INDEX_PREFIX) &&
|
|
990
|
+
!desired.some((index) => index.name === indexName)) {
|
|
991
|
+
drops.push(dropIndexDdl(indexName));
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
// `desiredIndexes` lists the declared indexes first, then the server ones.
|
|
995
|
+
for (const index of desired.slice(table.indexes.length)) {
|
|
996
|
+
if (!existing.has(index.name))
|
|
997
|
+
creates.push(index.ddl);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
return [...drops, ...creates];
|
|
1001
|
+
}
|
package/dist/sqlite-dialect.d.ts
CHANGED
|
@@ -36,7 +36,13 @@ import type { ScopeValueHistoryQuery, StoredChange, StoredCommit, StoredPushResu
|
|
|
36
36
|
* inverted index. Declared here so every SQLite-family storage (and D1,
|
|
37
37
|
* which applies this same DDL) keeps the same plan shape as Postgres.
|
|
38
38
|
*/
|
|
39
|
-
export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq INTEGER NOT NULL DEFAULT 0,\n horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required INTEGER NOT NULL DEFAULT 0,\n last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n writer_version INTEGER,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n last_failure TEXT,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,\n updated_at_ms INTEGER NOT NULL,\n PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq INTEGER NOT NULL,\n PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark INTEGER NOT NULL DEFAULT 0,\n owner_epoch INTEGER NOT NULL DEFAULT 0,\n observed_rows INTEGER NOT NULL DEFAULT 0,\n updated_at_ms INTEGER NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
39
|
+
export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq INTEGER NOT NULL DEFAULT 0,\n horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required INTEGER NOT NULL DEFAULT 0,\n last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n writer_version INTEGER,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n last_failure TEXT,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,\n updated_at_ms INTEGER NOT NULL,\n last_round_at_ms INTEGER, last_error_code TEXT,\n consecutive_failures INTEGER NOT NULL DEFAULT 0,\n PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq INTEGER NOT NULL,\n PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark INTEGER NOT NULL DEFAULT 0,\n owner_epoch INTEGER NOT NULL DEFAULT 0,\n observed_rows INTEGER NOT NULL DEFAULT 0,\n updated_at_ms INTEGER NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
40
|
+
/** Columns added to `sync_clients` after its first release, by column name. */
|
|
41
|
+
export declare const SYNC_CLIENT_ROUND_COLUMNS: {
|
|
42
|
+
readonly last_round_at_ms: "INTEGER";
|
|
43
|
+
readonly last_error_code: "TEXT";
|
|
44
|
+
readonly consecutive_failures: "INTEGER NOT NULL DEFAULT 0";
|
|
45
|
+
};
|
|
40
46
|
/** Split the DDL into individual statements (D1 applies them one by one). */
|
|
41
47
|
export declare function sqliteDdlStatements(): string[];
|
|
42
48
|
/** §4.6.1: rowIds of one table bound to one scope value, budgeted. */
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -89,6 +89,8 @@ CREATE TABLE IF NOT EXISTS sync_clients(
|
|
|
89
89
|
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
90
90
|
cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
|
|
91
91
|
updated_at_ms INTEGER NOT NULL,
|
|
92
|
+
last_round_at_ms INTEGER, last_error_code TEXT,
|
|
93
|
+
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
92
94
|
PRIMARY KEY(partition, client_id)
|
|
93
95
|
);
|
|
94
96
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
@@ -118,6 +120,12 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
|
|
|
118
120
|
required_writer_version INTEGER NOT NULL
|
|
119
121
|
);
|
|
120
122
|
`;
|
|
123
|
+
/** Columns added to `sync_clients` after its first release, by column name. */
|
|
124
|
+
export const SYNC_CLIENT_ROUND_COLUMNS = {
|
|
125
|
+
last_round_at_ms: 'INTEGER',
|
|
126
|
+
last_error_code: 'TEXT',
|
|
127
|
+
consecutive_failures: 'INTEGER NOT NULL DEFAULT 0',
|
|
128
|
+
};
|
|
121
129
|
/** Split the DDL into individual statements (D1 applies them one by one). */
|
|
122
130
|
export function sqliteDdlStatements() {
|
|
123
131
|
return SQLITE_DDL.split(';')
|
package/dist/sqlite-storage.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
2
2
|
import type { ScopeMap } from '@syncular/core';
|
|
3
3
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
4
4
|
import { type SqliteDatabase } from './sqlite-driver.js';
|
|
5
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
5
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
6
6
|
export declare class SqliteServerStorage implements ServerStorage {
|
|
7
7
|
#private;
|
|
8
8
|
readonly db: SqliteDatabase;
|
|
@@ -56,7 +56,8 @@ export declare class SqliteServerStorage implements ServerStorage {
|
|
|
56
56
|
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
57
57
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
58
58
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
59
|
-
putClientRecord(partition: string, record: ClientRecord): Promise<void>;
|
|
59
|
+
putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
|
|
60
|
+
recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
|
|
60
61
|
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
61
62
|
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
62
63
|
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
package/dist/sqlite-storage.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { retainRollbackFailure, rollbackAfterError } from './storage-errors.js';
|
|
2
2
|
import { validateCommitPruneQuery } from './prune.js';
|
|
3
3
|
import { StorageQueryError } from './storage-errors.js';
|
|
4
|
-
import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage.js';
|
|
4
|
+
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal, } from './storage.js';
|
|
5
5
|
import { serveNotReadyError } from './readiness.js';
|
|
6
6
|
/**
|
|
7
7
|
* SQLite server storage over the shared synchronous driver.
|
|
@@ -13,9 +13,9 @@ import { serveNotReadyError } from './readiness.js';
|
|
|
13
13
|
*/
|
|
14
14
|
import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
15
15
|
import { syncError } from './errors.js';
|
|
16
|
-
import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
16
|
+
import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
17
17
|
import { matchesEffective } from './scopes.js';
|
|
18
|
-
import { collectCommitWindowPage, deserializePushResult, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, placeholders, SQLITE_DDL, serializePushResult, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
|
|
18
|
+
import { collectCommitWindowPage, deserializePushResult, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, placeholders, SQLITE_DDL, SYNC_CLIENT_ROUND_COLUMNS, serializePushResult, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
|
|
19
19
|
import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
|
|
20
20
|
import { isSqliteConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
21
21
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
@@ -285,6 +285,11 @@ export class SqliteServerStorage {
|
|
|
285
285
|
if (!clientColumns.some((column) => column.name === 'wire_version')) {
|
|
286
286
|
this.db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1');
|
|
287
287
|
}
|
|
288
|
+
for (const [name, definition] of Object.entries(SYNC_CLIENT_ROUND_COLUMNS)) {
|
|
289
|
+
if (!clientColumns.some((column) => column.name === name)) {
|
|
290
|
+
this.db.exec(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
288
293
|
const commitColumns = this.db
|
|
289
294
|
.query('PRAGMA table_info("sync_commits")')
|
|
290
295
|
.all();
|
|
@@ -357,6 +362,29 @@ END`);
|
|
|
357
362
|
primaryKeyPosition: column.pk,
|
|
358
363
|
})), 'sqlite');
|
|
359
364
|
}
|
|
365
|
+
// Server-only indexes reconcile at every open of a database already at
|
|
366
|
+
// this version: they never reach a client, so no version bump is needed.
|
|
367
|
+
const reconcile = serverIndexDdl(schema, new Map([...schema.tables.values()].map((table) => [
|
|
368
|
+
table.name,
|
|
369
|
+
new Map(Object.entries(sqliteIndexSignatures(this.db
|
|
370
|
+
.query(SQLITE_TABLE_INDEXES_SQL)
|
|
371
|
+
.all(table.name)))),
|
|
372
|
+
])));
|
|
373
|
+
if (reconcile.length > 0) {
|
|
374
|
+
await this.#serializeWrite(() => {
|
|
375
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
376
|
+
try {
|
|
377
|
+
for (const statement of reconcile)
|
|
378
|
+
this.db.exec(statement);
|
|
379
|
+
this.db.exec('COMMIT');
|
|
380
|
+
}
|
|
381
|
+
catch (error) {
|
|
382
|
+
rollbackAfterError(() => {
|
|
383
|
+
this.db.exec('ROLLBACK');
|
|
384
|
+
}, error);
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
}
|
|
360
388
|
}
|
|
361
389
|
if (marker === null || marker.schema_version < schema.version) {
|
|
362
390
|
// Introspect existing app tables, then apply the migration subset
|
|
@@ -1133,7 +1161,7 @@ END`);
|
|
|
1133
1161
|
}
|
|
1134
1162
|
async getClientRecord(partition, clientId) {
|
|
1135
1163
|
const record = this.db
|
|
1136
|
-
.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
|
|
1164
|
+
.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?')
|
|
1137
1165
|
.get(partition, clientId);
|
|
1138
1166
|
if (record === null)
|
|
1139
1167
|
return undefined;
|
|
@@ -1144,12 +1172,32 @@ END`);
|
|
|
1144
1172
|
cursor: record.cursor,
|
|
1145
1173
|
updatedAtMs: record.updated_at_ms,
|
|
1146
1174
|
subscriptions: JSON.parse(record.subscriptions),
|
|
1175
|
+
...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
|
|
1147
1176
|
};
|
|
1148
1177
|
}
|
|
1149
|
-
async putClientRecord(partition, record) {
|
|
1178
|
+
async putClientRecord(partition, record, options) {
|
|
1150
1179
|
this.db
|
|
1151
|
-
.query(
|
|
1152
|
-
|
|
1180
|
+
.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
|
|
1181
|
+
ON CONFLICT(partition, client_id) DO UPDATE SET
|
|
1182
|
+
actor_id=excluded.actor_id,
|
|
1183
|
+
wire_version=excluded.wire_version,
|
|
1184
|
+
cursor=excluded.cursor,
|
|
1185
|
+
subscriptions=excluded.subscriptions,
|
|
1186
|
+
updated_at_ms=excluded.updated_at_ms,
|
|
1187
|
+
last_round_at_ms=excluded.last_round_at_ms${options?.keepRoundOutcome === true
|
|
1188
|
+
? ''
|
|
1189
|
+
: `,
|
|
1190
|
+
last_error_code=NULL,
|
|
1191
|
+
consecutive_failures=0`}`)
|
|
1192
|
+
.run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs, record.updatedAtMs);
|
|
1193
|
+
}
|
|
1194
|
+
async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
|
|
1195
|
+
this.db
|
|
1196
|
+
.query(`UPDATE sync_clients
|
|
1197
|
+
SET last_round_at_ms=?, last_error_code=?,
|
|
1198
|
+
consecutive_failures=consecutive_failures+1
|
|
1199
|
+
WHERE partition=? AND client_id=? AND actor_id=?`)
|
|
1200
|
+
.run(atMs, errorCode, partition, clientId, actorId);
|
|
1153
1201
|
}
|
|
1154
1202
|
async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
|
|
1155
1203
|
this.db
|
|
@@ -1241,7 +1289,7 @@ END`);
|
|
|
1241
1289
|
// -- admin/console read surface --------------------------------------------
|
|
1242
1290
|
async listClientRecords(partition) {
|
|
1243
1291
|
const records = this.db
|
|
1244
|
-
.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
|
|
1292
|
+
.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
|
|
1245
1293
|
.all(partition);
|
|
1246
1294
|
return records.map((record) => ({
|
|
1247
1295
|
clientId: record.client_id,
|
|
@@ -1250,6 +1298,7 @@ END`);
|
|
|
1250
1298
|
cursor: record.cursor,
|
|
1251
1299
|
updatedAtMs: record.updated_at_ms,
|
|
1252
1300
|
subscriptions: JSON.parse(record.subscriptions),
|
|
1301
|
+
...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
|
|
1253
1302
|
}));
|
|
1254
1303
|
}
|
|
1255
1304
|
async listCommitMetadata(partition, query) {
|
package/dist/storage.d.ts
CHANGED
|
@@ -235,7 +235,24 @@ export interface ClientRecord {
|
|
|
235
235
|
readonly cursor: number;
|
|
236
236
|
readonly updatedAtMs: number;
|
|
237
237
|
readonly subscriptions: readonly ClientSubscription[];
|
|
238
|
+
/** When this client's last sync round finished, successfully or not. */
|
|
239
|
+
readonly lastRoundAtMs?: number;
|
|
240
|
+
/** Code of the last round when it failed; absent after a successful round. */
|
|
241
|
+
readonly lastErrorCode?: string;
|
|
242
|
+
/** Rounds that failed in a row since the last successful round. */
|
|
243
|
+
readonly consecutiveFailures?: number;
|
|
238
244
|
}
|
|
245
|
+
/** `putClientRecord` options. */
|
|
246
|
+
export interface ClientRecordWriteOptions {
|
|
247
|
+
/**
|
|
248
|
+
* Leave the stored round outcome (`lastErrorCode`, `consecutiveFailures`)
|
|
249
|
+
* untouched. Set for a completed round that carried a `ROUND_FAILURE` report
|
|
250
|
+
* (§1.5, §4.5). `lastRoundAtMs` still moves to this round's completion.
|
|
251
|
+
*/
|
|
252
|
+
readonly keepRoundOutcome?: boolean;
|
|
253
|
+
}
|
|
254
|
+
/** The round-state fields of a stored `sync_clients` row. */
|
|
255
|
+
export declare function clientRoundState(lastRoundAtMs: number | null, lastErrorCode: string | null, consecutiveFailures: number): Pick<ClientRecord, 'lastRoundAtMs' | 'lastErrorCode' | 'consecutiveFailures'>;
|
|
239
256
|
export interface CommitWindowQuery {
|
|
240
257
|
readonly table: string;
|
|
241
258
|
/**
|
|
@@ -680,7 +697,20 @@ export interface ServerStorage {
|
|
|
680
697
|
*/
|
|
681
698
|
scanRowsByIndex?(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
682
699
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
683
|
-
|
|
700
|
+
/**
|
|
701
|
+
* Write the record for a completed round. The round-state fields reset: the
|
|
702
|
+
* stored last round is a success at `updatedAtMs` with no failures — unless
|
|
703
|
+
* `keepRoundOutcome` is set, which leaves the reported code and streak in
|
|
704
|
+
* place (the round carried a `ROUND_FAILURE` report, §1.5/§4.5).
|
|
705
|
+
*/
|
|
706
|
+
putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
|
|
707
|
+
/**
|
|
708
|
+
* Record a failed round on an existing record: set `lastRoundAtMs` and
|
|
709
|
+
* `lastErrorCode`, and increment `consecutiveFailures`. Requires a matching
|
|
710
|
+
* actor; a missing record or another actor's record is unchanged, and
|
|
711
|
+
* `updatedAtMs` (the pruning liveness signal) is never touched.
|
|
712
|
+
*/
|
|
713
|
+
recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
|
|
684
714
|
/**
|
|
685
715
|
* Atomically advance an existing record's cursor and activity timestamp.
|
|
686
716
|
* Preserve registration fields; require matching actor and current log epoch.
|
package/dist/storage.js
CHANGED
|
@@ -38,6 +38,14 @@ export function serveGateTokenChanged(before, after) {
|
|
|
38
38
|
}
|
|
39
39
|
return undefined;
|
|
40
40
|
}
|
|
41
|
+
/** The round-state fields of a stored `sync_clients` row. */
|
|
42
|
+
export function clientRoundState(lastRoundAtMs, lastErrorCode, consecutiveFailures) {
|
|
43
|
+
return {
|
|
44
|
+
...(lastRoundAtMs !== null ? { lastRoundAtMs } : {}),
|
|
45
|
+
...(lastErrorCode !== null ? { lastErrorCode } : {}),
|
|
46
|
+
...(consecutiveFailures > 0 ? { consecutiveFailures } : {}),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
41
49
|
/** Durable partition identity refreshed after host authentication (§2.1). */
|
|
42
50
|
/** Longest age of `lastAuthenticatedAtMs` that `touchPartition` leaves unwritten. */
|
|
43
51
|
export const PARTITION_TOUCH_INTERVAL_MS = 60_000;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.41.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.41.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/admin.ts
CHANGED
|
@@ -90,6 +90,12 @@ export interface AdminClient {
|
|
|
90
90
|
readonly connected?: boolean;
|
|
91
91
|
/** When the client's oldest open socket opened. */
|
|
92
92
|
readonly connectedSinceMs?: number;
|
|
93
|
+
/** When its last sync round finished, successfully or not. */
|
|
94
|
+
readonly lastRoundAtMs?: number;
|
|
95
|
+
/** Catalog code of its last round; present only while that round failed. */
|
|
96
|
+
readonly lastErrorCode?: string;
|
|
97
|
+
/** Rounds failed in a row since its last success; absent at zero. */
|
|
98
|
+
readonly consecutiveFailures?: number;
|
|
93
99
|
}
|
|
94
100
|
|
|
95
101
|
export interface AdminListClientsOptions {
|
|
@@ -1633,6 +1639,15 @@ export class SyncularAdmin {
|
|
|
1633
1639
|
? { connected: connectedSinceMs !== undefined }
|
|
1634
1640
|
: {}),
|
|
1635
1641
|
...(connectedSinceMs !== undefined ? { connectedSinceMs } : {}),
|
|
1642
|
+
...(record.lastRoundAtMs !== undefined
|
|
1643
|
+
? { lastRoundAtMs: record.lastRoundAtMs }
|
|
1644
|
+
: {}),
|
|
1645
|
+
...(record.lastErrorCode !== undefined
|
|
1646
|
+
? { lastErrorCode: record.lastErrorCode }
|
|
1647
|
+
: {}),
|
|
1648
|
+
...(record.consecutiveFailures !== undefined
|
|
1649
|
+
? { consecutiveFailures: record.consecutiveFailures }
|
|
1650
|
+
: {}),
|
|
1636
1651
|
};
|
|
1637
1652
|
};
|
|
1638
1653
|
}
|