@syncular/server 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/context.d.ts +15 -1
- package/dist/context.js +3 -0
- package/dist/d1-storage.js +10 -3
- package/dist/handler.js +38 -2
- package/dist/postgres-storage.js +4 -0
- package/dist/pull.d.ts +9 -1
- package/dist/pull.js +17 -11
- package/dist/push.js +71 -4
- package/dist/relational-rows.d.ts +31 -2
- package/dist/relational-rows.js +57 -6
- package/dist/schema-window.js +8 -2
- package/dist/schema.d.ts +3 -0
- package/dist/schema.js +15 -0
- package/dist/sqlite-storage.js +14 -1
- package/dist/validate.d.ts +25 -0
- package/package.json +2 -2
- package/src/context.ts +22 -1
- package/src/d1-storage.ts +20 -3
- package/src/handler.ts +38 -4
- package/src/postgres-storage.ts +6 -0
- package/src/pull.ts +41 -26
- package/src/push.ts +132 -2
- package/src/relational-rows.ts +83 -7
- package/src/schema-window.ts +13 -2
- package/src/schema.ts +25 -0
- package/src/sqlite-storage.ts +23 -0
- package/src/validate.ts +32 -0
package/src/relational-rows.ts
CHANGED
|
@@ -1106,8 +1106,9 @@ export function dropTableDdl(tableName: string): string {
|
|
|
1106
1106
|
|
|
1107
1107
|
/**
|
|
1108
1108
|
* Enforce the migration subset on a table's column list: the old layout
|
|
1109
|
-
* must be
|
|
1110
|
-
*
|
|
1109
|
+
* must be a prefix (same name and type) of the new one, a column may change
|
|
1110
|
+
* only from required to nullable (SPEC §2.4, relaxing a required column), and
|
|
1111
|
+
* appended columns must be nullable (there is no default to backfill).
|
|
1111
1112
|
* Returns `true` iff columns were appended.
|
|
1112
1113
|
*/
|
|
1113
1114
|
export function assertAppendOnlyMigration(
|
|
@@ -1128,10 +1129,10 @@ export function assertAppendOnlyMigration(
|
|
|
1128
1129
|
after === undefined ||
|
|
1129
1130
|
before.name !== after.name ||
|
|
1130
1131
|
before.type !== after.type ||
|
|
1131
|
-
before.nullable
|
|
1132
|
+
(before.nullable && !after.nullable)
|
|
1132
1133
|
) {
|
|
1133
1134
|
throw new Error(
|
|
1134
|
-
`table ${JSON.stringify(tableName)}: column ${i} changed (${JSON.stringify(before)} → ${JSON.stringify({ name: after?.name, type: after?.type, nullable: after?.nullable })}) — only appending nullable columns
|
|
1135
|
+
`table ${JSON.stringify(tableName)}: column ${i} changed (${JSON.stringify(before)} → ${JSON.stringify({ name: after?.name, type: after?.type, nullable: after?.nullable })}) — only appending nullable columns and relaxing a required column are supported`,
|
|
1135
1136
|
);
|
|
1136
1137
|
}
|
|
1137
1138
|
}
|
|
@@ -1251,16 +1252,19 @@ export function rewriteValues(
|
|
|
1251
1252
|
* - `backfill`: the table just gained physical projection columns whose
|
|
1252
1253
|
* values exist in stored payloads (materialization flipped on) — the
|
|
1253
1254
|
* projection refreshes from the payload, no re-encode.
|
|
1255
|
+
* - `relaxed`: existing projection columns the bump turns from required to
|
|
1256
|
+
* nullable (SPEC §2.4). Their payload bytes are unchanged; only the
|
|
1257
|
+
* physical `NOT NULL` goes.
|
|
1254
1258
|
*/
|
|
1255
1259
|
export function rewritePlan(
|
|
1256
1260
|
table: CompiledTable,
|
|
1257
1261
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
1258
1262
|
physicalColumnsBefore: ReadonlySet<string> | undefined,
|
|
1259
|
-
): { migrate: boolean; backfill: boolean } {
|
|
1263
|
+
): { migrate: boolean; backfill: boolean; relaxed: string[] } {
|
|
1260
1264
|
if (oldLayout === undefined || physicalColumnsBefore === undefined) {
|
|
1261
1265
|
// Fresh table (or a pre-layout database, which cannot be reasoned
|
|
1262
1266
|
// about): nothing stored to rewrite.
|
|
1263
|
-
return { migrate: false, backfill: false };
|
|
1267
|
+
return { migrate: false, backfill: false, relaxed: [] };
|
|
1264
1268
|
}
|
|
1265
1269
|
const migrate = assertAppendOnlyMigration(table.name, oldLayout, table);
|
|
1266
1270
|
const backfill =
|
|
@@ -1270,7 +1274,79 @@ export function rewritePlan(
|
|
|
1270
1274
|
index < oldLayout.length && // value exists in old payloads
|
|
1271
1275
|
!physicalColumnsBefore.has(column.name), // column was just added
|
|
1272
1276
|
);
|
|
1273
|
-
|
|
1277
|
+
const relaxed = table.materialize
|
|
1278
|
+
? table.columns
|
|
1279
|
+
.filter(
|
|
1280
|
+
(column, index) =>
|
|
1281
|
+
oldLayout[index]?.nullable === false &&
|
|
1282
|
+
column.nullable &&
|
|
1283
|
+
physicalColumnsBefore.has(column.name),
|
|
1284
|
+
)
|
|
1285
|
+
.map((column) => column.name)
|
|
1286
|
+
: [];
|
|
1287
|
+
return { migrate, backfill, relaxed };
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
/**
|
|
1291
|
+
* One physical column of a SQLite table plus the SQL of every index and
|
|
1292
|
+
* trigger on it, read by {@link SQLITE_TABLE_REBUILD_SQL}.
|
|
1293
|
+
*/
|
|
1294
|
+
export interface SqliteRebuildColumnRow {
|
|
1295
|
+
readonly name: string;
|
|
1296
|
+
readonly type: string;
|
|
1297
|
+
readonly not_null: number;
|
|
1298
|
+
readonly dflt_value: string | null;
|
|
1299
|
+
readonly pk: number;
|
|
1300
|
+
/** JSON array of the table's index and trigger SQL (same on every row). */
|
|
1301
|
+
readonly dependents: string;
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
/** Binds the table name once (`?1`); one statement, so D1 spends one query. */
|
|
1305
|
+
export const SQLITE_TABLE_REBUILD_SQL = `SELECT name, type, "notnull" AS not_null, dflt_value, pk,
|
|
1306
|
+
(SELECT json_group_array(sql) FROM sqlite_master
|
|
1307
|
+
WHERE tbl_name=?1 AND type IN ('index', 'trigger') AND sql IS NOT NULL) AS dependents
|
|
1308
|
+
FROM pragma_table_info(?1) ORDER BY cid`;
|
|
1309
|
+
|
|
1310
|
+
/**
|
|
1311
|
+
* SQLite has no `ALTER COLUMN … DROP NOT NULL` (SPEC §2.4, relaxing a
|
|
1312
|
+
* required column): rebuild the table as a copy that keeps every physical
|
|
1313
|
+
* column declaration except `NOT NULL` on `relaxed`, copy every row, swap
|
|
1314
|
+
* it in, and recreate the indexes and triggers the original carried
|
|
1315
|
+
* (operator-added ones included). Rows, payloads and stored scopes are copied
|
|
1316
|
+
* verbatim. Empty when no relaxed column is physically `NOT NULL`.
|
|
1317
|
+
*/
|
|
1318
|
+
export function sqliteRelaxRebuildDdl(
|
|
1319
|
+
tableName: string,
|
|
1320
|
+
columns: readonly SqliteRebuildColumnRow[],
|
|
1321
|
+
relaxed: readonly string[],
|
|
1322
|
+
): string[] {
|
|
1323
|
+
const relax = (column: SqliteRebuildColumnRow): boolean =>
|
|
1324
|
+
column.not_null === 1 && relaxed.includes(column.name);
|
|
1325
|
+
if (!columns.some(relax)) return [];
|
|
1326
|
+
const target = quoteIdent(tableName);
|
|
1327
|
+
// The `sync_` prefix is reserved for server storage, so no app table
|
|
1328
|
+
// collides with the copy.
|
|
1329
|
+
const copy = quoteIdent(`sync_rebuild_${tableName}`);
|
|
1330
|
+
const names = columns.map((column) => quoteIdent(column.name)).join(', ');
|
|
1331
|
+
const definitions = columns.map((column) =>
|
|
1332
|
+
[
|
|
1333
|
+
quoteIdent(column.name),
|
|
1334
|
+
...(column.type === '' ? [] : [column.type]),
|
|
1335
|
+
...(column.not_null === 1 && !relax(column) ? ['NOT NULL'] : []),
|
|
1336
|
+
...(column.dflt_value === null ? [] : [`DEFAULT ${column.dflt_value}`]),
|
|
1337
|
+
].join(' '),
|
|
1338
|
+
);
|
|
1339
|
+
const primaryKey = columns
|
|
1340
|
+
.filter((column) => column.pk > 0)
|
|
1341
|
+
.sort((a, b) => a.pk - b.pk)
|
|
1342
|
+
.map((column) => quoteIdent(column.name));
|
|
1343
|
+
return [
|
|
1344
|
+
`CREATE TABLE ${copy} (${[...definitions, `PRIMARY KEY (${primaryKey.join(', ')})`].join(', ')})`,
|
|
1345
|
+
`INSERT INTO ${copy} (${names}) SELECT ${names} FROM ${target}`,
|
|
1346
|
+
`DROP TABLE ${target}`,
|
|
1347
|
+
`ALTER TABLE ${copy} RENAME TO ${target}`,
|
|
1348
|
+
...(JSON.parse(columns[0]?.dependents ?? '[]') as string[]),
|
|
1349
|
+
];
|
|
1274
1350
|
}
|
|
1275
1351
|
|
|
1276
1352
|
/**
|
package/src/schema-window.ts
CHANGED
|
@@ -31,16 +31,27 @@ export function schemaWindowOf(
|
|
|
31
31
|
if (
|
|
32
32
|
next === undefined ||
|
|
33
33
|
table.columns.length > next.columns.length ||
|
|
34
|
+
// §6.12: a version may mark an existing column server-owned; the flag
|
|
35
|
+
// never reaches the wire.
|
|
34
36
|
table.columns.some(
|
|
35
37
|
(column, i) =>
|
|
36
|
-
JSON.stringify(column) !==
|
|
38
|
+
JSON.stringify({ ...column, serverOwned: undefined }) !==
|
|
39
|
+
JSON.stringify({ ...next.columns[i], serverOwned: undefined }),
|
|
37
40
|
) ||
|
|
38
41
|
next.columns
|
|
39
42
|
.slice(table.columns.length)
|
|
40
43
|
.some((column) => !column.nullable) ||
|
|
41
44
|
table.primaryKeyIndex !== next.primaryKeyIndex ||
|
|
45
|
+
// §6.12: a pattern over an appended server-owned column is new to the
|
|
46
|
+
// current version; an older client neither holds nor writes it.
|
|
42
47
|
JSON.stringify(table.scopePatterns) !==
|
|
43
|
-
JSON.stringify(
|
|
48
|
+
JSON.stringify(
|
|
49
|
+
next.scopePatterns.filter(
|
|
50
|
+
(pattern) =>
|
|
51
|
+
pattern.columnIndex < table.columns.length ||
|
|
52
|
+
!next.serverOwnedColumnIndices.includes(pattern.columnIndex),
|
|
53
|
+
),
|
|
54
|
+
) ||
|
|
44
55
|
JSON.stringify(table.references) !== JSON.stringify(next.references)
|
|
45
56
|
) {
|
|
46
57
|
throw new Error('sync.schema_window.incompatible_schema');
|
package/src/schema.ts
CHANGED
|
@@ -154,6 +154,9 @@ export interface CompiledTable {
|
|
|
154
154
|
* image copies ciphertext wholesale with no per-row decrypt pass, so an
|
|
155
155
|
* encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
|
|
156
156
|
readonly encryptedColumnIndices: readonly number[];
|
|
157
|
+
/** §6.12: column indices marked `serverOwned`. Empty ⇒ no server column
|
|
158
|
+
* hook output is accepted for the table. */
|
|
159
|
+
readonly serverOwnedColumnIndices: readonly number[];
|
|
157
160
|
}
|
|
158
161
|
|
|
159
162
|
export interface CompiledSchema {
|
|
@@ -381,6 +384,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
381
384
|
const blobRefColumnIndices: number[] = [];
|
|
382
385
|
const crdtColumns: { index: number; crdtType: string }[] = [];
|
|
383
386
|
const encryptedColumnIndices: number[] = [];
|
|
387
|
+
const serverOwnedColumnIndices: number[] = [];
|
|
384
388
|
const references: CompiledReference[] = [];
|
|
385
389
|
for (const reference of table.references ?? []) {
|
|
386
390
|
const index = columnIndex.get(reference.column);
|
|
@@ -400,6 +404,26 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
400
404
|
if (column.type === 'blob_ref') blobRefColumnIndices.push(index);
|
|
401
405
|
// §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
|
|
402
406
|
if (column.encrypted === true) encryptedColumnIndices.push(index);
|
|
407
|
+
// §6.12: the hook never changes who may write the row (owner set), and
|
|
408
|
+
// a client's optimistic row holds NULL until the server row arrives.
|
|
409
|
+
if (column.serverOwned === true) {
|
|
410
|
+
if (
|
|
411
|
+
index === primaryKeyIndex ||
|
|
412
|
+
!column.nullable ||
|
|
413
|
+
column.type === 'crdt' ||
|
|
414
|
+
column.encrypted === true ||
|
|
415
|
+
scopePatterns.some(
|
|
416
|
+
(pattern) =>
|
|
417
|
+
pattern.columnIndex === index &&
|
|
418
|
+
writeScopeSets[0]!.includes(pattern.variable),
|
|
419
|
+
)
|
|
420
|
+
) {
|
|
421
|
+
throw new Error(
|
|
422
|
+
`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.12)`,
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
serverOwnedColumnIndices.push(index);
|
|
426
|
+
}
|
|
403
427
|
if (column.type === 'crdt') {
|
|
404
428
|
// §5.10.1: a crdt column MUST name a crdtType (schema-compile-time
|
|
405
429
|
// requirement — a crdt column without one is a server bug).
|
|
@@ -443,6 +467,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
|
|
|
443
467
|
blobRefColumnIndices,
|
|
444
468
|
crdtColumns,
|
|
445
469
|
encryptedColumnIndices,
|
|
470
|
+
serverOwnedColumnIndices,
|
|
446
471
|
});
|
|
447
472
|
}
|
|
448
473
|
// §6.11: resolve incoming references and enforce the declaration rules the
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -33,6 +33,9 @@ import {
|
|
|
33
33
|
rekeyRowScopesSql,
|
|
34
34
|
retiredTableNames,
|
|
35
35
|
rewritePlan,
|
|
36
|
+
SQLITE_TABLE_REBUILD_SQL,
|
|
37
|
+
type SqliteRebuildColumnRow,
|
|
38
|
+
sqliteRelaxRebuildDdl,
|
|
36
39
|
rewriteRowSql,
|
|
37
40
|
rewriteValues,
|
|
38
41
|
SCHEMA_META_DDL_SQLITE,
|
|
@@ -655,6 +658,26 @@ END`);
|
|
|
655
658
|
.run(tableName);
|
|
656
659
|
this.db.exec(dropTableDdl(tableName));
|
|
657
660
|
}
|
|
661
|
+
// A relaxed column loses its NOT NULL before the bump's other DDL
|
|
662
|
+
// (SPEC §2.4), so the index diff below sees the rebuilt table.
|
|
663
|
+
for (const table of schema.tables.values()) {
|
|
664
|
+
const { relaxed } = rewritePlan(
|
|
665
|
+
table,
|
|
666
|
+
layouts[table.name],
|
|
667
|
+
existing.get(table.name),
|
|
668
|
+
);
|
|
669
|
+
if (relaxed.length === 0) continue;
|
|
670
|
+
const columns = this.db
|
|
671
|
+
.query<SqliteRebuildColumnRow, [string]>(SQLITE_TABLE_REBUILD_SQL)
|
|
672
|
+
.all(table.name);
|
|
673
|
+
for (const statement of sqliteRelaxRebuildDdl(
|
|
674
|
+
table.name,
|
|
675
|
+
columns,
|
|
676
|
+
relaxed,
|
|
677
|
+
)) {
|
|
678
|
+
this.db.exec(statement);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
658
681
|
for (const statement of schemaDdl(
|
|
659
682
|
schema,
|
|
660
683
|
existing,
|
package/src/validate.ts
CHANGED
|
@@ -315,3 +315,35 @@ export function toValidateRow(
|
|
|
315
315
|
});
|
|
316
316
|
return row;
|
|
317
317
|
}
|
|
318
|
+
|
|
319
|
+
/** §6.12: the accepted upsert a server column hook completes. `row` is the
|
|
320
|
+
* merged row (present columns over stored values, crdt columns merged) with
|
|
321
|
+
* every server-owned column as stored, or NULL on an insert. */
|
|
322
|
+
export interface ServerColumnsInput {
|
|
323
|
+
readonly table: string;
|
|
324
|
+
readonly rowId: string;
|
|
325
|
+
readonly row: ValidateRow;
|
|
326
|
+
readonly stored: ValidateRow | undefined;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
export interface ServerColumnsContext {
|
|
330
|
+
readonly actorId: string;
|
|
331
|
+
readonly partition: string;
|
|
332
|
+
readonly context?: string;
|
|
333
|
+
/** The §6.8 candidate-state reader of the push transaction. */
|
|
334
|
+
readonly read: CommitValidationReader;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* §6.12: returns values for the table's server-owned columns; a column it
|
|
339
|
+
* omits keeps its value in `row`. A key that names no server-owned column, a
|
|
340
|
+
* value the column's type refuses, or a changed scope column on an update
|
|
341
|
+
* rejects the commit with `sync.constraint_violation`, like any other throw
|
|
342
|
+
* that is not a {@link ValidationRejection}.
|
|
343
|
+
*/
|
|
344
|
+
export type ServerColumnsHook = (
|
|
345
|
+
input: ServerColumnsInput,
|
|
346
|
+
ctx: ServerColumnsContext,
|
|
347
|
+
) => ValidateRow | Promise<ValidateRow>;
|
|
348
|
+
|
|
349
|
+
export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;
|