@syncular/server 0.93.0 → 0.95.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1106,8 +1106,9 @@ export function dropTableDdl(tableName: string): string {
1106
1106
 
1107
1107
  /**
1108
1108
  * Enforce the migration subset on a table's column list: the old layout
1109
- * must be an exact prefix (same name, type, nullability) of the new one,
1110
- * and appended columns must be nullable (there is no default to backfill).
1109
+ * must be a prefix (same name and type) of the new one, a column may change
1110
+ * only from required to nullable (SPEC §2.4, relaxing a required column), and
1111
+ * appended columns must be nullable (there is no default to backfill).
1111
1112
  * Returns `true` iff columns were appended.
1112
1113
  */
1113
1114
  export function assertAppendOnlyMigration(
@@ -1128,10 +1129,10 @@ export function assertAppendOnlyMigration(
1128
1129
  after === undefined ||
1129
1130
  before.name !== after.name ||
1130
1131
  before.type !== after.type ||
1131
- before.nullable !== after.nullable
1132
+ (before.nullable && !after.nullable)
1132
1133
  ) {
1133
1134
  throw new Error(
1134
- `table ${JSON.stringify(tableName)}: column ${i} changed (${JSON.stringify(before)} → ${JSON.stringify({ name: after?.name, type: after?.type, nullable: after?.nullable })}) — only appending nullable columns is supported`,
1135
+ `table ${JSON.stringify(tableName)}: column ${i} changed (${JSON.stringify(before)} → ${JSON.stringify({ name: after?.name, type: after?.type, nullable: after?.nullable })}) — only appending nullable columns and relaxing a required column are supported`,
1135
1136
  );
1136
1137
  }
1137
1138
  }
@@ -1251,16 +1252,19 @@ export function rewriteValues(
1251
1252
  * - `backfill`: the table just gained physical projection columns whose
1252
1253
  * values exist in stored payloads (materialization flipped on) — the
1253
1254
  * projection refreshes from the payload, no re-encode.
1255
+ * - `relaxed`: existing projection columns the bump turns from required to
1256
+ * nullable (SPEC §2.4). Their payload bytes are unchanged; only the
1257
+ * physical `NOT NULL` goes.
1254
1258
  */
1255
1259
  export function rewritePlan(
1256
1260
  table: CompiledTable,
1257
1261
  oldLayout: readonly StoredColumnLayout[] | undefined,
1258
1262
  physicalColumnsBefore: ReadonlySet<string> | undefined,
1259
- ): { migrate: boolean; backfill: boolean } {
1263
+ ): { migrate: boolean; backfill: boolean; relaxed: string[] } {
1260
1264
  if (oldLayout === undefined || physicalColumnsBefore === undefined) {
1261
1265
  // Fresh table (or a pre-layout database, which cannot be reasoned
1262
1266
  // about): nothing stored to rewrite.
1263
- return { migrate: false, backfill: false };
1267
+ return { migrate: false, backfill: false, relaxed: [] };
1264
1268
  }
1265
1269
  const migrate = assertAppendOnlyMigration(table.name, oldLayout, table);
1266
1270
  const backfill =
@@ -1270,7 +1274,79 @@ export function rewritePlan(
1270
1274
  index < oldLayout.length && // value exists in old payloads
1271
1275
  !physicalColumnsBefore.has(column.name), // column was just added
1272
1276
  );
1273
- return { migrate, backfill };
1277
+ const relaxed = table.materialize
1278
+ ? table.columns
1279
+ .filter(
1280
+ (column, index) =>
1281
+ oldLayout[index]?.nullable === false &&
1282
+ column.nullable &&
1283
+ physicalColumnsBefore.has(column.name),
1284
+ )
1285
+ .map((column) => column.name)
1286
+ : [];
1287
+ return { migrate, backfill, relaxed };
1288
+ }
1289
+
1290
+ /**
1291
+ * One physical column of a SQLite table plus the SQL of every index and
1292
+ * trigger on it, read by {@link SQLITE_TABLE_REBUILD_SQL}.
1293
+ */
1294
+ export interface SqliteRebuildColumnRow {
1295
+ readonly name: string;
1296
+ readonly type: string;
1297
+ readonly not_null: number;
1298
+ readonly dflt_value: string | null;
1299
+ readonly pk: number;
1300
+ /** JSON array of the table's index and trigger SQL (same on every row). */
1301
+ readonly dependents: string;
1302
+ }
1303
+
1304
+ /** Binds the table name once (`?1`); one statement, so D1 spends one query. */
1305
+ export const SQLITE_TABLE_REBUILD_SQL = `SELECT name, type, "notnull" AS not_null, dflt_value, pk,
1306
+ (SELECT json_group_array(sql) FROM sqlite_master
1307
+ WHERE tbl_name=?1 AND type IN ('index', 'trigger') AND sql IS NOT NULL) AS dependents
1308
+ FROM pragma_table_info(?1) ORDER BY cid`;
1309
+
1310
+ /**
1311
+ * SQLite has no `ALTER COLUMN … DROP NOT NULL` (SPEC §2.4, relaxing a
1312
+ * required column): rebuild the table as a copy that keeps every physical
1313
+ * column declaration except `NOT NULL` on `relaxed`, copy every row, swap
1314
+ * it in, and recreate the indexes and triggers the original carried
1315
+ * (operator-added ones included). Rows, payloads and stored scopes are copied
1316
+ * verbatim. Empty when no relaxed column is physically `NOT NULL`.
1317
+ */
1318
+ export function sqliteRelaxRebuildDdl(
1319
+ tableName: string,
1320
+ columns: readonly SqliteRebuildColumnRow[],
1321
+ relaxed: readonly string[],
1322
+ ): string[] {
1323
+ const relax = (column: SqliteRebuildColumnRow): boolean =>
1324
+ column.not_null === 1 && relaxed.includes(column.name);
1325
+ if (!columns.some(relax)) return [];
1326
+ const target = quoteIdent(tableName);
1327
+ // The `sync_` prefix is reserved for server storage, so no app table
1328
+ // collides with the copy.
1329
+ const copy = quoteIdent(`sync_rebuild_${tableName}`);
1330
+ const names = columns.map((column) => quoteIdent(column.name)).join(', ');
1331
+ const definitions = columns.map((column) =>
1332
+ [
1333
+ quoteIdent(column.name),
1334
+ ...(column.type === '' ? [] : [column.type]),
1335
+ ...(column.not_null === 1 && !relax(column) ? ['NOT NULL'] : []),
1336
+ ...(column.dflt_value === null ? [] : [`DEFAULT ${column.dflt_value}`]),
1337
+ ].join(' '),
1338
+ );
1339
+ const primaryKey = columns
1340
+ .filter((column) => column.pk > 0)
1341
+ .sort((a, b) => a.pk - b.pk)
1342
+ .map((column) => quoteIdent(column.name));
1343
+ return [
1344
+ `CREATE TABLE ${copy} (${[...definitions, `PRIMARY KEY (${primaryKey.join(', ')})`].join(', ')})`,
1345
+ `INSERT INTO ${copy} (${names}) SELECT ${names} FROM ${target}`,
1346
+ `DROP TABLE ${target}`,
1347
+ `ALTER TABLE ${copy} RENAME TO ${target}`,
1348
+ ...(JSON.parse(columns[0]?.dependents ?? '[]') as string[]),
1349
+ ];
1274
1350
  }
1275
1351
 
1276
1352
  /**
@@ -31,16 +31,27 @@ export function schemaWindowOf(
31
31
  if (
32
32
  next === undefined ||
33
33
  table.columns.length > next.columns.length ||
34
+ // §6.12: a version may mark an existing column server-owned; the flag
35
+ // never reaches the wire.
34
36
  table.columns.some(
35
37
  (column, i) =>
36
- JSON.stringify(column) !== JSON.stringify(next.columns[i]),
38
+ JSON.stringify({ ...column, serverOwned: undefined }) !==
39
+ JSON.stringify({ ...next.columns[i], serverOwned: undefined }),
37
40
  ) ||
38
41
  next.columns
39
42
  .slice(table.columns.length)
40
43
  .some((column) => !column.nullable) ||
41
44
  table.primaryKeyIndex !== next.primaryKeyIndex ||
45
+ // §6.12: a pattern over an appended server-owned column is new to the
46
+ // current version; an older client neither holds nor writes it.
42
47
  JSON.stringify(table.scopePatterns) !==
43
- JSON.stringify(next.scopePatterns) ||
48
+ JSON.stringify(
49
+ next.scopePatterns.filter(
50
+ (pattern) =>
51
+ pattern.columnIndex < table.columns.length ||
52
+ !next.serverOwnedColumnIndices.includes(pattern.columnIndex),
53
+ ),
54
+ ) ||
44
55
  JSON.stringify(table.references) !== JSON.stringify(next.references)
45
56
  ) {
46
57
  throw new Error('sync.schema_window.incompatible_schema');
package/src/schema.ts CHANGED
@@ -154,6 +154,9 @@ export interface CompiledTable {
154
154
  * image copies ciphertext wholesale with no per-row decrypt pass, so an
155
155
  * encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
156
156
  readonly encryptedColumnIndices: readonly number[];
157
+ /** §6.12: column indices marked `serverOwned`. Empty ⇒ no server column
158
+ * hook output is accepted for the table. */
159
+ readonly serverOwnedColumnIndices: readonly number[];
157
160
  }
158
161
 
159
162
  export interface CompiledSchema {
@@ -381,6 +384,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
381
384
  const blobRefColumnIndices: number[] = [];
382
385
  const crdtColumns: { index: number; crdtType: string }[] = [];
383
386
  const encryptedColumnIndices: number[] = [];
387
+ const serverOwnedColumnIndices: number[] = [];
384
388
  const references: CompiledReference[] = [];
385
389
  for (const reference of table.references ?? []) {
386
390
  const index = columnIndex.get(reference.column);
@@ -400,6 +404,26 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
400
404
  if (column.type === 'blob_ref') blobRefColumnIndices.push(index);
401
405
  // §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
402
406
  if (column.encrypted === true) encryptedColumnIndices.push(index);
407
+ // §6.12: the hook never changes who may write the row (owner set), and
408
+ // a client's optimistic row holds NULL until the server row arrives.
409
+ if (column.serverOwned === true) {
410
+ if (
411
+ index === primaryKeyIndex ||
412
+ !column.nullable ||
413
+ column.type === 'crdt' ||
414
+ column.encrypted === true ||
415
+ scopePatterns.some(
416
+ (pattern) =>
417
+ pattern.columnIndex === index &&
418
+ writeScopeSets[0]!.includes(pattern.variable),
419
+ )
420
+ ) {
421
+ throw new Error(
422
+ `table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.12)`,
423
+ );
424
+ }
425
+ serverOwnedColumnIndices.push(index);
426
+ }
403
427
  if (column.type === 'crdt') {
404
428
  // §5.10.1: a crdt column MUST name a crdtType (schema-compile-time
405
429
  // requirement — a crdt column without one is a server bug).
@@ -443,6 +467,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
443
467
  blobRefColumnIndices,
444
468
  crdtColumns,
445
469
  encryptedColumnIndices,
470
+ serverOwnedColumnIndices,
446
471
  });
447
472
  }
448
473
  // §6.11: resolve incoming references and enforce the declaration rules the
@@ -33,6 +33,9 @@ import {
33
33
  rekeyRowScopesSql,
34
34
  retiredTableNames,
35
35
  rewritePlan,
36
+ SQLITE_TABLE_REBUILD_SQL,
37
+ type SqliteRebuildColumnRow,
38
+ sqliteRelaxRebuildDdl,
36
39
  rewriteRowSql,
37
40
  rewriteValues,
38
41
  SCHEMA_META_DDL_SQLITE,
@@ -655,6 +658,26 @@ END`);
655
658
  .run(tableName);
656
659
  this.db.exec(dropTableDdl(tableName));
657
660
  }
661
+ // A relaxed column loses its NOT NULL before the bump's other DDL
662
+ // (SPEC §2.4), so the index diff below sees the rebuilt table.
663
+ for (const table of schema.tables.values()) {
664
+ const { relaxed } = rewritePlan(
665
+ table,
666
+ layouts[table.name],
667
+ existing.get(table.name),
668
+ );
669
+ if (relaxed.length === 0) continue;
670
+ const columns = this.db
671
+ .query<SqliteRebuildColumnRow, [string]>(SQLITE_TABLE_REBUILD_SQL)
672
+ .all(table.name);
673
+ for (const statement of sqliteRelaxRebuildDdl(
674
+ table.name,
675
+ columns,
676
+ relaxed,
677
+ )) {
678
+ this.db.exec(statement);
679
+ }
680
+ }
658
681
  for (const statement of schemaDdl(
659
682
  schema,
660
683
  existing,
package/src/validate.ts CHANGED
@@ -315,3 +315,35 @@ export function toValidateRow(
315
315
  });
316
316
  return row;
317
317
  }
318
+
319
+ /** §6.12: the accepted upsert a server column hook completes. `row` is the
320
+ * merged row (present columns over stored values, crdt columns merged) with
321
+ * every server-owned column as stored, or NULL on an insert. */
322
+ export interface ServerColumnsInput {
323
+ readonly table: string;
324
+ readonly rowId: string;
325
+ readonly row: ValidateRow;
326
+ readonly stored: ValidateRow | undefined;
327
+ }
328
+
329
+ export interface ServerColumnsContext {
330
+ readonly actorId: string;
331
+ readonly partition: string;
332
+ readonly context?: string;
333
+ /** The §6.8 candidate-state reader of the push transaction. */
334
+ readonly read: CommitValidationReader;
335
+ }
336
+
337
+ /**
338
+ * §6.12: returns values for the table's server-owned columns; a column it
339
+ * omits keeps its value in `row`. A key that names no server-owned column, a
340
+ * value the column's type refuses, or a changed scope column on an update
341
+ * rejects the commit with `sync.constraint_violation`, like any other throw
342
+ * that is not a {@link ValidationRejection}.
343
+ */
344
+ export type ServerColumnsHook = (
345
+ input: ServerColumnsInput,
346
+ ctx: ServerColumnsContext,
347
+ ) => ValidateRow | Promise<ValidateRow>;
348
+
349
+ export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;