@syncular/server 0.93.0 → 0.95.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -194,7 +194,10 @@ UI. Unknown members, free-form tokens, malformed paths, and over-limit data
194
194
  fail at construction. Diagnostic prose stays in `message`; apps should map
195
195
  the stable code/details to localized copy instead of displaying that message.
196
196
 
197
- Validators are per-operation. A validator must not mutate the row it receives.
197
+ Validators are per-operation. A validator must not mutate the row it receives;
198
+ a value only the server decides belongs in a server-owned column, which a
199
+ `serverColumns` hook sets before the validator runs
200
+ ([Server-owned columns](https://syncular.dev/guide-server/#server-owned-columns)).
198
201
 
199
202
  A validator that decides from other rows (a membership, a role, a grant) calls
200
203
  `context.queryAuthoritative` with the request `storage.queryAuthoritative`
@@ -1420,6 +1423,8 @@ Pass `schemaWindow: [compileSchema(schema), compileSchema(previousSchema)]` to
1420
1423
  serve the prior codec through the current storage and validation rules. Entries
1421
1424
  are newest first, with the compiled current schema first. Added tables and
1422
1425
  appended nullable columns are supported. Removed/renamed columns, changed codecs,
1423
- primary keys, references or scopes reject the window before serving. Hosts must
1426
+ primary keys, references or scopes reject the window before serving; a
1427
+ required column relaxed to nullable is a changed column, so the window cannot
1428
+ hold the version before the relaxation (SPEC §2.4). Hosts must
1424
1429
  exclude semantically incompatible prior versions. The [schema upgrade guide](https://syncular.dev/concepts-schema-upgrades/#serving-a-compatibility-window)
1425
1430
  owns the pull, segment, push and realtime contract.
package/dist/context.d.ts CHANGED
@@ -16,7 +16,7 @@ import type { SegmentStore } from './segment-store.js';
16
16
  import type { BlobPresignConfig, BlobUploadPresignConfig, SegmentUrlConfig } from './signed-url.js';
17
17
  import type { SqliteImageBuilder } from './sqlite-image.js';
18
18
  import type { CheckpointDeclaration, PartitionTouch, ServerStorage, StoredCommit } from './storage.js';
19
- import type { CommitValidator, ValidatorRegistry } from './validate.js';
19
+ import type { CommitValidator, ServerColumnsRegistry, ValidatorRegistry } from './validate.js';
20
20
  /** SSP2 body content type (§1.1). */
21
21
  export declare const SSP2_CONTENT_TYPE = "application/vnd.syncular.sync.v2";
22
22
  /** Internal idempotency namespace. Ordinary SSP2 client IDs cannot use it. */
@@ -102,6 +102,12 @@ export interface ServerLimits {
102
102
  * bound gets an image of its own.
103
103
  */
104
104
  readonly maxImageSegmentBytes: number;
105
+ /**
106
+ * Segment puts one pull keeps in flight. The pull scans and encodes its
107
+ * pages in order and stores up to this many at once, so it holds this
108
+ * many encoded segments in memory.
109
+ */
110
+ readonly maxConcurrentSegmentPuts: number;
105
111
  }
106
112
  export declare const DEFAULT_LIMITS: ServerLimits;
107
113
  /** Receives applied commits for realtime fanout (§8.2). */
@@ -154,6 +160,14 @@ export interface SyncServerConfig {
154
160
  * process, never on the wire.
155
161
  */
156
162
  readonly validators?: ValidatorRegistry;
163
+ /**
164
+ * §6.12 per-table server column hooks (`table` → hook). Each accepted
165
+ * upsert on a listed table runs its hook inside the push transaction,
166
+ * after §3.4 authorization and before §6.7 validation; the returned values
167
+ * of the table's server-owned columns are stored and fanned out in the
168
+ * same commit.
169
+ */
170
+ readonly serverColumns?: ServerColumnsRegistry;
157
171
  /**
158
172
  * §6.8 whole-commit validator. When present, the storage serializes this
159
173
  * partition before operation reads/writes, then invokes the callback once
package/dist/context.js CHANGED
@@ -25,6 +25,9 @@ export const DEFAULT_LIMITS = {
25
25
  maxLiveSectionBytes: 16 * 1024 * 1024,
26
26
  // §5.3 DECISION (2026-11-17): a client holds about three images at once.
27
27
  maxImageSegmentBytes: 32 * 1024 * 1024,
28
+ // A Workers invocation holds six connections open at once, and an R2 put
29
+ // uses one; past six the runtime queues the put.
30
+ maxConcurrentSegmentPuts: 6,
28
31
  };
29
32
  /** §1.1: `authenticate` returned a context longer than 256 UTF-8 bytes. */
30
33
  export class AuthContextTooLargeError extends Error {
@@ -42,7 +42,7 @@ var _a;
42
42
  * rather than a lock D1 does not expose.
43
43
  */
44
44
  import { decodeRow } from '@syncular/core';
45
- import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanProbeSql, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
45
+ import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, SQLITE_TABLE_REBUILD_SQL, sqliteRelaxRebuildDdl, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanProbeSql, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
46
46
  import { matchesEffective } from './scopes.js';
47
47
  import { deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, } from './sqlite-dialect.js';
48
48
  import { asUint8Array, commitDeleteFloor, toStoredRow, toTombstone, } from './storage.js';
@@ -904,18 +904,23 @@ export class D1ServerStorage extends SqliteFamilyStorage {
904
904
  }
905
905
  statementsExecuted += 2;
906
906
  const columns = await this.#db
907
- .prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
907
+ .prepare(SQLITE_TABLE_REBUILD_SQL)
908
+ .bind(table.name)
908
909
  .all();
909
910
  const indexes = await this.#db
910
911
  .prepare(SQLITE_TABLE_INDEXES_SQL)
911
912
  .bind(table.name)
912
913
  .all();
913
914
  // Validate the old codec before any application DDL or row rewrite.
914
- rewritePlan(table, sourceLayouts[table.name], columns.results.length > 0
915
+ const { relaxed } = rewritePlan(table, sourceLayouts[table.name], columns.results.length > 0
915
916
  ? new Set(columns.results.map((column) => column.name))
916
917
  : undefined);
917
918
  if (!(await save({
918
919
  ...state,
920
+ rebuilds: [
921
+ ...(state.rebuilds ?? []),
922
+ ...sqliteRelaxRebuildDdl(table.name, columns.results, relaxed),
923
+ ],
919
924
  offset: state.offset + 1,
920
925
  afterPartition: '',
921
926
  afterRowId: '',
@@ -964,6 +969,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
964
969
  `DELETE FROM sync_blob_refs WHERE tbl='${name.replaceAll("'", "''")}'`,
965
970
  dropTableDdl(name),
966
971
  ]),
972
+ ...(state.rebuilds ?? []),
967
973
  ...schemaDdl(schema, columns, 'sqlite', indexes),
968
974
  ...rekeyCleanupSql(rekeyed),
969
975
  ];
@@ -972,6 +978,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
972
978
  phase: 'ddl',
973
979
  offset: 0,
974
980
  ddl,
981
+ rebuilds: [],
975
982
  rewrites,
976
983
  columns: {},
977
984
  indexes: {},
package/dist/handler.js CHANGED
@@ -615,6 +615,12 @@ async function* streamResponse(plan, ctx, schema, report) {
615
615
  // follows the configured limits, never the unit count.
616
616
  const bootstrapping = states.flatMap((state, index) => state.kind === 'bootstrap' ? [index] : []);
617
617
  const firstPages = Math.max(1, Math.floor(100_000 / (limits.limitSnapshotRows + 1)));
618
+ // The segment puts of every bootstrap section, in frame order. A
619
+ // pull holds the encoded bytes of the puts in flight.
620
+ const puts = [];
621
+ const inFlight = new Set();
622
+ const maxPuts = limitsOf(ctx).maxConcurrentSegmentPuts;
623
+ let failed = false;
618
624
  for (let next = 0; next < bootstrapping.length;) {
619
625
  const window = [];
620
626
  let pages = 0;
@@ -647,7 +653,30 @@ async function* streamResponse(plan, ctx, schema, report) {
647
653
  const segments = bootstrapSegments(ctx, schema, limits, bootstrap, traces[index], plan.logEpoch);
648
654
  let step = await segments.next();
649
655
  while (!step.done) {
650
- frames.push(step.value);
656
+ const frame = step.value;
657
+ if (frame.type === 'PENDING_REF') {
658
+ // A rows segment's put runs while later pages are scanned;
659
+ // the next page waits while `maxPuts` puts are in flight.
660
+ const settled = frame.ref.then(() => void inFlight.delete(settled), () => {
661
+ failed = true;
662
+ inFlight.delete(settled);
663
+ });
664
+ inFlight.add(settled);
665
+ puts.push(frame.ref);
666
+ frames.push(frame.ref);
667
+ while (inFlight.size >= maxPuts) {
668
+ await Promise.race(inFlight);
669
+ if (failed)
670
+ break;
671
+ }
672
+ // A failed put ends the pull before it scans another page.
673
+ if (failed)
674
+ for (const put of puts)
675
+ await put;
676
+ }
677
+ else {
678
+ frames.push(frame);
679
+ }
651
680
  step = await segments.next();
652
681
  }
653
682
  const restrict = units[index].restrict;
@@ -660,6 +689,13 @@ async function* streamResponse(plan, ctx, schema, report) {
660
689
  };
661
690
  }
662
691
  }
692
+ // Every put completes before the live read, as in a pull with one put
693
+ // in flight. A failed put fails the pull with the error of the first
694
+ // failed page, and the puts still in flight finish unobserved: each
695
+ // stores its bytes before its record, so no record points at missing
696
+ // bytes.
697
+ for (const put of puts)
698
+ await put;
663
699
  const bound = (state) => state.kind === 'live'
664
700
  ? R
665
701
  : state.kind === 'post'
@@ -757,7 +793,7 @@ async function* streamResponse(plan, ctx, schema, report) {
757
793
  }, plan.wireVersion);
758
794
  if (state.kind === 'bootstrap') {
759
795
  for (const frame of state.frames) {
760
- yield encodeResponseFrame(frame, plan.wireVersion);
796
+ yield encodeResponseFrame(await frame, plan.wireVersion);
761
797
  }
762
798
  }
763
799
  yield encodeResponseFrame(state.kind === 'bootstrap'
@@ -1751,6 +1751,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1751
1751
  for (const table of schema.tables.values()) {
1752
1752
  const oldLayout = layouts[table.name];
1753
1753
  const plan = rewritePlan(table, oldLayout, existing.get(table.name));
1754
+ // SPEC §2.4: payload bytes are unchanged; only the constraint goes.
1755
+ for (const column of plan.relaxed) {
1756
+ await client.query(`ALTER TABLE ${quoteIdent(table.name)} ALTER COLUMN ${quoteIdent(column)} DROP NOT NULL`);
1757
+ }
1754
1758
  const rekey = rekeyed.includes(table.name);
1755
1759
  if (!plan.migrate && !plan.backfill && !rekey)
1756
1760
  continue;
package/dist/pull.d.ts CHANGED
@@ -67,6 +67,14 @@ interface TableStart {
67
67
  readonly stored: SegmentRecord | undefined;
68
68
  };
69
69
  }
70
+ /**
71
+ * A section frame: a response frame, or a rows segment's `SEGMENT_REF`
72
+ * whose put is in flight. The consumer emits `ref` in its frame order.
73
+ */
74
+ type SectionFrame = ResponseFrame | {
75
+ readonly type: 'PENDING_REF';
76
+ readonly ref: Promise<ResponseFrame>;
77
+ };
70
78
  /** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
71
79
  export interface BootstrapPlan {
72
80
  readonly unit: UnitPlan;
@@ -98,7 +106,7 @@ export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledS
98
106
  * the resume token, or `undefined` once every table is exhausted. A pair the
99
107
  * wildcard fence revoked receives no segment (§3.3).
100
108
  */
101
- export declare function bootstrapSegments(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plan: BootstrapPlan, trace: PullSectionTrace | undefined, logEpoch: string): AsyncGenerator<ResponseFrame, string | undefined>;
109
+ export declare function bootstrapSegments(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plan: BootstrapPlan, trace: PullSectionTrace | undefined, logEpoch: string): AsyncGenerator<SectionFrame, string | undefined>;
102
110
  export interface PublishSqliteImageInput {
103
111
  /** The sync config the serving host runs, plus a `sqliteImageBuilder`. */
104
112
  readonly config: SyncServerConfig;
package/dist/pull.js CHANGED
@@ -473,7 +473,17 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
473
473
  yield { type: 'SEGMENT_INLINE', payload: bytes };
474
474
  }
475
475
  else {
476
- const record = await segments.put({
476
+ // The page's put runs while the next page is scanned; the caller keeps
477
+ // `maxConcurrentSegmentPuts` in flight and emits the frames in order.
478
+ trace?.segments.push({
479
+ mediaType: 'rows',
480
+ delivery: 'ref',
481
+ origin: 'built',
482
+ bytes: bytes.length,
483
+ rows: pageRows.length,
484
+ });
485
+ const ref = segments
486
+ .put({
477
487
  partition,
478
488
  logEpoch,
479
489
  table: pair.table.name,
@@ -484,15 +494,9 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
484
494
  rowCount: pageRows.length,
485
495
  rowCursor,
486
496
  nextRowCursor,
487
- }, bytes, now);
488
- trace?.segments.push({
489
- mediaType: 'rows',
490
- delivery: 'ref',
491
- origin: 'built',
492
- bytes: record.byteLength,
493
- rows: record.rowCount,
494
- });
495
- yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
497
+ }, bytes, now)
498
+ .then(async (record) => segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now)));
499
+ yield { type: 'PENDING_REF', ref };
496
500
  }
497
501
  if (!hasMore)
498
502
  return { pages: page + 1, rows, rowCursor: null, image: false };
@@ -650,7 +654,9 @@ export async function publishSqliteImage(input) {
650
654
  const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
651
655
  const refs = [];
652
656
  for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
653
- if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
657
+ if (frame.type === 'PENDING_REF')
658
+ await frame.ref;
659
+ else if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
654
660
  refs.push(frame);
655
661
  }
656
662
  }
package/dist/push.js CHANGED
@@ -133,6 +133,56 @@ async function runValidator(validators, table, op, rowId, values, storedValues,
133
133
  }
134
134
  return undefined;
135
135
  }
136
+ /**
137
+ * §6.12: complete an accepted upsert with the table's server column hook.
138
+ * The returned values land in `values` and are marked present, so their
139
+ * column versions advance. A `ValidationRejection` keeps its host code; any
140
+ * other failure (an undeclared key, a value the column's type refuses, a
141
+ * changed scope column on an update) is `sync.constraint_violation`.
142
+ */
143
+ async function runServerColumns(hooks, tx, schema, table, rowId, values, storedValues, present, opIndex, partition, actorId, context) {
144
+ const hook = hooks?.[table.name];
145
+ if (hook === undefined)
146
+ return undefined;
147
+ try {
148
+ const output = await hook({
149
+ table: table.name,
150
+ rowId,
151
+ row: toValidateRow(table.columns, values),
152
+ stored: storedValues !== undefined
153
+ ? toValidateRow(table.columns, storedValues)
154
+ : undefined,
155
+ }, {
156
+ actorId,
157
+ partition,
158
+ ...(context !== undefined ? { context } : {}),
159
+ read: commitValidationReader(tx, schema),
160
+ });
161
+ for (const [name, value] of Object.entries(output)) {
162
+ const index = table.columnIndex.get(name);
163
+ if (index === undefined ||
164
+ !table.serverOwnedColumnIndices.includes(index)) {
165
+ throw new Error('server column hook returned a column that is not server-owned');
166
+ }
167
+ // The column's own codec refuses a value of the wrong type.
168
+ encodeRow([table.columns[index]], [value]);
169
+ if (storedValues !== undefined &&
170
+ table.scopePatterns.some((pattern) => pattern.columnIndex === index) &&
171
+ value !== (storedValues[index] ?? null)) {
172
+ throw new Error('server column hook changed a scope column on update');
173
+ }
174
+ values[index] = value;
175
+ present[index] = true;
176
+ }
177
+ }
178
+ catch (error) {
179
+ if (error instanceof ValidationRejection) {
180
+ return errorRecord(opIndex, error.code, error.message, false, error.details);
181
+ }
182
+ return errorRecord(opIndex, 'sync.constraint_violation', 'server column hook failed');
183
+ }
184
+ return undefined;
185
+ }
136
186
  /**
137
187
  * §5.10.3: merge the row's `crdt` columns in place. For each crdt column,
138
188
  * replace the incoming value with `merge(stored, incoming)` (§5.10.2) —
@@ -175,7 +225,7 @@ async function mergeCrdtColumns(table, values, storedValues, present, opIndex, m
175
225
  }
176
226
  return { changed };
177
227
  }
178
- async function applyOperation(tx, schema, wireSchema, resolved, op, opIndex, blobCtx, crdtMergers, validators, partition, actorId, context,
228
+ async function applyOperation(tx, schema, wireSchema, resolved, op, opIndex, blobCtx, crdtMergers, validators, serverColumns, partition, actorId, context,
179
229
  /** §2.2: the partition's delete floor, raised by this commit's deletes. */
180
230
  deleteFloor) {
181
231
  const table = schema.tables.get(op.table);
@@ -410,6 +460,11 @@ deleteFloor) {
410
460
  catch (error) {
411
461
  return errorRecord(opIndex, 'sync.invalid_request', `row payload failed row-codec decode (§1.7): ${error instanceof Error ? error.message : String(error)}`);
412
462
  }
463
+ // §6.12: the server discards a pushed value of a server-owned column, so an
464
+ // older client that still writes one stays valid.
465
+ for (const index of table.serverOwnedColumnIndices) {
466
+ sparse[index] = stored === undefined ? null : undefined;
467
+ }
413
468
  const pkValue = renderScopeValue(sparse[table.primaryKeyIndex] ?? null);
414
469
  if (pkValue !== op.rowId) {
415
470
  return errorRecord(opIndex, 'sync.invalid_request', 'payload primary key does not match rowId');
@@ -483,6 +538,9 @@ deleteFloor) {
483
538
  const mergeOutcome = await mergeCrdtColumns(table, values, storedValues, present, opIndex, crdtMergers);
484
539
  if ('kind' in mergeOutcome)
485
540
  return mergeOutcome;
541
+ const serverReject = await runServerColumns(serverColumns, tx, schema, table, op.rowId, values, storedValues, present, opIndex, partition, actorId, context);
542
+ if (serverReject !== undefined)
543
+ return serverReject;
486
544
  const newVersion = stored.serverVersion + 1;
487
545
  const nextVersions = new Map();
488
546
  for (let i = 0; i < table.columns.length; i++) {
@@ -598,6 +656,15 @@ deleteFloor) {
598
656
  const insertMerge = await mergeCrdtColumns(table, values, undefined, present, opIndex, crdtMergers);
599
657
  if ('kind' in insertMerge)
600
658
  return insertMerge;
659
+ const serverReject = await runServerColumns(serverColumns, tx, schema, table, op.rowId, values, undefined, present, opIndex, partition, actorId, context);
660
+ if (serverReject !== undefined)
661
+ return serverReject;
662
+ // §6.12: the stored scopes and the change-scope index take the hook's
663
+ // values, so the row reaches a server-set scope's units in this commit.
664
+ const scoped = storedScopesForRow(table, values);
665
+ if ('missing' in scoped) {
666
+ return errorRecord(opIndex, 'sync.constraint_violation', 'server column hook failed');
667
+ }
601
668
  // The stored payload is the full-row codec (§2.4); every column is
602
669
  // present on an insert.
603
670
  const insertPayload = encodeRow(table.columns, values);
@@ -620,7 +687,7 @@ deleteFloor) {
620
687
  const newRow = {
621
688
  rowId: op.rowId,
622
689
  serverVersion,
623
- scopes: insert.scopes,
690
+ scopes: scoped.scopes,
624
691
  payload: insertPayload,
625
692
  };
626
693
  await tx.upsertRow(op.table, newRow, { opIndex });
@@ -632,7 +699,7 @@ deleteFloor) {
632
699
  rowId: op.rowId,
633
700
  op: 'upsert',
634
701
  rowVersion: serverVersion,
635
- scopes: insert.scopes,
702
+ scopes: scoped.scopes,
636
703
  payload: insertPayload,
637
704
  },
638
705
  ],
@@ -1401,7 +1468,7 @@ async function applyPushCommits(ctx, schema, resolved, clientId, commits, wireSc
1401
1468
  const op = operations[opIndex];
1402
1469
  if (op === undefined)
1403
1470
  continue;
1404
- const outcome = await applyOperation(tx, schema, wireSchema, resolved, op, opIndex, blobCtx, crdtMergers, validators, partition, ctx.actorId, ctx.context, deleteFloor);
1471
+ const outcome = await applyOperation(tx, schema, wireSchema, resolved, op, opIndex, blobCtx, crdtMergers, validators, ctx.serverColumns, partition, ctx.actorId, ctx.context, deleteFloor);
1405
1472
  if (outcome.kind === 'terminate') {
1406
1473
  terminated = outcome.record;
1407
1474
  break;
@@ -374,8 +374,9 @@ export declare function retiredTableNames(schema: CompiledSchema, storedLayouts:
374
374
  export declare function dropTableDdl(tableName: string): string;
375
375
  /**
376
376
  * Enforce the migration subset on a table's column list: the old layout
377
- * must be an exact prefix (same name, type, nullability) of the new one,
378
- * and appended columns must be nullable (there is no default to backfill).
377
+ * must be a prefix (same name and type) of the new one, a column may change
378
+ * only from required to nullable (SPEC §2.4, relaxing a required column), and
379
+ * appended columns must be nullable (there is no default to backfill).
379
380
  * Returns `true` iff columns were appended.
380
381
  */
381
382
  export declare function assertAppendOnlyMigration(tableName: string, oldLayout: readonly StoredColumnLayout[], table: CompiledTable): boolean;
@@ -418,11 +419,39 @@ export declare function rewriteValues(table: CompiledTable, partition: string, r
418
419
  * - `backfill`: the table just gained physical projection columns whose
419
420
  * values exist in stored payloads (materialization flipped on) — the
420
421
  * projection refreshes from the payload, no re-encode.
422
+ * - `relaxed`: existing projection columns the bump turns from required to
423
+ * nullable (SPEC §2.4). Their payload bytes are unchanged; only the
424
+ * physical `NOT NULL` goes.
421
425
  */
422
426
  export declare function rewritePlan(table: CompiledTable, oldLayout: readonly StoredColumnLayout[] | undefined, physicalColumnsBefore: ReadonlySet<string> | undefined): {
423
427
  migrate: boolean;
424
428
  backfill: boolean;
429
+ relaxed: string[];
425
430
  };
431
+ /**
432
+ * One physical column of a SQLite table plus the SQL of every index and
433
+ * trigger on it, read by {@link SQLITE_TABLE_REBUILD_SQL}.
434
+ */
435
+ export interface SqliteRebuildColumnRow {
436
+ readonly name: string;
437
+ readonly type: string;
438
+ readonly not_null: number;
439
+ readonly dflt_value: string | null;
440
+ readonly pk: number;
441
+ /** JSON array of the table's index and trigger SQL (same on every row). */
442
+ readonly dependents: string;
443
+ }
444
+ /** Binds the table name once (`?1`); one statement, so D1 spends one query. */
445
+ export declare const SQLITE_TABLE_REBUILD_SQL = "SELECT name, type, \"notnull\" AS not_null, dflt_value, pk,\n (SELECT json_group_array(sql) FROM sqlite_master\n WHERE tbl_name=?1 AND type IN ('index', 'trigger') AND sql IS NOT NULL) AS dependents\n FROM pragma_table_info(?1) ORDER BY cid";
446
+ /**
447
+ * SQLite has no `ALTER COLUMN … DROP NOT NULL` (SPEC §2.4, relaxing a
448
+ * required column): rebuild the table as a copy that keeps every physical
449
+ * column declaration except `NOT NULL` on `relaxed`, copy every row, swap
450
+ * it in, and recreate the indexes and triggers the original carried
451
+ * (operator-added ones included). Rows, payloads and stored scopes are copied
452
+ * verbatim. Empty when no relaxed column is physically `NOT NULL`.
453
+ */
454
+ export declare function sqliteRelaxRebuildDdl(tableName: string, columns: readonly SqliteRebuildColumnRow[], relaxed: readonly string[]): string[];
426
455
  /**
427
456
  * Every DDL statement to bring a database from the introspected relational
428
457
  * projections to `schema`: CREATE TABLE for absent tables, ADD COLUMN for
@@ -897,8 +897,9 @@ export function dropTableDdl(tableName) {
897
897
  }
898
898
  /**
899
899
  * Enforce the migration subset on a table's column list: the old layout
900
- * must be an exact prefix (same name, type, nullability) of the new one,
901
- * and appended columns must be nullable (there is no default to backfill).
900
+ * must be a prefix (same name and type) of the new one, a column may change
901
+ * only from required to nullable (SPEC §2.4, relaxing a required column), and
902
+ * appended columns must be nullable (there is no default to backfill).
902
903
  * Returns `true` iff columns were appended.
903
904
  */
904
905
  export function assertAppendOnlyMigration(tableName, oldLayout, table) {
@@ -912,8 +913,8 @@ export function assertAppendOnlyMigration(tableName, oldLayout, table) {
912
913
  after === undefined ||
913
914
  before.name !== after.name ||
914
915
  before.type !== after.type ||
915
- before.nullable !== after.nullable) {
916
- throw new Error(`table ${JSON.stringify(tableName)}: column ${i} changed (${JSON.stringify(before)} → ${JSON.stringify({ name: after?.name, type: after?.type, nullable: after?.nullable })}) — only appending nullable columns is supported`);
916
+ (before.nullable && !after.nullable)) {
917
+ throw new Error(`table ${JSON.stringify(tableName)}: column ${i} changed (${JSON.stringify(before)} → ${JSON.stringify({ name: after?.name, type: after?.type, nullable: after?.nullable })}) — only appending nullable columns and relaxing a required column are supported`);
917
918
  }
918
919
  }
919
920
  for (let i = oldLayout.length; i < table.columns.length; i++) {
@@ -1009,18 +1010,68 @@ export function rewriteValues(table, partition, rowId, payload, dialect, scopes)
1009
1010
  * - `backfill`: the table just gained physical projection columns whose
1010
1011
  * values exist in stored payloads (materialization flipped on) — the
1011
1012
  * projection refreshes from the payload, no re-encode.
1013
+ * - `relaxed`: existing projection columns the bump turns from required to
1014
+ * nullable (SPEC §2.4). Their payload bytes are unchanged; only the
1015
+ * physical `NOT NULL` goes.
1012
1016
  */
1013
1017
  export function rewritePlan(table, oldLayout, physicalColumnsBefore) {
1014
1018
  if (oldLayout === undefined || physicalColumnsBefore === undefined) {
1015
1019
  // Fresh table (or a pre-layout database, which cannot be reasoned
1016
1020
  // about): nothing stored to rewrite.
1017
- return { migrate: false, backfill: false };
1021
+ return { migrate: false, backfill: false, relaxed: [] };
1018
1022
  }
1019
1023
  const migrate = assertAppendOnlyMigration(table.name, oldLayout, table);
1020
1024
  const backfill = table.materialize &&
1021
1025
  table.columns.some((column, index) => index < oldLayout.length && // value exists in old payloads
1022
1026
  !physicalColumnsBefore.has(column.name));
1023
- return { migrate, backfill };
1027
+ const relaxed = table.materialize
1028
+ ? table.columns
1029
+ .filter((column, index) => oldLayout[index]?.nullable === false &&
1030
+ column.nullable &&
1031
+ physicalColumnsBefore.has(column.name))
1032
+ .map((column) => column.name)
1033
+ : [];
1034
+ return { migrate, backfill, relaxed };
1035
+ }
1036
+ /** Binds the table name once (`?1`); one statement, so D1 spends one query. */
1037
+ export const SQLITE_TABLE_REBUILD_SQL = `SELECT name, type, "notnull" AS not_null, dflt_value, pk,
1038
+ (SELECT json_group_array(sql) FROM sqlite_master
1039
+ WHERE tbl_name=?1 AND type IN ('index', 'trigger') AND sql IS NOT NULL) AS dependents
1040
+ FROM pragma_table_info(?1) ORDER BY cid`;
1041
+ /**
1042
+ * SQLite has no `ALTER COLUMN … DROP NOT NULL` (SPEC §2.4, relaxing a
1043
+ * required column): rebuild the table as a copy that keeps every physical
1044
+ * column declaration except `NOT NULL` on `relaxed`, copy every row, swap
1045
+ * it in, and recreate the indexes and triggers the original carried
1046
+ * (operator-added ones included). Rows, payloads and stored scopes are copied
1047
+ * verbatim. Empty when no relaxed column is physically `NOT NULL`.
1048
+ */
1049
+ export function sqliteRelaxRebuildDdl(tableName, columns, relaxed) {
1050
+ const relax = (column) => column.not_null === 1 && relaxed.includes(column.name);
1051
+ if (!columns.some(relax))
1052
+ return [];
1053
+ const target = quoteIdent(tableName);
1054
+ // The `sync_` prefix is reserved for server storage, so no app table
1055
+ // collides with the copy.
1056
+ const copy = quoteIdent(`sync_rebuild_${tableName}`);
1057
+ const names = columns.map((column) => quoteIdent(column.name)).join(', ');
1058
+ const definitions = columns.map((column) => [
1059
+ quoteIdent(column.name),
1060
+ ...(column.type === '' ? [] : [column.type]),
1061
+ ...(column.not_null === 1 && !relax(column) ? ['NOT NULL'] : []),
1062
+ ...(column.dflt_value === null ? [] : [`DEFAULT ${column.dflt_value}`]),
1063
+ ].join(' '));
1064
+ const primaryKey = columns
1065
+ .filter((column) => column.pk > 0)
1066
+ .sort((a, b) => a.pk - b.pk)
1067
+ .map((column) => quoteIdent(column.name));
1068
+ return [
1069
+ `CREATE TABLE ${copy} (${[...definitions, `PRIMARY KEY (${primaryKey.join(', ')})`].join(', ')})`,
1070
+ `INSERT INTO ${copy} (${names}) SELECT ${names} FROM ${target}`,
1071
+ `DROP TABLE ${target}`,
1072
+ `ALTER TABLE ${copy} RENAME TO ${target}`,
1073
+ ...JSON.parse(columns[0]?.dependents ?? '[]'),
1074
+ ];
1024
1075
  }
1025
1076
  /**
1026
1077
  * Every DDL statement to bring a database from the introspected relational
@@ -19,13 +19,19 @@ export function schemaWindowOf(current, configured) {
19
19
  const next = head.tables.get(table.name);
20
20
  if (next === undefined ||
21
21
  table.columns.length > next.columns.length ||
22
- table.columns.some((column, i) => JSON.stringify(column) !== JSON.stringify(next.columns[i])) ||
22
+ // §6.12: a version may mark an existing column server-owned; the flag
23
+ // never reaches the wire.
24
+ table.columns.some((column, i) => JSON.stringify({ ...column, serverOwned: undefined }) !==
25
+ JSON.stringify({ ...next.columns[i], serverOwned: undefined })) ||
23
26
  next.columns
24
27
  .slice(table.columns.length)
25
28
  .some((column) => !column.nullable) ||
26
29
  table.primaryKeyIndex !== next.primaryKeyIndex ||
30
+ // §6.12: a pattern over an appended server-owned column is new to the
31
+ // current version; an older client neither holds nor writes it.
27
32
  JSON.stringify(table.scopePatterns) !==
28
- JSON.stringify(next.scopePatterns) ||
33
+ JSON.stringify(next.scopePatterns.filter((pattern) => pattern.columnIndex < table.columns.length ||
34
+ !next.serverOwnedColumnIndices.includes(pattern.columnIndex))) ||
29
35
  JSON.stringify(table.references) !== JSON.stringify(next.references)) {
30
36
  throw new Error('sync.schema_window.incompatible_schema');
31
37
  }
package/dist/schema.d.ts CHANGED
@@ -141,6 +141,9 @@ export interface CompiledTable {
141
141
  * image copies ciphertext wholesale with no per-row decrypt pass, so an
142
142
  * encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
143
143
  readonly encryptedColumnIndices: readonly number[];
144
+ /** §6.12: column indices marked `serverOwned`. Empty ⇒ no server column
145
+ * hook output is accepted for the table. */
146
+ readonly serverOwnedColumnIndices: readonly number[];
144
147
  }
145
148
  export interface CompiledSchema {
146
149
  readonly version: number;
package/dist/schema.js CHANGED
@@ -154,6 +154,7 @@ export function compileSchema(schema) {
154
154
  const blobRefColumnIndices = [];
155
155
  const crdtColumns = [];
156
156
  const encryptedColumnIndices = [];
157
+ const serverOwnedColumnIndices = [];
157
158
  const references = [];
158
159
  for (const reference of table.references ?? []) {
159
160
  const index = columnIndex.get(reference.column);
@@ -173,6 +174,19 @@ export function compileSchema(schema) {
173
174
  // §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
174
175
  if (column.encrypted === true)
175
176
  encryptedColumnIndices.push(index);
177
+ // §6.12: the hook never changes who may write the row (owner set), and
178
+ // a client's optimistic row holds NULL until the server row arrives.
179
+ if (column.serverOwned === true) {
180
+ if (index === primaryKeyIndex ||
181
+ !column.nullable ||
182
+ column.type === 'crdt' ||
183
+ column.encrypted === true ||
184
+ scopePatterns.some((pattern) => pattern.columnIndex === index &&
185
+ writeScopeSets[0].includes(pattern.variable))) {
186
+ throw new Error(`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.12)`);
187
+ }
188
+ serverOwnedColumnIndices.push(index);
189
+ }
176
190
  if (column.type === 'crdt') {
177
191
  // §5.10.1: a crdt column MUST name a crdtType (schema-compile-time
178
192
  // requirement — a crdt column without one is a server bug).
@@ -207,6 +221,7 @@ export function compileSchema(schema) {
207
221
  blobRefColumnIndices,
208
222
  crdtColumns,
209
223
  encryptedColumnIndices,
224
+ serverOwnedColumnIndices,
210
225
  });
211
226
  }
212
227
  // §6.11: resolve incoming references and enforce the declaration rules the