@syncular/server 0.62.1 → 0.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -60,6 +60,7 @@ import type {
60
60
  ServerIndexSchema,
61
61
  } from './schema';
62
62
  import type { StoredRow } from './storage';
63
+ import { storedScopesForRow } from './scopes';
63
64
  import { StorageQueryError } from './storage-errors';
64
65
 
65
66
  export type RelationalDialect = 'sqlite' | 'postgres';
@@ -872,20 +873,120 @@ export type StoredLayouts = Record<string, readonly StoredColumnLayout[]>;
872
873
 
873
874
  /** The layouts JSON persisted alongside the schema version marker. */
874
875
  export function layoutsOf(schema: CompiledSchema): string {
875
- const out: Record<string, StoredColumnLayout[]> = {};
876
+ const out: Record<string, unknown> = {};
877
+ const scopes: Record<string, string[]> = {};
876
878
  for (const table of schema.tables.values()) {
877
879
  out[table.name] = table.columns.map((column) => ({
878
880
  name: column.name,
879
881
  type: column.type,
880
882
  nullable: column.nullable,
881
883
  }));
884
+ scopes[table.name] = scopePatternsOf(table);
882
885
  }
886
+ // `sync_` is a reserved table-name prefix, so this key names no table.
887
+ out.sync_scopes = scopes;
883
888
  return JSON.stringify(out);
884
889
  }
885
890
 
891
+ function scopePatternsOf(table: CompiledTable): string[] {
892
+ return table.scopePatterns
893
+ .map(({ prefix, variable, column }) => `${prefix}:{${variable}}@${column}`)
894
+ .sort();
895
+ }
896
+
886
897
  export function parseLayouts(json: string | null | undefined): StoredLayouts {
887
898
  if (json === null || json === undefined || json.length === 0) return {};
888
- return JSON.parse(json) as StoredLayouts;
899
+ const { sync_scopes: _scopes, ...layouts } = JSON.parse(json) as Record<
900
+ string,
901
+ unknown
902
+ >;
903
+ return layouts as StoredLayouts;
904
+ }
905
+
906
+ /**
907
+ * §3.1: the configured tables whose scope patterns differ from the recorded
908
+ * ones, or `undefined` when the marker predates recorded patterns (nothing to
909
+ * compare). A schema-version bump re-keys these tables.
910
+ */
911
+ export function changedScopePatterns(
912
+ schema: CompiledSchema,
913
+ storedJson: string | null | undefined,
914
+ ): { table: string; stored: string; configured: string }[] | undefined {
915
+ if (storedJson === null || storedJson === undefined || storedJson === '')
916
+ return [];
917
+ const stored = (
918
+ JSON.parse(storedJson) as {
919
+ sync_scopes?: Record<string, readonly string[]>;
920
+ }
921
+ ).sync_scopes;
922
+ if (stored === undefined) return undefined;
923
+ return [...schema.tables.values()].flatMap((table) => {
924
+ const before = stored[table.name];
925
+ const after = scopePatternsOf(table);
926
+ return before !== undefined && before.join('\n') !== after.join('\n')
927
+ ? [
928
+ {
929
+ table: table.name,
930
+ stored: before.join(' '),
931
+ configured: after.join(' '),
932
+ },
933
+ ]
934
+ : [];
935
+ });
936
+ }
937
+
938
+ /**
939
+ * §3.1: at the same schema version, refuse a configured table whose patterns
940
+ * differ from the recorded ones. Returns false when the marker predates
941
+ * recorded patterns, so the caller records them.
942
+ */
943
+ export function assertStoredScopePatterns(
944
+ schema: CompiledSchema,
945
+ storedJson: string | null | undefined,
946
+ ): boolean {
947
+ const changed = changedScopePatterns(schema, storedJson);
948
+ if (changed?.[0] !== undefined) {
949
+ throw new StorageQueryError(
950
+ 'sync.storage.scope_patterns_changed',
951
+ changed[0],
952
+ );
953
+ }
954
+ return changed !== undefined;
955
+ }
956
+
957
+ /**
958
+ * §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
959
+ * entries and tombstones, and raise every partition's horizon past the
960
+ * changes logged under the previous patterns. Empty when nothing re-keys.
961
+ */
962
+ export function rekeyCleanupSql(tables: readonly string[]): string[] {
963
+ if (tables.length === 0) return [];
964
+ const names = tables.map((name) => `'${name.replaceAll("'", "''")}'`);
965
+ return [
966
+ `DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
967
+ `DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
968
+ 'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
969
+ ];
970
+ }
971
+
972
+ /**
973
+ * §3.1 re-key: index the scopes of one rewritten page of rows. The entries
974
+ * bind as one JSON array of `[partition, variable, value, rowId]`, so the
975
+ * statement reads no row table (a planner that ranges a page over a declared
976
+ * index would scan the whole partition per page). Params: [entries].
977
+ */
978
+ export function rekeyRowScopesSql(
979
+ table: CompiledTable,
980
+ dialect: RelationalDialect,
981
+ ): string {
982
+ const name = `'${table.name.replaceAll("'", "''")}'`;
983
+ return dialect === 'sqlite'
984
+ ? `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
985
+ SELECT json_extract(e.value, '$[0]'), ${name}, json_extract(e.value, '$[1]'), json_extract(e.value, '$[2]'), json_extract(e.value, '$[3]')
986
+ FROM json_each(?) AS e`
987
+ : `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
988
+ SELECT e->>0, ${name}, e->>1, e->>2, e->>3
989
+ FROM jsonb_array_elements($1::text::jsonb) AS e`;
889
990
  }
890
991
 
891
992
  /**
@@ -1025,27 +1126,53 @@ export function selectRowsForRewriteSql(
1025
1126
  LIMIT ${p[2]}`;
1026
1127
  }
1027
1128
 
1129
+ /**
1130
+ * §3.1 re-key: the row's stored scopes under the table's current patterns. A
1131
+ * row without a value for one of them refuses the bump.
1132
+ */
1133
+ export function rekeyedScopes(
1134
+ table: CompiledTable,
1135
+ values: readonly RowValue[],
1136
+ rowId: string,
1137
+ ): Record<string, string> {
1138
+ const extracted = storedScopesForRow(table, values);
1139
+ if ('missing' in extracted) {
1140
+ throw new StorageQueryError('sync.storage.scope_value_missing', {
1141
+ table: table.name,
1142
+ variable: extracted.missing,
1143
+ rowId,
1144
+ });
1145
+ }
1146
+ return extracted.scopes;
1147
+ }
1148
+
1028
1149
  /**
1029
1150
  * SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
1030
- * materialized) and the payload. Bind with `rewriteValues`.
1151
+ * materialized), the stored scopes when the table re-keys (§3.1), and the
1152
+ * payload. Bind with `rewriteValues`.
1031
1153
  */
1032
- export function rewriteRowSql(table: CompiledTable): string {
1154
+ export function rewriteRowSql(table: CompiledTable, rekey: boolean): string {
1033
1155
  const sets = [
1034
1156
  ...(table.materialize
1035
1157
  ? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
1036
1158
  : []),
1159
+ ...(rekey ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=?`] : []),
1037
1160
  `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
1038
1161
  ];
1039
1162
  return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
1040
1163
  }
1041
1164
 
1042
- /** Bind values for `rewriteRowSql` from the (already migrated) payload. */
1165
+ /**
1166
+ * Bind values for `rewriteRowSql` from the (already migrated) payload:
1167
+ * projection columns, the re-keyed scopes JSON, payload, partition, row id.
1168
+ */
1043
1169
  export function rewriteValues(
1044
1170
  table: CompiledTable,
1045
1171
  partition: string,
1046
1172
  rowId: string,
1047
1173
  payload: Uint8Array,
1048
1174
  dialect: RelationalDialect,
1175
+ scopes: Record<string, string> | undefined,
1049
1176
  ): RowValue[] {
1050
1177
  const out: RowValue[] = [];
1051
1178
  if (table.materialize) {
@@ -1056,6 +1183,7 @@ export function rewriteValues(
1056
1183
  out.push(toSqlValue(column, values[i] ?? null, dialect));
1057
1184
  }
1058
1185
  }
1186
+ if (scopes !== undefined) out.push(JSON.stringify(scopes));
1059
1187
  out.push(payload, partition, rowId);
1060
1188
  return out;
1061
1189
  }
@@ -9,6 +9,7 @@
9
9
  * select candidates from the index and verify the full multi-variable
10
10
  * match against the stored scope map — never a log scan.
11
11
  */
12
+ import { decodeRow } from '@syncular/core';
12
13
  import {
13
14
  isSqliteConstraintError,
14
15
  retainRollbackFailure,
@@ -21,10 +22,15 @@ import { serveNotReadyError } from './readiness';
21
22
  import {
22
23
  assertPhysicalColumns,
23
24
  assertStoredLayouts,
25
+ assertStoredScopePatterns,
26
+ changedScopePatterns,
24
27
  dropTableDdl,
25
28
  layoutsOf,
26
29
  migratePayload,
27
30
  parseLayouts,
31
+ rekeyCleanupSql,
32
+ rekeyedScopes,
33
+ rekeyRowScopesSql,
28
34
  retiredTableNames,
29
35
  rewritePlan,
30
36
  rewriteRowSql,
@@ -507,6 +513,15 @@ END`);
507
513
  }
508
514
  if (marker !== null && marker.schema_version === schema.version) {
509
515
  assertStoredLayouts(schema, marker.layouts);
516
+ if (!assertStoredScopePatterns(schema, marker.layouts)) {
517
+ // A marker from before 0.63 records no scope patterns: record the
518
+ // configured ones, which every later start compares against.
519
+ this.db
520
+ .query(
521
+ 'UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?',
522
+ )
523
+ .run(layoutsOf(schema), schema.version);
524
+ }
510
525
  for (const table of schema.tables.values()) {
511
526
  const escapedTableName = table.name.replaceAll('"', '""');
512
527
  const columns = this.db
@@ -618,6 +633,9 @@ END`);
618
633
  this.schemaVersion = schema.version;
619
634
  return;
620
635
  }
636
+ const rekeyed = (
637
+ changedScopePatterns(schema, locked?.layouts) ?? []
638
+ ).map((change) => change.table);
621
639
  const layouts = parseLayouts(locked?.layouts);
622
640
  const retiredTables = retiredTableNames(schema, layouts);
623
641
  for (const tableName of retiredTables) {
@@ -637,11 +655,15 @@ END`);
637
655
  )) {
638
656
  this.db.exec(statement);
639
657
  }
658
+ for (const statement of rekeyCleanupSql(rekeyed)) {
659
+ this.db.exec(statement);
660
+ }
640
661
  for (const table of schema.tables.values()) {
641
662
  const oldLayout = layouts[table.name];
642
663
  const plan = rewritePlan(table, oldLayout, existing.get(table.name));
643
- if (!plan.migrate && !plan.backfill) continue;
644
- this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
664
+ const rekey = rekeyed.includes(table.name);
665
+ if (!plan.migrate && !plan.backfill && !rekey) continue;
666
+ this.#rewriteRows(table, plan.migrate ? oldLayout : undefined, rekey);
645
667
  }
646
668
  this.db
647
669
  .query(
@@ -688,14 +710,17 @@ END`);
688
710
  * Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
689
711
  * given every payload re-encodes under
690
712
  * the current columns, and the projection (when materialized) refreshes
691
- * from the payload either way.
713
+ * from the payload either way. `rekey` recomputes and indexes the stored
714
+ * scopes (§3.1).
692
715
  */
693
716
  #rewriteRows(
694
717
  table: CompiledTable,
695
718
  oldLayout: readonly StoredColumnLayout[] | undefined,
719
+ rekey: boolean,
696
720
  ): void {
697
721
  const select = selectRowsForRewriteSql(table, 'sqlite');
698
- const update = this.db.query(rewriteRowSql(table));
722
+ const update = this.db.query(rewriteRowSql(table, rekey));
723
+ const index = this.db.query(rekeyRowScopesSql(table, 'sqlite'));
699
724
  const BATCH = 500;
700
725
  let afterPartition = '';
701
726
  let afterRowId = '';
@@ -707,15 +732,30 @@ END`);
707
732
  >(select)
708
733
  .all(afterPartition, afterRowId, BATCH);
709
734
  if (rows.length === 0) break;
735
+ const entries: string[][] = [];
710
736
  for (const row of rows) {
711
737
  const payload =
712
738
  oldLayout !== undefined
713
739
  ? migratePayload(oldLayout, table, row.payload)
714
740
  : row.payload;
741
+ const scopes = rekey
742
+ ? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
743
+ : undefined;
744
+ for (const [variable, value] of Object.entries(scopes ?? {})) {
745
+ entries.push([row.partition, variable, value, row.row_id]);
746
+ }
715
747
  update.run(
716
- ...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'),
748
+ ...rewriteValues(
749
+ table,
750
+ row.partition,
751
+ row.row_id,
752
+ payload,
753
+ 'sqlite',
754
+ scopes,
755
+ ),
717
756
  );
718
757
  }
758
+ if (entries.length > 0) index.run(JSON.stringify(entries));
719
759
  const last = rows[rows.length - 1];
720
760
  if (last === undefined || rows.length < BATCH) break;
721
761
  afterPartition = last.partition;
@@ -32,6 +32,8 @@ export type StorageQueryErrorCode =
32
32
  | 'sync.storage.checkpoint_fence_missing'
33
33
  | 'sync.storage.checkpoint_incomplete'
34
34
  | 'sync.storage.stored_layout_mismatch'
35
+ | 'sync.storage.scope_patterns_changed'
36
+ | 'sync.storage.scope_value_missing'
35
37
  | 'sync.storage.physical_layout_mismatch'
36
38
  | 'sync.storage.transaction_query_unsupported'
37
39
  | 'sync.storage.partition_lock_unsupported'
@@ -65,6 +67,10 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
65
67
  'a declared backfill checkpoint is not activated and this caller did not declare it',
66
68
  'sync.storage.stored_layout_mismatch':
67
69
  'stored schema layouts disagree with the configured schema at the same version',
70
+ 'sync.storage.scope_patterns_changed':
71
+ 'a table declares scope patterns other than the ones its stored rows are indexed under',
72
+ 'sync.storage.scope_value_missing':
73
+ 'a stored row has no value for a scope column the schema bump re-keys it under',
68
74
  'sync.storage.physical_layout_mismatch':
69
75
  'a synced table does not match the storage layout the running code reads and writes',
70
76
  'sync.storage.transaction_query_unsupported':