@syncular/server 0.62.1 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +72 -7
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +83 -16
- package/dist/relational-rows.d.ts +42 -4
- package/dist/relational-rows.js +106 -5
- package/dist/sqlite-storage.js +31 -7
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +2 -0
- package/package.json +2 -2
- package/src/d1-storage.ts +108 -6
- package/src/postgres-storage.ts +66 -17
- package/src/relational-rows.ts +133 -5
- package/src/sqlite-storage.ts +45 -5
- package/src/storage-errors.ts +6 -0
package/src/relational-rows.ts
CHANGED
|
@@ -60,6 +60,7 @@ import type {
|
|
|
60
60
|
ServerIndexSchema,
|
|
61
61
|
} from './schema';
|
|
62
62
|
import type { StoredRow } from './storage';
|
|
63
|
+
import { storedScopesForRow } from './scopes';
|
|
63
64
|
import { StorageQueryError } from './storage-errors';
|
|
64
65
|
|
|
65
66
|
export type RelationalDialect = 'sqlite' | 'postgres';
|
|
@@ -872,20 +873,120 @@ export type StoredLayouts = Record<string, readonly StoredColumnLayout[]>;
|
|
|
872
873
|
|
|
873
874
|
/** The layouts JSON persisted alongside the schema version marker. */
|
|
874
875
|
export function layoutsOf(schema: CompiledSchema): string {
|
|
875
|
-
const out: Record<string,
|
|
876
|
+
const out: Record<string, unknown> = {};
|
|
877
|
+
const scopes: Record<string, string[]> = {};
|
|
876
878
|
for (const table of schema.tables.values()) {
|
|
877
879
|
out[table.name] = table.columns.map((column) => ({
|
|
878
880
|
name: column.name,
|
|
879
881
|
type: column.type,
|
|
880
882
|
nullable: column.nullable,
|
|
881
883
|
}));
|
|
884
|
+
scopes[table.name] = scopePatternsOf(table);
|
|
882
885
|
}
|
|
886
|
+
// `sync_` is a reserved table-name prefix, so this key names no table.
|
|
887
|
+
out.sync_scopes = scopes;
|
|
883
888
|
return JSON.stringify(out);
|
|
884
889
|
}
|
|
885
890
|
|
|
891
|
+
function scopePatternsOf(table: CompiledTable): string[] {
|
|
892
|
+
return table.scopePatterns
|
|
893
|
+
.map(({ prefix, variable, column }) => `${prefix}:{${variable}}@${column}`)
|
|
894
|
+
.sort();
|
|
895
|
+
}
|
|
896
|
+
|
|
886
897
|
export function parseLayouts(json: string | null | undefined): StoredLayouts {
|
|
887
898
|
if (json === null || json === undefined || json.length === 0) return {};
|
|
888
|
-
|
|
899
|
+
const { sync_scopes: _scopes, ...layouts } = JSON.parse(json) as Record<
|
|
900
|
+
string,
|
|
901
|
+
unknown
|
|
902
|
+
>;
|
|
903
|
+
return layouts as StoredLayouts;
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/**
|
|
907
|
+
* §3.1: the configured tables whose scope patterns differ from the recorded
|
|
908
|
+
* ones, or `undefined` when the marker predates recorded patterns (nothing to
|
|
909
|
+
* compare). A schema-version bump re-keys these tables.
|
|
910
|
+
*/
|
|
911
|
+
export function changedScopePatterns(
|
|
912
|
+
schema: CompiledSchema,
|
|
913
|
+
storedJson: string | null | undefined,
|
|
914
|
+
): { table: string; stored: string; configured: string }[] | undefined {
|
|
915
|
+
if (storedJson === null || storedJson === undefined || storedJson === '')
|
|
916
|
+
return [];
|
|
917
|
+
const stored = (
|
|
918
|
+
JSON.parse(storedJson) as {
|
|
919
|
+
sync_scopes?: Record<string, readonly string[]>;
|
|
920
|
+
}
|
|
921
|
+
).sync_scopes;
|
|
922
|
+
if (stored === undefined) return undefined;
|
|
923
|
+
return [...schema.tables.values()].flatMap((table) => {
|
|
924
|
+
const before = stored[table.name];
|
|
925
|
+
const after = scopePatternsOf(table);
|
|
926
|
+
return before !== undefined && before.join('\n') !== after.join('\n')
|
|
927
|
+
? [
|
|
928
|
+
{
|
|
929
|
+
table: table.name,
|
|
930
|
+
stored: before.join(' '),
|
|
931
|
+
configured: after.join(' '),
|
|
932
|
+
},
|
|
933
|
+
]
|
|
934
|
+
: [];
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/**
|
|
939
|
+
* §3.1: at the same schema version, refuse a configured table whose patterns
|
|
940
|
+
* differ from the recorded ones. Returns false when the marker predates
|
|
941
|
+
* recorded patterns, so the caller records them.
|
|
942
|
+
*/
|
|
943
|
+
export function assertStoredScopePatterns(
|
|
944
|
+
schema: CompiledSchema,
|
|
945
|
+
storedJson: string | null | undefined,
|
|
946
|
+
): boolean {
|
|
947
|
+
const changed = changedScopePatterns(schema, storedJson);
|
|
948
|
+
if (changed?.[0] !== undefined) {
|
|
949
|
+
throw new StorageQueryError(
|
|
950
|
+
'sync.storage.scope_patterns_changed',
|
|
951
|
+
changed[0],
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
return changed !== undefined;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
/**
|
|
958
|
+
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
959
|
+
* entries and tombstones, and raise every partition's horizon past the
|
|
960
|
+
* changes logged under the previous patterns. Empty when nothing re-keys.
|
|
961
|
+
*/
|
|
962
|
+
export function rekeyCleanupSql(tables: readonly string[]): string[] {
|
|
963
|
+
if (tables.length === 0) return [];
|
|
964
|
+
const names = tables.map((name) => `'${name.replaceAll("'", "''")}'`);
|
|
965
|
+
return [
|
|
966
|
+
`DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
|
|
967
|
+
`DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
|
|
968
|
+
'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
|
|
969
|
+
];
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* §3.1 re-key: index the scopes of one rewritten page of rows. The entries
|
|
974
|
+
* bind as one JSON array of `[partition, variable, value, rowId]`, so the
|
|
975
|
+
* statement reads no row table (a planner that ranges a page over a declared
|
|
976
|
+
* index would scan the whole partition per page). Params: [entries].
|
|
977
|
+
*/
|
|
978
|
+
export function rekeyRowScopesSql(
|
|
979
|
+
table: CompiledTable,
|
|
980
|
+
dialect: RelationalDialect,
|
|
981
|
+
): string {
|
|
982
|
+
const name = `'${table.name.replaceAll("'", "''")}'`;
|
|
983
|
+
return dialect === 'sqlite'
|
|
984
|
+
? `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
985
|
+
SELECT json_extract(e.value, '$[0]'), ${name}, json_extract(e.value, '$[1]'), json_extract(e.value, '$[2]'), json_extract(e.value, '$[3]')
|
|
986
|
+
FROM json_each(?) AS e`
|
|
987
|
+
: `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
988
|
+
SELECT e->>0, ${name}, e->>1, e->>2, e->>3
|
|
989
|
+
FROM jsonb_array_elements($1::text::jsonb) AS e`;
|
|
889
990
|
}
|
|
890
991
|
|
|
891
992
|
/**
|
|
@@ -1025,27 +1126,53 @@ export function selectRowsForRewriteSql(
|
|
|
1025
1126
|
LIMIT ${p[2]}`;
|
|
1026
1127
|
}
|
|
1027
1128
|
|
|
1129
|
+
/**
|
|
1130
|
+
* §3.1 re-key: the row's stored scopes under the table's current patterns. A
|
|
1131
|
+
* row without a value for one of them refuses the bump.
|
|
1132
|
+
*/
|
|
1133
|
+
export function rekeyedScopes(
|
|
1134
|
+
table: CompiledTable,
|
|
1135
|
+
values: readonly RowValue[],
|
|
1136
|
+
rowId: string,
|
|
1137
|
+
): Record<string, string> {
|
|
1138
|
+
const extracted = storedScopesForRow(table, values);
|
|
1139
|
+
if ('missing' in extracted) {
|
|
1140
|
+
throw new StorageQueryError('sync.storage.scope_value_missing', {
|
|
1141
|
+
table: table.name,
|
|
1142
|
+
variable: extracted.missing,
|
|
1143
|
+
rowId,
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
return extracted.scopes;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1028
1149
|
/**
|
|
1029
1150
|
* SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
|
|
1030
|
-
* materialized)
|
|
1151
|
+
* materialized), the stored scopes when the table re-keys (§3.1), and the
|
|
1152
|
+
* payload. Bind with `rewriteValues`.
|
|
1031
1153
|
*/
|
|
1032
|
-
export function rewriteRowSql(table: CompiledTable): string {
|
|
1154
|
+
export function rewriteRowSql(table: CompiledTable, rekey: boolean): string {
|
|
1033
1155
|
const sets = [
|
|
1034
1156
|
...(table.materialize
|
|
1035
1157
|
? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
|
|
1036
1158
|
: []),
|
|
1159
|
+
...(rekey ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=?`] : []),
|
|
1037
1160
|
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
|
|
1038
1161
|
];
|
|
1039
1162
|
return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
|
|
1040
1163
|
}
|
|
1041
1164
|
|
|
1042
|
-
/**
|
|
1165
|
+
/**
|
|
1166
|
+
* Bind values for `rewriteRowSql` from the (already migrated) payload:
|
|
1167
|
+
* projection columns, the re-keyed scopes JSON, payload, partition, row id.
|
|
1168
|
+
*/
|
|
1043
1169
|
export function rewriteValues(
|
|
1044
1170
|
table: CompiledTable,
|
|
1045
1171
|
partition: string,
|
|
1046
1172
|
rowId: string,
|
|
1047
1173
|
payload: Uint8Array,
|
|
1048
1174
|
dialect: RelationalDialect,
|
|
1175
|
+
scopes: Record<string, string> | undefined,
|
|
1049
1176
|
): RowValue[] {
|
|
1050
1177
|
const out: RowValue[] = [];
|
|
1051
1178
|
if (table.materialize) {
|
|
@@ -1056,6 +1183,7 @@ export function rewriteValues(
|
|
|
1056
1183
|
out.push(toSqlValue(column, values[i] ?? null, dialect));
|
|
1057
1184
|
}
|
|
1058
1185
|
}
|
|
1186
|
+
if (scopes !== undefined) out.push(JSON.stringify(scopes));
|
|
1059
1187
|
out.push(payload, partition, rowId);
|
|
1060
1188
|
return out;
|
|
1061
1189
|
}
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* select candidates from the index and verify the full multi-variable
|
|
10
10
|
* match against the stored scope map — never a log scan.
|
|
11
11
|
*/
|
|
12
|
+
import { decodeRow } from '@syncular/core';
|
|
12
13
|
import {
|
|
13
14
|
isSqliteConstraintError,
|
|
14
15
|
retainRollbackFailure,
|
|
@@ -21,10 +22,15 @@ import { serveNotReadyError } from './readiness';
|
|
|
21
22
|
import {
|
|
22
23
|
assertPhysicalColumns,
|
|
23
24
|
assertStoredLayouts,
|
|
25
|
+
assertStoredScopePatterns,
|
|
26
|
+
changedScopePatterns,
|
|
24
27
|
dropTableDdl,
|
|
25
28
|
layoutsOf,
|
|
26
29
|
migratePayload,
|
|
27
30
|
parseLayouts,
|
|
31
|
+
rekeyCleanupSql,
|
|
32
|
+
rekeyedScopes,
|
|
33
|
+
rekeyRowScopesSql,
|
|
28
34
|
retiredTableNames,
|
|
29
35
|
rewritePlan,
|
|
30
36
|
rewriteRowSql,
|
|
@@ -507,6 +513,15 @@ END`);
|
|
|
507
513
|
}
|
|
508
514
|
if (marker !== null && marker.schema_version === schema.version) {
|
|
509
515
|
assertStoredLayouts(schema, marker.layouts);
|
|
516
|
+
if (!assertStoredScopePatterns(schema, marker.layouts)) {
|
|
517
|
+
// A marker from before 0.63 records no scope patterns: record the
|
|
518
|
+
// configured ones, which every later start compares against.
|
|
519
|
+
this.db
|
|
520
|
+
.query(
|
|
521
|
+
'UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?',
|
|
522
|
+
)
|
|
523
|
+
.run(layoutsOf(schema), schema.version);
|
|
524
|
+
}
|
|
510
525
|
for (const table of schema.tables.values()) {
|
|
511
526
|
const escapedTableName = table.name.replaceAll('"', '""');
|
|
512
527
|
const columns = this.db
|
|
@@ -618,6 +633,9 @@ END`);
|
|
|
618
633
|
this.schemaVersion = schema.version;
|
|
619
634
|
return;
|
|
620
635
|
}
|
|
636
|
+
const rekeyed = (
|
|
637
|
+
changedScopePatterns(schema, locked?.layouts) ?? []
|
|
638
|
+
).map((change) => change.table);
|
|
621
639
|
const layouts = parseLayouts(locked?.layouts);
|
|
622
640
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
623
641
|
for (const tableName of retiredTables) {
|
|
@@ -637,11 +655,15 @@ END`);
|
|
|
637
655
|
)) {
|
|
638
656
|
this.db.exec(statement);
|
|
639
657
|
}
|
|
658
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
659
|
+
this.db.exec(statement);
|
|
660
|
+
}
|
|
640
661
|
for (const table of schema.tables.values()) {
|
|
641
662
|
const oldLayout = layouts[table.name];
|
|
642
663
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
643
|
-
|
|
644
|
-
|
|
664
|
+
const rekey = rekeyed.includes(table.name);
|
|
665
|
+
if (!plan.migrate && !plan.backfill && !rekey) continue;
|
|
666
|
+
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined, rekey);
|
|
645
667
|
}
|
|
646
668
|
this.db
|
|
647
669
|
.query(
|
|
@@ -688,14 +710,17 @@ END`);
|
|
|
688
710
|
* Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
|
|
689
711
|
* given every payload re-encodes under
|
|
690
712
|
* the current columns, and the projection (when materialized) refreshes
|
|
691
|
-
* from the payload either way.
|
|
713
|
+
* from the payload either way. `rekey` recomputes and indexes the stored
|
|
714
|
+
* scopes (§3.1).
|
|
692
715
|
*/
|
|
693
716
|
#rewriteRows(
|
|
694
717
|
table: CompiledTable,
|
|
695
718
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
719
|
+
rekey: boolean,
|
|
696
720
|
): void {
|
|
697
721
|
const select = selectRowsForRewriteSql(table, 'sqlite');
|
|
698
|
-
const update = this.db.query(rewriteRowSql(table));
|
|
722
|
+
const update = this.db.query(rewriteRowSql(table, rekey));
|
|
723
|
+
const index = this.db.query(rekeyRowScopesSql(table, 'sqlite'));
|
|
699
724
|
const BATCH = 500;
|
|
700
725
|
let afterPartition = '';
|
|
701
726
|
let afterRowId = '';
|
|
@@ -707,15 +732,30 @@ END`);
|
|
|
707
732
|
>(select)
|
|
708
733
|
.all(afterPartition, afterRowId, BATCH);
|
|
709
734
|
if (rows.length === 0) break;
|
|
735
|
+
const entries: string[][] = [];
|
|
710
736
|
for (const row of rows) {
|
|
711
737
|
const payload =
|
|
712
738
|
oldLayout !== undefined
|
|
713
739
|
? migratePayload(oldLayout, table, row.payload)
|
|
714
740
|
: row.payload;
|
|
741
|
+
const scopes = rekey
|
|
742
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
743
|
+
: undefined;
|
|
744
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
745
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
746
|
+
}
|
|
715
747
|
update.run(
|
|
716
|
-
...rewriteValues(
|
|
748
|
+
...rewriteValues(
|
|
749
|
+
table,
|
|
750
|
+
row.partition,
|
|
751
|
+
row.row_id,
|
|
752
|
+
payload,
|
|
753
|
+
'sqlite',
|
|
754
|
+
scopes,
|
|
755
|
+
),
|
|
717
756
|
);
|
|
718
757
|
}
|
|
758
|
+
if (entries.length > 0) index.run(JSON.stringify(entries));
|
|
719
759
|
const last = rows[rows.length - 1];
|
|
720
760
|
if (last === undefined || rows.length < BATCH) break;
|
|
721
761
|
afterPartition = last.partition;
|
package/src/storage-errors.ts
CHANGED
|
@@ -32,6 +32,8 @@ export type StorageQueryErrorCode =
|
|
|
32
32
|
| 'sync.storage.checkpoint_fence_missing'
|
|
33
33
|
| 'sync.storage.checkpoint_incomplete'
|
|
34
34
|
| 'sync.storage.stored_layout_mismatch'
|
|
35
|
+
| 'sync.storage.scope_patterns_changed'
|
|
36
|
+
| 'sync.storage.scope_value_missing'
|
|
35
37
|
| 'sync.storage.physical_layout_mismatch'
|
|
36
38
|
| 'sync.storage.transaction_query_unsupported'
|
|
37
39
|
| 'sync.storage.partition_lock_unsupported'
|
|
@@ -65,6 +67,10 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
|
|
|
65
67
|
'a declared backfill checkpoint is not activated and this caller did not declare it',
|
|
66
68
|
'sync.storage.stored_layout_mismatch':
|
|
67
69
|
'stored schema layouts disagree with the configured schema at the same version',
|
|
70
|
+
'sync.storage.scope_patterns_changed':
|
|
71
|
+
'a table declares scope patterns other than the ones its stored rows are indexed under',
|
|
72
|
+
'sync.storage.scope_value_missing':
|
|
73
|
+
'a stored row has no value for a scope column the schema bump re-keys it under',
|
|
68
74
|
'sync.storage.physical_layout_mismatch':
|
|
69
75
|
'a synced table does not match the storage layout the running code reads and writes',
|
|
70
76
|
'sync.storage.transaction_query_unsupported':
|