@syncular/server 0.15.48 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/dist/admin.js +1 -5
- package/dist/authoritative-query.d.ts +14 -6
- package/dist/authoritative-query.js +60 -82
- package/dist/d1-storage.d.ts +4 -1
- package/dist/d1-storage.js +59 -15
- package/dist/operations.d.ts +2 -0
- package/dist/operations.js +23 -4
- package/dist/postgres-storage.d.ts +4 -1
- package/dist/postgres-storage.js +39 -10
- package/dist/prune.d.ts +3 -1
- package/dist/prune.js +18 -14
- package/dist/pull.js +63 -32
- package/dist/push.js +5 -5
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +3 -3
- package/dist/sqlite-image.js +10 -6
- package/dist/sqlite-node.js +2 -2
- package/dist/sqlite-storage.d.ts +4 -1
- package/dist/sqlite-storage.js +67 -26
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +3 -0
- package/dist/storage.d.ts +20 -5
- package/package.json +2 -2
- package/src/admin.ts +4 -6
- package/src/authoritative-query.ts +89 -94
- package/src/d1-storage.ts +86 -17
- package/src/operations.ts +31 -3
- package/src/postgres-storage.ts +82 -25
- package/src/prune.ts +29 -15
- package/src/pull.ts +80 -42
- package/src/push.ts +5 -5
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +26 -15
- package/src/sqlite-node.ts +2 -2
- package/src/sqlite-storage.ts +86 -28
- package/src/storage-errors.ts +10 -1
- package/src/storage.ts +29 -5
|
@@ -14,28 +14,16 @@ export interface BoundAuthoritativeQuery {
|
|
|
14
14
|
|
|
15
15
|
const PARTITION_BIND = Symbol('syncular.authoritative_partition');
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
'natural',
|
|
28
|
-
'join',
|
|
29
|
-
'cross',
|
|
30
|
-
'using',
|
|
31
|
-
'limit',
|
|
32
|
-
'having',
|
|
33
|
-
]);
|
|
34
|
-
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
|
|
35
|
-
const TABLE_REF_RE = new RegExp(
|
|
36
|
-
`\\b(FROM|(?:NATURAL\\s+)?(?:(?:LEFT|RIGHT|FULL)(?:\\s+OUTER)?|INNER|CROSS)?\\s*JOIN)\\s+((?:\\(\\s*)*)(${IDENT})(?:\\s+(?:AS\\s+)?((?!(?:${[...RESERVED_ALIAS].join('|')})\\b)${IDENT}))?`,
|
|
37
|
-
'gi',
|
|
38
|
-
);
|
|
17
|
+
/** Compiler-proven occurrences in the exact generated positional statement. */
|
|
18
|
+
export interface AuthoritativeRelationPlan {
|
|
19
|
+
readonly sql: string;
|
|
20
|
+
readonly relations: readonly {
|
|
21
|
+
readonly table: string;
|
|
22
|
+
readonly start: number;
|
|
23
|
+
readonly end: number;
|
|
24
|
+
readonly alias?: string;
|
|
25
|
+
}[];
|
|
26
|
+
}
|
|
39
27
|
|
|
40
28
|
function protectedSqlEnd(sql: string, index: number): number | undefined {
|
|
41
29
|
const char = sql[index];
|
|
@@ -64,66 +52,57 @@ function protectedSqlEnd(sql: string, index: number): number | undefined {
|
|
|
64
52
|
return undefined;
|
|
65
53
|
}
|
|
66
54
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
while (index < sql.length) {
|
|
71
|
-
const end = protectedSqlEnd(sql, index);
|
|
72
|
-
if (end === undefined) out += sql[index];
|
|
73
|
-
else out += sql.slice(index, end).replace(/[^\n]/g, ' ');
|
|
74
|
-
index = end ?? index + 1;
|
|
75
|
-
}
|
|
76
|
-
return out;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
/**
|
|
80
|
-
* Turn generated local SQL into a partition-local authoritative statement.
|
|
81
|
-
* Only relations declared by the generated descriptor are rewritten. Values
|
|
82
|
-
* remain parameters; request data is never interpolated into SQL.
|
|
83
|
-
*/
|
|
84
|
-
export function prepareAuthoritativeQuery(
|
|
85
|
-
sql: string,
|
|
86
|
-
params: readonly AuthoritativeQueryValue[],
|
|
55
|
+
/** Validate trusted generated metadata before registration or storage execution. */
|
|
56
|
+
export function validateAuthoritativeRelationPlan(
|
|
57
|
+
plan: AuthoritativeRelationPlan,
|
|
87
58
|
declaredTables: readonly string[],
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
59
|
+
): void {
|
|
60
|
+
if (
|
|
61
|
+
plan === undefined ||
|
|
62
|
+
typeof plan.sql !== 'string' ||
|
|
63
|
+
!Array.isArray(plan.relations)
|
|
64
|
+
) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
'registered query requires generated relation plans; regenerate queries',
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const sql = plan.sql;
|
|
92
70
|
const declared = new Set(declaredTables);
|
|
93
|
-
const masked = maskedSql(sql);
|
|
94
|
-
const replacements: Array<{
|
|
95
|
-
readonly start: number;
|
|
96
|
-
readonly end: number;
|
|
97
|
-
readonly text: string;
|
|
98
|
-
}> = [];
|
|
99
71
|
const found = new Set<string>();
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
72
|
+
let previousEnd = 0;
|
|
73
|
+
for (const relation of plan.relations) {
|
|
74
|
+
if (
|
|
75
|
+
!Number.isSafeInteger(relation.start) ||
|
|
76
|
+
!Number.isSafeInteger(relation.end) ||
|
|
77
|
+
relation.start < previousEnd ||
|
|
78
|
+
relation.end <= relation.start ||
|
|
79
|
+
relation.end > sql.length
|
|
80
|
+
) {
|
|
81
|
+
throw new Error(
|
|
82
|
+
'registered query relation boundaries do not match its SQL; regenerate queries',
|
|
83
|
+
);
|
|
106
84
|
}
|
|
107
|
-
|
|
108
|
-
|
|
85
|
+
previousEnd = relation.end;
|
|
86
|
+
if (!declared.has(relation.table)) {
|
|
87
|
+
throw new Error('registered query table metadata does not match its SQL');
|
|
109
88
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
89
|
+
const spelling = sql.slice(relation.start, relation.end);
|
|
90
|
+
const quote = spelling[0];
|
|
91
|
+
const name =
|
|
92
|
+
quote === '['
|
|
93
|
+
? spelling.slice(1, -1)
|
|
94
|
+
: quote === '"' || quote === '`'
|
|
95
|
+
? spelling
|
|
96
|
+
.slice(1, -1)
|
|
97
|
+
.split(quote + quote)
|
|
98
|
+
.join(quote)
|
|
99
|
+
: spelling;
|
|
100
|
+
if (name.toLowerCase() !== relation.table.toLowerCase()) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
'registered query relation name does not match its SQL; regenerate queries',
|
|
103
|
+
);
|
|
113
104
|
}
|
|
114
|
-
|
|
115
|
-
const afterOperator = (match[1] as string).length;
|
|
116
|
-
const relative = match[0]
|
|
117
|
-
.toLowerCase()
|
|
118
|
-
.indexOf(rawTable.toLowerCase(), afterOperator);
|
|
119
|
-
const start = matchStart + relative;
|
|
120
|
-
const projection = table.columns.map((column) => quoteIdent(column.name));
|
|
121
|
-
replacements.push({
|
|
122
|
-
start,
|
|
123
|
-
end: start + rawTable.length,
|
|
124
|
-
text: `(SELECT ${projection.join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=/*syncular_partition*/?)${alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
|
|
125
|
-
});
|
|
126
|
-
found.add(table.name);
|
|
105
|
+
found.add(relation.table);
|
|
127
106
|
}
|
|
128
107
|
if (
|
|
129
108
|
found.size !== declared.size ||
|
|
@@ -131,42 +110,58 @@ export function prepareAuthoritativeQuery(
|
|
|
131
110
|
) {
|
|
132
111
|
throw new Error('registered query table metadata does not match its SQL');
|
|
133
112
|
}
|
|
134
|
-
|
|
135
|
-
for (const replacement of replacements.sort(
|
|
136
|
-
(left, right) => right.start - left.start,
|
|
137
|
-
)) {
|
|
138
|
-
rewritten =
|
|
139
|
-
rewritten.slice(0, replacement.start) +
|
|
140
|
-
replacement.text +
|
|
141
|
-
rewritten.slice(replacement.end);
|
|
142
|
-
}
|
|
113
|
+
}
|
|
143
114
|
|
|
115
|
+
/** Bind every compiler-proven table occurrence to the authenticated partition. */
|
|
116
|
+
export function prepareAuthoritativeQuery(
|
|
117
|
+
plan: AuthoritativeRelationPlan,
|
|
118
|
+
params: readonly AuthoritativeQueryValue[],
|
|
119
|
+
declaredTables: readonly string[],
|
|
120
|
+
tables: ReadonlyMap<string, CompiledTable>,
|
|
121
|
+
): PreparedAuthoritativeQuery {
|
|
122
|
+
validateAuthoritativeRelationPlan(plan, declaredTables);
|
|
123
|
+
const sql = plan.sql;
|
|
124
|
+
const replacements = plan.relations.map((relation) => {
|
|
125
|
+
const table = tables.get(relation.table);
|
|
126
|
+
if (table === undefined)
|
|
127
|
+
throw new Error('registered query targets an unknown table');
|
|
128
|
+
if (!table.materialize)
|
|
129
|
+
throw new Error('registered query targets a non-materialized table');
|
|
130
|
+
return {
|
|
131
|
+
start: relation.start,
|
|
132
|
+
end: relation.end,
|
|
133
|
+
text: `(SELECT ${table.columns.map((column) => quoteIdent(column.name)).join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?)${relation.alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
|
|
134
|
+
};
|
|
135
|
+
});
|
|
144
136
|
const bound: (AuthoritativeQueryValue | typeof PARTITION_BIND)[] = [];
|
|
145
137
|
let anonymousIndex = 0;
|
|
146
138
|
let rendered = '';
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
139
|
+
let nextRelation = 0;
|
|
140
|
+
for (let index = 0; index < sql.length; index += 1) {
|
|
141
|
+
const replacement = replacements[nextRelation];
|
|
142
|
+
if (replacement?.start === index) {
|
|
143
|
+
rendered += replacement.text;
|
|
150
144
|
bound.push(PARTITION_BIND);
|
|
151
|
-
index
|
|
145
|
+
index = replacement.end - 1;
|
|
146
|
+
nextRelation += 1;
|
|
152
147
|
continue;
|
|
153
148
|
}
|
|
154
|
-
const protectedEnd = protectedSqlEnd(
|
|
149
|
+
const protectedEnd = protectedSqlEnd(sql, index);
|
|
155
150
|
if (protectedEnd !== undefined) {
|
|
156
|
-
rendered +=
|
|
151
|
+
rendered += sql.slice(index, protectedEnd);
|
|
157
152
|
index = protectedEnd - 1;
|
|
158
153
|
continue;
|
|
159
154
|
}
|
|
160
|
-
const char =
|
|
155
|
+
const char = sql[index] as string;
|
|
161
156
|
if (char !== '?') {
|
|
162
157
|
rendered += char;
|
|
163
158
|
continue;
|
|
164
159
|
}
|
|
165
160
|
let end = index + 1;
|
|
166
|
-
while (end <
|
|
161
|
+
while (end < sql.length && /[0-9]/.test(sql[end] as string)) {
|
|
167
162
|
end += 1;
|
|
168
163
|
}
|
|
169
|
-
const numbered =
|
|
164
|
+
const numbered = sql.slice(index + 1, end);
|
|
170
165
|
const parameterIndex =
|
|
171
166
|
numbered.length > 0
|
|
172
167
|
? Number.parseInt(numbered, 10) - 1
|
package/src/d1-storage.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune';
|
|
2
|
+
import { StorageQueryError } from './storage-errors';
|
|
3
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
1
4
|
/**
|
|
2
5
|
* Cloudflare D1 server storage for Workers deployments.
|
|
3
6
|
*
|
|
@@ -1027,7 +1030,7 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1027
1030
|
}
|
|
1028
1031
|
const prepared = bindAuthoritativePartition(
|
|
1029
1032
|
prepareAuthoritativeQuery(
|
|
1030
|
-
query.
|
|
1033
|
+
query.plan,
|
|
1031
1034
|
query.params,
|
|
1032
1035
|
query.tables,
|
|
1033
1036
|
this.#tables,
|
|
@@ -1071,6 +1074,16 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1071
1074
|
};
|
|
1072
1075
|
}
|
|
1073
1076
|
|
|
1077
|
+
async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
|
|
1078
|
+
const row = await this.#db
|
|
1079
|
+
.prepare(
|
|
1080
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
|
|
1081
|
+
)
|
|
1082
|
+
.bind(partition)
|
|
1083
|
+
.first<{ log_epoch: string }>();
|
|
1084
|
+
return row?.log_epoch;
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1074
1087
|
async getHorizonSeq(partition: string): Promise<number> {
|
|
1075
1088
|
const row = await this.#db
|
|
1076
1089
|
.prepare('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
|
|
@@ -1082,33 +1095,76 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1082
1095
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
1083
1096
|
await this.#db
|
|
1084
1097
|
.prepare(
|
|
1085
|
-
'INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=excluded.horizon_seq',
|
|
1098
|
+
'INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)',
|
|
1086
1099
|
)
|
|
1087
1100
|
.bind(partition, seq)
|
|
1088
1101
|
.run();
|
|
1089
1102
|
}
|
|
1090
1103
|
|
|
1091
|
-
async pruneCommitsThrough(
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1104
|
+
async pruneCommitsThrough(
|
|
1105
|
+
partition: string,
|
|
1106
|
+
query: CommitPruneQuery,
|
|
1107
|
+
): Promise<CommitPruneResult> {
|
|
1108
|
+
validateCommitPruneQuery(query);
|
|
1109
|
+
if (!this.#pushApplySerialized)
|
|
1110
|
+
throw new Error(
|
|
1111
|
+
'D1 pruning requires externally serialized partition writes',
|
|
1112
|
+
);
|
|
1113
|
+
const epochGuard =
|
|
1114
|
+
'EXISTS (SELECT 1 FROM sync_partition_registry WHERE partition=? AND log_epoch=?)';
|
|
1115
|
+
const horizon =
|
|
1116
|
+
'(SELECT horizon_seq FROM sync_partitions WHERE partition=?)';
|
|
1117
|
+
const results = await this.#db.batch([
|
|
1099
1118
|
this.#db
|
|
1100
|
-
.prepare(
|
|
1101
|
-
|
|
1119
|
+
.prepare(
|
|
1120
|
+
`SELECT log_epoch, coalesce(${horizon},0) AS previous_horizon_seq FROM sync_partition_registry WHERE partition=?`,
|
|
1121
|
+
)
|
|
1122
|
+
.bind(partition, partition),
|
|
1102
1123
|
this.#db
|
|
1103
|
-
.prepare(
|
|
1104
|
-
|
|
1124
|
+
.prepare(`INSERT INTO sync_partitions(partition,horizon_seq) SELECT ?,? WHERE ${epochGuard}
|
|
1125
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
|
|
1126
|
+
.bind(partition, query.throughSeq, partition, query.logEpoch),
|
|
1105
1127
|
this.#db
|
|
1106
1128
|
.prepare(
|
|
1107
|
-
|
|
1129
|
+
`SELECT horizon_seq, (SELECT count(*) FROM sync_commits WHERE partition=? AND commit_seq<=${horizon}) AS removed_commits FROM sync_partitions WHERE partition=?`,
|
|
1108
1130
|
)
|
|
1109
|
-
.bind(partition,
|
|
1131
|
+
.bind(partition, partition, partition),
|
|
1132
|
+
...['sync_commits', 'sync_changes', 'sync_change_scopes'].map((table) =>
|
|
1133
|
+
this.#db
|
|
1134
|
+
.prepare(
|
|
1135
|
+
`DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
|
|
1136
|
+
)
|
|
1137
|
+
.bind(partition, partition, partition, query.logEpoch),
|
|
1138
|
+
),
|
|
1110
1139
|
]);
|
|
1111
|
-
|
|
1140
|
+
const [before, after] = [results[0], results[2]].map((result) => {
|
|
1141
|
+
if (
|
|
1142
|
+
typeof result !== 'object' ||
|
|
1143
|
+
result === null ||
|
|
1144
|
+
!('results' in result) ||
|
|
1145
|
+
!Array.isArray(result.results)
|
|
1146
|
+
) {
|
|
1147
|
+
throw new Error('D1 pruning returned an invalid batch result');
|
|
1148
|
+
}
|
|
1149
|
+
const row: unknown = result.results[0];
|
|
1150
|
+
return typeof row === 'object' && row !== null
|
|
1151
|
+
? (row as Readonly<Record<string, unknown>>)
|
|
1152
|
+
: undefined;
|
|
1153
|
+
});
|
|
1154
|
+
if (before?.log_epoch !== query.logEpoch)
|
|
1155
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
1156
|
+
if (
|
|
1157
|
+
typeof before.previous_horizon_seq !== 'number' ||
|
|
1158
|
+
typeof after?.horizon_seq !== 'number' ||
|
|
1159
|
+
typeof after.removed_commits !== 'number'
|
|
1160
|
+
) {
|
|
1161
|
+
throw new Error('D1 pruning returned invalid horizon metadata');
|
|
1162
|
+
}
|
|
1163
|
+
return {
|
|
1164
|
+
previousHorizonSeq: before.previous_horizon_seq,
|
|
1165
|
+
horizonSeq: after.horizon_seq,
|
|
1166
|
+
removedCommits: after.removed_commits,
|
|
1167
|
+
};
|
|
1112
1168
|
}
|
|
1113
1169
|
|
|
1114
1170
|
async getCommitSeqBefore(
|
|
@@ -1530,6 +1586,19 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1530
1586
|
.run();
|
|
1531
1587
|
}
|
|
1532
1588
|
|
|
1589
|
+
async getActiveClientCursorFloor(
|
|
1590
|
+
partition: string,
|
|
1591
|
+
cutoffMs: number,
|
|
1592
|
+
): Promise<number | null> {
|
|
1593
|
+
const row = await this.#db
|
|
1594
|
+
.prepare(
|
|
1595
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?',
|
|
1596
|
+
)
|
|
1597
|
+
.bind(partition, cutoffMs)
|
|
1598
|
+
.first<{ cursor: number | null }>();
|
|
1599
|
+
return row!.cursor;
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1533
1602
|
async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
|
|
1534
1603
|
const { results } = await this.#db
|
|
1535
1604
|
.prepare(
|
package/src/operations.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type AuthoritativeRelationPlan,
|
|
3
|
+
validateAuthoritativeRelationPlan,
|
|
4
|
+
} from './authoritative-query';
|
|
1
5
|
import {
|
|
2
6
|
decodeRow,
|
|
3
7
|
decodeRemoteOperationRequest,
|
|
@@ -42,6 +46,7 @@ export interface AuthoritativeQueryDescriptor<Params = undefined> {
|
|
|
42
46
|
readonly hasParams: boolean;
|
|
43
47
|
readonly sql: string;
|
|
44
48
|
readonly tables: readonly string[];
|
|
49
|
+
readonly relationPlans: readonly AuthoritativeRelationPlan[];
|
|
45
50
|
readonly resultColumns: readonly {
|
|
46
51
|
readonly name: string;
|
|
47
52
|
readonly type:
|
|
@@ -280,6 +285,9 @@ export function registerRemoteQuery<Params>(
|
|
|
280
285
|
options: RemoteQueryOptions<Params>,
|
|
281
286
|
): RegisteredRemoteQuery {
|
|
282
287
|
if (
|
|
288
|
+
!Array.isArray(descriptor.relationPlans) ||
|
|
289
|
+
!descriptor.relationPlans.some((plan) => plan.sql === descriptor.sql) ||
|
|
290
|
+
descriptor.relationPlans.some((plan) => !Array.isArray(plan.relations)) ||
|
|
283
291
|
descriptor.id.length === 0 ||
|
|
284
292
|
new Set(descriptor.tables).size !== descriptor.tables.length ||
|
|
285
293
|
!Array.isArray(descriptor.resultColumns) ||
|
|
@@ -288,9 +296,12 @@ export function registerRemoteQuery<Params>(
|
|
|
288
296
|
descriptor.resultColumns.length
|
|
289
297
|
) {
|
|
290
298
|
throw new Error(
|
|
291
|
-
'remote query requires a non-empty id and unique tables and result columns',
|
|
299
|
+
'remote query requires generated relation plans, a non-empty id, and unique tables and result columns; regenerate queries',
|
|
292
300
|
);
|
|
293
301
|
}
|
|
302
|
+
for (const plan of descriptor.relationPlans) {
|
|
303
|
+
validateAuthoritativeRelationPlan(plan, descriptor.tables);
|
|
304
|
+
}
|
|
294
305
|
if (
|
|
295
306
|
!Number.isSafeInteger(options.maxRows) ||
|
|
296
307
|
options.maxRows < 1 ||
|
|
@@ -394,12 +405,29 @@ export function registerRemoteQuery<Params>(
|
|
|
394
405
|
'configured storage does not implement authoritative queries',
|
|
395
406
|
);
|
|
396
407
|
}
|
|
397
|
-
await ctx.storage.ensureSchema(schema);
|
|
398
408
|
const selectedSql = descriptor.sqlFor?.(params) ?? descriptor.sql;
|
|
409
|
+
const plan = descriptor.relationPlans.find(
|
|
410
|
+
(candidate) => candidate.sql === selectedSql,
|
|
411
|
+
);
|
|
412
|
+
if (plan === undefined) {
|
|
413
|
+
throw syncError(
|
|
414
|
+
'operation.invalid_request',
|
|
415
|
+
'selected SQL has no generated relation plan; regenerate queries',
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
await ctx.storage.ensureSchema(schema);
|
|
419
|
+
const prefix = 'SELECT * FROM (';
|
|
399
420
|
let result;
|
|
400
421
|
try {
|
|
401
422
|
result = await ctx.storage.queryAuthoritative(ctx.partition, {
|
|
402
|
-
|
|
423
|
+
plan: {
|
|
424
|
+
sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
|
|
425
|
+
relations: plan.relations.map((relation) => ({
|
|
426
|
+
...relation,
|
|
427
|
+
start: relation.start + prefix.length,
|
|
428
|
+
end: relation.end + prefix.length,
|
|
429
|
+
})),
|
|
430
|
+
},
|
|
403
431
|
params: [...descriptor.bind(params), options.maxRows + 1],
|
|
404
432
|
tables: descriptor.tables,
|
|
405
433
|
});
|
package/src/postgres-storage.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune';
|
|
2
|
+
import { StorageQueryError } from './storage-errors';
|
|
3
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
1
4
|
/**
|
|
2
5
|
* Postgres server storage: the production database path.
|
|
3
6
|
*
|
|
@@ -223,6 +226,21 @@ interface SerializedResult {
|
|
|
223
226
|
details?: import('@syncular/core').RejectionDetails;
|
|
224
227
|
}
|
|
225
228
|
|
|
229
|
+
async function lockPartitionOn(
|
|
230
|
+
client: PgQueryable,
|
|
231
|
+
partition: string,
|
|
232
|
+
): Promise<void> {
|
|
233
|
+
await client.query(
|
|
234
|
+
`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
235
|
+
ON CONFLICT (partition) DO NOTHING`,
|
|
236
|
+
[partition],
|
|
237
|
+
);
|
|
238
|
+
await client.query(
|
|
239
|
+
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
240
|
+
[partition],
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
|
|
226
244
|
function toBase64(bytes: Uint8Array): string {
|
|
227
245
|
return Buffer.from(bytes).toString('base64');
|
|
228
246
|
}
|
|
@@ -649,15 +667,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
649
667
|
|
|
650
668
|
async lockPartitionForPush(): Promise<void> {
|
|
651
669
|
this.#assertOpen();
|
|
652
|
-
await this.#client
|
|
653
|
-
`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
654
|
-
ON CONFLICT (partition) DO NOTHING`,
|
|
655
|
-
[this.#partition],
|
|
656
|
-
);
|
|
657
|
-
await this.#client.query(
|
|
658
|
-
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
659
|
-
[this.#partition],
|
|
660
|
-
);
|
|
670
|
+
await lockPartitionOn(this.#client, this.#partition);
|
|
661
671
|
await this.#client.query('SAVEPOINT syncular_push_candidate');
|
|
662
672
|
this.#pushApplySavepoint = true;
|
|
663
673
|
}
|
|
@@ -1047,6 +1057,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1047
1057
|
): Promise<PartitionRegistryEntry> {
|
|
1048
1058
|
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
1049
1059
|
await this.#exec.transaction(async (client) => {
|
|
1060
|
+
await lockPartitionOn(client, partition);
|
|
1050
1061
|
await client.query(
|
|
1051
1062
|
`INSERT INTO sync_partition_registry(
|
|
1052
1063
|
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
@@ -1148,7 +1159,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1148
1159
|
}
|
|
1149
1160
|
const prepared = bindAuthoritativePartition(
|
|
1150
1161
|
prepareAuthoritativeQuery(
|
|
1151
|
-
query.
|
|
1162
|
+
query.plan,
|
|
1152
1163
|
query.params,
|
|
1153
1164
|
query.tables,
|
|
1154
1165
|
this.#tables,
|
|
@@ -1177,6 +1188,14 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1177
1188
|
});
|
|
1178
1189
|
}
|
|
1179
1190
|
|
|
1191
|
+
async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
|
|
1192
|
+
const { rows } = await this.#exec.query<{ log_epoch: string }>(
|
|
1193
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1',
|
|
1194
|
+
[partition],
|
|
1195
|
+
);
|
|
1196
|
+
return rows[0]?.log_epoch;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1180
1199
|
async getHorizonSeq(partition: string): Promise<number> {
|
|
1181
1200
|
const { rows } = await this.#exec.query<{ horizon_seq: unknown }>(
|
|
1182
1201
|
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
@@ -1188,25 +1207,52 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1188
1207
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
1189
1208
|
await this.#exec.query(
|
|
1190
1209
|
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
|
|
1191
|
-
ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`,
|
|
1210
|
+
ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`,
|
|
1192
1211
|
[partition, seq],
|
|
1193
1212
|
);
|
|
1194
1213
|
}
|
|
1195
1214
|
|
|
1196
|
-
async pruneCommitsThrough(
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
);
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
[
|
|
1208
|
-
|
|
1209
|
-
|
|
1215
|
+
async pruneCommitsThrough(
|
|
1216
|
+
partition: string,
|
|
1217
|
+
query: CommitPruneQuery,
|
|
1218
|
+
): Promise<CommitPruneResult> {
|
|
1219
|
+
validateCommitPruneQuery(query);
|
|
1220
|
+
return this.#exec.transaction(async (client) => {
|
|
1221
|
+
await lockPartitionOn(client, partition);
|
|
1222
|
+
const epoch = await client.query<{ log_epoch: string }>(
|
|
1223
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE',
|
|
1224
|
+
[partition],
|
|
1225
|
+
);
|
|
1226
|
+
if (epoch.rows[0]?.log_epoch !== query.logEpoch)
|
|
1227
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
1228
|
+
const previous = await client.query<{ horizon_seq: unknown }>(
|
|
1229
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
1230
|
+
[partition],
|
|
1231
|
+
);
|
|
1232
|
+
const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
|
|
1233
|
+
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
1234
|
+
await client.query(
|
|
1235
|
+
'UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1',
|
|
1236
|
+
[partition, horizonSeq],
|
|
1237
|
+
);
|
|
1238
|
+
const removed = await client.query(
|
|
1239
|
+
'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
|
|
1240
|
+
[partition, horizonSeq],
|
|
1241
|
+
);
|
|
1242
|
+
await client.query(
|
|
1243
|
+
'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
|
|
1244
|
+
[partition, horizonSeq],
|
|
1245
|
+
);
|
|
1246
|
+
await client.query(
|
|
1247
|
+
'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
|
|
1248
|
+
[partition, horizonSeq],
|
|
1249
|
+
);
|
|
1250
|
+
return {
|
|
1251
|
+
previousHorizonSeq,
|
|
1252
|
+
horizonSeq,
|
|
1253
|
+
removedCommits: removed.rowCount,
|
|
1254
|
+
};
|
|
1255
|
+
});
|
|
1210
1256
|
}
|
|
1211
1257
|
|
|
1212
1258
|
async getCommitSeqBefore(
|
|
@@ -1630,6 +1676,17 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1630
1676
|
);
|
|
1631
1677
|
}
|
|
1632
1678
|
|
|
1679
|
+
async getActiveClientCursorFloor(
|
|
1680
|
+
partition: string,
|
|
1681
|
+
cutoffMs: number,
|
|
1682
|
+
): Promise<number | null> {
|
|
1683
|
+
const { rows } = await this.#exec.query<{ cursor: unknown }>(
|
|
1684
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2',
|
|
1685
|
+
[partition, cutoffMs],
|
|
1686
|
+
);
|
|
1687
|
+
return rows[0]!.cursor === null ? null : asNumber(rows[0]!.cursor);
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1633
1690
|
async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
|
|
1634
1691
|
const { rows } = await this.#exec.query<{
|
|
1635
1692
|
client_id: string;
|
package/src/prune.ts
CHANGED
|
@@ -7,7 +7,20 @@
|
|
|
7
7
|
* least the newest `minRetainedCommits` commits are always retained.
|
|
8
8
|
*/
|
|
9
9
|
import { emitEvent, type SyncularServerEvents } from './events';
|
|
10
|
-
import type { ServerStorage } from './storage';
|
|
10
|
+
import type { CommitPruneQuery, ServerStorage } from './storage';
|
|
11
|
+
import { StorageQueryError } from './storage-errors';
|
|
12
|
+
|
|
13
|
+
/** Shared validation for the built-in atomic pruning adapters. */
|
|
14
|
+
export function validateCommitPruneQuery(query: CommitPruneQuery): void {
|
|
15
|
+
if (
|
|
16
|
+
!Number.isSafeInteger(query.throughSeq) ||
|
|
17
|
+
query.throughSeq < 0 ||
|
|
18
|
+
typeof query.logEpoch !== 'string' ||
|
|
19
|
+
query.logEpoch.length === 0
|
|
20
|
+
) {
|
|
21
|
+
throw new StorageQueryError('sync.storage.invalid_prune_cursor');
|
|
22
|
+
}
|
|
23
|
+
}
|
|
11
24
|
|
|
12
25
|
export interface RetentionPolicy {
|
|
13
26
|
/** Active window for laggard cursors (default 14 days). */
|
|
@@ -37,28 +50,29 @@ export interface PruneOptions {
|
|
|
37
50
|
export async function pruneCommitLog(options: PruneOptions): Promise<number> {
|
|
38
51
|
const { storage, partition, nowMs } = options;
|
|
39
52
|
const policy = { ...DEFAULT_RETENTION, ...options.retention };
|
|
53
|
+
const logEpoch = await storage.getPartitionLogEpoch(partition);
|
|
54
|
+
if (logEpoch === undefined)
|
|
55
|
+
throw new StorageQueryError('sync.storage.partition_unregistered');
|
|
40
56
|
const maxSeq = await storage.getMaxCommitSeq(partition);
|
|
41
|
-
const cursors = await storage.listClientCursors(partition);
|
|
42
|
-
const activeCursors = cursors
|
|
43
|
-
.filter((c) => c.updatedAtMs >= nowMs - policy.activeWindowMs)
|
|
44
|
-
.map((c) => c.cursor);
|
|
45
57
|
const cursorFloor =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
58
|
+
(await storage.getActiveClientCursorFloor(
|
|
59
|
+
partition,
|
|
60
|
+
nowMs - policy.activeWindowMs,
|
|
61
|
+
)) ?? Number.MAX_SAFE_INTEGER;
|
|
49
62
|
const forcedSeq = await storage.getCommitSeqBefore(
|
|
50
63
|
partition,
|
|
51
64
|
nowMs - policy.ageForceMs,
|
|
52
65
|
);
|
|
53
66
|
const retainFloor = maxSeq - policy.minRetainedCommits;
|
|
54
67
|
const target = Math.min(Math.max(cursorFloor, forcedSeq), retainFloor);
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
68
|
+
const {
|
|
69
|
+
previousHorizonSeq: current,
|
|
70
|
+
horizonSeq: horizon,
|
|
71
|
+
removedCommits,
|
|
72
|
+
} = await storage.pruneCommitsThrough(partition, {
|
|
73
|
+
logEpoch,
|
|
74
|
+
throughSeq: Math.max(0, target),
|
|
75
|
+
});
|
|
62
76
|
const events = options.events;
|
|
63
77
|
if (events !== undefined) {
|
|
64
78
|
emitEvent(events, {
|