@syncular/server 0.15.48 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -4
- package/dist/admin.js +1 -5
- package/dist/authoritative-query.d.ts +14 -6
- package/dist/authoritative-query.js +60 -82
- package/dist/d1-storage.d.ts +4 -1
- package/dist/d1-storage.js +59 -15
- package/dist/operations.d.ts +2 -0
- package/dist/operations.js +23 -4
- package/dist/postgres-storage.d.ts +4 -1
- package/dist/postgres-storage.js +39 -10
- package/dist/prune.d.ts +3 -1
- package/dist/prune.js +18 -14
- package/dist/pull.js +63 -32
- package/dist/push.js +5 -5
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +3 -3
- package/dist/sqlite-image.js +10 -6
- package/dist/sqlite-node.js +2 -2
- package/dist/sqlite-storage.d.ts +4 -1
- package/dist/sqlite-storage.js +67 -26
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +3 -0
- package/dist/storage.d.ts +20 -5
- package/package.json +2 -2
- package/src/admin.ts +4 -6
- package/src/authoritative-query.ts +89 -94
- package/src/d1-storage.ts +86 -17
- package/src/operations.ts +31 -3
- package/src/postgres-storage.ts +82 -25
- package/src/prune.ts +29 -15
- package/src/pull.ts +80 -42
- package/src/push.ts +5 -5
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +26 -15
- package/src/sqlite-node.ts +2 -2
- package/src/sqlite-storage.ts +86 -28
- package/src/storage-errors.ts +10 -1
- package/src/storage.ts +29 -5
package/README.md
CHANGED
|
@@ -15,6 +15,11 @@ registry, command mutations use the ordinary serialized push path, and
|
|
|
15
15
|
specified in [`docs/REMOTE.md`](../../docs/REMOTE.md) and the practical setup is
|
|
16
16
|
in the [remote operations guide](https://syncular.dev/guide-remote-operations/).
|
|
17
17
|
|
|
18
|
+
Remote query registration requires generated `relationPlans` for each selected
|
|
19
|
+
SQL statement. Run `syncular generate` before upgrading existing query modules.
|
|
20
|
+
The server uses those boundaries to bind every physical table occurrence to
|
|
21
|
+
the authenticated partition, including quoted self joins and CTE bodies.
|
|
22
|
+
|
|
18
23
|
Application intent belongs in immutable domain event rows written in the same
|
|
19
24
|
commit as the state change. `SyncularServerEvents` below remains operational
|
|
20
25
|
telemetry. See the [domain event guide](https://syncular.dev/guide-domain-events/).
|
|
@@ -927,10 +932,22 @@ at or below it. Nothing prunes automatically — the host schedules it.
|
|
|
927
932
|
|
|
928
933
|
**When to run.** A periodic job per partition — hourly to daily is the
|
|
929
934
|
sensible range; there is no benefit below the granularity of your
|
|
930
|
-
`activeWindowMs`.
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
935
|
+
`activeWindowMs`. Pass `events` to get `prune.completed` per pass.
|
|
936
|
+
|
|
937
|
+
Pruning verifies the captured log epoch and updates the horizon together with
|
|
938
|
+
commit/change/scope deletion in one transaction. Concurrent passes cannot
|
|
939
|
+
lower the horizon. A retry cleans up eligible records even when the horizon
|
|
940
|
+
already covers them. A restore invalidates a pending pass with
|
|
941
|
+
`sync.storage.prune_epoch_mismatch`; recompute its retention inputs.
|
|
942
|
+
Unregistered partitions cannot be pruned. D1 maintenance enters the owning
|
|
943
|
+
Durable Object's existing write queue through
|
|
944
|
+
`SyncularRealtimeHost.pruneCommitLog`.
|
|
945
|
+
|
|
946
|
+
Custom storage adapters must add `getPartitionLogEpoch(partition)` and replace
|
|
947
|
+
`pruneCommitsThrough(partition, seq)` with
|
|
948
|
+
`pruneCommitsThrough(partition, { logEpoch, throughSeq })`, returning
|
|
949
|
+
`{ previousHorizonSeq, horizonSeq, removedCommits }` from the transaction.
|
|
950
|
+
`setHorizonSeq` remains monotonic and is no longer used by the pruning helper.
|
|
934
951
|
|
|
935
952
|
**The retention floors (§4.6, encoded in `RetentionPolicy`).** The
|
|
936
953
|
horizon never advances past `min(cursor)` of *active* clients — clients
|
|
@@ -1149,3 +1166,23 @@ deterministic in-process sqlite loopback):
|
|
|
1149
1166
|
```sh
|
|
1150
1167
|
SYNCULAR_PG_URL=postgres://user:pass@localhost:5432/db bun run bench
|
|
1151
1168
|
```
|
|
1169
|
+
|
|
1170
|
+
Custom storage adapters must implement
|
|
1171
|
+
`getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum cursor
|
|
1172
|
+
whose `updatedAtMs >= cutoffMs`, or `null` when no client qualifies. Preserve
|
|
1173
|
+
negative bootstrap cursors. Pruning and admin horizon status use this scalar
|
|
1174
|
+
aggregate; `listClientCursors` remains the explicit listing interface.
|
|
1175
|
+
|
|
1176
|
+
SQLite image builders now return `Promise<Uint8Array>` and receive
|
|
1177
|
+
`rowBatches`, an iterable or async iterable of row arrays. Replace custom
|
|
1178
|
+
builders' `input.rows` loop with `for await (const rows of input.rowBatches)`,
|
|
1179
|
+
insert each batch into the dedicated image database, and count rows during
|
|
1180
|
+
consumption. Write the final row count into `_syncular_segment` before
|
|
1181
|
+
serialization. Await `buildSqliteImage(input)` when calling the built-in
|
|
1182
|
+
Bun or Node builder directly.
|
|
1183
|
+
|
|
1184
|
+
The server shares in-flight builds for the same storage pair and artifact
|
|
1185
|
+
identity after authorization. Sharing is local to one process. Signed URL
|
|
1186
|
+
grants remain per request. The first eligibility probe has at most
|
|
1187
|
+
`limitSnapshotRows + 1` rows; subsequent builder batches have at most 5,000
|
|
1188
|
+
rows. The image database and serialized output still consume memory.
|
package/dist/admin.js
CHANGED
|
@@ -173,11 +173,7 @@ export class SyncularAdmin {
|
|
|
173
173
|
const nowMs = this.#clock();
|
|
174
174
|
const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
|
|
175
175
|
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
176
|
-
const
|
|
177
|
-
const activeCursors = cursors
|
|
178
|
-
.filter((c) => c.updatedAtMs >= nowMs - this.#retention.activeWindowMs)
|
|
179
|
-
.map((c) => c.cursor);
|
|
180
|
-
const activeCursorFloor = activeCursors.length > 0 ? Math.min(...activeCursors) : null;
|
|
176
|
+
const activeCursorFloor = await this.#storage.getActiveClientCursorFloor(partition, nowMs - this.#retention.activeWindowMs);
|
|
181
177
|
const cursorFloor = activeCursorFloor ?? Number.MAX_SAFE_INTEGER;
|
|
182
178
|
const forcedSeq = await this.#storage.getCommitSeqBefore(partition, nowMs - this.#retention.ageForceMs);
|
|
183
179
|
const retainFloor = maxCommitSeq - this.#retention.minRetainedCommits;
|
|
@@ -9,12 +9,20 @@ export interface BoundAuthoritativeQuery {
|
|
|
9
9
|
readonly params: readonly AuthoritativeQueryValue[];
|
|
10
10
|
}
|
|
11
11
|
declare const PARTITION_BIND: unique symbol;
|
|
12
|
-
/**
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
/** Compiler-proven occurrences in the exact generated positional statement. */
|
|
13
|
+
export interface AuthoritativeRelationPlan {
|
|
14
|
+
readonly sql: string;
|
|
15
|
+
readonly relations: readonly {
|
|
16
|
+
readonly table: string;
|
|
17
|
+
readonly start: number;
|
|
18
|
+
readonly end: number;
|
|
19
|
+
readonly alias?: string;
|
|
20
|
+
}[];
|
|
21
|
+
}
|
|
22
|
+
/** Validate trusted generated metadata before registration or storage execution. */
|
|
23
|
+
export declare function validateAuthoritativeRelationPlan(plan: AuthoritativeRelationPlan, declaredTables: readonly string[]): void;
|
|
24
|
+
/** Bind every compiler-proven table occurrence to the authenticated partition. */
|
|
25
|
+
export declare function prepareAuthoritativeQuery(plan: AuthoritativeRelationPlan, params: readonly AuthoritativeQueryValue[], declaredTables: readonly string[], tables: ReadonlyMap<string, CompiledTable>): PreparedAuthoritativeQuery;
|
|
18
26
|
export declare function bindAuthoritativePartition(prepared: PreparedAuthoritativeQuery, partition: string): BoundAuthoritativeQuery;
|
|
19
27
|
export declare function postgresPlaceholders(sql: string): string;
|
|
20
28
|
export {};
|
|
@@ -1,24 +1,5 @@
|
|
|
1
1
|
import { quoteIdent, SYNC_PARTITION_COLUMN } from './relational-rows.js';
|
|
2
2
|
const PARTITION_BIND = Symbol('syncular.authoritative_partition');
|
|
3
|
-
const RESERVED_ALIAS = new Set([
|
|
4
|
-
'on',
|
|
5
|
-
'where',
|
|
6
|
-
'group',
|
|
7
|
-
'order',
|
|
8
|
-
'inner',
|
|
9
|
-
'left',
|
|
10
|
-
'right',
|
|
11
|
-
'full',
|
|
12
|
-
'outer',
|
|
13
|
-
'natural',
|
|
14
|
-
'join',
|
|
15
|
-
'cross',
|
|
16
|
-
'using',
|
|
17
|
-
'limit',
|
|
18
|
-
'having',
|
|
19
|
-
]);
|
|
20
|
-
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
|
|
21
|
-
const TABLE_REF_RE = new RegExp(`\\b(FROM|(?:NATURAL\\s+)?(?:(?:LEFT|RIGHT|FULL)(?:\\s+OUTER)?|INNER|CROSS)?\\s*JOIN)\\s+((?:\\(\\s*)*)(${IDENT})(?:\\s+(?:AS\\s+)?((?!(?:${[...RESERVED_ALIAS].join('|')})\\b)${IDENT}))?`, 'gi');
|
|
22
3
|
function protectedSqlEnd(sql, index) {
|
|
23
4
|
const char = sql[index];
|
|
24
5
|
const next = sql[index + 1];
|
|
@@ -48,97 +29,94 @@ function protectedSqlEnd(sql, index) {
|
|
|
48
29
|
}
|
|
49
30
|
return undefined;
|
|
50
31
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
out += sql[index];
|
|
58
|
-
else
|
|
59
|
-
out += sql.slice(index, end).replace(/[^\n]/g, ' ');
|
|
60
|
-
index = end ?? index + 1;
|
|
32
|
+
/** Validate trusted generated metadata before registration or storage execution. */
|
|
33
|
+
export function validateAuthoritativeRelationPlan(plan, declaredTables) {
|
|
34
|
+
if (plan === undefined ||
|
|
35
|
+
typeof plan.sql !== 'string' ||
|
|
36
|
+
!Array.isArray(plan.relations)) {
|
|
37
|
+
throw new Error('registered query requires generated relation plans; regenerate queries');
|
|
61
38
|
}
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Turn generated local SQL into a partition-local authoritative statement.
|
|
66
|
-
* Only relations declared by the generated descriptor are rewritten. Values
|
|
67
|
-
* remain parameters; request data is never interpolated into SQL.
|
|
68
|
-
*/
|
|
69
|
-
export function prepareAuthoritativeQuery(sql, params, declaredTables, tables) {
|
|
70
|
-
if (maskedSql(sql).includes(';'))
|
|
71
|
-
throw new Error('registered query must be one SELECT');
|
|
39
|
+
const sql = plan.sql;
|
|
72
40
|
const declared = new Set(declaredTables);
|
|
73
|
-
const masked = maskedSql(sql);
|
|
74
|
-
const replacements = [];
|
|
75
41
|
const found = new Set();
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
42
|
+
let previousEnd = 0;
|
|
43
|
+
for (const relation of plan.relations) {
|
|
44
|
+
if (!Number.isSafeInteger(relation.start) ||
|
|
45
|
+
!Number.isSafeInteger(relation.end) ||
|
|
46
|
+
relation.start < previousEnd ||
|
|
47
|
+
relation.end <= relation.start ||
|
|
48
|
+
relation.end > sql.length) {
|
|
49
|
+
throw new Error('registered query relation boundaries do not match its SQL; regenerate queries');
|
|
83
50
|
}
|
|
84
|
-
|
|
85
|
-
|
|
51
|
+
previousEnd = relation.end;
|
|
52
|
+
if (!declared.has(relation.table)) {
|
|
53
|
+
throw new Error('registered query table metadata does not match its SQL');
|
|
86
54
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
55
|
+
const spelling = sql.slice(relation.start, relation.end);
|
|
56
|
+
const quote = spelling[0];
|
|
57
|
+
const name = quote === '['
|
|
58
|
+
? spelling.slice(1, -1)
|
|
59
|
+
: quote === '"' || quote === '`'
|
|
60
|
+
? spelling
|
|
61
|
+
.slice(1, -1)
|
|
62
|
+
.split(quote + quote)
|
|
63
|
+
.join(quote)
|
|
64
|
+
: spelling;
|
|
65
|
+
if (name.toLowerCase() !== relation.table.toLowerCase()) {
|
|
66
|
+
throw new Error('registered query relation name does not match its SQL; regenerate queries');
|
|
90
67
|
}
|
|
91
|
-
|
|
92
|
-
const afterOperator = match[1].length;
|
|
93
|
-
const relative = match[0]
|
|
94
|
-
.toLowerCase()
|
|
95
|
-
.indexOf(rawTable.toLowerCase(), afterOperator);
|
|
96
|
-
const start = matchStart + relative;
|
|
97
|
-
const projection = table.columns.map((column) => quoteIdent(column.name));
|
|
98
|
-
replacements.push({
|
|
99
|
-
start,
|
|
100
|
-
end: start + rawTable.length,
|
|
101
|
-
text: `(SELECT ${projection.join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=/*syncular_partition*/?)${alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
|
|
102
|
-
});
|
|
103
|
-
found.add(table.name);
|
|
68
|
+
found.add(relation.table);
|
|
104
69
|
}
|
|
105
70
|
if (found.size !== declared.size ||
|
|
106
71
|
[...declared].some((table) => !found.has(table))) {
|
|
107
72
|
throw new Error('registered query table metadata does not match its SQL');
|
|
108
73
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
74
|
+
}
|
|
75
|
+
/** Bind every compiler-proven table occurrence to the authenticated partition. */
|
|
76
|
+
export function prepareAuthoritativeQuery(plan, params, declaredTables, tables) {
|
|
77
|
+
validateAuthoritativeRelationPlan(plan, declaredTables);
|
|
78
|
+
const sql = plan.sql;
|
|
79
|
+
const replacements = plan.relations.map((relation) => {
|
|
80
|
+
const table = tables.get(relation.table);
|
|
81
|
+
if (table === undefined)
|
|
82
|
+
throw new Error('registered query targets an unknown table');
|
|
83
|
+
if (!table.materialize)
|
|
84
|
+
throw new Error('registered query targets a non-materialized table');
|
|
85
|
+
return {
|
|
86
|
+
start: relation.start,
|
|
87
|
+
end: relation.end,
|
|
88
|
+
text: `(SELECT ${table.columns.map((column) => quoteIdent(column.name)).join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?)${relation.alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
|
|
89
|
+
};
|
|
90
|
+
});
|
|
116
91
|
const bound = [];
|
|
117
92
|
let anonymousIndex = 0;
|
|
118
93
|
let rendered = '';
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
94
|
+
let nextRelation = 0;
|
|
95
|
+
for (let index = 0; index < sql.length; index += 1) {
|
|
96
|
+
const replacement = replacements[nextRelation];
|
|
97
|
+
if (replacement?.start === index) {
|
|
98
|
+
rendered += replacement.text;
|
|
122
99
|
bound.push(PARTITION_BIND);
|
|
123
|
-
index
|
|
100
|
+
index = replacement.end - 1;
|
|
101
|
+
nextRelation += 1;
|
|
124
102
|
continue;
|
|
125
103
|
}
|
|
126
|
-
const protectedEnd = protectedSqlEnd(
|
|
104
|
+
const protectedEnd = protectedSqlEnd(sql, index);
|
|
127
105
|
if (protectedEnd !== undefined) {
|
|
128
|
-
rendered +=
|
|
106
|
+
rendered += sql.slice(index, protectedEnd);
|
|
129
107
|
index = protectedEnd - 1;
|
|
130
108
|
continue;
|
|
131
109
|
}
|
|
132
|
-
const char =
|
|
110
|
+
const char = sql[index];
|
|
133
111
|
if (char !== '?') {
|
|
134
112
|
rendered += char;
|
|
135
113
|
continue;
|
|
136
114
|
}
|
|
137
115
|
let end = index + 1;
|
|
138
|
-
while (end <
|
|
116
|
+
while (end < sql.length && /[0-9]/.test(sql[end])) {
|
|
139
117
|
end += 1;
|
|
140
118
|
}
|
|
141
|
-
const numbered =
|
|
119
|
+
const numbered = sql.slice(index + 1, end);
|
|
142
120
|
const parameterIndex = numbered.length > 0
|
|
143
121
|
? Number.parseInt(numbered, 10) - 1
|
|
144
122
|
: anonymousIndex++;
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
1
2
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
2
3
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
3
4
|
export interface D1PreparedStatement {
|
|
@@ -42,9 +43,10 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
42
43
|
begin(partition: string): Promise<StorageTransaction>;
|
|
43
44
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
44
45
|
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
46
|
+
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
45
47
|
getHorizonSeq(partition: string): Promise<number>;
|
|
46
48
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
47
|
-
pruneCommitsThrough(partition: string,
|
|
49
|
+
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
48
50
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
49
51
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
50
52
|
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
@@ -61,6 +63,7 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
61
63
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
62
64
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
63
65
|
putClientRecord(partition: string, record: ClientRecord): Promise<void>;
|
|
66
|
+
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
64
67
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
65
68
|
listRowsReferencingBlob(partition: string, blobId: string): Promise<{
|
|
66
69
|
readonly table: string;
|
package/dist/d1-storage.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
var _a;
|
|
2
|
+
import { validateCommitPruneQuery } from './prune.js';
|
|
3
|
+
import { StorageQueryError } from './storage-errors.js';
|
|
2
4
|
/**
|
|
3
5
|
* Cloudflare D1 server storage for Workers deployments.
|
|
4
6
|
*
|
|
@@ -676,7 +678,7 @@ export class D1ServerStorage {
|
|
|
676
678
|
if (this.#tables === undefined) {
|
|
677
679
|
throw new Error('ensureSchema(schema) must run before registered queries');
|
|
678
680
|
}
|
|
679
|
-
const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.
|
|
681
|
+
const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.plan, query.params, query.tables, this.#tables), partition);
|
|
680
682
|
const results = await this.#db.batch([
|
|
681
683
|
this.#db.prepare(prepared.sql).bind(...prepared.params),
|
|
682
684
|
this.#db
|
|
@@ -707,6 +709,13 @@ export class D1ServerStorage {
|
|
|
707
709
|
maxCommitSeq,
|
|
708
710
|
};
|
|
709
711
|
}
|
|
712
|
+
async getPartitionLogEpoch(partition) {
|
|
713
|
+
const row = await this.#db
|
|
714
|
+
.prepare('SELECT log_epoch FROM sync_partition_registry WHERE partition=?')
|
|
715
|
+
.bind(partition)
|
|
716
|
+
.first();
|
|
717
|
+
return row?.log_epoch;
|
|
718
|
+
}
|
|
710
719
|
async getHorizonSeq(partition) {
|
|
711
720
|
const row = await this.#db
|
|
712
721
|
.prepare('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
|
|
@@ -716,27 +725,55 @@ export class D1ServerStorage {
|
|
|
716
725
|
}
|
|
717
726
|
async setHorizonSeq(partition, seq) {
|
|
718
727
|
await this.#db
|
|
719
|
-
.prepare('INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=excluded.horizon_seq')
|
|
728
|
+
.prepare('INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)')
|
|
720
729
|
.bind(partition, seq)
|
|
721
730
|
.run();
|
|
722
731
|
}
|
|
723
|
-
async pruneCommitsThrough(partition,
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
732
|
+
async pruneCommitsThrough(partition, query) {
|
|
733
|
+
validateCommitPruneQuery(query);
|
|
734
|
+
if (!this.#pushApplySerialized)
|
|
735
|
+
throw new Error('D1 pruning requires externally serialized partition writes');
|
|
736
|
+
const epochGuard = 'EXISTS (SELECT 1 FROM sync_partition_registry WHERE partition=? AND log_epoch=?)';
|
|
737
|
+
const horizon = '(SELECT horizon_seq FROM sync_partitions WHERE partition=?)';
|
|
738
|
+
const results = await this.#db.batch([
|
|
729
739
|
this.#db
|
|
730
|
-
.prepare(
|
|
731
|
-
.bind(partition,
|
|
740
|
+
.prepare(`SELECT log_epoch, coalesce(${horizon},0) AS previous_horizon_seq FROM sync_partition_registry WHERE partition=?`)
|
|
741
|
+
.bind(partition, partition),
|
|
732
742
|
this.#db
|
|
733
|
-
.prepare(
|
|
734
|
-
|
|
743
|
+
.prepare(`INSERT INTO sync_partitions(partition,horizon_seq) SELECT ?,? WHERE ${epochGuard}
|
|
744
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
|
|
745
|
+
.bind(partition, query.throughSeq, partition, query.logEpoch),
|
|
735
746
|
this.#db
|
|
736
|
-
.prepare(
|
|
737
|
-
.bind(partition,
|
|
747
|
+
.prepare(`SELECT horizon_seq, (SELECT count(*) FROM sync_commits WHERE partition=? AND commit_seq<=${horizon}) AS removed_commits FROM sync_partitions WHERE partition=?`)
|
|
748
|
+
.bind(partition, partition, partition),
|
|
749
|
+
...['sync_commits', 'sync_changes', 'sync_change_scopes'].map((table) => this.#db
|
|
750
|
+
.prepare(`DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`)
|
|
751
|
+
.bind(partition, partition, partition, query.logEpoch)),
|
|
738
752
|
]);
|
|
739
|
-
|
|
753
|
+
const [before, after] = [results[0], results[2]].map((result) => {
|
|
754
|
+
if (typeof result !== 'object' ||
|
|
755
|
+
result === null ||
|
|
756
|
+
!('results' in result) ||
|
|
757
|
+
!Array.isArray(result.results)) {
|
|
758
|
+
throw new Error('D1 pruning returned an invalid batch result');
|
|
759
|
+
}
|
|
760
|
+
const row = result.results[0];
|
|
761
|
+
return typeof row === 'object' && row !== null
|
|
762
|
+
? row
|
|
763
|
+
: undefined;
|
|
764
|
+
});
|
|
765
|
+
if (before?.log_epoch !== query.logEpoch)
|
|
766
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
767
|
+
if (typeof before.previous_horizon_seq !== 'number' ||
|
|
768
|
+
typeof after?.horizon_seq !== 'number' ||
|
|
769
|
+
typeof after.removed_commits !== 'number') {
|
|
770
|
+
throw new Error('D1 pruning returned invalid horizon metadata');
|
|
771
|
+
}
|
|
772
|
+
return {
|
|
773
|
+
previousHorizonSeq: before.previous_horizon_seq,
|
|
774
|
+
horizonSeq: after.horizon_seq,
|
|
775
|
+
removedCommits: after.removed_commits,
|
|
776
|
+
};
|
|
740
777
|
}
|
|
741
778
|
async getCommitSeqBefore(partition, createdBeforeMs) {
|
|
742
779
|
const row = await this.#db
|
|
@@ -1000,6 +1037,13 @@ export class D1ServerStorage {
|
|
|
1000
1037
|
.bind(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs)
|
|
1001
1038
|
.run();
|
|
1002
1039
|
}
|
|
1040
|
+
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
1041
|
+
const row = await this.#db
|
|
1042
|
+
.prepare('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?')
|
|
1043
|
+
.bind(partition, cutoffMs)
|
|
1044
|
+
.first();
|
|
1045
|
+
return row.cursor;
|
|
1046
|
+
}
|
|
1003
1047
|
async listClientCursors(partition) {
|
|
1004
1048
|
const { results } = await this.#db
|
|
1005
1049
|
.prepare('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=?')
|
package/dist/operations.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type AuthoritativeRelationPlan } from './authoritative-query.js';
|
|
1
2
|
import { type RemoteOperationResponse, type RowValue } from '@syncular/core';
|
|
2
3
|
import type { SyncRequestContext } from './context.js';
|
|
3
4
|
import type { AuthoritativeQueryValue } from './storage.js';
|
|
@@ -20,6 +21,7 @@ export interface AuthoritativeQueryDescriptor<Params = undefined> {
|
|
|
20
21
|
readonly hasParams: boolean;
|
|
21
22
|
readonly sql: string;
|
|
22
23
|
readonly tables: readonly string[];
|
|
24
|
+
readonly relationPlans: readonly AuthoritativeRelationPlan[];
|
|
23
25
|
readonly resultColumns: readonly {
|
|
24
26
|
readonly name: string;
|
|
25
27
|
readonly type: 'string' | 'integer' | 'float' | 'boolean' | 'json' | 'bytes' | 'blob_ref' | 'crdt';
|
package/dist/operations.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { validateAuthoritativeRelationPlan, } from './authoritative-query.js';
|
|
1
2
|
import { decodeRow, decodeRemoteOperationRequest, encodeRow, encodeRemoteOperationResponse, } from '@syncular/core';
|
|
2
3
|
import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
3
4
|
import { SyncError, syncError } from './errors.js';
|
|
@@ -97,13 +98,19 @@ function normalizeQueryRows(rows, columns) {
|
|
|
97
98
|
}
|
|
98
99
|
/** Register one generated named query as an authoritative remote operation. */
|
|
99
100
|
export function registerRemoteQuery(descriptor, options) {
|
|
100
|
-
if (descriptor.
|
|
101
|
+
if (!Array.isArray(descriptor.relationPlans) ||
|
|
102
|
+
!descriptor.relationPlans.some((plan) => plan.sql === descriptor.sql) ||
|
|
103
|
+
descriptor.relationPlans.some((plan) => !Array.isArray(plan.relations)) ||
|
|
104
|
+
descriptor.id.length === 0 ||
|
|
101
105
|
new Set(descriptor.tables).size !== descriptor.tables.length ||
|
|
102
106
|
!Array.isArray(descriptor.resultColumns) ||
|
|
103
107
|
descriptor.resultColumns.length === 0 ||
|
|
104
108
|
new Set(descriptor.resultColumns.map((column) => column.name)).size !==
|
|
105
109
|
descriptor.resultColumns.length) {
|
|
106
|
-
throw new Error('remote query requires a non-empty id and unique tables and result columns');
|
|
110
|
+
throw new Error('remote query requires generated relation plans, a non-empty id, and unique tables and result columns; regenerate queries');
|
|
111
|
+
}
|
|
112
|
+
for (const plan of descriptor.relationPlans) {
|
|
113
|
+
validateAuthoritativeRelationPlan(plan, descriptor.tables);
|
|
107
114
|
}
|
|
108
115
|
if (!Number.isSafeInteger(options.maxRows) ||
|
|
109
116
|
options.maxRows < 1 ||
|
|
@@ -173,12 +180,24 @@ export function registerRemoteQuery(descriptor, options) {
|
|
|
173
180
|
if (ctx.storage.queryAuthoritative === undefined) {
|
|
174
181
|
throw syncError('operation.storage_unsupported', 'configured storage does not implement authoritative queries');
|
|
175
182
|
}
|
|
176
|
-
await ctx.storage.ensureSchema(schema);
|
|
177
183
|
const selectedSql = descriptor.sqlFor?.(params) ?? descriptor.sql;
|
|
184
|
+
const plan = descriptor.relationPlans.find((candidate) => candidate.sql === selectedSql);
|
|
185
|
+
if (plan === undefined) {
|
|
186
|
+
throw syncError('operation.invalid_request', 'selected SQL has no generated relation plan; regenerate queries');
|
|
187
|
+
}
|
|
188
|
+
await ctx.storage.ensureSchema(schema);
|
|
189
|
+
const prefix = 'SELECT * FROM (';
|
|
178
190
|
let result;
|
|
179
191
|
try {
|
|
180
192
|
result = await ctx.storage.queryAuthoritative(ctx.partition, {
|
|
181
|
-
|
|
193
|
+
plan: {
|
|
194
|
+
sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
|
|
195
|
+
relations: plan.relations.map((relation) => ({
|
|
196
|
+
...relation,
|
|
197
|
+
start: relation.start + prefix.length,
|
|
198
|
+
end: relation.end + prefix.length,
|
|
199
|
+
})),
|
|
200
|
+
},
|
|
182
201
|
params: [...descriptor.bind(params), options.maxRows + 1],
|
|
183
202
|
tables: descriptor.tables,
|
|
184
203
|
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
1
2
|
import { type PgExecutor } from './pg-executor.js';
|
|
2
3
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
3
4
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
@@ -46,9 +47,10 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
46
47
|
begin(partition: string): Promise<StorageTransaction>;
|
|
47
48
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
48
49
|
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
50
|
+
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
49
51
|
getHorizonSeq(partition: string): Promise<number>;
|
|
50
52
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
51
|
-
pruneCommitsThrough(partition: string,
|
|
53
|
+
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
52
54
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
53
55
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
54
56
|
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
@@ -65,6 +67,7 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
65
67
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
66
68
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
67
69
|
putClientRecord(partition: string, record: ClientRecord): Promise<void>;
|
|
70
|
+
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
68
71
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
69
72
|
listRowsReferencingBlob(partition: string, blobId: string): Promise<{
|
|
70
73
|
readonly table: string;
|
package/dist/postgres-storage.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune.js';
|
|
2
|
+
import { StorageQueryError } from './storage-errors.js';
|
|
1
3
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
2
4
|
import { syncError } from './errors.js';
|
|
3
5
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
@@ -107,6 +109,11 @@ CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
|
107
109
|
CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
|
|
108
110
|
ON sync_blob_refs(partition, blob_id);
|
|
109
111
|
`;
|
|
112
|
+
async function lockPartitionOn(client, partition) {
|
|
113
|
+
await client.query(`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
114
|
+
ON CONFLICT (partition) DO NOTHING`, [partition]);
|
|
115
|
+
await client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [partition]);
|
|
116
|
+
}
|
|
110
117
|
function toBase64(bytes) {
|
|
111
118
|
return Buffer.from(bytes).toString('base64');
|
|
112
119
|
}
|
|
@@ -381,9 +388,7 @@ class PostgresTransaction {
|
|
|
381
388
|
}
|
|
382
389
|
async lockPartitionForPush() {
|
|
383
390
|
this.#assertOpen();
|
|
384
|
-
await this.#client
|
|
385
|
-
ON CONFLICT (partition) DO NOTHING`, [this.#partition]);
|
|
386
|
-
await this.#client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [this.#partition]);
|
|
391
|
+
await lockPartitionOn(this.#client, this.#partition);
|
|
387
392
|
await this.#client.query('SAVEPOINT syncular_push_candidate');
|
|
388
393
|
this.#pushApplySavepoint = true;
|
|
389
394
|
}
|
|
@@ -661,6 +666,7 @@ export class PostgresServerStorage {
|
|
|
661
666
|
if (logEpoch.length === 0)
|
|
662
667
|
throw new Error('log epoch must be non-empty');
|
|
663
668
|
await this.#exec.transaction(async (client) => {
|
|
669
|
+
await lockPartitionOn(client, partition);
|
|
664
670
|
await client.query(`INSERT INTO sync_partition_registry(
|
|
665
671
|
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
666
672
|
) VALUES ($1,$2,TRUE,$3)
|
|
@@ -732,7 +738,7 @@ export class PostgresServerStorage {
|
|
|
732
738
|
if (this.#tables === undefined) {
|
|
733
739
|
throw new Error('ensureSchema(schema) must run before registered queries');
|
|
734
740
|
}
|
|
735
|
-
const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.
|
|
741
|
+
const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.plan, query.params, query.tables, this.#tables), partition);
|
|
736
742
|
return this.#exec.transaction(async (client) => {
|
|
737
743
|
await client.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY');
|
|
738
744
|
const result = await client.query(postgresPlaceholders(prepared.sql), prepared.params);
|
|
@@ -745,19 +751,38 @@ export class PostgresServerStorage {
|
|
|
745
751
|
};
|
|
746
752
|
});
|
|
747
753
|
}
|
|
754
|
+
async getPartitionLogEpoch(partition) {
|
|
755
|
+
const { rows } = await this.#exec.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=$1', [partition]);
|
|
756
|
+
return rows[0]?.log_epoch;
|
|
757
|
+
}
|
|
748
758
|
async getHorizonSeq(partition) {
|
|
749
759
|
const { rows } = await this.#exec.query('SELECT horizon_seq FROM sync_partitions WHERE partition=$1', [partition]);
|
|
750
760
|
return rows[0] === undefined ? 0 : asNumber(rows[0].horizon_seq);
|
|
751
761
|
}
|
|
752
762
|
async setHorizonSeq(partition, seq) {
|
|
753
763
|
await this.#exec.query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
|
|
754
|
-
ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`, [partition, seq]);
|
|
764
|
+
ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`, [partition, seq]);
|
|
755
765
|
}
|
|
756
|
-
async pruneCommitsThrough(partition,
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
766
|
+
async pruneCommitsThrough(partition, query) {
|
|
767
|
+
validateCommitPruneQuery(query);
|
|
768
|
+
return this.#exec.transaction(async (client) => {
|
|
769
|
+
await lockPartitionOn(client, partition);
|
|
770
|
+
const epoch = await client.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE', [partition]);
|
|
771
|
+
if (epoch.rows[0]?.log_epoch !== query.logEpoch)
|
|
772
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
773
|
+
const previous = await client.query('SELECT horizon_seq FROM sync_partitions WHERE partition=$1', [partition]);
|
|
774
|
+
const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
|
|
775
|
+
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
776
|
+
await client.query('UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1', [partition, horizonSeq]);
|
|
777
|
+
const removed = await client.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
778
|
+
await client.query('DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
779
|
+
await client.query('DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
780
|
+
return {
|
|
781
|
+
previousHorizonSeq,
|
|
782
|
+
horizonSeq,
|
|
783
|
+
removedCommits: removed.rowCount,
|
|
784
|
+
};
|
|
785
|
+
});
|
|
761
786
|
}
|
|
762
787
|
async getCommitSeqBefore(partition, createdBeforeMs) {
|
|
763
788
|
const { rows } = await this.#exec.query('SELECT max(commit_seq) AS seq FROM sync_commits WHERE partition=$1 AND created_at_ms<$2', [partition, createdBeforeMs]);
|
|
@@ -1069,6 +1094,10 @@ export class PostgresServerStorage {
|
|
|
1069
1094
|
record.updatedAtMs,
|
|
1070
1095
|
]);
|
|
1071
1096
|
}
|
|
1097
|
+
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
1098
|
+
const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2', [partition, cutoffMs]);
|
|
1099
|
+
return rows[0].cursor === null ? null : asNumber(rows[0].cursor);
|
|
1100
|
+
}
|
|
1072
1101
|
async listClientCursors(partition) {
|
|
1073
1102
|
const { rows } = await this.#exec.query('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=$1', [partition]);
|
|
1074
1103
|
return rows.map((r) => ({
|
package/dist/prune.d.ts
CHANGED
|
@@ -7,7 +7,9 @@
|
|
|
7
7
|
* least the newest `minRetainedCommits` commits are always retained.
|
|
8
8
|
*/
|
|
9
9
|
import { type SyncularServerEvents } from './events.js';
|
|
10
|
-
import type { ServerStorage } from './storage.js';
|
|
10
|
+
import type { CommitPruneQuery, ServerStorage } from './storage.js';
|
|
11
|
+
/** Shared validation for the built-in atomic pruning adapters. */
|
|
12
|
+
export declare function validateCommitPruneQuery(query: CommitPruneQuery): void;
|
|
11
13
|
export interface RetentionPolicy {
|
|
12
14
|
/** Active window for laggard cursors (default 14 days). */
|
|
13
15
|
readonly activeWindowMs: number;
|