@syncular/server 0.15.48 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -15,6 +15,11 @@ registry, command mutations use the ordinary serialized push path, and
15
15
  specified in [`docs/REMOTE.md`](../../docs/REMOTE.md) and the practical setup is
16
16
  in the [remote operations guide](https://syncular.dev/guide-remote-operations/).
17
17
 
18
+ Remote query registration requires generated `relationPlans` for each selected
19
+ SQL statement. Run `syncular generate` before upgrading existing query modules.
20
+ The server uses those boundaries to bind every physical table occurrence to
21
+ the authenticated partition, including quoted self joins and CTE bodies.
22
+
18
23
  Application intent belongs in immutable domain event rows written in the same
19
24
  commit as the state change. `SyncularServerEvents` below remains operational
20
25
  telemetry. See the [domain event guide](https://syncular.dev/guide-domain-events/).
@@ -927,10 +932,22 @@ at or below it. Nothing prunes automatically — the host schedules it.
927
932
 
928
933
  **When to run.** A periodic job per partition — hourly to daily is the
929
934
  sensible range; there is no benefit below the granularity of your
930
- `activeWindowMs`. Prune is cheap when there is nothing to do (one cursor
931
- scan + two point reads), so err on the side of running it often rather
932
- than letting a backlog build. Pass `events` to get `prune.completed`
933
- per pass.
935
+ `activeWindowMs`. Pass `events` to get `prune.completed` per pass.
936
+
937
+ Pruning verifies the captured log epoch and updates the horizon together with
938
+ commit/change/scope deletion in one transaction. Concurrent passes cannot
939
+ lower the horizon. A retry cleans up eligible records even when the horizon
940
+ already covers them. A restore invalidates a pending pass with
941
+ `sync.storage.prune_epoch_mismatch`; recompute its retention inputs.
942
+ Unregistered partitions cannot be pruned. D1 maintenance enters the owning
943
+ Durable Object's existing write queue through
944
+ `SyncularRealtimeHost.pruneCommitLog`.
945
+
946
+ Custom storage adapters must add `getPartitionLogEpoch(partition)` and replace
947
+ `pruneCommitsThrough(partition, seq)` with
948
+ `pruneCommitsThrough(partition, { logEpoch, throughSeq })`, returning
949
+ `{ previousHorizonSeq, horizonSeq, removedCommits }` from the transaction.
950
+ `setHorizonSeq` remains monotonic and is no longer used by the pruning helper.
934
951
 
935
952
  **The retention floors (§4.6, encoded in `RetentionPolicy`).** The
936
953
  horizon never advances past `min(cursor)` of *active* clients — clients
@@ -1149,3 +1166,23 @@ deterministic in-process sqlite loopback):
1149
1166
  ```sh
1150
1167
  SYNCULAR_PG_URL=postgres://user:pass@localhost:5432/db bun run bench
1151
1168
  ```
1169
+
1170
+ Custom storage adapters must implement
1171
+ `getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum cursor
1172
+ whose `updatedAtMs >= cutoffMs`, or `null` when no client qualifies. Preserve
1173
+ negative bootstrap cursors. Pruning and admin horizon status use this scalar
1174
+ aggregate; `listClientCursors` remains the explicit listing interface.
1175
+
1176
+ SQLite image builders now return `Promise<Uint8Array>` and receive
1177
+ `rowBatches`, an iterable or async iterable of row arrays. Replace custom
1178
+ builders' `input.rows` loop with `for await (const rows of input.rowBatches)`,
1179
+ insert each batch into the dedicated image database, and count rows during
1180
+ consumption. Write the final row count into `_syncular_segment` before
1181
+ serialization. Await `buildSqliteImage(input)` when calling the built-in
1182
+ Bun or Node builder directly.
1183
+
1184
+ The server shares in-flight builds for the same storage pair and artifact
1185
+ identity after authorization. Sharing is local to one process. Signed URL
1186
+ grants remain per request. The first eligibility probe has at most
1187
+ `limitSnapshotRows + 1` rows; subsequent builder batches have at most 5,000
1188
+ rows. The image database and serialized output still consume memory.
package/dist/admin.js CHANGED
@@ -173,11 +173,7 @@ export class SyncularAdmin {
173
173
  const nowMs = this.#clock();
174
174
  const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
175
175
  const horizonSeq = await this.#storage.getHorizonSeq(partition);
176
- const cursors = await this.#storage.listClientCursors(partition);
177
- const activeCursors = cursors
178
- .filter((c) => c.updatedAtMs >= nowMs - this.#retention.activeWindowMs)
179
- .map((c) => c.cursor);
180
- const activeCursorFloor = activeCursors.length > 0 ? Math.min(...activeCursors) : null;
176
+ const activeCursorFloor = await this.#storage.getActiveClientCursorFloor(partition, nowMs - this.#retention.activeWindowMs);
181
177
  const cursorFloor = activeCursorFloor ?? Number.MAX_SAFE_INTEGER;
182
178
  const forcedSeq = await this.#storage.getCommitSeqBefore(partition, nowMs - this.#retention.ageForceMs);
183
179
  const retainFloor = maxCommitSeq - this.#retention.minRetainedCommits;
@@ -9,12 +9,20 @@ export interface BoundAuthoritativeQuery {
9
9
  readonly params: readonly AuthoritativeQueryValue[];
10
10
  }
11
11
  declare const PARTITION_BIND: unique symbol;
12
- /**
13
- * Turn generated local SQL into a partition-local authoritative statement.
14
- * Only relations declared by the generated descriptor are rewritten. Values
15
- * remain parameters; request data is never interpolated into SQL.
16
- */
17
- export declare function prepareAuthoritativeQuery(sql: string, params: readonly AuthoritativeQueryValue[], declaredTables: readonly string[], tables: ReadonlyMap<string, CompiledTable>): PreparedAuthoritativeQuery;
12
+ /** Compiler-proven occurrences in the exact generated positional statement. */
13
+ export interface AuthoritativeRelationPlan {
14
+ readonly sql: string;
15
+ readonly relations: readonly {
16
+ readonly table: string;
17
+ readonly start: number;
18
+ readonly end: number;
19
+ readonly alias?: string;
20
+ }[];
21
+ }
22
+ /** Validate trusted generated metadata before registration or storage execution. */
23
+ export declare function validateAuthoritativeRelationPlan(plan: AuthoritativeRelationPlan, declaredTables: readonly string[]): void;
24
+ /** Bind every compiler-proven table occurrence to the authenticated partition. */
25
+ export declare function prepareAuthoritativeQuery(plan: AuthoritativeRelationPlan, params: readonly AuthoritativeQueryValue[], declaredTables: readonly string[], tables: ReadonlyMap<string, CompiledTable>): PreparedAuthoritativeQuery;
18
26
  export declare function bindAuthoritativePartition(prepared: PreparedAuthoritativeQuery, partition: string): BoundAuthoritativeQuery;
19
27
  export declare function postgresPlaceholders(sql: string): string;
20
28
  export {};
@@ -1,24 +1,5 @@
1
1
  import { quoteIdent, SYNC_PARTITION_COLUMN } from './relational-rows.js';
2
2
  const PARTITION_BIND = Symbol('syncular.authoritative_partition');
3
- const RESERVED_ALIAS = new Set([
4
- 'on',
5
- 'where',
6
- 'group',
7
- 'order',
8
- 'inner',
9
- 'left',
10
- 'right',
11
- 'full',
12
- 'outer',
13
- 'natural',
14
- 'join',
15
- 'cross',
16
- 'using',
17
- 'limit',
18
- 'having',
19
- ]);
20
- const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
21
- const TABLE_REF_RE = new RegExp(`\\b(FROM|(?:NATURAL\\s+)?(?:(?:LEFT|RIGHT|FULL)(?:\\s+OUTER)?|INNER|CROSS)?\\s*JOIN)\\s+((?:\\(\\s*)*)(${IDENT})(?:\\s+(?:AS\\s+)?((?!(?:${[...RESERVED_ALIAS].join('|')})\\b)${IDENT}))?`, 'gi');
22
3
  function protectedSqlEnd(sql, index) {
23
4
  const char = sql[index];
24
5
  const next = sql[index + 1];
@@ -48,97 +29,94 @@ function protectedSqlEnd(sql, index) {
48
29
  }
49
30
  return undefined;
50
31
  }
51
- function maskedSql(sql) {
52
- let out = '';
53
- let index = 0;
54
- while (index < sql.length) {
55
- const end = protectedSqlEnd(sql, index);
56
- if (end === undefined)
57
- out += sql[index];
58
- else
59
- out += sql.slice(index, end).replace(/[^\n]/g, ' ');
60
- index = end ?? index + 1;
32
+ /** Validate trusted generated metadata before registration or storage execution. */
33
+ export function validateAuthoritativeRelationPlan(plan, declaredTables) {
34
+ if (plan === undefined ||
35
+ typeof plan.sql !== 'string' ||
36
+ !Array.isArray(plan.relations)) {
37
+ throw new Error('registered query requires generated relation plans; regenerate queries');
61
38
  }
62
- return out;
63
- }
64
- /**
65
- * Turn generated local SQL into a partition-local authoritative statement.
66
- * Only relations declared by the generated descriptor are rewritten. Values
67
- * remain parameters; request data is never interpolated into SQL.
68
- */
69
- export function prepareAuthoritativeQuery(sql, params, declaredTables, tables) {
70
- if (maskedSql(sql).includes(';'))
71
- throw new Error('registered query must be one SELECT');
39
+ const sql = plan.sql;
72
40
  const declared = new Set(declaredTables);
73
- const masked = maskedSql(sql);
74
- const replacements = [];
75
41
  const found = new Set();
76
- for (const match of masked.matchAll(TABLE_REF_RE)) {
77
- const rawTable = match[3];
78
- const table = tables.get(rawTable);
79
- if (table === undefined)
80
- continue;
81
- if (!declared.has(table.name)) {
82
- throw new Error('registered query table metadata does not match its SQL');
42
+ let previousEnd = 0;
43
+ for (const relation of plan.relations) {
44
+ if (!Number.isSafeInteger(relation.start) ||
45
+ !Number.isSafeInteger(relation.end) ||
46
+ relation.start < previousEnd ||
47
+ relation.end <= relation.start ||
48
+ relation.end > sql.length) {
49
+ throw new Error('registered query relation boundaries do not match its SQL; regenerate queries');
83
50
  }
84
- if (!table.materialize) {
85
- throw new Error('registered query targets a non-materialized table');
51
+ previousEnd = relation.end;
52
+ if (!declared.has(relation.table)) {
53
+ throw new Error('registered query table metadata does not match its SQL');
86
54
  }
87
- let alias = match[4];
88
- if (alias !== undefined && RESERVED_ALIAS.has(alias.toLowerCase())) {
89
- alias = undefined;
55
+ const spelling = sql.slice(relation.start, relation.end);
56
+ const quote = spelling[0];
57
+ const name = quote === '['
58
+ ? spelling.slice(1, -1)
59
+ : quote === '"' || quote === '`'
60
+ ? spelling
61
+ .slice(1, -1)
62
+ .split(quote + quote)
63
+ .join(quote)
64
+ : spelling;
65
+ if (name.toLowerCase() !== relation.table.toLowerCase()) {
66
+ throw new Error('registered query relation name does not match its SQL; regenerate queries');
90
67
  }
91
- const matchStart = match.index ?? 0;
92
- const afterOperator = match[1].length;
93
- const relative = match[0]
94
- .toLowerCase()
95
- .indexOf(rawTable.toLowerCase(), afterOperator);
96
- const start = matchStart + relative;
97
- const projection = table.columns.map((column) => quoteIdent(column.name));
98
- replacements.push({
99
- start,
100
- end: start + rawTable.length,
101
- text: `(SELECT ${projection.join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=/*syncular_partition*/?)${alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
102
- });
103
- found.add(table.name);
68
+ found.add(relation.table);
104
69
  }
105
70
  if (found.size !== declared.size ||
106
71
  [...declared].some((table) => !found.has(table))) {
107
72
  throw new Error('registered query table metadata does not match its SQL');
108
73
  }
109
- let rewritten = sql;
110
- for (const replacement of replacements.sort((left, right) => right.start - left.start)) {
111
- rewritten =
112
- rewritten.slice(0, replacement.start) +
113
- replacement.text +
114
- rewritten.slice(replacement.end);
115
- }
74
+ }
75
+ /** Bind every compiler-proven table occurrence to the authenticated partition. */
76
+ export function prepareAuthoritativeQuery(plan, params, declaredTables, tables) {
77
+ validateAuthoritativeRelationPlan(plan, declaredTables);
78
+ const sql = plan.sql;
79
+ const replacements = plan.relations.map((relation) => {
80
+ const table = tables.get(relation.table);
81
+ if (table === undefined)
82
+ throw new Error('registered query targets an unknown table');
83
+ if (!table.materialize)
84
+ throw new Error('registered query targets a non-materialized table');
85
+ return {
86
+ start: relation.start,
87
+ end: relation.end,
88
+ text: `(SELECT ${table.columns.map((column) => quoteIdent(column.name)).join(', ')} FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?)${relation.alias === undefined ? ` AS ${quoteIdent(table.name)}` : ''}`,
89
+ };
90
+ });
116
91
  const bound = [];
117
92
  let anonymousIndex = 0;
118
93
  let rendered = '';
119
- for (let index = 0; index < rewritten.length; index += 1) {
120
- if (rewritten.startsWith('/*syncular_partition*/?', index)) {
121
- rendered += '?';
94
+ let nextRelation = 0;
95
+ for (let index = 0; index < sql.length; index += 1) {
96
+ const replacement = replacements[nextRelation];
97
+ if (replacement?.start === index) {
98
+ rendered += replacement.text;
122
99
  bound.push(PARTITION_BIND);
123
- index += '/*syncular_partition*/?'.length - 1;
100
+ index = replacement.end - 1;
101
+ nextRelation += 1;
124
102
  continue;
125
103
  }
126
- const protectedEnd = protectedSqlEnd(rewritten, index);
104
+ const protectedEnd = protectedSqlEnd(sql, index);
127
105
  if (protectedEnd !== undefined) {
128
- rendered += rewritten.slice(index, protectedEnd);
106
+ rendered += sql.slice(index, protectedEnd);
129
107
  index = protectedEnd - 1;
130
108
  continue;
131
109
  }
132
- const char = rewritten[index];
110
+ const char = sql[index];
133
111
  if (char !== '?') {
134
112
  rendered += char;
135
113
  continue;
136
114
  }
137
115
  let end = index + 1;
138
- while (end < rewritten.length && /[0-9]/.test(rewritten[end])) {
116
+ while (end < sql.length && /[0-9]/.test(sql[end])) {
139
117
  end += 1;
140
118
  }
141
- const numbered = rewritten.slice(index + 1, end);
119
+ const numbered = sql.slice(index + 1, end);
142
120
  const parameterIndex = numbered.length > 0
143
121
  ? Number.parseInt(numbered, 10) - 1
144
122
  : anonymousIndex++;
@@ -1,3 +1,4 @@
1
+ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
1
2
  import type { CompiledSchema, CompiledTable } from './schema.js';
2
3
  import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
3
4
  export interface D1PreparedStatement {
@@ -42,9 +43,10 @@ export declare class D1ServerStorage implements ServerStorage {
42
43
  begin(partition: string): Promise<StorageTransaction>;
43
44
  getMaxCommitSeq(partition: string): Promise<number>;
44
45
  queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
46
+ getPartitionLogEpoch(partition: string): Promise<string | undefined>;
45
47
  getHorizonSeq(partition: string): Promise<number>;
46
48
  setHorizonSeq(partition: string, seq: number): Promise<void>;
47
- pruneCommitsThrough(partition: string, seq: number): Promise<number>;
49
+ pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
48
50
  getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
49
51
  getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
50
52
  getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
@@ -61,6 +63,7 @@ export declare class D1ServerStorage implements ServerStorage {
61
63
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
62
64
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
63
65
  putClientRecord(partition: string, record: ClientRecord): Promise<void>;
66
+ getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
64
67
  listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
65
68
  listRowsReferencingBlob(partition: string, blobId: string): Promise<{
66
69
  readonly table: string;
@@ -1,4 +1,6 @@
1
1
  var _a;
2
+ import { validateCommitPruneQuery } from './prune.js';
3
+ import { StorageQueryError } from './storage-errors.js';
2
4
  /**
3
5
  * Cloudflare D1 server storage for Workers deployments.
4
6
  *
@@ -676,7 +678,7 @@ export class D1ServerStorage {
676
678
  if (this.#tables === undefined) {
677
679
  throw new Error('ensureSchema(schema) must run before registered queries');
678
680
  }
679
- const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.sql, query.params, query.tables, this.#tables), partition);
681
+ const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.plan, query.params, query.tables, this.#tables), partition);
680
682
  const results = await this.#db.batch([
681
683
  this.#db.prepare(prepared.sql).bind(...prepared.params),
682
684
  this.#db
@@ -707,6 +709,13 @@ export class D1ServerStorage {
707
709
  maxCommitSeq,
708
710
  };
709
711
  }
712
+ async getPartitionLogEpoch(partition) {
713
+ const row = await this.#db
714
+ .prepare('SELECT log_epoch FROM sync_partition_registry WHERE partition=?')
715
+ .bind(partition)
716
+ .first();
717
+ return row?.log_epoch;
718
+ }
710
719
  async getHorizonSeq(partition) {
711
720
  const row = await this.#db
712
721
  .prepare('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
@@ -716,27 +725,55 @@ export class D1ServerStorage {
716
725
  }
717
726
  async setHorizonSeq(partition, seq) {
718
727
  await this.#db
719
- .prepare('INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=excluded.horizon_seq')
728
+ .prepare('INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)')
720
729
  .bind(partition, seq)
721
730
  .run();
722
731
  }
723
- async pruneCommitsThrough(partition, seq) {
724
- const before = await this.#db
725
- .prepare('SELECT count(*) AS n FROM sync_commits WHERE partition=? AND commit_seq<=?')
726
- .bind(partition, seq)
727
- .first();
728
- await this.#db.batch([
732
+ async pruneCommitsThrough(partition, query) {
733
+ validateCommitPruneQuery(query);
734
+ if (!this.#pushApplySerialized)
735
+ throw new Error('D1 pruning requires externally serialized partition writes');
736
+ const epochGuard = 'EXISTS (SELECT 1 FROM sync_partition_registry WHERE partition=? AND log_epoch=?)';
737
+ const horizon = '(SELECT horizon_seq FROM sync_partitions WHERE partition=?)';
738
+ const results = await this.#db.batch([
729
739
  this.#db
730
- .prepare('DELETE FROM sync_commits WHERE partition=? AND commit_seq<=?')
731
- .bind(partition, seq),
740
+ .prepare(`SELECT log_epoch, coalesce(${horizon},0) AS previous_horizon_seq FROM sync_partition_registry WHERE partition=?`)
741
+ .bind(partition, partition),
732
742
  this.#db
733
- .prepare('DELETE FROM sync_changes WHERE partition=? AND commit_seq<=?')
734
- .bind(partition, seq),
743
+ .prepare(`INSERT INTO sync_partitions(partition,horizon_seq) SELECT ?,? WHERE ${epochGuard}
744
+ ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
745
+ .bind(partition, query.throughSeq, partition, query.logEpoch),
735
746
  this.#db
736
- .prepare('DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?')
737
- .bind(partition, seq),
747
+ .prepare(`SELECT horizon_seq, (SELECT count(*) FROM sync_commits WHERE partition=? AND commit_seq<=${horizon}) AS removed_commits FROM sync_partitions WHERE partition=?`)
748
+ .bind(partition, partition, partition),
749
+ ...['sync_commits', 'sync_changes', 'sync_change_scopes'].map((table) => this.#db
750
+ .prepare(`DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`)
751
+ .bind(partition, partition, partition, query.logEpoch)),
738
752
  ]);
739
- return before?.n ?? 0;
753
+ const [before, after] = [results[0], results[2]].map((result) => {
754
+ if (typeof result !== 'object' ||
755
+ result === null ||
756
+ !('results' in result) ||
757
+ !Array.isArray(result.results)) {
758
+ throw new Error('D1 pruning returned an invalid batch result');
759
+ }
760
+ const row = result.results[0];
761
+ return typeof row === 'object' && row !== null
762
+ ? row
763
+ : undefined;
764
+ });
765
+ if (before?.log_epoch !== query.logEpoch)
766
+ throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
767
+ if (typeof before.previous_horizon_seq !== 'number' ||
768
+ typeof after?.horizon_seq !== 'number' ||
769
+ typeof after.removed_commits !== 'number') {
770
+ throw new Error('D1 pruning returned invalid horizon metadata');
771
+ }
772
+ return {
773
+ previousHorizonSeq: before.previous_horizon_seq,
774
+ horizonSeq: after.horizon_seq,
775
+ removedCommits: after.removed_commits,
776
+ };
740
777
  }
741
778
  async getCommitSeqBefore(partition, createdBeforeMs) {
742
779
  const row = await this.#db
@@ -1000,6 +1037,13 @@ export class D1ServerStorage {
1000
1037
  .bind(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs)
1001
1038
  .run();
1002
1039
  }
1040
+ async getActiveClientCursorFloor(partition, cutoffMs) {
1041
+ const row = await this.#db
1042
+ .prepare('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?')
1043
+ .bind(partition, cutoffMs)
1044
+ .first();
1045
+ return row.cursor;
1046
+ }
1003
1047
  async listClientCursors(partition) {
1004
1048
  const { results } = await this.#db
1005
1049
  .prepare('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=?')
@@ -1,3 +1,4 @@
1
+ import { type AuthoritativeRelationPlan } from './authoritative-query.js';
1
2
  import { type RemoteOperationResponse, type RowValue } from '@syncular/core';
2
3
  import type { SyncRequestContext } from './context.js';
3
4
  import type { AuthoritativeQueryValue } from './storage.js';
@@ -20,6 +21,7 @@ export interface AuthoritativeQueryDescriptor<Params = undefined> {
20
21
  readonly hasParams: boolean;
21
22
  readonly sql: string;
22
23
  readonly tables: readonly string[];
24
+ readonly relationPlans: readonly AuthoritativeRelationPlan[];
23
25
  readonly resultColumns: readonly {
24
26
  readonly name: string;
25
27
  readonly type: 'string' | 'integer' | 'float' | 'boolean' | 'json' | 'bytes' | 'blob_ref' | 'crdt';
@@ -1,3 +1,4 @@
1
+ import { validateAuthoritativeRelationPlan, } from './authoritative-query.js';
1
2
  import { decodeRow, decodeRemoteOperationRequest, encodeRow, encodeRemoteOperationResponse, } from '@syncular/core';
2
3
  import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
3
4
  import { SyncError, syncError } from './errors.js';
@@ -97,13 +98,19 @@ function normalizeQueryRows(rows, columns) {
97
98
  }
98
99
  /** Register one generated named query as an authoritative remote operation. */
99
100
  export function registerRemoteQuery(descriptor, options) {
100
- if (descriptor.id.length === 0 ||
101
+ if (!Array.isArray(descriptor.relationPlans) ||
102
+ !descriptor.relationPlans.some((plan) => plan.sql === descriptor.sql) ||
103
+ descriptor.relationPlans.some((plan) => !Array.isArray(plan.relations)) ||
104
+ descriptor.id.length === 0 ||
101
105
  new Set(descriptor.tables).size !== descriptor.tables.length ||
102
106
  !Array.isArray(descriptor.resultColumns) ||
103
107
  descriptor.resultColumns.length === 0 ||
104
108
  new Set(descriptor.resultColumns.map((column) => column.name)).size !==
105
109
  descriptor.resultColumns.length) {
106
- throw new Error('remote query requires a non-empty id and unique tables and result columns');
110
+ throw new Error('remote query requires generated relation plans, a non-empty id, and unique tables and result columns; regenerate queries');
111
+ }
112
+ for (const plan of descriptor.relationPlans) {
113
+ validateAuthoritativeRelationPlan(plan, descriptor.tables);
107
114
  }
108
115
  if (!Number.isSafeInteger(options.maxRows) ||
109
116
  options.maxRows < 1 ||
@@ -173,12 +180,24 @@ export function registerRemoteQuery(descriptor, options) {
173
180
  if (ctx.storage.queryAuthoritative === undefined) {
174
181
  throw syncError('operation.storage_unsupported', 'configured storage does not implement authoritative queries');
175
182
  }
176
- await ctx.storage.ensureSchema(schema);
177
183
  const selectedSql = descriptor.sqlFor?.(params) ?? descriptor.sql;
184
+ const plan = descriptor.relationPlans.find((candidate) => candidate.sql === selectedSql);
185
+ if (plan === undefined) {
186
+ throw syncError('operation.invalid_request', 'selected SQL has no generated relation plan; regenerate queries');
187
+ }
188
+ await ctx.storage.ensureSchema(schema);
189
+ const prefix = 'SELECT * FROM (';
178
190
  let result;
179
191
  try {
180
192
  result = await ctx.storage.queryAuthoritative(ctx.partition, {
181
- sql: `SELECT * FROM (${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
193
+ plan: {
194
+ sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
195
+ relations: plan.relations.map((relation) => ({
196
+ ...relation,
197
+ start: relation.start + prefix.length,
198
+ end: relation.end + prefix.length,
199
+ })),
200
+ },
182
201
  params: [...descriptor.bind(params), options.maxRows + 1],
183
202
  tables: descriptor.tables,
184
203
  });
@@ -1,3 +1,4 @@
1
+ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
1
2
  import { type PgExecutor } from './pg-executor.js';
2
3
  import type { CompiledSchema, CompiledTable } from './schema.js';
3
4
  import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
@@ -46,9 +47,10 @@ export declare class PostgresServerStorage implements ServerStorage {
46
47
  begin(partition: string): Promise<StorageTransaction>;
47
48
  getMaxCommitSeq(partition: string): Promise<number>;
48
49
  queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
50
+ getPartitionLogEpoch(partition: string): Promise<string | undefined>;
49
51
  getHorizonSeq(partition: string): Promise<number>;
50
52
  setHorizonSeq(partition: string, seq: number): Promise<void>;
51
- pruneCommitsThrough(partition: string, seq: number): Promise<number>;
53
+ pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
52
54
  getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
53
55
  getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
54
56
  getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
@@ -65,6 +67,7 @@ export declare class PostgresServerStorage implements ServerStorage {
65
67
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
66
68
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
67
69
  putClientRecord(partition: string, record: ClientRecord): Promise<void>;
70
+ getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
68
71
  listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
69
72
  listRowsReferencingBlob(partition: string, blobId: string): Promise<{
70
73
  readonly table: string;
@@ -1,3 +1,5 @@
1
+ import { validateCommitPruneQuery } from './prune.js';
2
+ import { StorageQueryError } from './storage-errors.js';
1
3
  import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
2
4
  import { syncError } from './errors.js';
3
5
  import { asBytes, asNumber, } from './pg-executor.js';
@@ -107,6 +109,11 @@ CREATE TABLE IF NOT EXISTS sync_blob_refs(
107
109
  CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
108
110
  ON sync_blob_refs(partition, blob_id);
109
111
  `;
112
+ async function lockPartitionOn(client, partition) {
113
+ await client.query(`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
114
+ ON CONFLICT (partition) DO NOTHING`, [partition]);
115
+ await client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [partition]);
116
+ }
110
117
  function toBase64(bytes) {
111
118
  return Buffer.from(bytes).toString('base64');
112
119
  }
@@ -381,9 +388,7 @@ class PostgresTransaction {
381
388
  }
382
389
  async lockPartitionForPush() {
383
390
  this.#assertOpen();
384
- await this.#client.query(`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
385
- ON CONFLICT (partition) DO NOTHING`, [this.#partition]);
386
- await this.#client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [this.#partition]);
391
+ await lockPartitionOn(this.#client, this.#partition);
387
392
  await this.#client.query('SAVEPOINT syncular_push_candidate');
388
393
  this.#pushApplySavepoint = true;
389
394
  }
@@ -661,6 +666,7 @@ export class PostgresServerStorage {
661
666
  if (logEpoch.length === 0)
662
667
  throw new Error('log epoch must be non-empty');
663
668
  await this.#exec.transaction(async (client) => {
669
+ await lockPartitionOn(client, partition);
664
670
  await client.query(`INSERT INTO sync_partition_registry(
665
671
  partition, log_epoch, epoch_required, last_authenticated_at_ms
666
672
  ) VALUES ($1,$2,TRUE,$3)
@@ -732,7 +738,7 @@ export class PostgresServerStorage {
732
738
  if (this.#tables === undefined) {
733
739
  throw new Error('ensureSchema(schema) must run before registered queries');
734
740
  }
735
- const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.sql, query.params, query.tables, this.#tables), partition);
741
+ const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.plan, query.params, query.tables, this.#tables), partition);
736
742
  return this.#exec.transaction(async (client) => {
737
743
  await client.query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY');
738
744
  const result = await client.query(postgresPlaceholders(prepared.sql), prepared.params);
@@ -745,19 +751,38 @@ export class PostgresServerStorage {
745
751
  };
746
752
  });
747
753
  }
754
+ async getPartitionLogEpoch(partition) {
755
+ const { rows } = await this.#exec.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=$1', [partition]);
756
+ return rows[0]?.log_epoch;
757
+ }
748
758
  async getHorizonSeq(partition) {
749
759
  const { rows } = await this.#exec.query('SELECT horizon_seq FROM sync_partitions WHERE partition=$1', [partition]);
750
760
  return rows[0] === undefined ? 0 : asNumber(rows[0].horizon_seq);
751
761
  }
752
762
  async setHorizonSeq(partition, seq) {
753
763
  await this.#exec.query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
754
- ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`, [partition, seq]);
764
+ ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`, [partition, seq]);
755
765
  }
756
- async pruneCommitsThrough(partition, seq) {
757
- const removed = await this.#exec.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2', [partition, seq]);
758
- await this.#exec.query('DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2', [partition, seq]);
759
- await this.#exec.query('DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2', [partition, seq]);
760
- return removed.rowCount;
766
+ async pruneCommitsThrough(partition, query) {
767
+ validateCommitPruneQuery(query);
768
+ return this.#exec.transaction(async (client) => {
769
+ await lockPartitionOn(client, partition);
770
+ const epoch = await client.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE', [partition]);
771
+ if (epoch.rows[0]?.log_epoch !== query.logEpoch)
772
+ throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
773
+ const previous = await client.query('SELECT horizon_seq FROM sync_partitions WHERE partition=$1', [partition]);
774
+ const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
775
+ const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
776
+ await client.query('UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1', [partition, horizonSeq]);
777
+ const removed = await client.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
778
+ await client.query('DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
779
+ await client.query('DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
780
+ return {
781
+ previousHorizonSeq,
782
+ horizonSeq,
783
+ removedCommits: removed.rowCount,
784
+ };
785
+ });
761
786
  }
762
787
  async getCommitSeqBefore(partition, createdBeforeMs) {
763
788
  const { rows } = await this.#exec.query('SELECT max(commit_seq) AS seq FROM sync_commits WHERE partition=$1 AND created_at_ms<$2', [partition, createdBeforeMs]);
@@ -1069,6 +1094,10 @@ export class PostgresServerStorage {
1069
1094
  record.updatedAtMs,
1070
1095
  ]);
1071
1096
  }
1097
+ async getActiveClientCursorFloor(partition, cutoffMs) {
1098
+ const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2', [partition, cutoffMs]);
1099
+ return rows[0].cursor === null ? null : asNumber(rows[0].cursor);
1100
+ }
1072
1101
  async listClientCursors(partition) {
1073
1102
  const { rows } = await this.#exec.query('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=$1', [partition]);
1074
1103
  return rows.map((r) => ({
package/dist/prune.d.ts CHANGED
@@ -7,7 +7,9 @@
7
7
  * least the newest `minRetainedCommits` commits are always retained.
8
8
  */
9
9
  import { type SyncularServerEvents } from './events.js';
10
- import type { ServerStorage } from './storage.js';
10
+ import type { CommitPruneQuery, ServerStorage } from './storage.js';
11
+ /** Shared validation for the built-in atomic pruning adapters. */
12
+ export declare function validateCommitPruneQuery(query: CommitPruneQuery): void;
11
13
  export interface RetentionPolicy {
12
14
  /** Active window for laggard cursors (default 14 days). */
13
15
  readonly activeWindowMs: number;