@syncular/server 0.82.3 → 0.84.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,18 @@ import { validateCommitPruneQuery } from './prune';
2
2
  import { StorageQueryError } from './storage-errors';
3
3
  import type { CommitPruneQuery, CommitPruneResult } from './storage';
4
4
  import {
5
- clientRoundState,
5
+ asJson,
6
+ CLIENT_RECORD_COLUMNS,
7
+ pushResultFromJson,
8
+ pushResultToJson,
9
+ type StoredChangeRecord,
10
+ type StoredReactionRecord,
11
+ type StoredRowRecord,
12
+ toClientRecord,
13
+ toStoredChange,
14
+ toStoredReaction,
15
+ toStoredRow,
16
+ toTombstone,
6
17
  PARTITION_TOUCH_INTERVAL_MS,
7
18
  commitDeleteFloor,
8
19
  rowsLiveAfterDelete,
@@ -46,11 +57,7 @@ import { serveNotReadyError } from './readiness';
46
57
  * would leave gaps on rollback, which the pull-window arithmetic in §4.5
47
58
  * does not tolerate). Cross-partition pushes never contend.
48
59
  */
49
- import {
50
- decodeRow,
51
- type PushOperationResult,
52
- type UnitScopes,
53
- } from '@syncular/core';
60
+ import { decodeRow, type UnitScopes } from '@syncular/core';
54
61
  import {
55
62
  bindAuthoritativePartition,
56
63
  type BoundAuthoritativeQuery,
@@ -121,7 +128,6 @@ import type {
121
128
  CommitDetail,
122
129
  CommitMetadata,
123
130
  CommitMetadataQuery,
124
- DurableJsonValue,
125
131
  IndexRowScanQuery,
126
132
  LiveWindow,
127
133
  LiveWindowQuery,
@@ -136,7 +142,6 @@ import type {
136
142
  PushApplyCommit,
137
143
  PushWatermark,
138
144
  ReactionClaimQuery,
139
- ReactionFailure,
140
145
  ReactionFailureUpdate,
141
146
  ReactionListQuery,
142
147
  ReactionPruneQuery,
@@ -151,7 +156,6 @@ import type {
151
156
  ServerStorage,
152
157
  ServeGate,
153
158
  StorageTransaction,
154
- StoredChange,
155
159
  StoredCheckpoint,
156
160
  StoredPushResult,
157
161
  StoredReaction,
@@ -347,18 +351,6 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
347
351
  );
348
352
  `;
349
353
 
350
- interface SerializedResult {
351
- opIndex: number;
352
- status: string;
353
- code?: string;
354
- message?: string;
355
- serverVersion?: number;
356
- serverRow?: string;
357
- conflictColumns?: string;
358
- retryable?: boolean;
359
- details?: import('@syncular/core').RejectionDetails;
360
- }
361
-
362
354
  /** Lock the partition row; returns its `max_commit_seq`, which the lock freezes. */
363
355
  async function lockPartitionOn(
364
356
  client: PgQueryable,
@@ -727,10 +719,6 @@ function toBase64(bytes: Uint8Array): string {
727
719
  return Buffer.from(bytes).toString('base64');
728
720
  }
729
721
 
730
- function fromBase64(text: string): Uint8Array {
731
- return new Uint8Array(Buffer.from(text, 'base64'));
732
- }
733
-
734
722
  interface PostgresCheckpointRecord {
735
723
  partition: string;
736
724
  name: string;
@@ -757,260 +745,6 @@ function toStoredCheckpoint(
757
745
  };
758
746
  }
759
747
 
760
- interface PostgresReactionRecord {
761
- idempotency_key: string;
762
- type: string;
763
- version: unknown;
764
- payload: unknown;
765
- source_client_id: string;
766
- source_client_commit_id: string;
767
- source_commit_seq: unknown;
768
- created_at_ms: unknown;
769
- available_at_ms: unknown;
770
- status: StoredReaction['status'];
771
- attempts: unknown;
772
- max_attempts: unknown;
773
- lease_owner: string | null;
774
- lease_expires_at_ms: unknown | null;
775
- completed_at_ms: unknown | null;
776
- last_failure: unknown | null;
777
- }
778
-
779
- function toStoredReaction(record: PostgresReactionRecord): StoredReaction {
780
- return {
781
- idempotencyKey: record.idempotency_key,
782
- type: record.type,
783
- version: asNumber(record.version),
784
- payload: asJson(record.payload) as DurableJsonValue,
785
- sourceClientId: record.source_client_id,
786
- sourceClientCommitId: record.source_client_commit_id,
787
- sourceCommitSeq: asNumber(record.source_commit_seq),
788
- createdAtMs: asNumber(record.created_at_ms),
789
- maxAttempts: asNumber(record.max_attempts),
790
- status: record.status,
791
- attempts: asNumber(record.attempts),
792
- availableAtMs: asNumber(record.available_at_ms),
793
- ...(record.lease_owner !== null ? { leaseOwner: record.lease_owner } : {}),
794
- ...(record.lease_expires_at_ms !== null
795
- ? { leaseExpiresAtMs: asNumber(record.lease_expires_at_ms) }
796
- : {}),
797
- ...(record.completed_at_ms !== null
798
- ? { completedAtMs: asNumber(record.completed_at_ms) }
799
- : {}),
800
- ...(record.last_failure !== null
801
- ? { lastFailure: asJson(record.last_failure) as ReactionFailure }
802
- : {}),
803
- };
804
- }
805
-
806
- /**
807
- * Serialize a push result to a JSON-able object (stored in a JSONB column).
808
- * `serverRow` bytes are base64-encoded — JSONB cannot hold raw bytes.
809
- */
810
- function serializePushResult(result: StoredPushResult): unknown {
811
- return {
812
- status: result.status,
813
- ...(result.commitSeq !== undefined ? { commitSeq: result.commitSeq } : {}),
814
- ...(result.recordedAtMs !== undefined
815
- ? { recordedAtMs: result.recordedAtMs }
816
- : {}),
817
- ...(result.cacheIdentity !== undefined
818
- ? { cacheIdentity: result.cacheIdentity }
819
- : {}),
820
- ...(result.conflictAuthorization !== undefined
821
- ? { conflictAuthorization: result.conflictAuthorization }
822
- : {}),
823
- results: result.results.map((record) => {
824
- if (record.status === 'conflict') {
825
- return {
826
- opIndex: record.opIndex,
827
- status: record.status,
828
- code: record.code,
829
- message: record.message,
830
- serverVersion: record.serverVersion,
831
- serverRow: toBase64(record.serverRow),
832
- conflictColumns: toBase64(record.conflictColumns),
833
- };
834
- }
835
- if (record.status === 'error') {
836
- return {
837
- opIndex: record.opIndex,
838
- status: record.status,
839
- code: record.code,
840
- message: record.message,
841
- retryable: record.retryable,
842
- ...(record.details !== undefined ? { details: record.details } : {}),
843
- };
844
- }
845
- return { opIndex: record.opIndex, status: record.status };
846
- }),
847
- };
848
- }
849
-
850
- function deserializePushResult(value: unknown): StoredPushResult {
851
- const parsed = value as {
852
- status: 'applied' | 'rejected';
853
- commitSeq?: number;
854
- recordedAtMs?: number;
855
- cacheIdentity?: string;
856
- conflictAuthorization?: StoredPushResult['conflictAuthorization'];
857
- results: SerializedResult[];
858
- };
859
- const results: PushOperationResult[] = parsed.results.map((record) => {
860
- if (record.status === 'conflict') {
861
- return {
862
- opIndex: record.opIndex,
863
- status: 'conflict',
864
- code: record.code ?? '',
865
- message: record.message ?? '',
866
- serverVersion: record.serverVersion ?? 0,
867
- serverRow: fromBase64(record.serverRow ?? ''),
868
- conflictColumns: fromBase64(record.conflictColumns ?? ''),
869
- };
870
- }
871
- if (record.status === 'error') {
872
- return {
873
- opIndex: record.opIndex,
874
- status: 'error',
875
- code: record.code ?? '',
876
- message: record.message ?? '',
877
- retryable: record.retryable ?? false,
878
- ...(record.details !== undefined ? { details: record.details } : {}),
879
- };
880
- }
881
- return { opIndex: record.opIndex, status: 'applied' };
882
- });
883
- return {
884
- status: parsed.status,
885
- ...(parsed.commitSeq !== undefined ? { commitSeq: parsed.commitSeq } : {}),
886
- ...(parsed.recordedAtMs !== undefined
887
- ? { recordedAtMs: parsed.recordedAtMs }
888
- : {}),
889
- ...(parsed.cacheIdentity !== undefined
890
- ? { cacheIdentity: parsed.cacheIdentity }
891
- : {}),
892
- ...(parsed.conflictAuthorization !== undefined
893
- ? { conflictAuthorization: parsed.conflictAuthorization }
894
- : {}),
895
- results,
896
- };
897
- }
898
-
899
- /**
900
- * Some drivers return a JSONB column already parsed (pglite, node-postgres);
901
- * accept a string too for defensiveness.
902
- */
903
- function asJson<T>(value: unknown): T {
904
- if (typeof value === 'string') return JSON.parse(value) as T;
905
- return value as T;
906
- }
907
-
908
- interface RowRecord {
909
- row_id: string;
910
- server_version: unknown;
911
- scopes: unknown;
912
- payload: unknown;
913
- column_versions: unknown;
914
- }
915
-
916
- interface ChangeRecord {
917
- tbl: string;
918
- row_id: string;
919
- op: unknown;
920
- row_version: unknown;
921
- scopes: unknown;
922
- payload: unknown;
923
- }
924
-
925
- function toStoredRow(record: RowRecord): StoredRow {
926
- return {
927
- rowId: record.row_id,
928
- serverVersion: asNumber(record.server_version),
929
- scopes: asJson<Record<string, string>>(record.scopes),
930
- payload: asBytes(record.payload),
931
- ...(record.column_versions === null || record.column_versions === undefined
932
- ? {}
933
- : { columnVersions: asBytes(record.column_versions) }),
934
- };
935
- }
936
-
937
- function toTombstone(
938
- seq: unknown,
939
- version: unknown,
940
- scopes: unknown,
941
- ): Tombstone | null {
942
- if (seq === null || seq === undefined) return null;
943
- return {
944
- commitSeq: asNumber(seq),
945
- ...(version === null || version === undefined
946
- ? {}
947
- : { serverVersion: asNumber(version) }),
948
- ...(scopes === null || scopes === undefined
949
- ? {}
950
- : { scopes: asJson<Record<string, string>>(scopes) }),
951
- };
952
- }
953
-
954
- const CLIENT_RECORD_COLUMNS =
955
- 'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
956
-
957
- function toClientRecord(record: {
958
- client_id: string;
959
- actor_id: string;
960
- wire_version: unknown;
961
- cursor: unknown;
962
- updated_at_ms: unknown;
963
- schema_version: unknown;
964
- units: unknown;
965
- registration_digest: unknown;
966
- catching_up: unknown;
967
- last_round_at_ms: unknown;
968
- last_error_code: string | null;
969
- consecutive_failures: unknown;
970
- }): ClientRecord {
971
- return {
972
- clientId: record.client_id,
973
- actorId: record.actor_id,
974
- wireVersion: asNumber(record.wire_version),
975
- cursor: asNumber(record.cursor),
976
- updatedAtMs: asNumber(record.updated_at_ms),
977
- ...(record.schema_version === null
978
- ? {}
979
- : { schemaVersion: asNumber(record.schema_version) }),
980
- ...(record.units === null || record.registration_digest === null
981
- ? {}
982
- : {
983
- registration: {
984
- units: asJson<UnitScopes[]>(record.units),
985
- digest: asBytes(record.registration_digest),
986
- },
987
- }),
988
- catchingUp: asJson<ClientRecord['catchingUp']>(record.catching_up),
989
- ...clientRoundState(
990
- record.last_round_at_ms === null
991
- ? null
992
- : asNumber(record.last_round_at_ms),
993
- record.last_error_code,
994
- asNumber(record.consecutive_failures),
995
- ),
996
- };
997
- }
998
-
999
- function toStoredChange(record: ChangeRecord): StoredChange {
1000
- return {
1001
- table: record.tbl,
1002
- rowId: record.row_id,
1003
- op: asNumber(record.op) === 1 ? 'upsert' : 'delete',
1004
- ...(record.row_version !== null && record.row_version !== undefined
1005
- ? { rowVersion: asNumber(record.row_version) }
1006
- : {}),
1007
- scopes: asJson<Record<string, string>>(record.scopes),
1008
- ...(record.payload !== null && record.payload !== undefined
1009
- ? { payload: asBytes(record.payload) }
1010
- : {}),
1011
- };
1012
- }
1013
-
1014
748
  /**
1015
749
  * Migration rewrite: keyset-paged walk of a row table inside the migration
1016
750
  * transaction. When `oldLayout` is
@@ -1120,7 +854,7 @@ async function getRowOn(
1120
854
  partition: string,
1121
855
  rowId: string,
1122
856
  ): Promise<StoredRow | undefined> {
1123
- const { rows } = await q.query<RowRecord>(
857
+ const { rows } = await q.query<StoredRowRecord>(
1124
858
  selectRowSql(compiled, 'postgres'),
1125
859
  [partition, rowId],
1126
860
  );
@@ -1144,7 +878,10 @@ async function scanRowsByIndexOn(
1144
878
  query.limit,
1145
879
  'postgres',
1146
880
  );
1147
- const { rows } = await q.query<RowRecord>(statement.sql, statement.params);
881
+ const { rows } = await q.query<StoredRowRecord>(
882
+ statement.sql,
883
+ statement.params,
884
+ );
1148
885
  return rows.map(toStoredRow);
1149
886
  }
1150
887
 
@@ -1455,7 +1192,7 @@ async function getPushResultOn(
1455
1192
  function toPersistedPushResult(value: unknown): StoredPushResult | undefined {
1456
1193
  if (value === null || value === undefined) return undefined;
1457
1194
  try {
1458
- return deserializePushResult(asJson(value));
1195
+ return pushResultFromJson(asJson(value));
1459
1196
  } catch {
1460
1197
  throw syncError(
1461
1198
  'sync.idempotency_cache_miss',
@@ -1821,7 +1558,7 @@ class PostgresTransaction implements StorageTransaction {
1821
1558
  if (missing.length === 0) return rows;
1822
1559
  // One statement for every id neither the staged writes nor the
1823
1560
  // admission read answered (see `getRow`).
1824
- const { rows: records } = await this.#serial.query<RowRecord>(
1561
+ const { rows: records } = await this.#serial.query<StoredRowRecord>(
1825
1562
  selectRowsSql(this.#resolveTable(table), 'postgres'),
1826
1563
  [this.#partition, JSON.stringify(missing)],
1827
1564
  );
@@ -1870,11 +1607,13 @@ class PostgresTransaction implements StorageTransaction {
1870
1607
  );
1871
1608
  this.#prefetchedTombstones.set(
1872
1609
  key,
1873
- toTombstone(
1874
- record.tombstone_seq,
1875
- record.tombstone_version,
1876
- record.tombstone_scopes,
1877
- ),
1610
+ record.tombstone_seq === null || record.tombstone_seq === undefined
1611
+ ? null
1612
+ : toTombstone({
1613
+ commit_seq: record.tombstone_seq,
1614
+ server_version: record.tombstone_version,
1615
+ scopes: record.tombstone_scopes,
1616
+ }),
1878
1617
  );
1879
1618
  }
1880
1619
  }
@@ -1895,10 +1634,9 @@ class PostgresTransaction implements StorageTransaction {
1895
1634
  [this.#partition, table, rowId],
1896
1635
  );
1897
1636
  const row = rows[0];
1898
- return (
1899
- toTombstone(row?.commit_seq, row?.server_version, row?.scopes) ??
1900
- undefined
1901
- );
1637
+ return row === undefined || row.commit_seq === null
1638
+ ? undefined
1639
+ : toTombstone(row);
1902
1640
  }
1903
1641
 
1904
1642
  async clearTombstone(table: string, rowId: string): Promise<void> {
@@ -1937,7 +1675,7 @@ class PostgresTransaction implements StorageTransaction {
1937
1675
  let afterRowId = query.afterRowId ?? '';
1938
1676
  const batchSize = Math.max(64, query.limit);
1939
1677
  while (rows.length < query.limit) {
1940
- const { rows: records } = await client.query<RowRecord>(sql, [
1678
+ const { rows: records } = await client.query<StoredRowRecord>(sql, [
1941
1679
  this.#partition,
1942
1680
  query.table,
1943
1681
  variable,
@@ -2130,7 +1868,7 @@ class PostgresTransaction implements StorageTransaction {
2130
1868
  this.#pushResults.push({
2131
1869
  client_id: clientId,
2132
1870
  client_commit_id: clientCommitId,
2133
- result: serializePushResult(result),
1871
+ result: pushResultToJson(result),
2134
1872
  outbox_id: sequence?.outboxId ?? null,
2135
1873
  client_seq: sequence?.clientSeq ?? null,
2136
1874
  });
@@ -2620,9 +2358,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2620
2358
  ? undefined
2621
2359
  : asNumber(marker.schema_version);
2622
2360
  if (stored !== undefined && stored > schema.version) {
2623
- throw new Error(
2624
- `stored schema version ${stored} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
2625
- );
2361
+ throw new StorageQueryError('sync.storage.schema_changed', {
2362
+ storedVersion: String(stored),
2363
+ configuredVersion: String(schema.version),
2364
+ });
2626
2365
  }
2627
2366
  if (stored !== undefined && stored === schema.version) {
2628
2367
  const storedLayouts =
@@ -2710,9 +2449,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2710
2449
  ? undefined
2711
2450
  : asNumber(lockedMarker[0].schema_version);
2712
2451
  if (lockedVersion !== undefined && lockedVersion > schema.version) {
2713
- throw new Error(
2714
- `stored schema version ${lockedVersion} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
2715
- );
2452
+ throw new StorageQueryError('sync.storage.schema_changed', {
2453
+ storedVersion: String(lockedVersion),
2454
+ configuredVersion: String(schema.version),
2455
+ });
2716
2456
  }
2717
2457
  if (lockedVersion === schema.version) return;
2718
2458
  const lockedLayouts =
@@ -3309,7 +3049,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3309
3049
  const limitParam = nowParam + 1;
3310
3050
  const workerParam = limitParam + 1;
3311
3051
  const expiresParam = workerParam + 1;
3312
- const { rows } = await this.#exec.query<PostgresReactionRecord>(
3052
+ const { rows } = await this.#exec.query<StoredReactionRecord>(
3313
3053
  `WITH due AS (
3314
3054
  SELECT partition, idempotency_key
3315
3055
  FROM sync_reactions
@@ -3427,7 +3167,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3427
3167
  partition: string,
3428
3168
  idempotencyKey: string,
3429
3169
  ): Promise<StoredReaction | undefined> {
3430
- const { rows } = await this.#exec.query<PostgresReactionRecord>(
3170
+ const { rows } = await this.#exec.query<StoredReactionRecord>(
3431
3171
  'SELECT * FROM sync_reactions WHERE partition=$1 AND idempotency_key=$2',
3432
3172
  [partition, idempotencyKey],
3433
3173
  );
@@ -3455,7 +3195,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3455
3195
  params.push(...query.types);
3456
3196
  }
3457
3197
  params.push(query.limit);
3458
- const { rows } = await this.#exec.query<PostgresReactionRecord>(
3198
+ const { rows } = await this.#exec.query<StoredReactionRecord>(
3459
3199
  `SELECT * FROM sync_reactions WHERE ${where.join(' AND ')}
3460
3200
  ORDER BY created_at_ms DESC, idempotency_key DESC LIMIT $${params.length}`,
3461
3201
  params,
@@ -3598,7 +3338,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3598
3338
  },
3599
3339
  async (keys, commitSeqs) => {
3600
3340
  const { rows } = await this.#exec.query<
3601
- ChangeRecord & { commit_seq: unknown }
3341
+ StoredChangeRecord & { commit_seq: unknown }
3602
3342
  >(
3603
3343
  `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
3604
3344
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
@@ -3646,7 +3386,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3646
3386
  let afterRowId = query.afterRowId ?? '';
3647
3387
  const batchSize = Math.max(64, query.limit);
3648
3388
  while (rows.length < query.limit) {
3649
- const records = await this.#batchedPage<RowRecord>(
3389
+ const records = await this.#batchedPage<StoredRowRecord>(
3650
3390
  'scanRows',
3651
3391
  (specs) =>
3652
3392
  scanRowBatchSql(
@@ -4070,7 +3810,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
4070
3810
  );
4071
3811
  const meta = rows[0];
4072
3812
  if (meta === undefined) return undefined;
4073
- const { rows: changeRows } = await this.#exec.query<ChangeRecord>(
3813
+ const { rows: changeRows } = await this.#exec.query<StoredChangeRecord>(
4074
3814
  'SELECT tbl, row_id, op, row_version, scopes, payload FROM sync_changes WHERE partition=$1 AND commit_seq=$2 ORDER BY idx',
4075
3815
  [partition, commitSeq],
4076
3816
  );
@@ -4124,7 +3864,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
4124
3864
  async listRows(partition: string, query: RowListQuery): Promise<StoredRow[]> {
4125
3865
  // Keyset page over the row table's (partition, rowId) primary key: an
4126
3866
  // index range read, no scope filter, no per-row lookup.
4127
- const { rows } = await this.#exec.query<RowRecord>(
3867
+ const { rows } = await this.#exec.query<StoredRowRecord>(
4128
3868
  `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
4129
3869
  FROM ${quoteIdent(this.table(query.table).name)}
4130
3870
  WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>$2
@@ -4159,7 +3899,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
4159
3899
  try {
4160
3900
  return {
4161
3901
  clientId: row.client_id,
4162
- result: deserializePushResult(asJson(row.result)),
3902
+ result: pushResultFromJson(asJson(row.result)),
4163
3903
  };
4164
3904
  } catch {
4165
3905
  throw syncError(