@syncular/server 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +49 -36
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +46 -40
  4. package/dist/blob-handlers.d.ts +25 -31
  5. package/dist/blob-handlers.js +63 -41
  6. package/dist/blob-store.d.ts +42 -24
  7. package/dist/blob-store.js +0 -0
  8. package/dist/bucket-blob-store.d.ts +12 -8
  9. package/dist/bucket-blob-store.js +54 -45
  10. package/dist/bucket-segment-store.js +18 -16
  11. package/dist/content-encoding.d.ts +5 -2
  12. package/dist/content-encoding.js +42 -13
  13. package/dist/context.d.ts +19 -5
  14. package/dist/context.js +19 -1
  15. package/dist/d1-storage.d.ts +14 -98
  16. package/dist/d1-storage.js +249 -1119
  17. package/dist/errors.js +16 -0
  18. package/dist/events.d.ts +36 -18
  19. package/dist/frame-bytes.js +24 -6
  20. package/dist/handler.js +503 -208
  21. package/dist/local-authority.d.ts +2 -0
  22. package/dist/local-authority.js +24 -12
  23. package/dist/object-bucket.d.ts +2 -0
  24. package/dist/operations-realtime.js +69 -50
  25. package/dist/operations.d.ts +5 -2
  26. package/dist/operations.js +20 -29
  27. package/dist/pg-executor-pglite.d.ts +5 -6
  28. package/dist/pg-executor-pglite.js +22 -14
  29. package/dist/pg-executor.d.ts +4 -1
  30. package/dist/postgres-storage.d.ts +24 -19
  31. package/dist/postgres-storage.js +420 -262
  32. package/dist/pull.d.ts +70 -47
  33. package/dist/pull.js +270 -301
  34. package/dist/push.d.ts +12 -3
  35. package/dist/push.js +361 -173
  36. package/dist/realtime.d.ts +85 -39
  37. package/dist/realtime.js +355 -219
  38. package/dist/relational-rows.d.ts +21 -66
  39. package/dist/relational-rows.js +28 -109
  40. package/dist/s3-blob-store.d.ts +11 -12
  41. package/dist/s3-blob-store.js +20 -13
  42. package/dist/s3-object-bucket.d.ts +8 -1
  43. package/dist/s3-object-bucket.js +16 -3
  44. package/dist/schema.d.ts +14 -0
  45. package/dist/schema.js +23 -0
  46. package/dist/scopes.d.ts +6 -2
  47. package/dist/scopes.js +22 -4
  48. package/dist/seed.js +10 -16
  49. package/dist/segment-download.d.ts +4 -1
  50. package/dist/segment-download.js +22 -6
  51. package/dist/signed-url.d.ts +2 -2
  52. package/dist/signed-url.js +9 -2
  53. package/dist/sigv4.d.ts +8 -2
  54. package/dist/sigv4.js +8 -4
  55. package/dist/sqlite-blob-store.d.ts +1 -1
  56. package/dist/sqlite-blob-store.js +23 -22
  57. package/dist/sqlite-dialect.d.ts +164 -84
  58. package/dist/sqlite-dialect.js +1145 -150
  59. package/dist/sqlite-lease-store.js +13 -18
  60. package/dist/sqlite-storage.d.ts +9 -62
  61. package/dist/sqlite-storage.js +303 -1123
  62. package/dist/storage-query.d.ts +30 -1
  63. package/dist/storage-query.js +68 -0
  64. package/dist/storage.d.ts +185 -63
  65. package/dist/storage.js +15 -0
  66. package/package.json +2 -2
  67. package/src/admin.ts +99 -57
  68. package/src/blob-handlers.ts +91 -72
  69. package/src/blob-store.ts +0 -0
  70. package/src/bucket-blob-store.ts +75 -48
  71. package/src/bucket-segment-store.ts +20 -15
  72. package/src/content-encoding.ts +48 -16
  73. package/src/context.ts +33 -6
  74. package/src/d1-storage.ts +350 -1700
  75. package/src/errors.ts +16 -0
  76. package/src/events.ts +37 -18
  77. package/src/frame-bytes.ts +26 -6
  78. package/src/handler.ts +654 -261
  79. package/src/local-authority.ts +27 -14
  80. package/src/object-bucket.ts +2 -0
  81. package/src/operations-realtime.ts +85 -61
  82. package/src/operations.ts +44 -42
  83. package/src/pg-executor-pglite.ts +27 -23
  84. package/src/pg-executor.ts +5 -1
  85. package/src/postgres-storage.ts +557 -354
  86. package/src/pull.ts +406 -440
  87. package/src/push.ts +466 -220
  88. package/src/realtime.ts +447 -290
  89. package/src/relational-rows.ts +43 -121
  90. package/src/s3-blob-store.ts +23 -13
  91. package/src/s3-object-bucket.ts +20 -2
  92. package/src/schema.ts +31 -0
  93. package/src/scopes.ts +23 -3
  94. package/src/seed.ts +13 -20
  95. package/src/segment-download.ts +33 -7
  96. package/src/signed-url.ts +10 -4
  97. package/src/sigv4.ts +17 -4
  98. package/src/sqlite-blob-store.ts +27 -26
  99. package/src/sqlite-dialect.ts +1851 -199
  100. package/src/sqlite-lease-store.ts +12 -23
  101. package/src/sqlite-storage.ts +390 -1821
  102. package/src/storage-query.ts +102 -0
  103. package/src/storage.ts +215 -67
package/dist/context.d.ts CHANGED
@@ -44,7 +44,7 @@ export type ResolverOutage = typeof RESOLVER_OUTAGE;
44
44
  /**
45
45
  * Host callback: actor → allowed scopes (§3.2 step 3). Resolved at most
46
46
  * once per request and memoized (§3.4 step 1). `'*'` means "any value for
47
- * this variable". A throwing resolver revokes subscriptions and rejects
47
+ * this variable". A throwing resolver revokes units and rejects
48
48
  * writes with `sync.forbidden` — fail loud, never leak. Returning
49
49
  * `RESOLVER_OUTAGE` instead opts the request into lease authorization
50
50
  * (§7.3.3) — a deliberate outage signal, not a failure.
@@ -64,16 +64,25 @@ export interface ServerLimits {
64
64
  readonly inlineSegmentMaxBytes: number;
65
65
  /** §6.11 declared-reference cascade cap per commit (reference default). */
66
66
  readonly maxCascadeOperationsPerCommit: number;
67
+ /**
68
+ * §1.7 cap on a request body's decoded bytes: an HTTP body after gzip
69
+ * decoding, and a socket round's assembled request (§8.7).
70
+ */
71
+ readonly maxRequestBytes: number;
72
+ /** §1.7 cap on `UNIT` frames per request: the units of a traveling list. */
73
+ readonly maxSubscriptionsPerRequest: number;
74
+ /** Cap on concurrent registered-query watches per watch session. */
75
+ readonly maxWatchesPerSession: number;
67
76
  }
68
77
  export declare const DEFAULT_LIMITS: ServerLimits;
69
78
  /** Receives applied commits for realtime fanout (§8.2). */
70
79
  export interface RealtimeNotifier {
71
80
  notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
72
81
  /**
73
- * A pull changed the client's persisted subscription list (§8.7). A hub
74
- * re-registers that client's live sockets, so a client that runs its
75
- * rounds over HTTP while a socket is open receives deltas for the new
76
- * subscriptions.
82
+ * A pull changed the client's registration list or live set (§8.1). A
83
+ * hub re-registers that client's live sockets, so a client that runs its
84
+ * rounds over HTTP while a socket is open receives deltas for the units
85
+ * that joined.
77
86
  */
78
87
  notifySubscriptions?(partition: string, clientId: string): void | Promise<void>;
79
88
  }
@@ -202,6 +211,11 @@ export interface SyncRequestContext extends SyncServerConfig {
202
211
  readonly actorId: string;
203
212
  }
204
213
  export declare function clockOf(ctx: SyncServerConfig): () => number;
214
+ /**
215
+ * The configured limits over the defaults. A cap that is not a positive safe
216
+ * integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
217
+ * `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
218
+ */
205
219
  export declare function limitsOf(ctx: SyncServerConfig): ServerLimits;
206
220
  /** Refresh the registry after host authentication and return log continuity. */
207
221
  export declare function touchAuthenticatedPartition(ctx: SyncRequestContext): Promise<PartitionRegistryEntry>;
package/dist/context.js CHANGED
@@ -15,12 +15,30 @@ export const DEFAULT_LIMITS = {
15
15
  maxOperationsPerRequest: 500,
16
16
  inlineSegmentMaxBytes: 256 * 1024,
17
17
  maxCascadeOperationsPerCommit: 1_000,
18
+ // The size above which adapters already refuse to buffer a segment body.
19
+ maxRequestBytes: 16 * 1024 * 1024,
20
+ // One unit per pushable operation: the existing per-request cap.
21
+ maxSubscriptionsPerRequest: 500,
22
+ // Every commit reruns a session's affected watches one after another.
23
+ maxWatchesPerSession: 100,
18
24
  };
19
25
  export function clockOf(ctx) {
20
26
  return ctx.clock ?? Date.now;
21
27
  }
28
+ /**
29
+ * The configured limits over the defaults. A cap that is not a positive safe
30
+ * integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
31
+ * `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
32
+ */
22
33
  export function limitsOf(ctx) {
23
- return { ...DEFAULT_LIMITS, ...ctx.limits };
34
+ const limits = { ...DEFAULT_LIMITS, ...ctx.limits };
35
+ for (const [name, value] of Object.entries(limits)) {
36
+ const min = name === 'inlineSegmentMaxBytes' ? 0 : 1;
37
+ if (!Number.isSafeInteger(value) || value < min) {
38
+ throw new Error(`ServerLimits.${name} is out of range`);
39
+ }
40
+ }
41
+ return limits;
24
42
  }
25
43
  /** Refresh the registry after host authentication and return log continuity. */
26
44
  export function touchAuthenticatedPartition(ctx) {
@@ -1,48 +1,7 @@
1
- import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
2
- /**
3
- * Cloudflare D1 server storage for Workers deployments.
4
- *
5
- * D1 *is* SQLite exposed over an async, statement-at-a-time API
6
- * (`prepare(sql).bind(...).all()` / `.first()` / `.run()`, plus `batch([…])`
7
- * for atomic multi-statement writes). So this storage shares the schema and
8
- * value codecs with `SqliteServerStorage` via `sqlite-dialect.ts` — same
9
- * DDL, same `?` placeholders, same JSON-scope / base64-push-result
10
- * serialization — and differs only in the execution shape (async, no
11
- * interactive transaction).
12
- *
13
- * ## Transaction model (the one real divergence)
14
- *
15
- * D1 has **no interactive transaction**: you cannot `BEGIN`, run
16
- * conditional reads and writes on a pinned connection, then `COMMIT`. The
17
- * only atomic primitive is `db.batch([stmt, …])` — a set of statements
18
- * applied in one implicit transaction, all-or-nothing.
19
- *
20
- * The push handler (§6) reads first (conflict detection via `getRow`) and
21
- * only then writes (`upsertRow`/`deleteRow`/`appendCommit`/
22
- * `putPushResult`). `D1Transaction` exploits exactly that ordering:
23
- *
24
- * - reads (`getRow`) execute immediately against D1 (autocommit);
25
- * - writes are **buffered** as prepared statements;
26
- * - `commit()` flushes the whole buffer as one `db.batch(...)` — the same
27
- * atomicity §6.4 requires (a rejected op rolls back by simply never
28
- * flushing);
29
- * - `rollback()` drops the buffer unsent.
30
- *
31
- * `appendCommit` allocates the dense per-partition `commitSeq` (§2.1) by
32
- * reading `max_commit_seq` live and buffering the `+1` write. Under a single
33
- * Worker request this is exact. Two concurrent pushes to one partition need an
34
- * external serialization point (normally a per-partition Durable Object); a
35
- * realtime notifier alone does not serialize HTTP writes. Every deployment
36
- * that accepts D1 pushes MUST front same-partition sync rounds with a
37
- * coordinating primitive (a DO or a Queue). The adapter fails closed unless
38
- * that coordinator explicitly sets `pushApplySerialized`, because D1 cannot
39
- * provide the required pre-operation lock.
40
- * This mirrors PostgreSQL's per-partition row lock, achieved by placement
41
- * rather than a lock D1 does not expose.
42
- */
43
- import { type ScopeMap } from '@syncular/core';
44
- import type { CompiledSchema, CompiledTable } from './schema.js';
45
- import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
1
+ import type { CompiledSchema } from './schema.js';
2
+ import { type SqliteStatementSpec, SqliteFamilyStorage } from './sqlite-dialect.js';
3
+ import type { SqliteValue } from './sqlite-driver.js';
4
+ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, CommitPruneQuery, CommitPruneResult, ScopeValueHistoryQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint } from './storage.js';
46
5
  export interface D1PreparedStatement {
47
6
  bind(...values: unknown[]): D1PreparedStatement;
48
7
  first<T = Record<string, unknown>>(): Promise<T | null>;
@@ -71,13 +30,20 @@ export interface D1ServerStorageOptions {
71
30
  */
72
31
  readonly commitValidationSerialized?: boolean;
73
32
  }
74
- export declare class D1ServerStorage implements ServerStorage {
33
+ export declare class D1ServerStorage extends SqliteFamilyStorage implements ServerStorage {
75
34
  #private;
35
+ /**
36
+ * D1 caps bound parameters at 100 per statement; the relational upsert
37
+ * binds one per app column plus the five `_sync_*` meta columns.
38
+ */
39
+ protected readonly maxBindParams = 100;
76
40
  constructor(db: D1Database, options?: D1ServerStorageOptions);
41
+ protected all<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
42
+ /** Queue one read into the current turn's `db.batch` (see `#pendingReads`). */
43
+ protected guardedAll<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
44
+ protected atomic(statements: readonly SqliteStatementSpec[]): Promise<unknown[][]>;
77
45
  /** Apply the schema DDL (idempotent). Call once before use. */
78
46
  migrate(): Promise<void>;
79
- /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
80
- table(name: string): CompiledTable;
81
47
  ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
82
48
  /** Run once per Worker invocation. Resume an incomplete result in another invocation. */
83
49
  migrateSchema(schema: CompiledSchema, options?: {
@@ -86,64 +52,14 @@ export declare class D1ServerStorage implements ServerStorage {
86
52
  readonly complete: boolean;
87
53
  readonly statementsExecuted: number;
88
54
  }>;
89
- touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
90
- rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
91
- listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
92
55
  begin(partition: string): Promise<StorageTransaction>;
93
- getMaxCommitSeq(partition: string): Promise<number>;
94
56
  queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, _checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
95
- getPartitionLogEpoch(partition: string): Promise<string | undefined>;
96
- getHorizonSeq(partition: string): Promise<number>;
97
- setHorizonSeq(partition: string, seq: number): Promise<void>;
98
57
  pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
99
- getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
100
58
  readCheckpoints(_partition: string): Promise<StoredCheckpoint[]>;
101
59
  readServeGate(partition: string, _runningSchemaVersion: number): Promise<ServeGate>;
102
60
  declareCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
103
61
  activateCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _sources: readonly string[], _nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
104
62
  claimCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
105
63
  advanceCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _observedRows: number, _nowMs: number): Promise<boolean>;
106
- sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
107
- hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
108
- getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
109
- getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
110
- claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
111
- completeReaction(partition: string, idempotencyKey: string, leaseOwner: string, completedAtMs: number): Promise<boolean>;
112
- extendReactionLease(partition: string, idempotencyKey: string, leaseOwner: string, leaseExpiresAtMs: number): Promise<boolean>;
113
- failReaction(partition: string, idempotencyKey: string, update: ReactionFailureUpdate): Promise<boolean>;
114
- retryReaction(partition: string, idempotencyKey: string, nowMs: number): Promise<boolean>;
115
- getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
116
- listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
117
- pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
118
- latestScopedChangeSeq(partition: string, query: {
119
- readonly table: string;
120
- readonly scopeFilter: ScopeMap;
121
- }): Promise<number>;
122
- readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
123
- scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
124
- scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
125
- getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
126
- putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
127
- recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
128
- advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
129
- updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
130
- getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
131
- listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
132
- listRowsReferencingBlob(partition: string, blobId: string): Promise<{
133
- readonly table: string;
134
- readonly rowId: string;
135
- readonly scopes: Record<string, string>;
136
- }[]>;
137
- listReferencedBlobIds(partition: string): Promise<string[]>;
138
- listScopeValueRows(partition: string, query: ScopeValueQuery): Promise<string[]>;
139
64
  eraseScopeValueHistory(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
140
- listClientRecords(partition: string): Promise<ClientRecord[]>;
141
- listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
142
- getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
143
- listRowChanges(partition: string, query: RowChangesQuery): Promise<RowChangeRecord[]>;
144
- listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
145
- countRows(partition: string, table: string): Promise<number>;
146
- listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
147
- scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
148
- listPartitions(): Promise<string[]>;
149
65
  }