@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/dist/context.d.ts
CHANGED
|
@@ -44,7 +44,7 @@ export type ResolverOutage = typeof RESOLVER_OUTAGE;
|
|
|
44
44
|
/**
|
|
45
45
|
* Host callback: actor → allowed scopes (§3.2 step 3). Resolved at most
|
|
46
46
|
* once per request and memoized (§3.4 step 1). `'*'` means "any value for
|
|
47
|
-
* this variable". A throwing resolver revokes
|
|
47
|
+
* this variable". A throwing resolver revokes units and rejects
|
|
48
48
|
* writes with `sync.forbidden` — fail loud, never leak. Returning
|
|
49
49
|
* `RESOLVER_OUTAGE` instead opts the request into lease authorization
|
|
50
50
|
* (§7.3.3) — a deliberate outage signal, not a failure.
|
|
@@ -64,16 +64,25 @@ export interface ServerLimits {
|
|
|
64
64
|
readonly inlineSegmentMaxBytes: number;
|
|
65
65
|
/** §6.11 declared-reference cascade cap per commit (reference default). */
|
|
66
66
|
readonly maxCascadeOperationsPerCommit: number;
|
|
67
|
+
/**
|
|
68
|
+
* §1.7 cap on a request body's decoded bytes: an HTTP body after gzip
|
|
69
|
+
* decoding, and a socket round's assembled request (§8.7).
|
|
70
|
+
*/
|
|
71
|
+
readonly maxRequestBytes: number;
|
|
72
|
+
/** §1.7 cap on `UNIT` frames per request: the units of a traveling list. */
|
|
73
|
+
readonly maxSubscriptionsPerRequest: number;
|
|
74
|
+
/** Cap on concurrent registered-query watches per watch session. */
|
|
75
|
+
readonly maxWatchesPerSession: number;
|
|
67
76
|
}
|
|
68
77
|
export declare const DEFAULT_LIMITS: ServerLimits;
|
|
69
78
|
/** Receives applied commits for realtime fanout (§8.2). */
|
|
70
79
|
export interface RealtimeNotifier {
|
|
71
80
|
notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
|
|
72
81
|
/**
|
|
73
|
-
* A pull changed the client's
|
|
74
|
-
* re-registers that client's live sockets, so a client that runs its
|
|
75
|
-
* rounds over HTTP while a socket is open receives deltas for the
|
|
76
|
-
*
|
|
82
|
+
* A pull changed the client's registration list or live set (§8.1). A
|
|
83
|
+
* hub re-registers that client's live sockets, so a client that runs its
|
|
84
|
+
* rounds over HTTP while a socket is open receives deltas for the units
|
|
85
|
+
* that joined.
|
|
77
86
|
*/
|
|
78
87
|
notifySubscriptions?(partition: string, clientId: string): void | Promise<void>;
|
|
79
88
|
}
|
|
@@ -202,6 +211,11 @@ export interface SyncRequestContext extends SyncServerConfig {
|
|
|
202
211
|
readonly actorId: string;
|
|
203
212
|
}
|
|
204
213
|
export declare function clockOf(ctx: SyncServerConfig): () => number;
|
|
214
|
+
/**
|
|
215
|
+
* The configured limits over the defaults. A cap that is not a positive safe
|
|
216
|
+
* integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
|
|
217
|
+
* `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
|
|
218
|
+
*/
|
|
205
219
|
export declare function limitsOf(ctx: SyncServerConfig): ServerLimits;
|
|
206
220
|
/** Refresh the registry after host authentication and return log continuity. */
|
|
207
221
|
export declare function touchAuthenticatedPartition(ctx: SyncRequestContext): Promise<PartitionRegistryEntry>;
|
package/dist/context.js
CHANGED
|
@@ -15,12 +15,30 @@ export const DEFAULT_LIMITS = {
|
|
|
15
15
|
maxOperationsPerRequest: 500,
|
|
16
16
|
inlineSegmentMaxBytes: 256 * 1024,
|
|
17
17
|
maxCascadeOperationsPerCommit: 1_000,
|
|
18
|
+
// The size above which adapters already refuse to buffer a segment body.
|
|
19
|
+
maxRequestBytes: 16 * 1024 * 1024,
|
|
20
|
+
// One unit per pushable operation: the existing per-request cap.
|
|
21
|
+
maxSubscriptionsPerRequest: 500,
|
|
22
|
+
// Every commit reruns a session's affected watches one after another.
|
|
23
|
+
maxWatchesPerSession: 100,
|
|
18
24
|
};
|
|
19
25
|
export function clockOf(ctx) {
|
|
20
26
|
return ctx.clock ?? Date.now;
|
|
21
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* The configured limits over the defaults. A cap that is not a positive safe
|
|
30
|
+
* integer (`NaN`, `0`, `Infinity`) would disable or jam it, so it throws;
|
|
31
|
+
* `inlineSegmentMaxBytes` is a threshold and may be `0` (inline nothing).
|
|
32
|
+
*/
|
|
22
33
|
export function limitsOf(ctx) {
|
|
23
|
-
|
|
34
|
+
const limits = { ...DEFAULT_LIMITS, ...ctx.limits };
|
|
35
|
+
for (const [name, value] of Object.entries(limits)) {
|
|
36
|
+
const min = name === 'inlineSegmentMaxBytes' ? 0 : 1;
|
|
37
|
+
if (!Number.isSafeInteger(value) || value < min) {
|
|
38
|
+
throw new Error(`ServerLimits.${name} is out of range`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return limits;
|
|
24
42
|
}
|
|
25
43
|
/** Refresh the registry after host authentication and return log continuity. */
|
|
26
44
|
export function touchAuthenticatedPartition(ctx) {
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -1,48 +1,7 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* D1 *is* SQLite exposed over an async, statement-at-a-time API
|
|
6
|
-
* (`prepare(sql).bind(...).all()` / `.first()` / `.run()`, plus `batch([…])`
|
|
7
|
-
* for atomic multi-statement writes). So this storage shares the schema and
|
|
8
|
-
* value codecs with `SqliteServerStorage` via `sqlite-dialect.ts` — same
|
|
9
|
-
* DDL, same `?` placeholders, same JSON-scope / base64-push-result
|
|
10
|
-
* serialization — and differs only in the execution shape (async, no
|
|
11
|
-
* interactive transaction).
|
|
12
|
-
*
|
|
13
|
-
* ## Transaction model (the one real divergence)
|
|
14
|
-
*
|
|
15
|
-
* D1 has **no interactive transaction**: you cannot `BEGIN`, run
|
|
16
|
-
* conditional reads and writes on a pinned connection, then `COMMIT`. The
|
|
17
|
-
* only atomic primitive is `db.batch([stmt, …])` — a set of statements
|
|
18
|
-
* applied in one implicit transaction, all-or-nothing.
|
|
19
|
-
*
|
|
20
|
-
* The push handler (§6) reads first (conflict detection via `getRow`) and
|
|
21
|
-
* only then writes (`upsertRow`/`deleteRow`/`appendCommit`/
|
|
22
|
-
* `putPushResult`). `D1Transaction` exploits exactly that ordering:
|
|
23
|
-
*
|
|
24
|
-
* - reads (`getRow`) execute immediately against D1 (autocommit);
|
|
25
|
-
* - writes are **buffered** as prepared statements;
|
|
26
|
-
* - `commit()` flushes the whole buffer as one `db.batch(...)` — the same
|
|
27
|
-
* atomicity §6.4 requires (a rejected op rolls back by simply never
|
|
28
|
-
* flushing);
|
|
29
|
-
* - `rollback()` drops the buffer unsent.
|
|
30
|
-
*
|
|
31
|
-
* `appendCommit` allocates the dense per-partition `commitSeq` (§2.1) by
|
|
32
|
-
* reading `max_commit_seq` live and buffering the `+1` write. Under a single
|
|
33
|
-
* Worker request this is exact. Two concurrent pushes to one partition need an
|
|
34
|
-
* external serialization point (normally a per-partition Durable Object); a
|
|
35
|
-
* realtime notifier alone does not serialize HTTP writes. Every deployment
|
|
36
|
-
* that accepts D1 pushes MUST front same-partition sync rounds with a
|
|
37
|
-
* coordinating primitive (a DO or a Queue). The adapter fails closed unless
|
|
38
|
-
* that coordinator explicitly sets `pushApplySerialized`, because D1 cannot
|
|
39
|
-
* provide the required pre-operation lock.
|
|
40
|
-
* This mirrors PostgreSQL's per-partition row lock, achieved by placement
|
|
41
|
-
* rather than a lock D1 does not expose.
|
|
42
|
-
*/
|
|
43
|
-
import { type ScopeMap } from '@syncular/core';
|
|
44
|
-
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
45
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
1
|
+
import type { CompiledSchema } from './schema.js';
|
|
2
|
+
import { type SqliteStatementSpec, SqliteFamilyStorage } from './sqlite-dialect.js';
|
|
3
|
+
import type { SqliteValue } from './sqlite-driver.js';
|
|
4
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, CommitPruneQuery, CommitPruneResult, ScopeValueHistoryQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint } from './storage.js';
|
|
46
5
|
export interface D1PreparedStatement {
|
|
47
6
|
bind(...values: unknown[]): D1PreparedStatement;
|
|
48
7
|
first<T = Record<string, unknown>>(): Promise<T | null>;
|
|
@@ -71,13 +30,20 @@ export interface D1ServerStorageOptions {
|
|
|
71
30
|
*/
|
|
72
31
|
readonly commitValidationSerialized?: boolean;
|
|
73
32
|
}
|
|
74
|
-
export declare class D1ServerStorage implements ServerStorage {
|
|
33
|
+
export declare class D1ServerStorage extends SqliteFamilyStorage implements ServerStorage {
|
|
75
34
|
#private;
|
|
35
|
+
/**
|
|
36
|
+
* D1 caps bound parameters at 100 per statement; the relational upsert
|
|
37
|
+
* binds one per app column plus the five `_sync_*` meta columns.
|
|
38
|
+
*/
|
|
39
|
+
protected readonly maxBindParams = 100;
|
|
76
40
|
constructor(db: D1Database, options?: D1ServerStorageOptions);
|
|
41
|
+
protected all<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
|
|
42
|
+
/** Queue one read into the current turn's `db.batch` (see `#pendingReads`). */
|
|
43
|
+
protected guardedAll<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
|
|
44
|
+
protected atomic(statements: readonly SqliteStatementSpec[]): Promise<unknown[][]>;
|
|
77
45
|
/** Apply the schema DDL (idempotent). Call once before use. */
|
|
78
46
|
migrate(): Promise<void>;
|
|
79
|
-
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
80
|
-
table(name: string): CompiledTable;
|
|
81
47
|
ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
|
|
82
48
|
/** Run once per Worker invocation. Resume an incomplete result in another invocation. */
|
|
83
49
|
migrateSchema(schema: CompiledSchema, options?: {
|
|
@@ -86,64 +52,14 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
86
52
|
readonly complete: boolean;
|
|
87
53
|
readonly statementsExecuted: number;
|
|
88
54
|
}>;
|
|
89
|
-
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
90
|
-
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
91
|
-
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
92
55
|
begin(partition: string): Promise<StorageTransaction>;
|
|
93
|
-
getMaxCommitSeq(partition: string): Promise<number>;
|
|
94
56
|
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, _checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
|
|
95
|
-
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
96
|
-
getHorizonSeq(partition: string): Promise<number>;
|
|
97
|
-
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
98
57
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
99
|
-
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
100
58
|
readCheckpoints(_partition: string): Promise<StoredCheckpoint[]>;
|
|
101
59
|
readServeGate(partition: string, _runningSchemaVersion: number): Promise<ServeGate>;
|
|
102
60
|
declareCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
|
|
103
61
|
activateCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _sources: readonly string[], _nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
|
|
104
62
|
claimCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
|
|
105
63
|
advanceCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _observedRows: number, _nowMs: number): Promise<boolean>;
|
|
106
|
-
sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
|
|
107
|
-
hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
|
|
108
|
-
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
109
|
-
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
110
|
-
claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
|
|
111
|
-
completeReaction(partition: string, idempotencyKey: string, leaseOwner: string, completedAtMs: number): Promise<boolean>;
|
|
112
|
-
extendReactionLease(partition: string, idempotencyKey: string, leaseOwner: string, leaseExpiresAtMs: number): Promise<boolean>;
|
|
113
|
-
failReaction(partition: string, idempotencyKey: string, update: ReactionFailureUpdate): Promise<boolean>;
|
|
114
|
-
retryReaction(partition: string, idempotencyKey: string, nowMs: number): Promise<boolean>;
|
|
115
|
-
getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
|
|
116
|
-
listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
|
|
117
|
-
pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
118
|
-
latestScopedChangeSeq(partition: string, query: {
|
|
119
|
-
readonly table: string;
|
|
120
|
-
readonly scopeFilter: ScopeMap;
|
|
121
|
-
}): Promise<number>;
|
|
122
|
-
readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
|
|
123
|
-
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
124
|
-
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
125
|
-
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
126
|
-
putClientRecord(partition: string, record: ClientRecord, options?: ClientRecordWriteOptions): Promise<void>;
|
|
127
|
-
recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
|
|
128
|
-
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
129
|
-
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
130
|
-
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
131
|
-
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
132
|
-
listRowsReferencingBlob(partition: string, blobId: string): Promise<{
|
|
133
|
-
readonly table: string;
|
|
134
|
-
readonly rowId: string;
|
|
135
|
-
readonly scopes: Record<string, string>;
|
|
136
|
-
}[]>;
|
|
137
|
-
listReferencedBlobIds(partition: string): Promise<string[]>;
|
|
138
|
-
listScopeValueRows(partition: string, query: ScopeValueQuery): Promise<string[]>;
|
|
139
64
|
eraseScopeValueHistory(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
|
|
140
|
-
listClientRecords(partition: string): Promise<ClientRecord[]>;
|
|
141
|
-
listCommitMetadata(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
142
|
-
getCommit(partition: string, commitSeq: number): Promise<CommitDetail | undefined>;
|
|
143
|
-
listRowChanges(partition: string, query: RowChangesQuery): Promise<RowChangeRecord[]>;
|
|
144
|
-
listRows(partition: string, query: RowListQuery): Promise<StoredRow[]>;
|
|
145
|
-
countRows(partition: string, table: string): Promise<number>;
|
|
146
|
-
listPushResults(partition: string, clientCommitId: string): Promise<ClientPushResult[]>;
|
|
147
|
-
scopeActivity(partition: string, query: ScopeActivityQuery): Promise<ScopeCommitActivity[]>;
|
|
148
|
-
listPartitions(): Promise<string[]>;
|
|
149
65
|
}
|