@syncular/server 0.26.5 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-encoding.d.ts +17 -0
- package/dist/content-encoding.js +22 -0
- package/dist/context.d.ts +9 -0
- package/dist/d1-storage.d.ts +46 -0
- package/dist/d1-storage.js +12 -1
- package/dist/errors.d.ts +19 -0
- package/dist/errors.js +29 -0
- package/dist/handler.js +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/operations-realtime.js +4 -1
- package/dist/operations.js +6 -1
- package/dist/postgres-storage.d.ts +42 -0
- package/dist/postgres-storage.js +12 -1
- package/dist/pull.d.ts +46 -6
- package/dist/pull.js +129 -34
- package/dist/realtime.d.ts +3 -0
- package/dist/realtime.js +32 -26
- package/dist/relational-rows.d.ts +13 -0
- package/dist/relational-rows.js +25 -0
- package/dist/s3-segment-store.d.ts +3 -1
- package/dist/s3-segment-store.js +16 -0
- package/dist/segment-download.d.ts +16 -0
- package/dist/segment-download.js +71 -21
- package/dist/segment-store.d.ts +16 -0
- package/dist/segment-store.js +19 -0
- package/dist/sqlite-storage.d.ts +5 -0
- package/dist/sqlite-storage.js +18 -1
- package/dist/storage.d.ts +12 -0
- package/package.json +2 -2
- package/src/content-encoding.ts +35 -0
- package/src/context.ts +9 -0
- package/src/d1-storage.ts +20 -1
- package/src/errors.ts +56 -0
- package/src/handler.ts +2 -0
- package/src/index.ts +1 -0
- package/src/operations-realtime.ts +4 -1
- package/src/operations.ts +6 -1
- package/src/postgres-storage.ts +22 -1
- package/src/pull.ts +212 -39
- package/src/realtime.ts +45 -32
- package/src/relational-rows.ts +29 -0
- package/src/s3-segment-store.ts +15 -0
- package/src/segment-download.ts +129 -21
- package/src/segment-store.ts +36 -0
- package/src/sqlite-storage.ts +27 -0
- package/src/storage.ts +13 -0
|
@@ -25,3 +25,20 @@ export interface EncodedSegmentBody {
|
|
|
25
25
|
* tiny, or the runtime has no codec.
|
|
26
26
|
*/
|
|
27
27
|
export declare function encodeSegmentBody(bytes: Uint8Array, acceptEncoding: string | undefined): EncodedSegmentBody;
|
|
28
|
+
/**
|
|
29
|
+
* Segments above this size are relayed as a stream instead of being
|
|
30
|
+
* buffered and compressed in memory: a sqlite image can exceed the memory of
|
|
31
|
+
* a Workers isolate (128 MB).
|
|
32
|
+
*/
|
|
33
|
+
export declare const SEGMENT_STREAM_THRESHOLD_BYTES: number;
|
|
34
|
+
export interface EncodedSegmentStream {
|
|
35
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
36
|
+
/** Absent ⇒ identity (no `Content-Encoding` header). */
|
|
37
|
+
readonly contentEncoding?: 'gzip';
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Compress a streamed segment body per §5.8 with the runtime's
|
|
41
|
+
* `CompressionStream`: gzip when the client accepts it, identity otherwise
|
|
42
|
+
* (zstd has no standard streaming codec).
|
|
43
|
+
*/
|
|
44
|
+
export declare function encodeSegmentStream(body: ReadableStream<Uint8Array<ArrayBuffer>>, acceptEncoding: string | undefined): EncodedSegmentStream;
|
package/dist/content-encoding.js
CHANGED
|
@@ -67,3 +67,25 @@ export function encodeSegmentBody(bytes, acceptEncoding) {
|
|
|
67
67
|
}
|
|
68
68
|
return { bytes };
|
|
69
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Segments above this size are relayed as a stream instead of being
|
|
72
|
+
* buffered and compressed in memory: a sqlite image can exceed the memory of
|
|
73
|
+
* a Workers isolate (128 MB).
|
|
74
|
+
*/
|
|
75
|
+
export const SEGMENT_STREAM_THRESHOLD_BYTES = 16 * 1024 * 1024;
|
|
76
|
+
/**
|
|
77
|
+
* Compress a streamed segment body per §5.8 with the runtime's
|
|
78
|
+
* `CompressionStream`: gzip when the client accepts it, identity otherwise
|
|
79
|
+
* (zstd has no standard streaming codec).
|
|
80
|
+
*/
|
|
81
|
+
export function encodeSegmentStream(body, acceptEncoding) {
|
|
82
|
+
if (acceptEncoding === undefined ||
|
|
83
|
+
!accepts(acceptEncoding, 'gzip') ||
|
|
84
|
+
typeof CompressionStream !== 'function') {
|
|
85
|
+
return { body };
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
body: body.pipeThrough(new CompressionStream('gzip')),
|
|
89
|
+
contentEncoding: 'gzip',
|
|
90
|
+
};
|
|
91
|
+
}
|
package/dist/context.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import type { ScopeMap } from '@syncular/core';
|
|
8
8
|
import type { BlobStore } from './blob-store.js';
|
|
9
9
|
import type { CrdtMergerRegistry } from './crdt-merger.js';
|
|
10
|
+
import type { SyncularErrorHandler } from './errors.js';
|
|
10
11
|
import type { SyncularServerEvents } from './events.js';
|
|
11
12
|
import type { LeaseStore } from './lease-store.js';
|
|
12
13
|
import type { AnyReactionPlanner } from './reactions.js';
|
|
@@ -167,6 +168,14 @@ export interface SyncServerConfig {
|
|
|
167
168
|
* event objects are built. A throwing sink never affects processing.
|
|
168
169
|
*/
|
|
169
170
|
readonly events?: SyncularServerEvents;
|
|
171
|
+
/**
|
|
172
|
+
* Host error reporting (§10.2 `sync.internal_error`). Receives the original
|
|
173
|
+
* exception whenever a request, socket round, segment or blob download,
|
|
174
|
+
* or remote operation fails with something other than a `SyncError`; the
|
|
175
|
+
* client sees only the catalog code. Like the events sink, a throwing
|
|
176
|
+
* handler never changes the response.
|
|
177
|
+
*/
|
|
178
|
+
readonly onError?: SyncularErrorHandler;
|
|
170
179
|
}
|
|
171
180
|
/** Per-request context: the config plus host-authenticated identity. */
|
|
172
181
|
export interface SyncRequestContext extends SyncServerConfig {
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -1,4 +1,46 @@
|
|
|
1
1
|
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
2
|
+
/**
|
|
3
|
+
* Cloudflare D1 server storage for Workers deployments.
|
|
4
|
+
*
|
|
5
|
+
* D1 *is* SQLite exposed over an async, statement-at-a-time API
|
|
6
|
+
* (`prepare(sql).bind(...).all()` / `.first()` / `.run()`, plus `batch([…])`
|
|
7
|
+
* for atomic multi-statement writes). So this storage shares the schema and
|
|
8
|
+
* value codecs with `SqliteServerStorage` via `sqlite-dialect.ts` — same
|
|
9
|
+
* DDL, same `?` placeholders, same JSON-scope / base64-push-result
|
|
10
|
+
* serialization — and differs only in the execution shape (async, no
|
|
11
|
+
* interactive transaction).
|
|
12
|
+
*
|
|
13
|
+
* ## Transaction model (the one real divergence)
|
|
14
|
+
*
|
|
15
|
+
* D1 has **no interactive transaction**: you cannot `BEGIN`, run
|
|
16
|
+
* conditional reads and writes on a pinned connection, then `COMMIT`. The
|
|
17
|
+
* only atomic primitive is `db.batch([stmt, …])` — a set of statements
|
|
18
|
+
* applied in one implicit transaction, all-or-nothing.
|
|
19
|
+
*
|
|
20
|
+
* The push handler (§6) reads first (conflict detection via `getRow`) and
|
|
21
|
+
* only then writes (`upsertRow`/`deleteRow`/`appendCommit`/
|
|
22
|
+
* `putPushResult`). `D1Transaction` exploits exactly that ordering:
|
|
23
|
+
*
|
|
24
|
+
* - reads (`getRow`) execute immediately against D1 (autocommit);
|
|
25
|
+
* - writes are **buffered** as prepared statements;
|
|
26
|
+
* - `commit()` flushes the whole buffer as one `db.batch(...)` — the same
|
|
27
|
+
* atomicity §6.4 requires (a rejected op rolls back by simply never
|
|
28
|
+
* flushing);
|
|
29
|
+
* - `rollback()` drops the buffer unsent.
|
|
30
|
+
*
|
|
31
|
+
* `appendCommit` allocates the dense per-partition `commitSeq` (§2.1) by
|
|
32
|
+
* reading `max_commit_seq` live and buffering the `+1` write. Under a single
|
|
33
|
+
* Worker request this is exact. Two concurrent pushes to one partition need an
|
|
34
|
+
* external serialization point (normally a per-partition Durable Object); a
|
|
35
|
+
* realtime notifier alone does not serialize HTTP writes. Every deployment
|
|
36
|
+
* that accepts D1 pushes MUST front same-partition sync rounds with a
|
|
37
|
+
* coordinating primitive (a DO or a Queue). The adapter fails closed unless
|
|
38
|
+
* that coordinator explicitly sets `pushApplySerialized`, because D1 cannot
|
|
39
|
+
* provide the required pre-operation lock.
|
|
40
|
+
* This mirrors PostgreSQL's per-partition row lock, achieved by placement
|
|
41
|
+
* rather than a lock D1 does not expose.
|
|
42
|
+
*/
|
|
43
|
+
import { type ScopeMap } from '@syncular/core';
|
|
2
44
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
3
45
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
4
46
|
export interface D1PreparedStatement {
|
|
@@ -73,6 +115,10 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
73
115
|
getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
|
|
74
116
|
listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
|
|
75
117
|
pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
118
|
+
latestScopedChangeSeq(partition: string, query: {
|
|
119
|
+
readonly table: string;
|
|
120
|
+
readonly scopeFilter: ScopeMap;
|
|
121
|
+
}): Promise<number>;
|
|
76
122
|
readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
|
|
77
123
|
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
78
124
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
package/dist/d1-storage.js
CHANGED
|
@@ -45,7 +45,7 @@ import { StorageQueryError } from './storage-errors.js';
|
|
|
45
45
|
import { decodeRow } from '@syncular/core';
|
|
46
46
|
import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
47
47
|
import { syncError } from './errors.js';
|
|
48
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
|
|
48
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, latestScopedChangeSeqSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
|
|
49
49
|
import { matchesEffective } from './scopes.js';
|
|
50
50
|
import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, serializePushResult, sqliteDdlStatements, toStoredRow, } from './sqlite-dialect.js';
|
|
51
51
|
import { isD1ConstraintError, StorageConstraintError } from './storage-errors.js';
|
|
@@ -1439,6 +1439,17 @@ export class D1ServerStorage {
|
|
|
1439
1439
|
deadLetter: results.filter((row) => row.status === 'dead-letter').length,
|
|
1440
1440
|
};
|
|
1441
1441
|
}
|
|
1442
|
+
async latestScopedChangeSeq(partition, query) {
|
|
1443
|
+
const firstVariable = Object.keys(query.scopeFilter).sort()[0];
|
|
1444
|
+
const values = firstVariable === undefined
|
|
1445
|
+
? []
|
|
1446
|
+
: (query.scopeFilter[firstVariable] ?? []);
|
|
1447
|
+
if (firstVariable === undefined || values.length === 0)
|
|
1448
|
+
return 0;
|
|
1449
|
+
// Concurrent reads of a pull share one `db.batch` round trip.
|
|
1450
|
+
const [row] = await this.#batchedAll(latestScopedChangeSeqSql(values.length), values.flatMap((value) => [partition, query.table, firstVariable, value]));
|
|
1451
|
+
return row?.seq ?? 0;
|
|
1452
|
+
}
|
|
1442
1453
|
async readCommitWindow(partition, query) {
|
|
1443
1454
|
const variables = Object.keys(query.scopeFilter).sort();
|
|
1444
1455
|
const firstVariable = variables[0];
|
package/dist/errors.d.ts
CHANGED
|
@@ -34,3 +34,22 @@ export declare function errorBody(error: SyncError): {
|
|
|
34
34
|
message: string;
|
|
35
35
|
details?: unknown;
|
|
36
36
|
};
|
|
37
|
+
/** The adapter surface that caught an exception reported through `onError`. */
|
|
38
|
+
export type SyncularErrorRoute = 'sync' | 'operations' | 'segments' | 'blobs' | 'admin' | 'realtime';
|
|
39
|
+
/**
|
|
40
|
+
* Host error reporting: receives every exception that is not a `SyncError`
|
|
41
|
+
* before the adapter answers `sync.internal_error` (or the remote-operation
|
|
42
|
+
* `operation.*` failure code), with the surface that caught it.
|
|
43
|
+
*/
|
|
44
|
+
export type SyncularErrorHandler = (error: unknown, context: {
|
|
45
|
+
readonly route: SyncularErrorRoute;
|
|
46
|
+
}) => void;
|
|
47
|
+
/**
|
|
48
|
+
* The protocol error for an exception an adapter caught. A `SyncError`
|
|
49
|
+
* passes through unchanged. Anything else goes to `onError` and becomes
|
|
50
|
+
* `sync.internal_error` (§10.2), whose message never carries the exception
|
|
51
|
+
* text.
|
|
52
|
+
*/
|
|
53
|
+
export declare function adapterSyncError(error: unknown, onError: SyncularErrorHandler | undefined, route: SyncularErrorRoute): SyncError;
|
|
54
|
+
/** Hand `error` to the host's `onError`; a throwing handler is contained. */
|
|
55
|
+
export declare function reportError(error: unknown, onError: SyncularErrorHandler | undefined, route: SyncularErrorRoute): void;
|
package/dist/errors.js
CHANGED
|
@@ -163,6 +163,14 @@ export const ERROR_CATALOG = {
|
|
|
163
163
|
// checkpoint is incomplete or the database is newer than the running build.
|
|
164
164
|
// Formerly host-only; promoted to the §10.2 catalogue so an in-flight
|
|
165
165
|
// request can be refused with a distinguishable identity.
|
|
166
|
+
// An exception outside this catalog (host code, storage, network) reached
|
|
167
|
+
// an adapter; the original goes to `onError`, never onto the wire.
|
|
168
|
+
'sync.internal_error': {
|
|
169
|
+
category: 'internal',
|
|
170
|
+
retryable: true,
|
|
171
|
+
recommendedAction: 'retryLater',
|
|
172
|
+
httpStatus: 500,
|
|
173
|
+
},
|
|
166
174
|
'sync.schema_not_ready': {
|
|
167
175
|
category: 'internal',
|
|
168
176
|
retryable: true,
|
|
@@ -288,3 +296,24 @@ export function errorBody(error) {
|
|
|
288
296
|
: {}),
|
|
289
297
|
};
|
|
290
298
|
}
|
|
299
|
+
/**
|
|
300
|
+
* The protocol error for an exception an adapter caught. A `SyncError`
|
|
301
|
+
* passes through unchanged. Anything else goes to `onError` and becomes
|
|
302
|
+
* `sync.internal_error` (§10.2), whose message never carries the exception
|
|
303
|
+
* text.
|
|
304
|
+
*/
|
|
305
|
+
export function adapterSyncError(error, onError, route) {
|
|
306
|
+
if (error instanceof SyncError)
|
|
307
|
+
return error;
|
|
308
|
+
reportError(error, onError, route);
|
|
309
|
+
return new SyncError('sync.internal_error', 'internal server error');
|
|
310
|
+
}
|
|
311
|
+
/** Hand `error` to the host's `onError`; a throwing handler is contained. */
|
|
312
|
+
export function reportError(error, onError, route) {
|
|
313
|
+
try {
|
|
314
|
+
onError?.(error, { route });
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
// The report is best effort; the response stays the catalog error.
|
|
318
|
+
}
|
|
319
|
+
}
|
package/dist/handler.js
CHANGED
|
@@ -353,7 +353,7 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
353
353
|
const limits = clampPullLimits(plan.pull);
|
|
354
354
|
const maxSeq = await ctx.storage.getMaxCommitSeq(ctx.partition);
|
|
355
355
|
const horizonSeq = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
356
|
-
const prepared = await prepareSections(ctx, limits, plan.subscriptions, maxSeq, horizonSeq);
|
|
356
|
+
const prepared = await prepareSections(ctx, schema, limits, plan.subscriptions, maxSeq, horizonSeq, plan.logEpoch);
|
|
357
357
|
for (const [index, subscription] of plan.subscriptions.entries()) {
|
|
358
358
|
const trace = summaries !== undefined ? { segments: [] } : undefined;
|
|
359
359
|
const reads = prepared[index];
|
package/dist/index.d.ts
CHANGED
|
@@ -44,7 +44,7 @@ export * from './segment-store.js';
|
|
|
44
44
|
export * from './signed-url.js';
|
|
45
45
|
export * from './sqlite-dialect.js';
|
|
46
46
|
export * from './sqlite-driver.js';
|
|
47
|
-
export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, type SqliteImageBuilder, type SqliteImageInput, } from './sqlite-image.js';
|
|
47
|
+
export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, type SqliteImageBuilder, type SqliteImageInput, writeSqliteImage, } from './sqlite-image.js';
|
|
48
48
|
export * from './sqlite-blob-store.js';
|
|
49
49
|
export * from './sqlite-lease-store.js';
|
|
50
50
|
export * from './sqlite-segment-store.js';
|
package/dist/index.js
CHANGED
|
@@ -48,7 +48,7 @@ export * from './segment-store.js';
|
|
|
48
48
|
export * from './signed-url.js';
|
|
49
49
|
export * from './sqlite-dialect.js';
|
|
50
50
|
export * from './sqlite-driver.js';
|
|
51
|
-
export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, } from './sqlite-image.js';
|
|
51
|
+
export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, writeSqliteImage, } from './sqlite-image.js';
|
|
52
52
|
export * from './sqlite-blob-store.js';
|
|
53
53
|
export * from './sqlite-lease-store.js';
|
|
54
54
|
export * from './sqlite-segment-store.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { decodeRemoteOperationRealtimeMessage, encodeRemoteOperationRealtimeMessage, } from '@syncular/core';
|
|
2
2
|
import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context.js';
|
|
3
|
-
import { SyncError, syncError } from './errors.js';
|
|
3
|
+
import { reportError, SyncError, syncError } from './errors.js';
|
|
4
4
|
class WatchSession {
|
|
5
5
|
#ctx;
|
|
6
6
|
#registry;
|
|
@@ -122,6 +122,9 @@ class WatchSession {
|
|
|
122
122
|
if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
|
|
123
123
|
return;
|
|
124
124
|
}
|
|
125
|
+
if (!(error instanceof SyncError)) {
|
|
126
|
+
reportError(error, this.#ctx.onError, 'operations');
|
|
127
|
+
}
|
|
125
128
|
this.#sendError(state.watchId, error instanceof SyncError
|
|
126
129
|
? error
|
|
127
130
|
: syncError('operation.query_failed'));
|
package/dist/operations.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { validateAuthoritativeRelationPlan, } from './authoritative-query.js';
|
|
2
2
|
import { decodeRow, decodeRemoteOperationRequest, encodeRemoteOperationResponse, encodeSparseRow, } from '@syncular/core';
|
|
3
3
|
import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
4
|
-
import { SyncError, syncError } from './errors.js';
|
|
4
|
+
import { reportError, SyncError, syncError } from './errors.js';
|
|
5
5
|
import { processPushOperationsWithTrace } from './push.js';
|
|
6
6
|
import { compileSchema } from './schema.js';
|
|
7
7
|
import { authorizeWrite } from './scopes.js';
|
|
@@ -206,6 +206,7 @@ export function registerRemoteQuery(descriptor, options) {
|
|
|
206
206
|
catch (error) {
|
|
207
207
|
if (error instanceof SyncError)
|
|
208
208
|
throw error;
|
|
209
|
+
reportError(error, ctx.onError, 'operations');
|
|
209
210
|
throw syncError('operation.query_failed', 'registered query execution failed');
|
|
210
211
|
}
|
|
211
212
|
if (result.rows.length > options.maxRows) {
|
|
@@ -218,6 +219,7 @@ export function registerRemoteQuery(descriptor, options) {
|
|
|
218
219
|
catch (error) {
|
|
219
220
|
if (error instanceof SyncError)
|
|
220
221
|
throw error;
|
|
222
|
+
reportError(error, ctx.onError, 'operations');
|
|
221
223
|
throw syncError('operation.query_failed', 'registered query result decoding failed');
|
|
222
224
|
}
|
|
223
225
|
return {
|
|
@@ -409,6 +411,9 @@ export async function handleRemoteOperation(bytes, ctx, registry) {
|
|
|
409
411
|
return encodeRemoteOperationResponse(await operation.run(ctx, request.clientId, request.requestId, request.params));
|
|
410
412
|
}
|
|
411
413
|
catch (error) {
|
|
414
|
+
if (!(error instanceof SyncError)) {
|
|
415
|
+
reportError(error, ctx.onError, 'operations');
|
|
416
|
+
}
|
|
412
417
|
return encodeRemoteOperationError(error, 'operation.execution_failed');
|
|
413
418
|
}
|
|
414
419
|
}
|
|
@@ -1,4 +1,42 @@
|
|
|
1
1
|
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
2
|
+
/**
|
|
3
|
+
* Postgres server storage: the production database path.
|
|
4
|
+
*
|
|
5
|
+
* Semantics mirror `SqliteServerStorage` exactly (both run the shared
|
|
6
|
+
* storage contract in `test/storage-contract.ts`); the difference is that
|
|
7
|
+
* scope fanout MUST survive contact with Postgres — no scan-before-LIMIT,
|
|
8
|
+
* ever (performance-by-construction):
|
|
9
|
+
*
|
|
10
|
+
* - both the commit log and the current-row table carry a (table, var,
|
|
11
|
+
* value) inverted scope index;
|
|
12
|
+
* - `readCommitWindow`/`scanRows` select candidates through that index
|
|
13
|
+
* (`sync_change_scopes` / `sync_row_scopes`), ordered + LIMITed at the
|
|
14
|
+
* index, then verify the full multi-variable match against the stored
|
|
15
|
+
* scope map — never a log or table scan;
|
|
16
|
+
* - the candidate indexes are *covering* for their selection: the ordered
|
|
17
|
+
* column (`commit_seq` / `row_id`) sits in the index key after
|
|
18
|
+
* (tbl, var, value), so the planner does an index range scan and returns
|
|
19
|
+
* already-ordered candidates without a heap sort. `test/postgres-explain
|
|
20
|
+
* .test.ts` asserts an `Index` node (never `Seq Scan`) so the regression
|
|
21
|
+
* cannot silently return.
|
|
22
|
+
*
|
|
23
|
+
* Storage is written against the `PgExecutor` seam (zero runtime deps); the
|
|
24
|
+
* production driver (Bun.sql / node-postgres) and the test driver (pglite)
|
|
25
|
+
* are wired by the host. See `pg-executor.ts` and the server README.
|
|
26
|
+
*
|
|
27
|
+
* ## commitSeq allocation under concurrency
|
|
28
|
+
*
|
|
29
|
+
* Per-partition `commitSeq` is a dense, gap-free counter (§2.1), allocated
|
|
30
|
+
* inside the push transaction by `UPDATE sync_partitions SET
|
|
31
|
+
* max_commit_seq = max_commit_seq + 1 … RETURNING`. The `UPDATE` takes a
|
|
32
|
+
* row-level write lock on the partition row for the duration of the
|
|
33
|
+
* transaction, so two concurrent pushes to the same partition serialize on
|
|
34
|
+
* that row: the second blocks until the first commits, then reads the
|
|
35
|
+
* advanced counter. This keeps the sequence dense (a Postgres `SEQUENCE`
|
|
36
|
+
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
37
|
+
* does not tolerate). Cross-partition pushes never contend.
|
|
38
|
+
*/
|
|
39
|
+
import type { ScopeMap } from '@syncular/core';
|
|
2
40
|
import { type PgExecutor } from './pg-executor.js';
|
|
3
41
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
4
42
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
@@ -77,6 +115,10 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
77
115
|
getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
|
|
78
116
|
listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
|
|
79
117
|
pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
|
|
118
|
+
latestScopedChangeSeq(partition: string, query: {
|
|
119
|
+
readonly table: string;
|
|
120
|
+
readonly scopeFilter: ScopeMap;
|
|
121
|
+
}): Promise<number>;
|
|
80
122
|
readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
|
|
81
123
|
scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
|
|
82
124
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
package/dist/postgres-storage.js
CHANGED
|
@@ -5,7 +5,7 @@ import { serveNotReadyError } from './readiness.js';
|
|
|
5
5
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_POSTGRES, scanRowBatchSql, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
8
|
+
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_POSTGRES, scanRowBatchSql, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
9
|
import { matchesEffective } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
11
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
@@ -1402,6 +1402,17 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1402
1402
|
deadLetter: rows.filter((row) => row.status === 'dead-letter').length,
|
|
1403
1403
|
};
|
|
1404
1404
|
}
|
|
1405
|
+
async latestScopedChangeSeq(partition, query) {
|
|
1406
|
+
const firstVariable = Object.keys(query.scopeFilter).sort()[0];
|
|
1407
|
+
const values = firstVariable === undefined
|
|
1408
|
+
? []
|
|
1409
|
+
: (query.scopeFilter[firstVariable] ?? []);
|
|
1410
|
+
if (firstVariable === undefined || values.length === 0)
|
|
1411
|
+
return 0;
|
|
1412
|
+
// Concurrent reads of one table in a pull share one statement.
|
|
1413
|
+
const [row] = await this.#batchedPage(LATEST_SCOPED_CHANGE_BATCH_SQL, partition, query.table, { var: firstVariable, vals: values });
|
|
1414
|
+
return row === undefined ? 0 : asNumber(row.seq);
|
|
1415
|
+
}
|
|
1405
1416
|
async readCommitWindow(partition, query) {
|
|
1406
1417
|
const variables = Object.keys(query.scopeFilter).sort();
|
|
1407
1418
|
const firstVariable = variables[0];
|
package/dist/pull.d.ts
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
* (SPEC.md §4, §5).
|
|
4
4
|
*/
|
|
5
5
|
import { type PullHeaderFrame, type ResponseFrame, type ScopeMap, type SubscriptionFrame } from '@syncular/core';
|
|
6
|
-
import type { SyncRequestContext } from './context.js';
|
|
6
|
+
import type { SyncRequestContext, SyncServerConfig } from './context.js';
|
|
7
7
|
import type { PullSegmentSummary } from './events.js';
|
|
8
|
-
import type
|
|
8
|
+
import { type CompiledSchema, type CompiledTable } from './schema.js';
|
|
9
|
+
import type { SegmentRecord } from './segment-store.js';
|
|
9
10
|
import type { StoredCommit, StoredRow } from './storage.js';
|
|
10
11
|
/** §4.2 accept bitmask. */
|
|
11
12
|
export declare const ACCEPT_INLINE_ROWS: number;
|
|
@@ -41,19 +42,26 @@ export interface PullSectionTrace {
|
|
|
41
42
|
/**
|
|
42
43
|
* The storage reads a section needs before its first frame. A pull starts
|
|
43
44
|
* them for every subscription before awaiting any, so a storage that batches
|
|
44
|
-
* concurrent page reads (PostgreSQL) answers all subscriptions of one
|
|
45
|
-
* with one statement, and the horizon re-check runs once per pull.
|
|
45
|
+
* concurrent page reads (PostgreSQL, D1) answers all subscriptions of one
|
|
46
|
+
* table with one statement, and the horizon re-check runs once per pull.
|
|
46
47
|
*/
|
|
47
48
|
export interface PreparedSection {
|
|
48
49
|
/** §4.5 window read for an incremental cursor; empty when none was read. */
|
|
49
50
|
readonly commits: StoredCommit[];
|
|
50
51
|
/** Horizon to judge the cursor against, re-read after any window read. */
|
|
51
52
|
readonly horizonSeq: number;
|
|
52
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* A bootstrap's first snapshot page (`limitSnapshotRows + 1` rows from the
|
|
55
|
+
* resume cursor), which is also the §5.3 image eligibility probe.
|
|
56
|
+
*/
|
|
53
57
|
readonly firstPage: StoredRow[] | undefined;
|
|
58
|
+
/** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
|
|
59
|
+
readonly pin: number | undefined;
|
|
60
|
+
/** §5.3 stored image for a sqlite-lane bootstrap's reuse key, if any. */
|
|
61
|
+
readonly image: SegmentRecord | undefined;
|
|
54
62
|
}
|
|
55
63
|
/** Run the pre-frame storage reads of every subscription concurrently. */
|
|
56
|
-
export declare function prepareSections(ctx: SyncRequestContext, limits: PullLimits, plans: readonly SubscriptionPlan[], maxSeq: number, horizonSeq: number): Promise<PreparedSection[]>;
|
|
64
|
+
export declare function prepareSections(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plans: readonly SubscriptionPlan[], maxSeq: number, horizonSeq: number, logEpoch: string): Promise<PreparedSection[]>;
|
|
57
65
|
/**
|
|
58
66
|
* Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
|
|
59
67
|
* returning the cursor recorded for the retention watermark (§4.5).
|
|
@@ -61,3 +69,35 @@ export declare function prepareSections(ctx: SyncRequestContext, limits: PullLim
|
|
|
61
69
|
export declare function subscriptionSection(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plan: SubscriptionPlan, maxSeq: number, horizonSeq: number, trace?: PullSectionTrace, logEpoch?: string): AsyncGenerator<ResponseFrame, SubscriptionResult>;
|
|
62
70
|
/** The section of one subscription whose reads `prepareSections` ran. */
|
|
63
71
|
export declare function preparedSection(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plan: SubscriptionPlan, maxSeq: number, prepared: PreparedSection, trace?: PullSectionTrace, logEpoch?: string): AsyncGenerator<ResponseFrame, SubscriptionResult>;
|
|
72
|
+
export interface PublishSqliteImageInput {
|
|
73
|
+
/** The sync config the serving host runs, plus a `sqliteImageBuilder`. */
|
|
74
|
+
readonly config: SyncServerConfig;
|
|
75
|
+
readonly partition: string;
|
|
76
|
+
readonly table: string;
|
|
77
|
+
/**
|
|
78
|
+
* The effective scopes the subscribing clients hold for `table` (§3.2):
|
|
79
|
+
* the reuse key carries their digest, so publish once per distinct scope
|
|
80
|
+
* set that clients subscribe with.
|
|
81
|
+
*/
|
|
82
|
+
readonly scopes: ScopeMap;
|
|
83
|
+
}
|
|
84
|
+
export interface PublishedSqliteImage {
|
|
85
|
+
readonly segmentId: string;
|
|
86
|
+
/** The §4.7 scope pin the image is stored under. */
|
|
87
|
+
readonly asOfCommitSeq: number;
|
|
88
|
+
readonly scopeDigest: string;
|
|
89
|
+
readonly rowCount: number;
|
|
90
|
+
readonly byteLength: number;
|
|
91
|
+
/** `reused` when an unexpired image for the same reuse key already existed. */
|
|
92
|
+
readonly origin: 'built' | 'reused';
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
|
|
96
|
+
* up, from a host with a SQLite engine (a Bun process with the production
|
|
97
|
+
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
98
|
+
* reuse key, the build), so the stored identity is exactly the one a
|
|
99
|
+
* serving host without a builder finds. The image stays current until a
|
|
100
|
+
* change reaches the scope; its lifetime is the segment store's TTL at
|
|
101
|
+
* publication.
|
|
102
|
+
*/
|
|
103
|
+
export declare function publishSqliteImage(input: PublishSqliteImageInput): Promise<PublishedSqliteImage>;
|