@syncular/server 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +3 -1
- package/dist/admin.js +7 -1
- package/dist/blob-handlers.js +1 -1
- package/dist/context.d.ts +8 -1
- package/dist/d1-storage.d.ts +11 -3
- package/dist/d1-storage.js +84 -2
- package/dist/errors.js +10 -0
- package/dist/handler.js +50 -4
- package/dist/operations.js +3 -3
- package/dist/postgres-storage.d.ts +12 -4
- package/dist/postgres-storage.js +266 -6
- package/dist/push.js +18 -0
- package/dist/readiness.d.ts +9 -2
- package/dist/readiness.js +27 -2
- package/dist/realtime.js +20 -1
- package/dist/segment-download.js +8 -0
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +15 -0
- package/dist/sqlite-storage.d.ts +11 -3
- package/dist/sqlite-storage.js +296 -33
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +4 -0
- package/dist/storage.d.ts +127 -2
- package/dist/storage.js +40 -1
- package/package.json +2 -2
- package/src/admin.ts +13 -1
- package/src/blob-handlers.ts +1 -1
- package/src/context.ts +8 -0
- package/src/d1-storage.ts +143 -1
- package/src/errors.ts +10 -0
- package/src/handler.ts +62 -4
- package/src/operations.ts +17 -13
- package/src/postgres-storage.ts +430 -3
- package/src/push.ts +18 -0
- package/src/readiness.ts +39 -4
- package/src/realtime.ts +22 -1
- package/src/segment-download.ts +16 -0
- package/src/sqlite-dialect.ts +15 -0
- package/src/sqlite-storage.ts +439 -36
- package/src/storage-errors.ts +13 -1
- package/src/storage.ts +218 -1
package/dist/admin.d.ts
CHANGED
|
@@ -21,7 +21,7 @@ import type { LeaseRecord, LeaseStore } from './lease-store.js';
|
|
|
21
21
|
import { type RetentionPolicy } from './prune.js';
|
|
22
22
|
import { type ServerSchema } from './schema.js';
|
|
23
23
|
import type { SegmentStore, SegmentStoreStats } from './segment-store.js';
|
|
24
|
-
import type { CommitMetadata, ReactionStatus, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
|
|
24
|
+
import type { CheckpointDeclaration, CommitMetadata, ReactionStatus, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
|
|
25
25
|
/** A connected/known client as the console sees it (§4.5, §8.1). */
|
|
26
26
|
export interface AdminClient {
|
|
27
27
|
readonly clientId: string;
|
|
@@ -139,6 +139,8 @@ export interface SyncularAdminOptions {
|
|
|
139
139
|
* a storage instance that has not served a sync request yet.
|
|
140
140
|
*/
|
|
141
141
|
readonly schema?: ServerSchema;
|
|
142
|
+
/** RFC 0007: declarations `inspectRow` carries into `ensureSchema`. */
|
|
143
|
+
readonly checkpoints?: readonly CheckpointDeclaration[];
|
|
142
144
|
/** The event ring feeding the event tail. Absent ⇒ `events()` is empty. */
|
|
143
145
|
readonly ring?: RingBufferEvents;
|
|
144
146
|
readonly segments?: SegmentStore;
|
package/dist/admin.js
CHANGED
|
@@ -41,6 +41,7 @@ function toAdminClient(record, active, maxCommitSeq) {
|
|
|
41
41
|
export class SyncularAdmin {
|
|
42
42
|
#storage;
|
|
43
43
|
#schema;
|
|
44
|
+
#checkpoints;
|
|
44
45
|
#ring;
|
|
45
46
|
#segments;
|
|
46
47
|
#blobs;
|
|
@@ -51,6 +52,8 @@ export class SyncularAdmin {
|
|
|
51
52
|
this.#storage = options.storage;
|
|
52
53
|
if (options.schema !== undefined)
|
|
53
54
|
this.#schema = options.schema;
|
|
55
|
+
if (options.checkpoints !== undefined)
|
|
56
|
+
this.#checkpoints = options.checkpoints;
|
|
54
57
|
if (options.ring !== undefined)
|
|
55
58
|
this.#ring = options.ring;
|
|
56
59
|
if (options.segments !== undefined)
|
|
@@ -71,6 +74,9 @@ export class SyncularAdmin {
|
|
|
71
74
|
return new SyncularAdmin({
|
|
72
75
|
storage: config.storage,
|
|
73
76
|
schema: config.schema,
|
|
77
|
+
...(config.checkpoints !== undefined
|
|
78
|
+
? { checkpoints: config.checkpoints }
|
|
79
|
+
: {}),
|
|
74
80
|
segments: config.segments,
|
|
75
81
|
...(config.blobs !== undefined ? { blobs: config.blobs } : {}),
|
|
76
82
|
...(config.leases !== undefined ? { leases: config.leases.store } : {}),
|
|
@@ -138,7 +144,7 @@ export class SyncularAdmin {
|
|
|
138
144
|
async inspectRow(partition, table, rowId) {
|
|
139
145
|
const read = required(this.#storage.getRowScopes?.bind(this.#storage), 'storage');
|
|
140
146
|
if (this.#schema !== undefined) {
|
|
141
|
-
await this.#storage.ensureSchema(compileSchema(this.#schema));
|
|
147
|
+
await this.#storage.ensureSchema(compileSchema(this.#schema), this.#checkpoints);
|
|
142
148
|
}
|
|
143
149
|
const row = await read(partition, table, rowId);
|
|
144
150
|
if (row === undefined) {
|
package/dist/blob-handlers.js
CHANGED
|
@@ -149,7 +149,7 @@ async function downloadBlob(ctx, blobId) {
|
|
|
149
149
|
const schema = compileSchema(ctx.schema);
|
|
150
150
|
// A blob download can hit a fresh storage instance before any sync
|
|
151
151
|
// request has run (Workers): make sure the row tables exist first.
|
|
152
|
-
await ctx.storage.ensureSchema(schema);
|
|
152
|
+
await ctx.storage.ensureSchema(schema, ctx.checkpoints);
|
|
153
153
|
const rows = await listRows.call(ctx.storage, ctx.partition, blobId);
|
|
154
154
|
let authorized = false;
|
|
155
155
|
for (const row of rows) {
|
package/dist/context.d.ts
CHANGED
|
@@ -14,7 +14,7 @@ import type { ServerSchema } from './schema.js';
|
|
|
14
14
|
import type { SegmentStore } from './segment-store.js';
|
|
15
15
|
import type { BlobPresignConfig, BlobUploadPresignConfig, SegmentUrlConfig } from './signed-url.js';
|
|
16
16
|
import type { SqliteImageBuilder } from './sqlite-image.js';
|
|
17
|
-
import type { PartitionRegistryEntry, ServerStorage, StoredCommit } from './storage.js';
|
|
17
|
+
import type { CheckpointDeclaration, PartitionRegistryEntry, ServerStorage, StoredCommit } from './storage.js';
|
|
18
18
|
import type { CommitValidator, ValidatorRegistry } from './validate.js';
|
|
19
19
|
/** SSP2 body content type (§1.1). */
|
|
20
20
|
export declare const SSP2_CONTENT_TYPE = "application/vnd.syncular.sync.v2";
|
|
@@ -72,6 +72,13 @@ export interface RealtimeNotifier {
|
|
|
72
72
|
export interface SyncServerConfig {
|
|
73
73
|
readonly schema: ServerSchema;
|
|
74
74
|
readonly storage: ServerStorage;
|
|
75
|
+
/**
|
|
76
|
+
* RFC 0007: the backfill checkpoints this host installs. Every readiness
|
|
77
|
+
* path carries this set into `ensureSchema`. A process that omits it
|
|
78
|
+
* declares nothing, and the storage refuses to serve a partition whose
|
|
79
|
+
* stored state shows a checkpoint that is not activated.
|
|
80
|
+
*/
|
|
81
|
+
readonly checkpoints?: readonly CheckpointDeclaration[];
|
|
75
82
|
readonly segments: SegmentStore;
|
|
76
83
|
/**
|
|
77
84
|
* Blob store (§5.9). Absent ⇒ blobs unsupported: a table with a
|
package/dist/d1-storage.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
2
2
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
3
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
3
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
4
4
|
export interface D1PreparedStatement {
|
|
5
5
|
bind(...values: unknown[]): D1PreparedStatement;
|
|
6
6
|
first<T = Record<string, unknown>>(): Promise<T | null>;
|
|
@@ -36,7 +36,7 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
36
36
|
migrate(): Promise<void>;
|
|
37
37
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
38
38
|
table(name: string): CompiledTable;
|
|
39
|
-
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
39
|
+
ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
|
|
40
40
|
/** Run once per Worker invocation. Resume an incomplete result in another invocation. */
|
|
41
41
|
migrateSchema(schema: CompiledSchema, options?: {
|
|
42
42
|
readonly maxStatements?: number;
|
|
@@ -49,12 +49,20 @@ export declare class D1ServerStorage implements ServerStorage {
|
|
|
49
49
|
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
50
50
|
begin(partition: string): Promise<StorageTransaction>;
|
|
51
51
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
52
|
-
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
52
|
+
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, _checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
|
|
53
53
|
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
54
54
|
getHorizonSeq(partition: string): Promise<number>;
|
|
55
55
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
56
56
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
57
57
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
58
|
+
readCheckpoints(_partition: string): Promise<StoredCheckpoint[]>;
|
|
59
|
+
readServeGate(partition: string, _runningSchemaVersion: number): Promise<ServeGate>;
|
|
60
|
+
declareCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
|
|
61
|
+
activateCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _sources: readonly string[], _nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
|
|
62
|
+
claimCheckpoint(_partition: string, _name: string, _schemaVersion: number, _nowMs: number): Promise<StoredCheckpoint>;
|
|
63
|
+
advanceCheckpoint(_partition: string, _name: string, _ownerEpoch: number, _watermark: number, _observedRows: number, _nowMs: number): Promise<boolean>;
|
|
64
|
+
sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
|
|
65
|
+
hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
|
|
58
66
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
59
67
|
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
60
68
|
claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
|
package/dist/d1-storage.js
CHANGED
|
@@ -264,6 +264,21 @@ class D1Transaction {
|
|
|
264
264
|
// write for this partition is already serialized (normally by its DO).
|
|
265
265
|
this.#pushApplyCheckpoint = this.#buffer.length;
|
|
266
266
|
}
|
|
267
|
+
advanceCheckpoint(_name, _ownerEpoch, _watermark, _observedRows, _nowMs) {
|
|
268
|
+
// D1 installs no fence and declares no checkpoints in this release.
|
|
269
|
+
return Promise.reject(new StorageQueryError('sync.storage.checkpoint_unsupported'));
|
|
270
|
+
}
|
|
271
|
+
async readServeGate(_runningSchemaVersion) {
|
|
272
|
+
this.#assertOpen();
|
|
273
|
+
const marker = await this.#db
|
|
274
|
+
.prepare('SELECT schema_version FROM sync_schema_meta WHERE id=1')
|
|
275
|
+
.first();
|
|
276
|
+
return {
|
|
277
|
+
storedSchemaVersion: marker?.schema_version ?? 0,
|
|
278
|
+
logEpoch: undefined,
|
|
279
|
+
pending: [],
|
|
280
|
+
};
|
|
281
|
+
}
|
|
267
282
|
async commitRejectedPushResult(clientId, clientCommitId, result) {
|
|
268
283
|
this.#assertOpen();
|
|
269
284
|
const checkpoint = this.#pushApplyCheckpoint;
|
|
@@ -527,7 +542,12 @@ export class D1ServerStorage {
|
|
|
527
542
|
}
|
|
528
543
|
return table;
|
|
529
544
|
}
|
|
530
|
-
async ensureSchema(schema) {
|
|
545
|
+
async ensureSchema(schema, checkpoints) {
|
|
546
|
+
if (checkpoints !== undefined && checkpoints.length > 0) {
|
|
547
|
+
// No database-side fence can be installed atomically with schema
|
|
548
|
+
// visibility here; refuse instead of accepting a weaker guarantee.
|
|
549
|
+
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
550
|
+
}
|
|
531
551
|
if (this.#schemaVersion === schema.version) {
|
|
532
552
|
await this.#schemaDatabase().batch([]);
|
|
533
553
|
return;
|
|
@@ -972,7 +992,10 @@ export class D1ServerStorage {
|
|
|
972
992
|
.first();
|
|
973
993
|
return row?.max_commit_seq ?? 0;
|
|
974
994
|
}
|
|
975
|
-
async queryAuthoritative(partition, query) {
|
|
995
|
+
async queryAuthoritative(partition, query, _checkpoints) {
|
|
996
|
+
// D1 declares no checkpoints. `#schemaDatabase` prepends a guard that
|
|
997
|
+
// re-checks the published marker inside this same batch, so condition 2
|
|
998
|
+
// is already evaluated with the query (§2.4 D1 schema readiness).
|
|
976
999
|
const db = this.#schemaDatabase();
|
|
977
1000
|
if (this.#tables === undefined) {
|
|
978
1001
|
throw new Error('ensureSchema(schema) must run before registered queries');
|
|
@@ -1086,6 +1109,65 @@ export class D1ServerStorage {
|
|
|
1086
1109
|
.first();
|
|
1087
1110
|
return row?.seq ?? 0;
|
|
1088
1111
|
}
|
|
1112
|
+
async readCheckpoints(_partition) {
|
|
1113
|
+
// D1 supports no declared backfill checkpoints in this release: migrations
|
|
1114
|
+
// run out of band and no database-side fence can be installed atomically
|
|
1115
|
+
// with schema visibility. No checkpoint row can exist.
|
|
1116
|
+
return [];
|
|
1117
|
+
}
|
|
1118
|
+
async readServeGate(partition, _runningSchemaVersion) {
|
|
1119
|
+
// D1 has no checkpoints. The published marker is the only gate input;
|
|
1120
|
+
// `#schemaDatabase` already re-checks it per protected batch.
|
|
1121
|
+
const marker = await this.#db
|
|
1122
|
+
.prepare('SELECT schema_version FROM sync_schema_meta WHERE id=1')
|
|
1123
|
+
.first();
|
|
1124
|
+
return {
|
|
1125
|
+
storedSchemaVersion: marker?.schema_version ?? 0,
|
|
1126
|
+
logEpoch: await this.getPartitionLogEpoch(partition),
|
|
1127
|
+
pending: [],
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
1130
|
+
async declareCheckpoint(_partition, _name, _schemaVersion, _nowMs) {
|
|
1131
|
+
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
1132
|
+
}
|
|
1133
|
+
async activateCheckpoint(_partition, _name, _ownerEpoch, _watermark, _sources, _nowMs) {
|
|
1134
|
+
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
1135
|
+
}
|
|
1136
|
+
async claimCheckpoint(_partition, _name, _schemaVersion, _nowMs) {
|
|
1137
|
+
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
1138
|
+
}
|
|
1139
|
+
async advanceCheckpoint(_partition, _name, _ownerEpoch, _watermark, _observedRows, _nowMs) {
|
|
1140
|
+
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
1141
|
+
}
|
|
1142
|
+
async sourceCoverageSeq(partition, tables) {
|
|
1143
|
+
if (tables.length === 0)
|
|
1144
|
+
return 0;
|
|
1145
|
+
const tableParams = tables.map(() => '?').join(',');
|
|
1146
|
+
const row = await this.#db
|
|
1147
|
+
.prepare(`SELECT max(commit_seq) AS seq FROM sync_changes
|
|
1148
|
+
WHERE partition=? AND tbl IN (${tableParams})`)
|
|
1149
|
+
.bind(partition, ...tables)
|
|
1150
|
+
.first();
|
|
1151
|
+
return row?.seq ?? 0;
|
|
1152
|
+
}
|
|
1153
|
+
async hasSourceChangesAbove(partition, tables, seq) {
|
|
1154
|
+
// Read the horizon before the window scan: a horizon past `seq` means the
|
|
1155
|
+
// history that would answer the question is gone, so an empty scan proves
|
|
1156
|
+
// nothing and must not be reported as `clean`.
|
|
1157
|
+
const horizon = await this.getHorizonSeq(partition);
|
|
1158
|
+
if (tables.length === 0)
|
|
1159
|
+
return 'clean';
|
|
1160
|
+
const tableParams = tables.map(() => '?').join(',');
|
|
1161
|
+
const hit = await this.#db
|
|
1162
|
+
.prepare(`SELECT 1 AS hit FROM sync_changes
|
|
1163
|
+
WHERE partition=? AND tbl IN (${tableParams}) AND commit_seq>?
|
|
1164
|
+
LIMIT 1`)
|
|
1165
|
+
.bind(partition, ...tables, seq)
|
|
1166
|
+
.first();
|
|
1167
|
+
if (hit !== null)
|
|
1168
|
+
return 'changed';
|
|
1169
|
+
return horizon > seq ? 'unverifiable' : 'clean';
|
|
1170
|
+
}
|
|
1089
1171
|
async getRow(partition, table, rowId) {
|
|
1090
1172
|
const db = this.#schemaDatabase();
|
|
1091
1173
|
const record = await db
|
package/dist/errors.js
CHANGED
|
@@ -159,6 +159,16 @@ export const ERROR_CATALOG = {
|
|
|
159
159
|
recommendedAction: 'retryLater',
|
|
160
160
|
httpStatus: 500,
|
|
161
161
|
},
|
|
162
|
+
// §2.4 on a non-D1 backend: the server refuses while a declared backfill
|
|
163
|
+
// checkpoint is incomplete or the database is newer than the running build.
|
|
164
|
+
// Formerly host-only; promoted to the §10.2 catalogue so an in-flight
|
|
165
|
+
// request can be refused with a distinguishable identity.
|
|
166
|
+
'sync.schema_not_ready': {
|
|
167
|
+
category: 'internal',
|
|
168
|
+
retryable: true,
|
|
169
|
+
recommendedAction: 'retryLater',
|
|
170
|
+
httpStatus: 503,
|
|
171
|
+
},
|
|
162
172
|
'sync.too_many_operations': {
|
|
163
173
|
category: 'invalid-request',
|
|
164
174
|
retryable: false,
|
package/dist/handler.js
CHANGED
|
@@ -16,8 +16,11 @@ import { emitEvent, } from './events.js';
|
|
|
16
16
|
import { END_FRAME_BYTES, encodeResponseFrame, responseEnvelopeHeader, } from './frame-bytes.js';
|
|
17
17
|
import { ACCEPT_EXTERNAL_ROWS, ACCEPT_INLINE_ROWS, clampPullLimits, subscriptionSection, } from './pull.js';
|
|
18
18
|
import { processPushCommitWithTrace } from './push.js';
|
|
19
|
+
import { serveNotReadyError } from './readiness.js';
|
|
19
20
|
import { compileSchema } from './schema.js';
|
|
20
21
|
import { computeEffective } from './scopes.js';
|
|
22
|
+
import { StorageQueryError } from './storage-errors.js';
|
|
23
|
+
import { serveGateRefusal, serveGateTokenChanged, } from './storage.js';
|
|
21
24
|
function validateResolvedKeys(allowed, schema) {
|
|
22
25
|
for (const key of Object.keys(allowed)) {
|
|
23
26
|
if (!schema.declaredVariables.has(key)) {
|
|
@@ -500,13 +503,56 @@ async function createStreamCore(bytes, ctx, events, startedAtMs = 0) {
|
|
|
500
503
|
const schema = compileSchema(ctx.schema);
|
|
501
504
|
// Relational row tables: create/
|
|
502
505
|
// migrate on first contact; memoized per storage instance thereafter.
|
|
503
|
-
|
|
506
|
+
try {
|
|
507
|
+
await ctx.storage.ensureSchema(schema, ctx.checkpoints);
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
if (error instanceof StorageQueryError &&
|
|
511
|
+
error.code === 'sync.storage.checkpoint_incomplete') {
|
|
512
|
+
const refusal = serveGateRefusal(await ctx.storage.readServeGate(ctx.partition, schema.version), schema.version, ctx.checkpoints);
|
|
513
|
+
throw refusal !== undefined ? serveNotReadyError(refusal) : error;
|
|
514
|
+
}
|
|
515
|
+
throw error;
|
|
516
|
+
}
|
|
517
|
+
// RFC 0007 serve gate: re-read stored state on every request so an
|
|
518
|
+
// already-serving process notices a migration or a newly declared
|
|
519
|
+
// checkpoint. Read after `touchAuthenticatedPartition` establishes the
|
|
520
|
+
// partition's log epoch, so the entry token is the epoch this request
|
|
521
|
+
// serves under. Nothing has escaped at this point; the push transaction
|
|
522
|
+
// re-evaluates the gate under the partition lock.
|
|
504
523
|
const registry = await touchAuthenticatedPartition(ctx);
|
|
524
|
+
const gate = await ctx.storage.readServeGate(ctx.partition, schema.version);
|
|
525
|
+
const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
|
|
526
|
+
if (refusal !== undefined)
|
|
527
|
+
throw serveNotReadyError(refusal);
|
|
505
528
|
const plan = await planRequest(request, ctx, schema, registry);
|
|
506
|
-
if (events === undefined)
|
|
507
|
-
return streamResponse(plan, ctx, schema);
|
|
508
529
|
const report = { outcome: 'ok' };
|
|
509
|
-
|
|
530
|
+
// RFC 0007 read-verify-refuse for the streamed read path: the generator
|
|
531
|
+
// builds the whole response (its data reads) before yielding the first
|
|
532
|
+
// frame, then compares the gate token across those reads. Nothing escapes
|
|
533
|
+
// before the verification, and the buffered size is bounded by the request's
|
|
534
|
+
// pull limits and the server's inline segment cap. Every request verifies,
|
|
535
|
+
// including a mixed push+pull: a write that already applied when the token
|
|
536
|
+
// changed is durable and replayable under the same commit id / idempotency
|
|
537
|
+
// key, so refusing the request is safe (proved in the serve-gate lane).
|
|
538
|
+
const verified = async function* () {
|
|
539
|
+
const buffered = [];
|
|
540
|
+
for await (const chunk of streamResponse(plan, ctx, schema, report)) {
|
|
541
|
+
buffered.push(chunk);
|
|
542
|
+
}
|
|
543
|
+
const after = await ctx.storage.readServeGate(ctx.partition, schema.version);
|
|
544
|
+
const changed = serveGateTokenChanged(gate, after);
|
|
545
|
+
const afterRefusal = serveGateRefusal(after, schema.version, ctx.checkpoints);
|
|
546
|
+
if (afterRefusal !== undefined)
|
|
547
|
+
throw serveNotReadyError(afterRefusal);
|
|
548
|
+
if (changed !== undefined)
|
|
549
|
+
throw serveNotReadyError(changed);
|
|
550
|
+
for (const chunk of buffered)
|
|
551
|
+
yield chunk;
|
|
552
|
+
};
|
|
553
|
+
if (events === undefined)
|
|
554
|
+
return verified();
|
|
555
|
+
return instrumentedStream(verified(), ctx, events, plan, report, bytes.length, startedAtMs);
|
|
510
556
|
}
|
|
511
557
|
/** Counts response bytes and emits `request.handled` when the stream
|
|
512
558
|
* finishes — including a thrown host failure mid-stream (fail loud). */
|
package/dist/operations.js
CHANGED
|
@@ -185,7 +185,7 @@ export function registerRemoteQuery(descriptor, options) {
|
|
|
185
185
|
if (plan === undefined) {
|
|
186
186
|
throw syncError('operation.invalid_request', 'selected SQL has no generated relation plan; regenerate queries');
|
|
187
187
|
}
|
|
188
|
-
await ctx.storage.ensureSchema(schema);
|
|
188
|
+
await ctx.storage.ensureSchema(schema, ctx.checkpoints);
|
|
189
189
|
const prefix = 'SELECT * FROM (';
|
|
190
190
|
let result;
|
|
191
191
|
try {
|
|
@@ -200,7 +200,7 @@ export function registerRemoteQuery(descriptor, options) {
|
|
|
200
200
|
},
|
|
201
201
|
params: [...descriptor.bind(params), options.maxRows + 1],
|
|
202
202
|
tables: descriptor.tables,
|
|
203
|
-
});
|
|
203
|
+
}, ctx.checkpoints);
|
|
204
204
|
}
|
|
205
205
|
catch (error) {
|
|
206
206
|
if (error instanceof SyncError)
|
|
@@ -318,7 +318,7 @@ export function registerRemoteCommand(descriptor, options) {
|
|
|
318
318
|
}
|
|
319
319
|
const resolved = { ok: true, allowed };
|
|
320
320
|
const schema = compileSchema(ctx.schema);
|
|
321
|
-
await ctx.storage.ensureSchema(schema);
|
|
321
|
+
await ctx.storage.ensureSchema(schema, ctx.checkpoints);
|
|
322
322
|
const processed = await processPushOperationsWithTrace(ctx, schema, resolved, JSON.stringify(['remote-command', ctx.actorId, clientId]), JSON.stringify([descriptor.id, requestId]), async (tx) => commandOperations(await options.run(commandContext(ctx, tx, resolved, schema, clientId, descriptor.id, requestId), input), schema));
|
|
323
323
|
return {
|
|
324
324
|
revision: 1,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
2
2
|
import { type PgExecutor } from './pg-executor.js';
|
|
3
3
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
4
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
4
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
5
5
|
/**
|
|
6
6
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
7
7
|
*
|
|
@@ -31,7 +31,7 @@ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorI
|
|
|
31
31
|
* set, §5.9.5) as an index range, never a scan. `postgres-explain
|
|
32
32
|
* .test.ts` asserts an `Index` node here too.
|
|
33
33
|
*/
|
|
34
|
-
export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id)\n);\n";
|
|
34
|
+
export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n partition TEXT PRIMARY KEY,\n max_commit_seq BIGINT NOT NULL DEFAULT 0,\n horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n partition TEXT PRIMARY KEY,\n log_epoch TEXT NOT NULL,\n epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n writer_version BIGINT,\n PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n partition TEXT NOT NULL, tbl TEXT NOT NULL,\n var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n partition TEXT NOT NULL, client_id TEXT NOT NULL,\n client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n last_failure JSONB,\n PRIMARY KEY(partition, idempotency_key),\n CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n wire_version INTEGER NOT NULL DEFAULT 1,\n cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n blob_id TEXT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n commit_seq BIGINT NOT NULL,\n PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n partition TEXT NOT NULL, name TEXT NOT NULL,\n schema_version INTEGER NOT NULL,\n state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n watermark BIGINT NOT NULL DEFAULT 0,\n owner_epoch BIGINT NOT NULL DEFAULT 0,\n observed_rows BIGINT NOT NULL DEFAULT 0,\n updated_at_ms BIGINT NOT NULL,\n PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n partition TEXT PRIMARY KEY,\n required_writer_version INTEGER NOT NULL\n);\n";
|
|
35
35
|
export declare class PostgresServerStorage implements ServerStorage {
|
|
36
36
|
#private;
|
|
37
37
|
constructor(exec: PgExecutor);
|
|
@@ -39,7 +39,7 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
39
39
|
migrate(): Promise<void>;
|
|
40
40
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
41
41
|
table(name: string): CompiledTable;
|
|
42
|
-
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
42
|
+
ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
|
|
43
43
|
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
44
44
|
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
45
45
|
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
@@ -53,12 +53,20 @@ export declare class PostgresServerStorage implements ServerStorage {
|
|
|
53
53
|
*/
|
|
54
54
|
begin(partition: string): Promise<StorageTransaction>;
|
|
55
55
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
56
|
-
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
56
|
+
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest, checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
|
|
57
57
|
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
58
58
|
getHorizonSeq(partition: string): Promise<number>;
|
|
59
59
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
60
60
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
61
61
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
62
|
+
readCheckpoints(partition: string): Promise<StoredCheckpoint[]>;
|
|
63
|
+
readServeGate(partition: string, runningSchemaVersion: number): Promise<ServeGate>;
|
|
64
|
+
declareCheckpoint(partition: string, name: string, schemaVersion: number, nowMs: number): Promise<StoredCheckpoint>;
|
|
65
|
+
activateCheckpoint(partition: string, name: string, ownerEpoch: number, watermark: number, sources: readonly string[], nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
|
|
66
|
+
claimCheckpoint(partition: string, name: string, schemaVersion: number, nowMs: number): Promise<StoredCheckpoint>;
|
|
67
|
+
advanceCheckpoint(partition: string, name: string, ownerEpoch: number, watermark: number, observedRows: number, nowMs: number): Promise<boolean>;
|
|
68
|
+
sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
|
|
69
|
+
hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
|
|
62
70
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
63
71
|
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
64
72
|
claimReactions(partition: string, query: ReactionClaimQuery): Promise<StoredReaction[]>;
|