lakesync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/adapter.d.ts +369 -0
- package/dist/adapter.js +39 -0
- package/dist/adapter.js.map +1 -0
- package/dist/analyst.d.ts +268 -0
- package/dist/analyst.js +495 -0
- package/dist/analyst.js.map +1 -0
- package/dist/auth-CAVutXzx.d.ts +30 -0
- package/dist/base-poller-Qo_SmCZs.d.ts +82 -0
- package/dist/catalogue.d.ts +65 -0
- package/dist/catalogue.js +17 -0
- package/dist/catalogue.js.map +1 -0
- package/dist/chunk-4ARO6KTJ.js +257 -0
- package/dist/chunk-4ARO6KTJ.js.map +1 -0
- package/dist/chunk-5YOFCJQ7.js +1115 -0
- package/dist/chunk-5YOFCJQ7.js.map +1 -0
- package/dist/chunk-7D4SUZUM.js +38 -0
- package/dist/chunk-7D4SUZUM.js.map +1 -0
- package/dist/chunk-BNJOGBYK.js +335 -0
- package/dist/chunk-BNJOGBYK.js.map +1 -0
- package/dist/chunk-ICNT7I3K.js +1180 -0
- package/dist/chunk-ICNT7I3K.js.map +1 -0
- package/dist/chunk-P5DRFKIT.js +413 -0
- package/dist/chunk-P5DRFKIT.js.map +1 -0
- package/dist/chunk-X3RO5SYJ.js +880 -0
- package/dist/chunk-X3RO5SYJ.js.map +1 -0
- package/dist/client.d.ts +428 -0
- package/dist/client.js +2048 -0
- package/dist/client.js.map +1 -0
- package/dist/compactor.d.ts +342 -0
- package/dist/compactor.js +793 -0
- package/dist/compactor.js.map +1 -0
- package/dist/coordinator-CxckTzYW.d.ts +396 -0
- package/dist/db-types-BR6Kt4uf.d.ts +29 -0
- package/dist/gateway-D5SaaMvT.d.ts +337 -0
- package/dist/gateway-server.d.ts +306 -0
- package/dist/gateway-server.js +4663 -0
- package/dist/gateway-server.js.map +1 -0
- package/dist/gateway.d.ts +196 -0
- package/dist/gateway.js +79 -0
- package/dist/gateway.js.map +1 -0
- package/dist/hlc-DiD8QNG3.d.ts +70 -0
- package/dist/index.d.ts +245 -0
- package/dist/index.js +102 -0
- package/dist/index.js.map +1 -0
- package/dist/json-dYtqiL0F.d.ts +18 -0
- package/dist/nessie-client-DrNikVXy.d.ts +160 -0
- package/dist/parquet.d.ts +78 -0
- package/dist/parquet.js +15 -0
- package/dist/parquet.js.map +1 -0
- package/dist/proto.d.ts +434 -0
- package/dist/proto.js +67 -0
- package/dist/proto.js.map +1 -0
- package/dist/react.d.ts +147 -0
- package/dist/react.js +224 -0
- package/dist/react.js.map +1 -0
- package/dist/resolver-C3Wphi6O.d.ts +10 -0
- package/dist/result-CojzlFE2.d.ts +64 -0
- package/dist/src-QU2YLPZY.js +383 -0
- package/dist/src-QU2YLPZY.js.map +1 -0
- package/dist/src-WYBF5LOI.js +102 -0
- package/dist/src-WYBF5LOI.js.map +1 -0
- package/dist/src-WZNPHANQ.js +426 -0
- package/dist/src-WZNPHANQ.js.map +1 -0
- package/dist/types-Bs-QyOe-.d.ts +143 -0
- package/dist/types-DAQL_vU_.d.ts +118 -0
- package/dist/types-DSC_EiwR.d.ts +45 -0
- package/dist/types-V_jVu2sA.d.ts +73 -0
- package/package.json +119 -0
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
import { g as ActionHandler, i as ActionPush, l as AuthContext, j as ActionResponse, a as ActionValidationError, d as ActionDiscovery, b as SyncRulesContext } from './types-Bs-QyOe-.js';
|
|
2
|
+
import { H as HLCTimestamp, R as Result, S as SchemaError, b as ClockDriftError, B as BackpressureError, a as AdapterNotFoundError, A as AdapterError, F as FlushError } from './result-CojzlFE2.js';
|
|
3
|
+
import { I as IngestTarget } from './base-poller-Qo_SmCZs.js';
|
|
4
|
+
import { R as RowDelta, a as RowKey, T as TableSchema, b as SyncPush, S as SyncPull, c as SyncResponse } from './types-V_jVu2sA.js';
|
|
5
|
+
import { D as DatabaseAdapter } from './db-types-BR6Kt4uf.js';
|
|
6
|
+
import { L as LakeAdapter } from './types-DSC_EiwR.js';
|
|
7
|
+
import { N as NessieCatalogueClient } from './nessie-client-DrNikVXy.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Dispatches imperative actions to registered handlers.
|
|
11
|
+
*
|
|
12
|
+
* Manages idempotency via actionId deduplication and idempotencyKey mapping.
|
|
13
|
+
* Completely decoupled from the HLC clock — takes a callback for timestamp generation.
|
|
14
|
+
*/
|
|
15
|
+
declare class ActionDispatcher {
|
|
16
|
+
private actionHandlers;
|
|
17
|
+
private executedActions;
|
|
18
|
+
private idempotencyMap;
|
|
19
|
+
constructor(handlers?: Record<string, ActionHandler>);
|
|
20
|
+
/**
|
|
21
|
+
* Dispatch an action push to registered handlers.
|
|
22
|
+
*
|
|
23
|
+
* Iterates over actions, dispatches each to the registered ActionHandler
|
|
24
|
+
* by connector name. Supports idempotency via actionId deduplication and
|
|
25
|
+
* idempotencyKey mapping.
|
|
26
|
+
*
|
|
27
|
+
* @param msg - The action push containing one or more actions.
|
|
28
|
+
* @param hlcNow - Callback to get the current server HLC timestamp.
|
|
29
|
+
* @param context - Optional auth context for permission checks.
|
|
30
|
+
* @returns A `Result` containing results for each action.
|
|
31
|
+
*/
|
|
32
|
+
dispatch(msg: ActionPush, hlcNow: () => HLCTimestamp, context?: AuthContext): Promise<Result<ActionResponse, ActionValidationError>>;
|
|
33
|
+
/**
|
|
34
|
+
* Register a named action handler.
|
|
35
|
+
*
|
|
36
|
+
* @param name - Connector name (matches `Action.connector`).
|
|
37
|
+
* @param handler - The action handler to register.
|
|
38
|
+
*/
|
|
39
|
+
registerHandler(name: string, handler: ActionHandler): void;
|
|
40
|
+
/**
|
|
41
|
+
* Unregister a named action handler.
|
|
42
|
+
*
|
|
43
|
+
* @param name - The connector name to remove.
|
|
44
|
+
*/
|
|
45
|
+
unregisterHandler(name: string): void;
|
|
46
|
+
/**
|
|
47
|
+
* List all registered action handler names.
|
|
48
|
+
*
|
|
49
|
+
* @returns Array of registered connector names.
|
|
50
|
+
*/
|
|
51
|
+
listHandlers(): string[];
|
|
52
|
+
/**
|
|
53
|
+
* Describe all registered action handlers and their supported actions.
|
|
54
|
+
*
|
|
55
|
+
* Returns a map of connector name to its {@link ActionDescriptor} array,
|
|
56
|
+
* enabling frontend discovery of available actions.
|
|
57
|
+
*
|
|
58
|
+
* @returns An {@link ActionDiscovery} object listing connectors and their actions.
|
|
59
|
+
*/
|
|
60
|
+
describe(): ActionDiscovery;
|
|
61
|
+
/** Cache an action result for idempotency deduplication. */
|
|
62
|
+
private cacheActionResult;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Dual-structure delta buffer.
|
|
67
|
+
*
|
|
68
|
+
* Maintains an append-only log for event streaming (pull) and flush,
|
|
69
|
+
* plus a row-level index for O(1) conflict resolution lookups.
|
|
70
|
+
*/
|
|
71
|
+
declare class DeltaBuffer {
|
|
72
|
+
private log;
|
|
73
|
+
private index;
|
|
74
|
+
private deltaIds;
|
|
75
|
+
private estimatedBytes;
|
|
76
|
+
private createdAt;
|
|
77
|
+
private tableBytes;
|
|
78
|
+
private tableLog;
|
|
79
|
+
/** Append a delta to the log and upsert the index (post-conflict-resolution). */
|
|
80
|
+
append(delta: RowDelta): void;
|
|
81
|
+
/** Get the current merged state for a row (for conflict resolution). */
|
|
82
|
+
getRow(key: RowKey): RowDelta | undefined;
|
|
83
|
+
/** Check if a delta with this ID already exists in the log (for idempotency). */
|
|
84
|
+
hasDelta(deltaId: string): boolean;
|
|
85
|
+
/** Return change events from the log since a given HLC. */
|
|
86
|
+
getEventsSince(hlc: HLCTimestamp, limit: number): {
|
|
87
|
+
deltas: RowDelta[];
|
|
88
|
+
hasMore: boolean;
|
|
89
|
+
};
|
|
90
|
+
/** Check if the buffer should be flushed based on size or age thresholds. */
|
|
91
|
+
shouldFlush(config: {
|
|
92
|
+
maxBytes: number;
|
|
93
|
+
maxAgeMs: number;
|
|
94
|
+
}): boolean;
|
|
95
|
+
/** Per-table buffer statistics. */
|
|
96
|
+
tableStats(): Array<{
|
|
97
|
+
table: string;
|
|
98
|
+
byteSize: number;
|
|
99
|
+
deltaCount: number;
|
|
100
|
+
}>;
|
|
101
|
+
/** Drain only the specified table's deltas, leaving other tables intact. */
|
|
102
|
+
drainTable(table: string): RowDelta[];
|
|
103
|
+
/** Drain the log for flush. Returns log entries and clears both structures. */
|
|
104
|
+
drain(): RowDelta[];
|
|
105
|
+
/** Number of log entries */
|
|
106
|
+
get logSize(): number;
|
|
107
|
+
/** Number of unique rows in the index */
|
|
108
|
+
get indexSize(): number;
|
|
109
|
+
/** Estimated byte size of the buffer */
|
|
110
|
+
get byteSize(): number;
|
|
111
|
+
/** Average byte size per delta in the buffer (0 if empty). */
|
|
112
|
+
get averageDeltaBytes(): number;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Manages schema versioning and validation for the gateway.
|
|
117
|
+
*
|
|
118
|
+
* Validates incoming deltas against the current schema and supports
|
|
119
|
+
* safe schema evolution (adding nullable columns only).
|
|
120
|
+
*/
|
|
121
|
+
declare class SchemaManager {
|
|
122
|
+
private currentSchema;
|
|
123
|
+
private version;
|
|
124
|
+
private allowedColumns;
|
|
125
|
+
constructor(schema: TableSchema, version?: number);
|
|
126
|
+
/** Get the current schema and version. */
|
|
127
|
+
getSchema(): {
|
|
128
|
+
schema: TableSchema;
|
|
129
|
+
version: number;
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* Validate that a delta's columns are compatible with the current schema.
|
|
133
|
+
*
|
|
134
|
+
* Unknown columns result in a SchemaError. Missing columns are fine (sparse deltas).
|
|
135
|
+
* DELETE ops with empty columns are always valid.
|
|
136
|
+
*/
|
|
137
|
+
validateDelta(delta: RowDelta): Result<void, SchemaError>;
|
|
138
|
+
/**
|
|
139
|
+
* Evolve the schema by adding new nullable columns.
|
|
140
|
+
*
|
|
141
|
+
* Only adding columns is allowed. Removing columns or changing types
|
|
142
|
+
* returns a SchemaError.
|
|
143
|
+
*/
|
|
144
|
+
evolveSchema(newSchema: TableSchema): Result<{
|
|
145
|
+
version: number;
|
|
146
|
+
}, SchemaError>;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Result returned by {@link SyncGateway.handlePush}. */
|
|
150
|
+
interface HandlePushResult {
|
|
151
|
+
/** Server HLC after processing the push. */
|
|
152
|
+
serverHlc: HLCTimestamp;
|
|
153
|
+
/** Number of deltas accepted (including idempotent re-pushes). */
|
|
154
|
+
accepted: number;
|
|
155
|
+
/** Deltas actually ingested (excludes idempotent re-pushes). */
|
|
156
|
+
deltas: RowDelta[];
|
|
157
|
+
}
|
|
158
|
+
/** Configuration for buffer thresholds (subset of GatewayConfig). */
|
|
159
|
+
interface BufferConfig {
|
|
160
|
+
/** Maximum buffer size in bytes before triggering flush. */
|
|
161
|
+
maxBufferBytes: number;
|
|
162
|
+
/** Maximum buffer age in milliseconds before triggering flush. */
|
|
163
|
+
maxBufferAgeMs: number;
|
|
164
|
+
/** Adaptive buffer configuration for wide-column deltas. */
|
|
165
|
+
adaptiveBufferConfig?: {
|
|
166
|
+
/** Average delta byte threshold above which flush triggers earlier. */
|
|
167
|
+
wideColumnThreshold: number;
|
|
168
|
+
/** Factor to reduce effective maxBufferBytes (0-1). */
|
|
169
|
+
reductionFactor: number;
|
|
170
|
+
};
|
|
171
|
+
/** Maximum buffer bytes before rejecting pushes (default: 2 × maxBufferBytes). */
|
|
172
|
+
maxBackpressureBytes?: number;
|
|
173
|
+
/** Maximum buffer bytes per table before auto-flushing that table. */
|
|
174
|
+
perTableBudgetBytes?: number;
|
|
175
|
+
}
|
|
176
|
+
/** Configuration for the sync gateway */
|
|
177
|
+
interface GatewayConfig extends BufferConfig {
|
|
178
|
+
/** Unique gateway identifier */
|
|
179
|
+
gatewayId: string;
|
|
180
|
+
/** Flush output format. Defaults to "parquet". */
|
|
181
|
+
flushFormat?: "json" | "parquet";
|
|
182
|
+
/** Table schema — required for Parquet flush. */
|
|
183
|
+
tableSchema?: TableSchema;
|
|
184
|
+
/** Optional Nessie catalogue client for Iceberg snapshot registration. */
|
|
185
|
+
catalogue?: NessieCatalogueClient;
|
|
186
|
+
/** Optional schema manager for delta validation. */
|
|
187
|
+
schemaManager?: SchemaManager;
|
|
188
|
+
/** Optional storage adapter — LakeAdapter (S3/R2) or DatabaseAdapter (Postgres/MySQL). */
|
|
189
|
+
adapter?: LakeAdapter | DatabaseAdapter;
|
|
190
|
+
/** Named source adapters for adapter-sourced pulls. */
|
|
191
|
+
sourceAdapters?: Record<string, DatabaseAdapter>;
|
|
192
|
+
/** Named action handlers for imperative action execution. */
|
|
193
|
+
actionHandlers?: Record<string, ActionHandler>;
|
|
194
|
+
}
|
|
195
|
+
/** Gateway runtime state */
|
|
196
|
+
interface GatewayState {
|
|
197
|
+
/** Current server HLC */
|
|
198
|
+
hlc: HLCTimestamp;
|
|
199
|
+
/** Whether a flush is currently in progress */
|
|
200
|
+
flushing: boolean;
|
|
201
|
+
}
|
|
202
|
+
/** Versioned envelope for flushed data */
|
|
203
|
+
interface FlushEnvelope {
|
|
204
|
+
/** Envelope format version */
|
|
205
|
+
version: 1;
|
|
206
|
+
/** Gateway that produced this flush */
|
|
207
|
+
gatewayId: string;
|
|
208
|
+
/** ISO 8601 creation timestamp */
|
|
209
|
+
createdAt: string;
|
|
210
|
+
/** Range of HLC timestamps in this flush */
|
|
211
|
+
hlcRange: {
|
|
212
|
+
min: HLCTimestamp;
|
|
213
|
+
max: HLCTimestamp;
|
|
214
|
+
};
|
|
215
|
+
/** Number of deltas in this flush */
|
|
216
|
+
deltaCount: number;
|
|
217
|
+
/** Estimated byte size */
|
|
218
|
+
byteSize: number;
|
|
219
|
+
/** The flushed deltas */
|
|
220
|
+
deltas: RowDelta[];
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Sync gateway -- coordinates delta ingestion, conflict resolution, and flush.
|
|
225
|
+
*
|
|
226
|
+
* Thin facade composing ActionDispatcher, DeltaBuffer, and flushEntries.
|
|
227
|
+
*/
|
|
228
|
+
declare class SyncGateway implements IngestTarget {
|
|
229
|
+
private hlc;
|
|
230
|
+
readonly buffer: DeltaBuffer;
|
|
231
|
+
readonly actions: ActionDispatcher;
|
|
232
|
+
private config;
|
|
233
|
+
private adapter;
|
|
234
|
+
private flushing;
|
|
235
|
+
constructor(config: GatewayConfig, adapter?: LakeAdapter | DatabaseAdapter);
|
|
236
|
+
/** Restore drained entries back to the buffer for retry. */
|
|
237
|
+
private restoreEntries;
|
|
238
|
+
/**
|
|
239
|
+
* Handle an incoming push from a client.
|
|
240
|
+
*
|
|
241
|
+
* Validates HLC drift, resolves conflicts via LWW, and appends to the buffer.
|
|
242
|
+
*
|
|
243
|
+
* @param msg - The push message containing client deltas.
|
|
244
|
+
* @returns A `Result` with the new server HLC and accepted count,
|
|
245
|
+
* or a `ClockDriftError` if the client clock is too far ahead.
|
|
246
|
+
*/
|
|
247
|
+
handlePush(msg: SyncPush): Result<HandlePushResult, ClockDriftError | SchemaError | BackpressureError>;
|
|
248
|
+
/**
|
|
249
|
+
* Handle a pull request from a client.
|
|
250
|
+
*
|
|
251
|
+
* When `msg.source` is set, pulls deltas from the named source adapter
|
|
252
|
+
* instead of the in-memory buffer. Otherwise, returns change events
|
|
253
|
+
* from the log since the given HLC. When a {@link SyncRulesContext} is
|
|
254
|
+
* provided, deltas are post-filtered by the client's bucket definitions
|
|
255
|
+
* and JWT claims. The buffer path over-fetches (3x the requested limit)
|
|
256
|
+
* and retries up to 5 times to fill the page.
|
|
257
|
+
*
|
|
258
|
+
* @param msg - The pull message specifying the cursor and limit.
|
|
259
|
+
* @param context - Optional sync rules context for row-level filtering.
|
|
260
|
+
* @returns A `Result` containing the matching deltas, server HLC, and pagination flag.
|
|
261
|
+
*/
|
|
262
|
+
handlePull(msg: SyncPull & {
|
|
263
|
+
source: string;
|
|
264
|
+
}, context?: SyncRulesContext): Promise<Result<SyncResponse, AdapterNotFoundError | AdapterError>>;
|
|
265
|
+
handlePull(msg: SyncPull, context?: SyncRulesContext): Result<SyncResponse, never>;
|
|
266
|
+
/** Pull from the in-memory buffer (original path). */
|
|
267
|
+
private handleBufferPull;
|
|
268
|
+
/** Pull from a named source adapter. */
|
|
269
|
+
private handleAdapterPull;
|
|
270
|
+
/**
|
|
271
|
+
* Flush the buffer to the configured adapter.
|
|
272
|
+
*
|
|
273
|
+
* Writes deltas as either a Parquet file (default) or a JSON
|
|
274
|
+
* {@link FlushEnvelope} to the adapter, depending on
|
|
275
|
+
* `config.flushFormat`. If the write fails, the buffer entries
|
|
276
|
+
* are restored so they can be retried.
|
|
277
|
+
*
|
|
278
|
+
* @returns A `Result` indicating success or a `FlushError`.
|
|
279
|
+
*/
|
|
280
|
+
flush(): Promise<Result<void, FlushError>>;
|
|
281
|
+
/**
|
|
282
|
+
* Flush a single table's deltas from the buffer.
|
|
283
|
+
*
|
|
284
|
+
* Drains only the specified table's deltas and flushes them,
|
|
285
|
+
* leaving other tables in the buffer.
|
|
286
|
+
*/
|
|
287
|
+
flushTable(table: string): Promise<Result<void, FlushError>>;
|
|
288
|
+
/** Handle an incoming action push from a client. */
|
|
289
|
+
handleAction(msg: ActionPush, context?: AuthContext): Promise<Result<ActionResponse, ActionValidationError>>;
|
|
290
|
+
/** Register a named action handler. */
|
|
291
|
+
registerActionHandler(name: string, handler: ActionHandler): void;
|
|
292
|
+
/** Unregister a named action handler. */
|
|
293
|
+
unregisterActionHandler(name: string): void;
|
|
294
|
+
/** List all registered action handler names. */
|
|
295
|
+
listActionHandlers(): string[];
|
|
296
|
+
/** Describe all registered action handlers and their supported actions. */
|
|
297
|
+
describeActions(): ActionDiscovery;
|
|
298
|
+
/**
|
|
299
|
+
* Register a named source adapter for adapter-sourced pulls.
|
|
300
|
+
*
|
|
301
|
+
* @param name - Unique source name (used as the `source` parameter in pull requests).
|
|
302
|
+
* @param adapter - The database adapter to register.
|
|
303
|
+
*/
|
|
304
|
+
registerSource(name: string, adapter: DatabaseAdapter): void;
|
|
305
|
+
/**
|
|
306
|
+
* Unregister a named source adapter.
|
|
307
|
+
*
|
|
308
|
+
* @param name - The source name to remove.
|
|
309
|
+
*/
|
|
310
|
+
unregisterSource(name: string): void;
|
|
311
|
+
/**
|
|
312
|
+
* List all registered source adapter names.
|
|
313
|
+
*
|
|
314
|
+
* @returns Array of registered source adapter names.
|
|
315
|
+
*/
|
|
316
|
+
listSources(): string[];
|
|
317
|
+
/** Get per-table buffer statistics. */
|
|
318
|
+
get tableStats(): Array<{
|
|
319
|
+
table: string;
|
|
320
|
+
byteSize: number;
|
|
321
|
+
deltaCount: number;
|
|
322
|
+
}>;
|
|
323
|
+
/**
|
|
324
|
+
* Get tables that exceed the per-table budget.
|
|
325
|
+
*/
|
|
326
|
+
getTablesExceedingBudget(): string[];
|
|
327
|
+
/** Check if the buffer should be flushed based on config thresholds. */
|
|
328
|
+
shouldFlush(): boolean;
|
|
329
|
+
/** Get buffer statistics for monitoring. */
|
|
330
|
+
get bufferStats(): {
|
|
331
|
+
logSize: number;
|
|
332
|
+
indexSize: number;
|
|
333
|
+
byteSize: number;
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
export { ActionDispatcher as A, type BufferConfig as B, DeltaBuffer as D, type FlushEnvelope as F, type GatewayConfig as G, type HandlePushResult as H, SyncGateway as S, type GatewayState as a, SchemaManager as b };
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
export { A as AuthClaims, a as AuthError, v as verifyToken } from './auth-CAVutXzx.js';
|
|
2
|
+
import { D as DatabaseAdapter } from './db-types-BR6Kt4uf.js';
|
|
3
|
+
import { S as SyncGateway } from './gateway-D5SaaMvT.js';
|
|
4
|
+
import { R as RowDelta, c as SyncResponse } from './types-V_jVu2sA.js';
|
|
5
|
+
import { L as LakeAdapter } from './types-DSC_EiwR.js';
|
|
6
|
+
import { H as HLCTimestamp } from './result-CojzlFE2.js';
|
|
7
|
+
import './types-Bs-QyOe-.js';
|
|
8
|
+
import './base-poller-Qo_SmCZs.js';
|
|
9
|
+
import './hlc-DiD8QNG3.js';
|
|
10
|
+
import './nessie-client-DrNikVXy.js';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Interface for distributed locking across gateway-server instances.
|
|
14
|
+
*
|
|
15
|
+
* Used to coordinate exclusive operations (e.g. flush) across
|
|
16
|
+
* multiple instances behind a load balancer.
|
|
17
|
+
*/
|
|
18
|
+
interface DistributedLock {
|
|
19
|
+
/** Attempt to acquire a lock. Returns true if acquired. */
|
|
20
|
+
acquire(key: string, ttlMs: number): Promise<boolean>;
|
|
21
|
+
/** Release a previously acquired lock. */
|
|
22
|
+
release(key: string): Promise<void>;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Database-backed distributed lock using an advisory lock row.
|
|
26
|
+
*
|
|
27
|
+
* Uses a `lakesync_locks` table with columns: key, holder, expires_at.
|
|
28
|
+
* Lock acquisition is atomic via INSERT ... ON CONFLICT or UPDATE WHERE expires_at < NOW().
|
|
29
|
+
*/
|
|
30
|
+
declare class AdapterBasedLock implements DistributedLock {
|
|
31
|
+
private readonly adapter;
|
|
32
|
+
private readonly instanceId;
|
|
33
|
+
constructor(adapter: DatabaseAdapter, instanceId?: string);
|
|
34
|
+
acquire(key: string, ttlMs: number): Promise<boolean>;
|
|
35
|
+
release(key: string): Promise<void>;
|
|
36
|
+
private makeLockDelta;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Generic query function — abstracts any SQL database. */
|
|
40
|
+
type QueryFn = (sql: string, params?: unknown[]) => Promise<Record<string, unknown>[]>;
|
|
41
|
+
/** Configuration for a single polling ingest source. */
|
|
42
|
+
interface IngestSourceConfig {
|
|
43
|
+
/** Unique name for this source (used as clientId prefix). */
|
|
44
|
+
name: string;
|
|
45
|
+
/** Generic query function for the source database. */
|
|
46
|
+
queryFn: QueryFn;
|
|
47
|
+
/** Tables to poll. */
|
|
48
|
+
tables: IngestTableConfig[];
|
|
49
|
+
/** Poll interval in ms. Default 10_000 (10s). */
|
|
50
|
+
intervalMs?: number;
|
|
51
|
+
}
|
|
52
|
+
/** Configuration for a single table within an ingest source. */
|
|
53
|
+
interface IngestTableConfig {
|
|
54
|
+
/** Target table name in LakeSync (where deltas appear). */
|
|
55
|
+
table: string;
|
|
56
|
+
/** SQL query to fetch rows. Must return a row ID column + data columns. */
|
|
57
|
+
query: string;
|
|
58
|
+
/** Column used as the unique row identifier. Default: "id". */
|
|
59
|
+
rowIdColumn?: string;
|
|
60
|
+
/** Change detection strategy. */
|
|
61
|
+
strategy: CursorStrategy | DiffStrategy;
|
|
62
|
+
}
|
|
63
|
+
/** Cursor-based change detection — fast, requires a monotonically increasing column. */
|
|
64
|
+
interface CursorStrategy {
|
|
65
|
+
type: "cursor";
|
|
66
|
+
/** Column name for cursor (e.g. "updated_at"). Must be monotonically increasing. */
|
|
67
|
+
cursorColumn: string;
|
|
68
|
+
/** Look-back overlap in ms to catch late-committing transactions. Default 5000. */
|
|
69
|
+
lookbackMs?: number;
|
|
70
|
+
}
|
|
71
|
+
/** Full-diff change detection — slower, detects deletes, no schema requirement. */
|
|
72
|
+
interface DiffStrategy {
|
|
73
|
+
type: "diff";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Polls an external data source and pushes detected changes into a
|
|
78
|
+
* {@link SyncGateway} via `handlePush()`.
|
|
79
|
+
*
|
|
80
|
+
* Supports two change detection strategies:
|
|
81
|
+
* - **cursor**: fast incremental polling using a monotonically increasing column
|
|
82
|
+
* - **diff**: full-table comparison detecting inserts, updates, and deletes
|
|
83
|
+
*/
|
|
84
|
+
declare class SourcePoller {
|
|
85
|
+
private readonly config;
|
|
86
|
+
private readonly gateway;
|
|
87
|
+
private readonly hlc;
|
|
88
|
+
private readonly clientId;
|
|
89
|
+
private timer;
|
|
90
|
+
private running;
|
|
91
|
+
/** Cursor state per table (keyed by table name). */
|
|
92
|
+
private cursorStates;
|
|
93
|
+
/** Diff snapshot per table (keyed by table name). */
|
|
94
|
+
private diffStates;
|
|
95
|
+
constructor(config: IngestSourceConfig, gateway: SyncGateway);
|
|
96
|
+
/** Start the polling loop. */
|
|
97
|
+
start(): void;
|
|
98
|
+
/** Stop the polling loop. */
|
|
99
|
+
stop(): void;
|
|
100
|
+
/** Whether the poller is currently running. */
|
|
101
|
+
get isRunning(): boolean;
|
|
102
|
+
private schedulePoll;
|
|
103
|
+
/** Execute a single poll cycle across all configured tables. */
|
|
104
|
+
poll(): Promise<void>;
|
|
105
|
+
private pollCursor;
|
|
106
|
+
private pollDiff;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Persistence interface for buffering unflushed deltas across restarts.
|
|
111
|
+
*
|
|
112
|
+
* Implementations must be synchronous (no async) to avoid race conditions
|
|
113
|
+
* during the push-then-flush cycle.
|
|
114
|
+
*/
|
|
115
|
+
interface DeltaPersistence {
|
|
116
|
+
/** Append a batch of deltas to the persistence store. */
|
|
117
|
+
appendBatch(deltas: RowDelta[]): void;
|
|
118
|
+
/** Load all persisted deltas. */
|
|
119
|
+
loadAll(): RowDelta[];
|
|
120
|
+
/** Clear all persisted deltas (after successful flush). */
|
|
121
|
+
clear(): void;
|
|
122
|
+
/** Release resources. */
|
|
123
|
+
close(): void;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* In-memory persistence (no durability across restarts).
|
|
127
|
+
* Used as the default when `persistence` is "memory".
|
|
128
|
+
*/
|
|
129
|
+
declare class MemoryPersistence implements DeltaPersistence {
|
|
130
|
+
private buffer;
|
|
131
|
+
appendBatch(deltas: RowDelta[]): void;
|
|
132
|
+
loadAll(): RowDelta[];
|
|
133
|
+
clear(): void;
|
|
134
|
+
close(): void;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* SQLite-backed persistence using `better-sqlite3`.
|
|
138
|
+
*
|
|
139
|
+
* Stores deltas as JSON rows in a single table. On startup, loads all
|
|
140
|
+
* rows back as RowDeltas. On flush success, truncates the table.
|
|
141
|
+
*/
|
|
142
|
+
declare class SqlitePersistence implements DeltaPersistence {
|
|
143
|
+
private db;
|
|
144
|
+
constructor(path: string);
|
|
145
|
+
appendBatch(deltas: RowDelta[]): void;
|
|
146
|
+
loadAll(): RowDelta[];
|
|
147
|
+
clear(): void;
|
|
148
|
+
close(): void;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Configuration for the self-hosted gateway server. */
|
|
152
|
+
interface GatewayServerConfig {
|
|
153
|
+
/** Port to listen on (default 3000). */
|
|
154
|
+
port?: number;
|
|
155
|
+
/** Unique gateway identifier. */
|
|
156
|
+
gatewayId: string;
|
|
157
|
+
/** Storage adapter — LakeAdapter (S3/R2) or DatabaseAdapter (Postgres/MySQL). */
|
|
158
|
+
adapter?: LakeAdapter | DatabaseAdapter;
|
|
159
|
+
/** Maximum buffer size in bytes before triggering flush (default 4 MiB). */
|
|
160
|
+
maxBufferBytes?: number;
|
|
161
|
+
/** Maximum buffer age in milliseconds before triggering flush (default 30s). */
|
|
162
|
+
maxBufferAgeMs?: number;
|
|
163
|
+
/** HMAC-SHA256 secret for JWT verification. When omitted, auth is disabled. */
|
|
164
|
+
jwtSecret?: string;
|
|
165
|
+
/** Interval in milliseconds between periodic flushes (default 30s). */
|
|
166
|
+
flushIntervalMs?: number;
|
|
167
|
+
/** CORS allowed origins. When omitted, all origins are reflected. */
|
|
168
|
+
allowedOrigins?: string[];
|
|
169
|
+
/** DeltaBuffer persistence strategy (default "memory"). */
|
|
170
|
+
persistence?: "memory" | "sqlite";
|
|
171
|
+
/** Path to the SQLite file when `persistence` is "sqlite" (default "./lakesync-buffer.sqlite"). */
|
|
172
|
+
sqlitePath?: string;
|
|
173
|
+
/** Polling ingest sources. Each source is polled independently. */
|
|
174
|
+
ingestSources?: IngestSourceConfig[];
|
|
175
|
+
/** Optional clustering configuration for multi-instance deployment. */
|
|
176
|
+
cluster?: {
|
|
177
|
+
/** Distributed lock for coordinated flush. */
|
|
178
|
+
lock: DistributedLock;
|
|
179
|
+
/** Shared database adapter for cross-instance visibility. */
|
|
180
|
+
sharedAdapter: DatabaseAdapter;
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Self-hosted HTTP gateway server wrapping {@link SyncGateway}.
|
|
185
|
+
*
|
|
186
|
+
* Provides the same route surface as the Cloudflare Workers gateway-worker
|
|
187
|
+
* but runs as a standalone Node/Bun HTTP server. Supports optional JWT
|
|
188
|
+
* authentication, CORS, periodic flush, and SQLite-based buffer persistence.
|
|
189
|
+
*
|
|
190
|
+
* @example
|
|
191
|
+
* ```ts
|
|
192
|
+
* const server = new GatewayServer({
|
|
193
|
+
* gatewayId: "my-gateway",
|
|
194
|
+
* port: 3000,
|
|
195
|
+
* adapter: new PostgresAdapter({ connectionString: "..." }),
|
|
196
|
+
* jwtSecret: process.env.JWT_SECRET,
|
|
197
|
+
* });
|
|
198
|
+
* await server.start();
|
|
199
|
+
* ```
|
|
200
|
+
*/
|
|
201
|
+
declare class GatewayServer {
|
|
202
|
+
private gateway;
|
|
203
|
+
private config;
|
|
204
|
+
private httpServer;
|
|
205
|
+
private flushTimer;
|
|
206
|
+
private configStore;
|
|
207
|
+
private persistence;
|
|
208
|
+
private resolvedPort;
|
|
209
|
+
private wss;
|
|
210
|
+
private wsClients;
|
|
211
|
+
private pollers;
|
|
212
|
+
private connectorAdapters;
|
|
213
|
+
private connectorPollers;
|
|
214
|
+
private sharedBuffer;
|
|
215
|
+
constructor(config: GatewayServerConfig);
|
|
216
|
+
/**
|
|
217
|
+
* Start the HTTP server and periodic flush timer.
|
|
218
|
+
*
|
|
219
|
+
* Rehydrates unflushed deltas from the persistence layer before
|
|
220
|
+
* accepting connections.
|
|
221
|
+
*/
|
|
222
|
+
start(): Promise<void>;
|
|
223
|
+
/**
|
|
224
|
+
* Handle HTTP -> WebSocket upgrade.
|
|
225
|
+
*
|
|
226
|
+
* Authenticates via Bearer token in Authorization header or `?token=` query param.
|
|
227
|
+
*/
|
|
228
|
+
private handleUpgrade;
|
|
229
|
+
/**
|
|
230
|
+
* Handle an incoming WebSocket message (binary protobuf).
|
|
231
|
+
*/
|
|
232
|
+
private handleWsMessage;
|
|
233
|
+
/**
|
|
234
|
+
* Build sync rules context from WebSocket client claims.
|
|
235
|
+
*/
|
|
236
|
+
private buildWsSyncRulesContext;
|
|
237
|
+
/**
|
|
238
|
+
* Broadcast ingested deltas to all connected WebSocket clients except the sender.
|
|
239
|
+
*/
|
|
240
|
+
private broadcastDeltas;
|
|
241
|
+
private broadcastDeltasAsync;
|
|
242
|
+
/** Stop the server and clear the flush timer. */
|
|
243
|
+
stop(): Promise<void>;
|
|
244
|
+
/** The port the server is listening on (available after start). */
|
|
245
|
+
get port(): number;
|
|
246
|
+
/** The underlying SyncGateway instance for direct access. */
|
|
247
|
+
get gatewayInstance(): SyncGateway;
|
|
248
|
+
private handleRequest;
|
|
249
|
+
/**
|
|
250
|
+
* Handle `POST /sync/:gatewayId/push` -- ingest client deltas.
|
|
251
|
+
*/
|
|
252
|
+
private handlePush;
|
|
253
|
+
/**
|
|
254
|
+
* Handle `GET /sync/:gatewayId/pull` -- retrieve deltas since a given HLC.
|
|
255
|
+
*/
|
|
256
|
+
private handlePull;
|
|
257
|
+
/**
|
|
258
|
+
* Handle `POST /sync/:gatewayId/action` -- execute imperative actions.
|
|
259
|
+
*/
|
|
260
|
+
private handleAction;
|
|
261
|
+
/** Handle `GET /sync/:gatewayId/actions` -- describe available action handlers. */
|
|
262
|
+
private handleDescribeActions;
|
|
263
|
+
/** Handle `POST /admin/flush/:gatewayId` -- manual flush. */
|
|
264
|
+
private handleFlush;
|
|
265
|
+
/** Handle `POST /admin/schema/:gatewayId` -- save table schema. */
|
|
266
|
+
private handleSaveSchemaRoute;
|
|
267
|
+
/** Handle `POST /admin/sync-rules/:gatewayId` -- save sync rules. */
|
|
268
|
+
private handleSaveSyncRulesRoute;
|
|
269
|
+
/** Handle `POST /admin/connectors/:gatewayId` -- register a new connector. */
|
|
270
|
+
private handleRegisterConnector;
|
|
271
|
+
/** Handle `DELETE /admin/connectors/:gatewayId/:name` -- unregister a connector. */
|
|
272
|
+
private handleUnregisterConnector;
|
|
273
|
+
/** Handle `GET /admin/connectors/:gatewayId` -- list registered connectors. */
|
|
274
|
+
private handleListConnectorsRoute;
|
|
275
|
+
/** Handle `GET /admin/metrics/:gatewayId` -- return buffer stats and process memory. */
|
|
276
|
+
private handleMetricsRoute;
|
|
277
|
+
private periodicFlush;
|
|
278
|
+
private corsHeaders;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Write-through buffer that pushes to both the in-memory gateway buffer
|
|
283
|
+
* and a shared database adapter for cross-instance visibility.
|
|
284
|
+
*
|
|
285
|
+
* Pull merges in-memory buffer results with adapter query results,
|
|
286
|
+
* deduplicating by deltaId.
|
|
287
|
+
*/
|
|
288
|
+
declare class SharedBuffer {
|
|
289
|
+
private readonly sharedAdapter;
|
|
290
|
+
constructor(sharedAdapter: DatabaseAdapter);
|
|
291
|
+
/**
|
|
292
|
+
* Write-through push: write to shared adapter for cross-instance visibility.
|
|
293
|
+
*
|
|
294
|
+
* Gateway buffer handles fast reads; shared adapter handles
|
|
295
|
+
* cross-instance visibility and durability.
|
|
296
|
+
*/
|
|
297
|
+
writeThroughPush(deltas: RowDelta[]): Promise<void>;
|
|
298
|
+
/**
|
|
299
|
+
* Merge pull: combine local buffer results with shared adapter results.
|
|
300
|
+
*
|
|
301
|
+
* Deduplicates by deltaId to avoid returning the same delta twice.
|
|
302
|
+
*/
|
|
303
|
+
mergePull(localResult: SyncResponse, sinceHlc: HLCTimestamp): Promise<SyncResponse>;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export { AdapterBasedLock, type CursorStrategy, type DeltaPersistence, type DiffStrategy, type DistributedLock, GatewayServer, type GatewayServerConfig, type IngestSourceConfig, type IngestTableConfig, MemoryPersistence, type QueryFn, SharedBuffer, SourcePoller, SqlitePersistence };
|