lakesync 0.1.5 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter.d.ts +199 -19
- package/dist/adapter.js +19 -3
- package/dist/analyst.js +2 -2
- package/dist/{base-poller-CBvhdvcj.d.ts → base-poller-Bj9kX9dv.d.ts} +76 -19
- package/dist/catalogue.d.ts +1 -1
- package/dist/catalogue.js +3 -3
- package/dist/chunk-DGUM43GV.js +11 -0
- package/dist/{chunk-PWGQ3PXE.js → chunk-JI4C4R5H.js} +280 -140
- package/dist/chunk-JI4C4R5H.js.map +1 -0
- package/dist/{chunk-L4ZL5JA7.js → chunk-KVSWLIJR.js} +2 -2
- package/dist/{chunk-7UBS6MFH.js → chunk-LDFFCG2K.js} +377 -247
- package/dist/chunk-LDFFCG2K.js.map +1 -0
- package/dist/{chunk-Z7FGLEQU.js → chunk-LPWXOYNS.js} +376 -287
- package/dist/chunk-LPWXOYNS.js.map +1 -0
- package/dist/{chunk-SZSGSTVZ.js → chunk-PYRS74YP.js} +15 -4
- package/dist/{chunk-SZSGSTVZ.js.map → chunk-PYRS74YP.js.map} +1 -1
- package/dist/{chunk-TVLTXHW6.js → chunk-QNITY4F6.js} +30 -7
- package/dist/{chunk-TVLTXHW6.js.map → chunk-QNITY4F6.js.map} +1 -1
- package/dist/{chunk-46CKACNC.js → chunk-SSICS5KI.js} +2 -2
- package/dist/{chunk-B3QEUG6E.js → chunk-TMLG32QV.js} +2 -2
- package/dist/client.d.ts +164 -13
- package/dist/client.js +310 -163
- package/dist/client.js.map +1 -1
- package/dist/compactor.d.ts +1 -1
- package/dist/compactor.js +4 -4
- package/dist/connector-jira.d.ts +2 -2
- package/dist/connector-jira.js +3 -3
- package/dist/connector-salesforce.d.ts +2 -2
- package/dist/connector-salesforce.js +3 -3
- package/dist/{coordinator-DN8D8C7W.d.ts → coordinator-NXy6tA0h.d.ts} +23 -16
- package/dist/{db-types-B6_JKQWK.d.ts → db-types-CfLMUBfW.d.ts} +1 -1
- package/dist/gateway-server.d.ts +158 -64
- package/dist/gateway-server.js +482 -4003
- package/dist/gateway-server.js.map +1 -1
- package/dist/gateway.d.ts +61 -104
- package/dist/gateway.js +12 -6
- package/dist/index.d.ts +45 -10
- package/dist/index.js +14 -2
- package/dist/parquet.d.ts +1 -1
- package/dist/parquet.js +3 -3
- package/dist/proto.d.ts +1 -1
- package/dist/proto.js +3 -3
- package/dist/react.d.ts +47 -10
- package/dist/react.js +88 -40
- package/dist/react.js.map +1 -1
- package/dist/{registry-BN_9spxE.d.ts → registry-BcspAtZI.d.ts} +19 -4
- package/dist/{gateway-CvO7Xy3T.d.ts → request-handler-pUvL7ozF.d.ts} +139 -10
- package/dist/{resolver-BZURzdlL.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
- package/dist/{src-RR7I76OL.js → src-B6NLV3FP.js} +4 -4
- package/dist/{src-SLVE5567.js → src-ROW4XLO7.js} +15 -3
- package/dist/{src-V2CTPR7V.js → src-ZRHKG42A.js} +4 -4
- package/dist/{types-GGBfZBKQ.d.ts → types-BdGBv2ba.d.ts} +23 -2
- package/dist/{types-D-E0VrfS.d.ts → types-BrcD1oJg.d.ts} +26 -19
- package/package.json +1 -1
- package/dist/chunk-7D4SUZUM.js +0 -38
- package/dist/chunk-7UBS6MFH.js.map +0 -1
- package/dist/chunk-PWGQ3PXE.js.map +0 -1
- package/dist/chunk-Z7FGLEQU.js.map +0 -1
- /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
- /package/dist/{chunk-L4ZL5JA7.js.map → chunk-KVSWLIJR.js.map} +0 -0
- /package/dist/{chunk-46CKACNC.js.map → chunk-SSICS5KI.js.map} +0 -0
- /package/dist/{chunk-B3QEUG6E.js.map → chunk-TMLG32QV.js.map} +0 -0
- /package/dist/{src-RR7I76OL.js.map → src-B6NLV3FP.js.map} +0 -0
- /package/dist/{src-SLVE5567.js.map → src-ROW4XLO7.js.map} +0 -0
- /package/dist/{src-V2CTPR7V.js.map → src-ZRHKG42A.js.map} +0 -0
package/dist/adapter.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { BigQuery } from '@google-cloud/bigquery';
|
|
2
2
|
import { R as Result, A as AdapterError, H as HLCTimestamp } from './result-CojzlFE2.js';
|
|
3
|
-
import { R as RowDelta, T as TableSchema,
|
|
4
|
-
import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-
|
|
5
|
-
export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-
|
|
6
|
-
import { C as ConnectorConfig } from './types-
|
|
3
|
+
import { R as RowDelta, T as TableSchema, a as ColumnDelta } from './types-BdGBv2ba.js';
|
|
4
|
+
import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-CfLMUBfW.js';
|
|
5
|
+
export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-CfLMUBfW.js';
|
|
6
|
+
import { C as ConnectorConfig } from './types-BrcD1oJg.js';
|
|
7
7
|
import { L as LakeAdapter, A as AdapterConfig, O as ObjectInfo } from './types-DSC_EiwR.js';
|
|
8
8
|
import mysql from 'mysql2/promise';
|
|
9
9
|
import { Pool } from 'pg';
|
|
@@ -39,6 +39,20 @@ interface Materialisable {
|
|
|
39
39
|
* Uses duck-typing (same pattern as `isDatabaseAdapter`).
|
|
40
40
|
*/
|
|
41
41
|
declare function isMaterialisable(adapter: unknown): adapter is Materialisable;
|
|
42
|
+
/**
|
|
43
|
+
* Resolve the primary key columns for a table schema.
|
|
44
|
+
* Defaults to `["row_id"]` when not explicitly set.
|
|
45
|
+
*/
|
|
46
|
+
declare function resolvePrimaryKey(schema: TableSchema): string[];
|
|
47
|
+
/**
|
|
48
|
+
* Resolve the conflict columns used for upsert ON CONFLICT targeting.
|
|
49
|
+
* When `externalIdColumn` is set, upserts resolve on that column instead of the PK.
|
|
50
|
+
*/
|
|
51
|
+
declare function resolveConflictColumns(schema: TableSchema): string[];
|
|
52
|
+
/**
|
|
53
|
+
* Whether tombstoned rows should be soft-deleted (default) or hard-deleted.
|
|
54
|
+
*/
|
|
55
|
+
declare function isSoftDelete(schema: TableSchema): boolean;
|
|
42
56
|
/**
|
|
43
57
|
* Group deltas by their table name, collecting the set of affected row IDs per table.
|
|
44
58
|
*
|
|
@@ -56,6 +70,56 @@ declare function groupDeltasByTable(deltas: ReadonlyArray<RowDelta>): Map<string
|
|
|
56
70
|
* @returns A map from source table name to schema.
|
|
57
71
|
*/
|
|
58
72
|
declare function buildSchemaIndex(schemas: ReadonlyArray<TableSchema>): Map<string, TableSchema>;
|
|
73
|
+
/** Minimal query interface for executing SQL against a database. */
|
|
74
|
+
interface QueryExecutor {
|
|
75
|
+
query(sql: string, params: unknown[]): Promise<void>;
|
|
76
|
+
queryRows(sql: string, params: unknown[]): Promise<Array<{
|
|
77
|
+
row_id: string;
|
|
78
|
+
columns: string | ColumnDelta[];
|
|
79
|
+
op: string;
|
|
80
|
+
}>>;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* SQL dialect interface — encapsulates the syntactic differences between
|
|
84
|
+
* Postgres, MySQL, and BigQuery for the materialise algorithm.
|
|
85
|
+
*/
|
|
86
|
+
interface SqlDialect {
|
|
87
|
+
/** Generate CREATE TABLE IF NOT EXISTS for the destination table. */
|
|
88
|
+
createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
|
|
89
|
+
sql: string;
|
|
90
|
+
params: unknown[];
|
|
91
|
+
};
|
|
92
|
+
/** Generate a query to fetch delta history for a set of affected row IDs. */
|
|
93
|
+
queryDeltaHistory(sourceTable: string, rowIds: string[]): {
|
|
94
|
+
sql: string;
|
|
95
|
+
params: unknown[];
|
|
96
|
+
};
|
|
97
|
+
/** Generate an upsert statement for the merged row states. */
|
|
98
|
+
buildUpsert(dest: string, schema: TableSchema, conflictCols: string[], softDelete: boolean, upserts: Array<{
|
|
99
|
+
rowId: string;
|
|
100
|
+
state: Record<string, unknown>;
|
|
101
|
+
}>): {
|
|
102
|
+
sql: string;
|
|
103
|
+
params: unknown[];
|
|
104
|
+
};
|
|
105
|
+
/** Generate a delete (hard or soft) statement for tombstoned row IDs. */
|
|
106
|
+
buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
|
|
107
|
+
sql: string;
|
|
108
|
+
params: unknown[];
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Execute the shared materialise algorithm using the provided dialect and executor.
|
|
113
|
+
*
|
|
114
|
+
* Algorithm: group by table -> build schema index -> for each table:
|
|
115
|
+
* create dest table -> query history -> merge -> upsert -> delete.
|
|
116
|
+
*
|
|
117
|
+
* @param executor - Executes SQL statements against the database.
|
|
118
|
+
* @param dialect - Generates dialect-specific SQL.
|
|
119
|
+
* @param deltas - The deltas that were just flushed.
|
|
120
|
+
* @param schemas - Table schemas defining destination tables and column mappings.
|
|
121
|
+
*/
|
|
122
|
+
declare function executeMaterialise(executor: QueryExecutor, dialect: SqlDialect, deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
|
|
59
123
|
|
|
60
124
|
/**
|
|
61
125
|
* Configuration for the BigQuery adapter.
|
|
@@ -71,6 +135,40 @@ interface BigQueryAdapterConfig {
|
|
|
71
135
|
/** Dataset location (default: "US"). */
|
|
72
136
|
location?: string;
|
|
73
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* BigQuery SQL dialect for the shared materialise algorithm.
|
|
140
|
+
*
|
|
141
|
+
* Uses `@name` named parameters, `MERGE` for upserts, `JSON` type,
|
|
142
|
+
* `TIMESTAMP` types, and `CLUSTER BY` for table creation.
|
|
143
|
+
*
|
|
144
|
+
* BigQuery's parameterised query API uses named params in a flat object,
|
|
145
|
+
* so this dialect encodes params as named keys and returns them as an
|
|
146
|
+
* array of `[key, value]` pairs that the executor unpacks into a
|
|
147
|
+
* `Record<string, unknown>`.
|
|
148
|
+
*/
|
|
149
|
+
declare class BigQuerySqlDialect implements SqlDialect {
|
|
150
|
+
private readonly dataset;
|
|
151
|
+
constructor(dataset: string);
|
|
152
|
+
createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
|
|
153
|
+
sql: string;
|
|
154
|
+
params: unknown[];
|
|
155
|
+
};
|
|
156
|
+
queryDeltaHistory(sourceTable: string, rowIds: string[]): {
|
|
157
|
+
sql: string;
|
|
158
|
+
params: unknown[];
|
|
159
|
+
};
|
|
160
|
+
buildUpsert(dest: string, schema: TableSchema, conflictCols: string[], softDelete: boolean, upserts: Array<{
|
|
161
|
+
rowId: string;
|
|
162
|
+
state: Record<string, unknown>;
|
|
163
|
+
}>): {
|
|
164
|
+
sql: string;
|
|
165
|
+
params: unknown[];
|
|
166
|
+
};
|
|
167
|
+
buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
|
|
168
|
+
sql: string;
|
|
169
|
+
params: unknown[];
|
|
170
|
+
};
|
|
171
|
+
}
|
|
74
172
|
/**
|
|
75
173
|
* BigQuery database adapter for LakeSync.
|
|
76
174
|
*
|
|
@@ -89,6 +187,8 @@ declare class BigQueryAdapter implements DatabaseAdapter, Materialisable {
|
|
|
89
187
|
readonly dataset: string;
|
|
90
188
|
/** @internal */
|
|
91
189
|
readonly location: string;
|
|
190
|
+
private readonly dialect;
|
|
191
|
+
private readonly executor;
|
|
92
192
|
constructor(config: BigQueryAdapterConfig);
|
|
93
193
|
/**
|
|
94
194
|
* Insert deltas into the database in a single batch.
|
|
@@ -113,10 +213,8 @@ declare class BigQueryAdapter implements DatabaseAdapter, Materialisable {
|
|
|
113
213
|
/**
|
|
114
214
|
* Materialise deltas into destination tables.
|
|
115
215
|
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
* deletes tombstoned rows. The consumer-owned `props` column is never
|
|
119
|
-
* touched on UPDATE.
|
|
216
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
217
|
+
* BigQuery SQL dialect.
|
|
120
218
|
*/
|
|
121
219
|
materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
|
|
122
220
|
/**
|
|
@@ -165,7 +263,7 @@ declare class CompositeAdapter implements DatabaseAdapter {
|
|
|
165
263
|
*
|
|
166
264
|
* Switches on `config.type` and creates the matching adapter using
|
|
167
265
|
* the type-specific connection configuration. Returns an {@link AdapterError}
|
|
168
|
-
* if the type
|
|
266
|
+
* if the type is unsupported or the adapter constructor throws.
|
|
169
267
|
*
|
|
170
268
|
* @param config - Validated connector configuration.
|
|
171
269
|
* @returns The instantiated adapter or an error.
|
|
@@ -185,8 +283,14 @@ interface FanOutAdapterConfig {
|
|
|
185
283
|
*
|
|
186
284
|
* Secondary failures are silently caught and never affect the return value.
|
|
187
285
|
* Use case: write to Postgres (fast, operational), replicate to BigQuery (analytics).
|
|
286
|
+
*
|
|
287
|
+
* **Materialisation:** This adapter exposes a `materialise()` method for
|
|
288
|
+
* duck-type compatibility with `isMaterialisable()`. When the primary adapter
|
|
289
|
+
* is itself materialisable, materialisation is delegated to it; otherwise
|
|
290
|
+
* the method is a graceful no-op returning `Ok`. Materialisable secondaries
|
|
291
|
+
* receive fire-and-forget replication.
|
|
188
292
|
*/
|
|
189
|
-
declare class FanOutAdapter implements DatabaseAdapter
|
|
293
|
+
declare class FanOutAdapter implements DatabaseAdapter {
|
|
190
294
|
private readonly primary;
|
|
191
295
|
private readonly secondaries;
|
|
192
296
|
constructor(config: FanOutAdapterConfig);
|
|
@@ -227,8 +331,14 @@ interface LifecycleAdapterConfig {
|
|
|
227
331
|
*
|
|
228
332
|
* Use {@link migrateToTier} as a background job to copy aged-out deltas
|
|
229
333
|
* from hot to cold.
|
|
334
|
+
*
|
|
335
|
+
* **Materialisation:** This adapter exposes a `materialise()` method for
|
|
336
|
+
* duck-type compatibility with `isMaterialisable()`. Materialisation is
|
|
337
|
+
* delegated to the hot tier only — cold tier stores archived deltas, not
|
|
338
|
+
* destination tables. When the hot adapter is not materialisable, the
|
|
339
|
+
* method is a graceful no-op returning `Ok`.
|
|
230
340
|
*/
|
|
231
|
-
declare class LifecycleAdapter implements DatabaseAdapter
|
|
341
|
+
declare class LifecycleAdapter implements DatabaseAdapter {
|
|
232
342
|
private readonly hot;
|
|
233
343
|
private readonly cold;
|
|
234
344
|
private readonly maxAgeMs;
|
|
@@ -331,6 +441,33 @@ declare class MinIOAdapter implements LakeAdapter {
|
|
|
331
441
|
deleteObjects(paths: string[]): Promise<Result<void, AdapterError>>;
|
|
332
442
|
}
|
|
333
443
|
|
|
444
|
+
/**
|
|
445
|
+
* MySQL SQL dialect for the shared materialise algorithm.
|
|
446
|
+
*
|
|
447
|
+
* Uses `?` positional parameters, `ON DUPLICATE KEY UPDATE`,
|
|
448
|
+
* `JSON` type, and `TIMESTAMP` types.
|
|
449
|
+
*/
|
|
450
|
+
declare class MySqlDialect implements SqlDialect {
|
|
451
|
+
createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
|
|
452
|
+
sql: string;
|
|
453
|
+
params: unknown[];
|
|
454
|
+
};
|
|
455
|
+
queryDeltaHistory(sourceTable: string, rowIds: string[]): {
|
|
456
|
+
sql: string;
|
|
457
|
+
params: unknown[];
|
|
458
|
+
};
|
|
459
|
+
buildUpsert(dest: string, schema: TableSchema, _conflictCols: string[], softDelete: boolean, upserts: Array<{
|
|
460
|
+
rowId: string;
|
|
461
|
+
state: Record<string, unknown>;
|
|
462
|
+
}>): {
|
|
463
|
+
sql: string;
|
|
464
|
+
params: unknown[];
|
|
465
|
+
};
|
|
466
|
+
buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
|
|
467
|
+
sql: string;
|
|
468
|
+
params: unknown[];
|
|
469
|
+
};
|
|
470
|
+
}
|
|
334
471
|
/**
|
|
335
472
|
* MySQL database adapter for LakeSync.
|
|
336
473
|
*
|
|
@@ -341,7 +478,9 @@ declare class MinIOAdapter implements LakeAdapter {
|
|
|
341
478
|
declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
|
|
342
479
|
/** @internal */
|
|
343
480
|
readonly pool: mysql.Pool;
|
|
481
|
+
private readonly dialect;
|
|
344
482
|
constructor(config: DatabaseAdapterConfig);
|
|
483
|
+
private get executor();
|
|
345
484
|
/**
|
|
346
485
|
* Insert deltas into the database in a single batch.
|
|
347
486
|
* Uses INSERT IGNORE for idempotent writes — duplicate deltaIds are silently skipped.
|
|
@@ -365,15 +504,41 @@ declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
|
|
|
365
504
|
/**
|
|
366
505
|
* Materialise deltas into destination tables.
|
|
367
506
|
*
|
|
368
|
-
*
|
|
369
|
-
*
|
|
370
|
-
* rows are deleted. The `props` column is never touched.
|
|
507
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
508
|
+
* MySQL SQL dialect.
|
|
371
509
|
*/
|
|
372
510
|
materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
|
|
373
511
|
/** Close the database connection pool and release resources. */
|
|
374
512
|
close(): Promise<void>;
|
|
375
513
|
}
|
|
376
514
|
|
|
515
|
+
/**
|
|
516
|
+
* Postgres SQL dialect for the shared materialise algorithm.
|
|
517
|
+
*
|
|
518
|
+
* Uses `$N` positional parameters, `ON CONFLICT DO UPDATE`, `JSONB`,
|
|
519
|
+
* and `TIMESTAMPTZ` types.
|
|
520
|
+
*/
|
|
521
|
+
declare class PostgresSqlDialect implements SqlDialect {
|
|
522
|
+
createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
|
|
523
|
+
sql: string;
|
|
524
|
+
params: unknown[];
|
|
525
|
+
};
|
|
526
|
+
queryDeltaHistory(sourceTable: string, rowIds: string[]): {
|
|
527
|
+
sql: string;
|
|
528
|
+
params: unknown[];
|
|
529
|
+
};
|
|
530
|
+
buildUpsert(dest: string, schema: TableSchema, conflictCols: string[], softDelete: boolean, upserts: Array<{
|
|
531
|
+
rowId: string;
|
|
532
|
+
state: Record<string, unknown>;
|
|
533
|
+
}>): {
|
|
534
|
+
sql: string;
|
|
535
|
+
params: unknown[];
|
|
536
|
+
};
|
|
537
|
+
buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
|
|
538
|
+
sql: string;
|
|
539
|
+
params: unknown[];
|
|
540
|
+
};
|
|
541
|
+
}
|
|
377
542
|
/**
|
|
378
543
|
* PostgreSQL database adapter for LakeSync.
|
|
379
544
|
*
|
|
@@ -383,7 +548,9 @@ declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
|
|
|
383
548
|
declare class PostgresAdapter implements DatabaseAdapter, Materialisable {
|
|
384
549
|
/** @internal */
|
|
385
550
|
readonly pool: Pool;
|
|
551
|
+
private readonly dialect;
|
|
386
552
|
constructor(config: DatabaseAdapterConfig);
|
|
553
|
+
private get executor();
|
|
387
554
|
/**
|
|
388
555
|
* Insert deltas into the database in a single batch.
|
|
389
556
|
* Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
|
|
@@ -407,9 +574,8 @@ declare class PostgresAdapter implements DatabaseAdapter, Materialisable {
|
|
|
407
574
|
/**
|
|
408
575
|
* Materialise deltas into destination tables.
|
|
409
576
|
*
|
|
410
|
-
*
|
|
411
|
-
*
|
|
412
|
-
* rows are deleted. The `props` column is never touched.
|
|
577
|
+
* Delegates to the shared `executeMaterialise` algorithm with the
|
|
578
|
+
* Postgres SQL dialect.
|
|
413
579
|
*/
|
|
414
580
|
materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
|
|
415
581
|
/** Close the database connection pool and release resources. */
|
|
@@ -423,13 +589,27 @@ type QueryFn = (sql: string, params?: unknown[]) => Promise<Record<string, unkno
|
|
|
423
589
|
*
|
|
424
590
|
* Uses dynamic imports so the database drivers (pg, mysql2) are only
|
|
425
591
|
* loaded when actually needed. Returns `null` for connector types that
|
|
426
|
-
* do not support the standard SQL polling model (e.g. BigQuery).
|
|
592
|
+
* do not support the standard SQL polling model (e.g. BigQuery, Jira, Salesforce).
|
|
427
593
|
*
|
|
428
594
|
* @param config - Validated connector configuration.
|
|
429
595
|
* @returns A query function or `null` if the connector type is unsupported.
|
|
430
596
|
*/
|
|
431
597
|
declare function createQueryFn(config: ConnectorConfig): Promise<QueryFn | null>;
|
|
432
598
|
|
|
599
|
+
/**
|
|
600
|
+
* Group raw delta rows by row_id, merge to latest state, and partition into upserts and deletes.
|
|
601
|
+
*/
|
|
602
|
+
declare function groupAndMerge(rows: Array<{
|
|
603
|
+
row_id: string;
|
|
604
|
+
columns: string | ColumnDelta[];
|
|
605
|
+
op: string;
|
|
606
|
+
}>): {
|
|
607
|
+
upserts: Array<{
|
|
608
|
+
rowId: string;
|
|
609
|
+
state: Record<string, unknown>;
|
|
610
|
+
}>;
|
|
611
|
+
deleteIds: string[];
|
|
612
|
+
};
|
|
433
613
|
/** Normalise a caught value into an Error or undefined. */
|
|
434
614
|
declare function toCause(error: unknown): Error | undefined;
|
|
435
615
|
/** Execute an async operation and wrap errors into an AdapterError Result. */
|
|
@@ -444,4 +624,4 @@ declare function mergeLatestState(rows: Array<{
|
|
|
444
624
|
op: string;
|
|
445
625
|
}>): Record<string, unknown> | null;
|
|
446
626
|
|
|
447
|
-
export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, ObjectInfo, PostgresAdapter, type QueryFn, buildSchemaIndex, createDatabaseAdapter, createQueryFn, groupDeltasByTable, isMaterialisable, mergeLatestState, migrateAdapter, migrateToTier, toCause, wrapAsync };
|
|
627
|
+
export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, BigQuerySqlDialect, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, MySqlDialect, ObjectInfo, PostgresAdapter, PostgresSqlDialect, type QueryExecutor, type QueryFn, type SqlDialect, buildSchemaIndex, createDatabaseAdapter, createQueryFn, executeMaterialise, groupAndMerge, groupDeltasByTable, isMaterialisable, isSoftDelete, mergeLatestState, migrateAdapter, migrateToTier, resolveConflictColumns, resolvePrimaryKey, toCause, wrapAsync };
|
package/dist/adapter.js
CHANGED
|
@@ -1,44 +1,60 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BigQueryAdapter,
|
|
3
|
+
BigQuerySqlDialect,
|
|
3
4
|
CompositeAdapter,
|
|
4
5
|
FanOutAdapter,
|
|
5
6
|
LifecycleAdapter,
|
|
6
7
|
MinIOAdapter,
|
|
7
8
|
MySQLAdapter,
|
|
9
|
+
MySqlDialect,
|
|
8
10
|
PostgresAdapter,
|
|
11
|
+
PostgresSqlDialect,
|
|
9
12
|
buildSchemaIndex,
|
|
10
13
|
createDatabaseAdapter,
|
|
11
14
|
createQueryFn,
|
|
15
|
+
executeMaterialise,
|
|
16
|
+
groupAndMerge,
|
|
12
17
|
groupDeltasByTable,
|
|
13
18
|
isDatabaseAdapter,
|
|
14
19
|
isMaterialisable,
|
|
20
|
+
isSoftDelete,
|
|
15
21
|
lakeSyncTypeToBigQuery,
|
|
16
22
|
mergeLatestState,
|
|
17
23
|
migrateAdapter,
|
|
18
24
|
migrateToTier,
|
|
25
|
+
resolveConflictColumns,
|
|
26
|
+
resolvePrimaryKey,
|
|
19
27
|
toCause,
|
|
20
28
|
wrapAsync
|
|
21
|
-
} from "./chunk-
|
|
22
|
-
import "./chunk-
|
|
23
|
-
import "./chunk-
|
|
29
|
+
} from "./chunk-LPWXOYNS.js";
|
|
30
|
+
import "./chunk-LDFFCG2K.js";
|
|
31
|
+
import "./chunk-DGUM43GV.js";
|
|
24
32
|
export {
|
|
25
33
|
BigQueryAdapter,
|
|
34
|
+
BigQuerySqlDialect,
|
|
26
35
|
CompositeAdapter,
|
|
27
36
|
FanOutAdapter,
|
|
28
37
|
LifecycleAdapter,
|
|
29
38
|
MinIOAdapter,
|
|
30
39
|
MySQLAdapter,
|
|
40
|
+
MySqlDialect,
|
|
31
41
|
PostgresAdapter,
|
|
42
|
+
PostgresSqlDialect,
|
|
32
43
|
buildSchemaIndex,
|
|
33
44
|
createDatabaseAdapter,
|
|
34
45
|
createQueryFn,
|
|
46
|
+
executeMaterialise,
|
|
47
|
+
groupAndMerge,
|
|
35
48
|
groupDeltasByTable,
|
|
36
49
|
isDatabaseAdapter,
|
|
37
50
|
isMaterialisable,
|
|
51
|
+
isSoftDelete,
|
|
38
52
|
lakeSyncTypeToBigQuery,
|
|
39
53
|
mergeLatestState,
|
|
40
54
|
migrateAdapter,
|
|
41
55
|
migrateToTier,
|
|
56
|
+
resolveConflictColumns,
|
|
57
|
+
resolvePrimaryKey,
|
|
42
58
|
toCause,
|
|
43
59
|
wrapAsync
|
|
44
60
|
};
|
package/dist/analyst.js
CHANGED
|
@@ -1,11 +1,78 @@
|
|
|
1
|
-
import { S as SyncPush, R as RowDelta } from './types-
|
|
1
|
+
import { S as SyncPush, R as RowDelta } from './types-BdGBv2ba.js';
|
|
2
2
|
import { H as HLC } from './hlc-DiD8QNG3.js';
|
|
3
3
|
import { R as Result, F as FlushError } from './result-CojzlFE2.js';
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Target that supports flush and buffer inspection.
|
|
7
|
+
* Implemented by SyncGateway so pollers can trigger flushes to relieve memory pressure.
|
|
8
|
+
*/
|
|
9
|
+
interface FlushableTarget {
|
|
10
|
+
flush(): Promise<Result<void, FlushError>>;
|
|
11
|
+
shouldFlush(): boolean;
|
|
12
|
+
readonly bufferStats: {
|
|
13
|
+
logSize: number;
|
|
14
|
+
indexSize: number;
|
|
15
|
+
byteSize: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Monitors buffer pressure on a {@link FlushableTarget} and triggers
|
|
20
|
+
* flushes when thresholds are exceeded. Created at construction time
|
|
21
|
+
* only when the target supports flushing — no runtime type checks needed.
|
|
22
|
+
*/
|
|
23
|
+
declare class PressureManager {
|
|
24
|
+
private readonly target;
|
|
25
|
+
private readonly memoryBudgetBytes;
|
|
26
|
+
private readonly flushThreshold;
|
|
27
|
+
constructor(config: {
|
|
28
|
+
target: FlushableTarget;
|
|
29
|
+
memoryBudgetBytes?: number;
|
|
30
|
+
flushThreshold?: number;
|
|
31
|
+
});
|
|
32
|
+
/** Check buffer pressure and flush if thresholds are exceeded. */
|
|
33
|
+
checkAndFlush(): Promise<void>;
|
|
34
|
+
/** Force a flush regardless of current pressure. */
|
|
35
|
+
forceFlush(): Promise<void>;
|
|
36
|
+
private shouldFlush;
|
|
37
|
+
}
|
|
38
|
+
|
|
5
39
|
/** Minimal interface for a push target (avoids depending on @lakesync/gateway). */
|
|
6
40
|
interface PushTarget {
|
|
7
41
|
handlePush(push: SyncPush): unknown;
|
|
8
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Manages chunked pushing of deltas to a {@link PushTarget}.
|
|
45
|
+
* Handles backpressure retry (flush + retry once) when a
|
|
46
|
+
* {@link PressureManager} is present.
|
|
47
|
+
*
|
|
48
|
+
* No runtime type checks — the PressureManager is either provided
|
|
49
|
+
* at construction (target supports flush) or null.
|
|
50
|
+
*/
|
|
51
|
+
declare class ChunkedPusher {
|
|
52
|
+
private readonly target;
|
|
53
|
+
private readonly clientId;
|
|
54
|
+
private readonly chunkSize;
|
|
55
|
+
private readonly pressure;
|
|
56
|
+
private pendingDeltas;
|
|
57
|
+
constructor(config: {
|
|
58
|
+
target: PushTarget;
|
|
59
|
+
clientId: string;
|
|
60
|
+
chunkSize: number;
|
|
61
|
+
pressure: PressureManager | null;
|
|
62
|
+
});
|
|
63
|
+
/**
|
|
64
|
+
* Accumulate a single delta. When `chunkSize` is reached, the pending
|
|
65
|
+
* deltas are automatically pushed (and flushed if needed).
|
|
66
|
+
*/
|
|
67
|
+
accumulate(delta: RowDelta): Promise<void>;
|
|
68
|
+
/** Flush any remaining accumulated deltas. */
|
|
69
|
+
flush(): Promise<void>;
|
|
70
|
+
/** Push deltas directly (single-shot, backward compat). */
|
|
71
|
+
pushImmediate(deltas: RowDelta[]): void;
|
|
72
|
+
private pushPendingChunk;
|
|
73
|
+
private pushChunkWithFlush;
|
|
74
|
+
}
|
|
75
|
+
|
|
9
76
|
/**
|
|
10
77
|
* Extended push target that supports flush and buffer inspection.
|
|
11
78
|
* Implemented by SyncGateway so pollers can trigger flushes to relieve memory pressure.
|
|
@@ -32,19 +99,18 @@ interface PollerMemoryConfig {
|
|
|
32
99
|
}
|
|
33
100
|
/**
|
|
34
101
|
* Base class for source pollers that poll an external API and push deltas
|
|
35
|
-
* to a SyncGateway.
|
|
102
|
+
* to a SyncGateway.
|
|
103
|
+
*
|
|
104
|
+
* Composes {@link PollingScheduler} (lifecycle), {@link ChunkedPusher}
|
|
105
|
+
* (chunked push with backpressure), and {@link PressureManager}
|
|
106
|
+
* (memory-budget flush decisions).
|
|
36
107
|
*/
|
|
37
108
|
declare abstract class BaseSourcePoller {
|
|
38
109
|
protected readonly gateway: PushTarget;
|
|
39
110
|
protected readonly hlc: HLC;
|
|
40
111
|
protected readonly clientId: string;
|
|
41
|
-
private readonly
|
|
42
|
-
private
|
|
43
|
-
private running;
|
|
44
|
-
private readonly chunkSize;
|
|
45
|
-
private readonly memoryBudgetBytes;
|
|
46
|
-
private readonly flushThreshold;
|
|
47
|
-
private pendingDeltas;
|
|
112
|
+
private readonly scheduler;
|
|
113
|
+
private readonly pusher;
|
|
48
114
|
constructor(config: {
|
|
49
115
|
name: string;
|
|
50
116
|
intervalMs: number;
|
|
@@ -77,15 +143,6 @@ declare abstract class BaseSourcePoller {
|
|
|
77
143
|
protected accumulateDelta(delta: RowDelta): Promise<void>;
|
|
78
144
|
/** Flush any remaining accumulated deltas. Call at the end of `poll()`. */
|
|
79
145
|
protected flushAccumulator(): Promise<void>;
|
|
80
|
-
/**
|
|
81
|
-
* Push a chunk of pending deltas. If the gateway is an IngestTarget,
|
|
82
|
-
* checks memory pressure and flushes before/after push when needed.
|
|
83
|
-
* On backpressure, flushes once and retries.
|
|
84
|
-
*/
|
|
85
|
-
private pushPendingChunk;
|
|
86
|
-
private pushChunkWithFlush;
|
|
87
|
-
private shouldFlushTarget;
|
|
88
|
-
private schedulePoll;
|
|
89
146
|
}
|
|
90
147
|
|
|
91
|
-
export { BaseSourcePoller as B, type IngestTarget as I, type PushTarget as P, type PollerMemoryConfig as a, isIngestTarget as i };
|
|
148
|
+
export { BaseSourcePoller as B, ChunkedPusher as C, type FlushableTarget as F, type IngestTarget as I, type PushTarget as P, type PollerMemoryConfig as a, PressureManager as b, isIngestTarget as i };
|
package/dist/catalogue.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { I as IcebergSchema, P as PartitionSpec } from './nessie-client-DrNikVXy.js';
|
|
2
2
|
export { C as CatalogueConfig, a as CatalogueError, D as DataFile, b as IcebergField, N as NessieCatalogueClient, S as Snapshot, T as TableMetadata } from './nessie-client-DrNikVXy.js';
|
|
3
|
-
import { T as TableSchema } from './types-
|
|
3
|
+
import { T as TableSchema } from './types-BdGBv2ba.js';
|
|
4
4
|
import './result-CojzlFE2.js';
|
|
5
5
|
|
|
6
6
|
/**
|
package/dist/catalogue.js
CHANGED
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
buildPartitionSpec,
|
|
5
5
|
lakeSyncTableName,
|
|
6
6
|
tableSchemaToIceberg
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-TMLG32QV.js";
|
|
8
|
+
import "./chunk-LDFFCG2K.js";
|
|
9
|
+
import "./chunk-DGUM43GV.js";
|
|
10
10
|
export {
|
|
11
11
|
CatalogueError,
|
|
12
12
|
NessieCatalogueClient,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
2
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
3
|
+
}) : x)(function(x) {
|
|
4
|
+
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
5
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
export {
|
|
9
|
+
__require
|
|
10
|
+
};
|
|
11
|
+
//# sourceMappingURL=chunk-DGUM43GV.js.map
|