lakesync 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-types-DwsQGQS4.d.ts +94 -0
- package/dist/adapter.d.ts +202 -63
- package/dist/adapter.js +20 -5
- package/dist/analyst.js +2 -2
- package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Y7ORYgUv.d.ts} +78 -19
- package/dist/catalogue.d.ts +1 -1
- package/dist/catalogue.js +3 -3
- package/dist/{chunk-P3FT7QCW.js → chunk-4SG66H5K.js} +395 -252
- package/dist/chunk-4SG66H5K.js.map +1 -0
- package/dist/{chunk-GUJWMK5P.js → chunk-C4KD6YKP.js} +419 -380
- package/dist/chunk-C4KD6YKP.js.map +1 -0
- package/dist/chunk-DGUM43GV.js +11 -0
- package/dist/{chunk-IRJ4QRWV.js → chunk-FIIHPQMQ.js} +396 -209
- package/dist/chunk-FIIHPQMQ.js.map +1 -0
- package/dist/{chunk-UAUQGP3B.js → chunk-U2NV4DUX.js} +2 -2
- package/dist/{chunk-NCZYFZ3B.js → chunk-XVP5DJJ7.js} +44 -18
- package/dist/{chunk-NCZYFZ3B.js.map → chunk-XVP5DJJ7.js.map} +1 -1
- package/dist/{chunk-FHVTUKXL.js → chunk-YHYBLU6W.js} +2 -2
- package/dist/{chunk-QMS7TGFL.js → chunk-ZNY4DSFU.js} +29 -15
- package/dist/{chunk-QMS7TGFL.js.map → chunk-ZNY4DSFU.js.map} +1 -1
- package/dist/{chunk-SF7Y6ZUA.js → chunk-ZU7RC7CT.js} +2 -2
- package/dist/client.d.ts +186 -17
- package/dist/client.js +456 -188
- package/dist/client.js.map +1 -1
- package/dist/compactor.d.ts +2 -2
- package/dist/compactor.js +4 -4
- package/dist/connector-jira.d.ts +13 -3
- package/dist/connector-jira.js +7 -3
- package/dist/connector-salesforce.d.ts +13 -3
- package/dist/connector-salesforce.js +7 -3
- package/dist/{coordinator-D32a5rNk.d.ts → coordinator-eGmZMnJ_.d.ts} +120 -30
- package/dist/create-poller-Cc2MGfhh.d.ts +55 -0
- package/dist/factory-DFfR-030.d.ts +33 -0
- package/dist/gateway-server.d.ts +516 -119
- package/dist/gateway-server.js +1201 -4035
- package/dist/gateway-server.js.map +1 -1
- package/dist/gateway.d.ts +69 -106
- package/dist/gateway.js +13 -6
- package/dist/index.d.ts +65 -58
- package/dist/index.js +18 -4
- package/dist/parquet.d.ts +1 -1
- package/dist/parquet.js +3 -3
- package/dist/proto.d.ts +1 -1
- package/dist/proto.js +3 -3
- package/dist/react.d.ts +47 -10
- package/dist/react.js +88 -40
- package/dist/react.js.map +1 -1
- package/dist/{registry-CPTgO9jv.d.ts → registry-Dd8JuW8T.d.ts} +19 -4
- package/dist/{gateway-Bpvatd9n.d.ts → request-handler-B1I5xDOx.d.ts} +193 -20
- package/dist/{resolver-CbuXm3nB.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
- package/dist/{src-RHKJFQKR.js → src-WU7IBVC4.js} +19 -5
- package/dist/{types-CLlD4XOy.d.ts → types-BdGBv2ba.d.ts} +17 -2
- package/dist/{types-D-E0VrfS.d.ts → types-D2C9jTbL.d.ts} +39 -22
- package/package.json +1 -1
- package/dist/auth-CAVutXzx.d.ts +0 -30
- package/dist/chunk-7D4SUZUM.js +0 -38
- package/dist/chunk-GUJWMK5P.js.map +0 -1
- package/dist/chunk-IRJ4QRWV.js.map +0 -1
- package/dist/chunk-P3FT7QCW.js.map +0 -1
- package/dist/db-types-BlN-4KbQ.d.ts +0 -29
- package/dist/src-CLCALYDT.js +0 -25
- package/dist/src-FPJQYQNA.js +0 -27
- package/dist/src-FPJQYQNA.js.map +0 -1
- package/dist/src-RHKJFQKR.js.map +0 -1
- package/dist/types-DSC_EiwR.d.ts +0 -45
- /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
- /package/dist/{chunk-UAUQGP3B.js.map → chunk-U2NV4DUX.js.map} +0 -0
- /package/dist/{chunk-FHVTUKXL.js.map → chunk-YHYBLU6W.js.map} +0 -0
- /package/dist/{chunk-SF7Y6ZUA.js.map → chunk-ZU7RC7CT.js.map} +0 -0
- /package/dist/{src-CLCALYDT.js.map → src-WU7IBVC4.js.map} +0 -0
package/dist/gateway.d.ts
CHANGED
|
@@ -1,47 +1,16 @@
|
|
|
1
1
|
import { S as SyncRulesConfig, R as ResolvedClaims, b as SyncRulesContext, i as ActionPush } from './types-Bs-QyOe-.js';
|
|
2
2
|
export { d as ActionDiscovery, g as ActionHandler, j as ActionResponse, l as AuthContext } from './types-Bs-QyOe-.js';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
export {
|
|
3
|
+
import { T as TableSchema, R as RowDelta, e as SyncPull, S as SyncPush } from './types-BdGBv2ba.js';
|
|
4
|
+
export { f as SyncResponse } from './types-BdGBv2ba.js';
|
|
5
|
+
export { I as IngestTarget, i as isIngestTarget } from './base-poller-Y7ORYgUv.js';
|
|
6
6
|
export { b as bigintReplacer, a as bigintReviver } from './json-dYtqiL0F.js';
|
|
7
|
-
import {
|
|
8
|
-
export { A as ActionDispatcher, B as BufferConfig,
|
|
9
|
-
import { C as ConnectorConfig } from './types-D-E0VrfS.js';
|
|
10
|
-
import { R as Result, F as FlushError, H as HLCTimestamp } from './result-CojzlFE2.js';
|
|
11
|
-
import { D as DatabaseAdapter } from './db-types-BlN-4KbQ.js';
|
|
12
|
-
import { L as LakeAdapter } from './types-DSC_EiwR.js';
|
|
7
|
+
import { D as DeltaBuffer } from './request-handler-B1I5xDOx.js';
|
|
8
|
+
export { A as ActionCacheConfig, a as ActionDispatcher, B as BufferConfig, C as CachedActionResult, b as ConfigStore, F as FlushEnvelope, G as GatewayConfig, c as GatewayState, H as HandlePushResult, d as HandlerResult, I as IdempotencyCache, e as IdempotencyCacheConfig, M as MemoryConfigStore, f as MemoryIdempotencyCache, S as SchemaManager, g as SyncGateway, h as handleActionRequest, i as handleFlushRequest, j as handleListConnectorTypes, k as handleListConnectors, l as handleMetrics, m as handlePullRequest, n as handlePushRequest, o as handleRegisterConnector, p as handleSaveSchema, q as handleSaveSyncRules, r as handleUnregisterConnector } from './request-handler-B1I5xDOx.js';
|
|
13
9
|
import { N as NessieCatalogueClient } from './nessie-client-DrNikVXy.js';
|
|
10
|
+
import { R as Result, F as FlushError, H as HLCTimestamp } from './result-CojzlFE2.js';
|
|
11
|
+
import { L as LakeAdapter, D as DatabaseAdapter } from './adapter-types-DwsQGQS4.js';
|
|
14
12
|
import './hlc-DiD8QNG3.js';
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Platform-agnostic configuration storage interface.
|
|
18
|
-
*
|
|
19
|
-
* Implemented by MemoryConfigStore (tests, gateway-server) and
|
|
20
|
-
* DurableStorageConfigStore (gateway-worker).
|
|
21
|
-
*/
|
|
22
|
-
interface ConfigStore {
|
|
23
|
-
getSchema(gatewayId: string): Promise<TableSchema | undefined>;
|
|
24
|
-
setSchema(gatewayId: string, schema: TableSchema): Promise<void>;
|
|
25
|
-
getSyncRules(gatewayId: string): Promise<SyncRulesConfig | undefined>;
|
|
26
|
-
setSyncRules(gatewayId: string, rules: SyncRulesConfig): Promise<void>;
|
|
27
|
-
getConnectors(): Promise<Record<string, ConnectorConfig>>;
|
|
28
|
-
setConnectors(connectors: Record<string, ConnectorConfig>): Promise<void>;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* In-memory implementation of ConfigStore.
|
|
32
|
-
* Used by tests and gateway-server.
|
|
33
|
-
*/
|
|
34
|
-
declare class MemoryConfigStore implements ConfigStore {
|
|
35
|
-
private schemas;
|
|
36
|
-
private syncRules;
|
|
37
|
-
private connectors;
|
|
38
|
-
getSchema(gatewayId: string): Promise<TableSchema | undefined>;
|
|
39
|
-
setSchema(gatewayId: string, schema: TableSchema): Promise<void>;
|
|
40
|
-
getSyncRules(gatewayId: string): Promise<SyncRulesConfig | undefined>;
|
|
41
|
-
setSyncRules(gatewayId: string, rules: SyncRulesConfig): Promise<void>;
|
|
42
|
-
getConnectors(): Promise<Record<string, ConnectorConfig>>;
|
|
43
|
-
setConnectors(connectors: Record<string, ConnectorConfig>): Promise<void>;
|
|
44
|
-
}
|
|
13
|
+
import './types-D2C9jTbL.js';
|
|
45
14
|
|
|
46
15
|
/** Maximum push payload size (1 MiB). */
|
|
47
16
|
declare const MAX_PUSH_PAYLOAD_BYTES = 1048576;
|
|
@@ -51,8 +20,8 @@ declare const MAX_DELTAS_PER_PUSH = 10000;
|
|
|
51
20
|
declare const MAX_PULL_LIMIT = 10000;
|
|
52
21
|
/** Default number of deltas returned in a pull when no limit is specified. */
|
|
53
22
|
declare const DEFAULT_PULL_LIMIT = 100;
|
|
54
|
-
/** Allowed column types for schema validation. */
|
|
55
|
-
declare const VALID_COLUMN_TYPES:
|
|
23
|
+
/** Allowed column types for schema validation. Derived from core COLUMN_TYPES. */
|
|
24
|
+
declare const VALID_COLUMN_TYPES: ReadonlySet<string>;
|
|
56
25
|
/** Default maximum buffer size before triggering flush (4 MiB). */
|
|
57
26
|
declare const DEFAULT_MAX_BUFFER_BYTES: number;
|
|
58
27
|
/** Default maximum buffer age before triggering flush (30 seconds). */
|
|
@@ -64,6 +33,8 @@ interface FlushConfig {
|
|
|
64
33
|
flushFormat?: "json" | "parquet";
|
|
65
34
|
tableSchema?: TableSchema;
|
|
66
35
|
catalogue?: NessieCatalogueClient;
|
|
36
|
+
/** Optional callback invoked when materialisation fails. Useful for metrics/alerting. */
|
|
37
|
+
onMaterialisationFailure?: (table: string, deltaCount: number, error: Error) => void;
|
|
67
38
|
}
|
|
68
39
|
/** Dependencies injected into flush operations. */
|
|
69
40
|
interface FlushDeps {
|
|
@@ -77,6 +48,11 @@ declare function hlcRange(entries: RowDelta[]): {
|
|
|
77
48
|
min: HLCTimestamp;
|
|
78
49
|
max: HLCTimestamp;
|
|
79
50
|
};
|
|
51
|
+
/** Strategy for flushing deltas to a specific adapter type. */
|
|
52
|
+
interface FlushStrategy {
|
|
53
|
+
/** Flush entries to the target adapter. */
|
|
54
|
+
flush(entries: RowDelta[], byteSize: number, deps: FlushDeps, keyPrefix?: string): Promise<Result<void, FlushError>>;
|
|
55
|
+
}
|
|
80
56
|
/**
|
|
81
57
|
* Flush a set of entries to the configured adapter.
|
|
82
58
|
*
|
|
@@ -92,80 +68,67 @@ declare function flushEntries(entries: RowDelta[], byteSize: number, deps: Flush
|
|
|
92
68
|
*/
|
|
93
69
|
declare function commitToCatalogue(objectKey: string, fileSizeInBytes: number, recordCount: number, catalogue: NessieCatalogueClient, schema: TableSchema): Promise<void>;
|
|
94
70
|
|
|
95
|
-
/**
|
|
96
|
-
interface
|
|
97
|
-
|
|
98
|
-
|
|
71
|
+
/** Dependencies for flush operations. */
|
|
72
|
+
interface FlushCoordinatorDeps {
|
|
73
|
+
/** Gateway configuration for flush. */
|
|
74
|
+
config: FlushConfig;
|
|
75
|
+
/** Table schemas for materialisation after flush. */
|
|
76
|
+
schemas?: ReadonlyArray<TableSchema>;
|
|
99
77
|
}
|
|
100
78
|
/**
|
|
101
|
-
*
|
|
79
|
+
* Coordinates flush operations from the buffer to the adapter.
|
|
102
80
|
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
declare function handleActionRequest(gateway: SyncGateway, raw: string, headerClientId?: string | null, claims?: ResolvedClaims): Promise<HandlerResult>;
|
|
129
|
-
/**
|
|
130
|
-
* Handle a flush request.
|
|
131
|
-
*/
|
|
132
|
-
declare function handleFlushRequest(gateway: SyncGateway, opts?: {
|
|
133
|
-
clearPersistence?: () => void;
|
|
134
|
-
}): Promise<HandlerResult>;
|
|
135
|
-
/**
|
|
136
|
-
* Handle saving a table schema.
|
|
137
|
-
*/
|
|
138
|
-
declare function handleSaveSchema(raw: string, store: ConfigStore, gatewayId: string): Promise<HandlerResult>;
|
|
139
|
-
/**
|
|
140
|
-
* Handle saving sync rules.
|
|
141
|
-
*/
|
|
142
|
-
declare function handleSaveSyncRules(raw: string, store: ConfigStore, gatewayId: string): Promise<HandlerResult>;
|
|
143
|
-
/**
|
|
144
|
-
* Handle registering a connector.
|
|
145
|
-
*/
|
|
146
|
-
declare function handleRegisterConnector(raw: string, store: ConfigStore): Promise<HandlerResult>;
|
|
147
|
-
/**
|
|
148
|
-
* Handle unregistering a connector.
|
|
149
|
-
*/
|
|
150
|
-
declare function handleUnregisterConnector(name: string, store: ConfigStore): Promise<HandlerResult>;
|
|
151
|
-
/**
|
|
152
|
-
* Handle listing connectors.
|
|
153
|
-
*/
|
|
154
|
-
declare function handleListConnectors(store: ConfigStore): Promise<HandlerResult>;
|
|
155
|
-
/**
|
|
156
|
-
* Handle listing available connector types (static metadata).
|
|
157
|
-
*/
|
|
158
|
-
declare function handleListConnectorTypes(): HandlerResult;
|
|
81
|
+
* Owns the flushing state to prevent concurrent flushes and handles
|
|
82
|
+
* entry restoration on failure.
|
|
83
|
+
*/
|
|
84
|
+
declare class FlushCoordinator {
|
|
85
|
+
private flushing;
|
|
86
|
+
/** Whether a flush is currently in progress. */
|
|
87
|
+
get isFlushing(): boolean;
|
|
88
|
+
/**
|
|
89
|
+
* Flush all entries from the buffer to the adapter.
|
|
90
|
+
*
|
|
91
|
+
* Drains the buffer first, then writes to the adapter. On failure,
|
|
92
|
+
* entries are restored to the buffer.
|
|
93
|
+
*/
|
|
94
|
+
flush(buffer: DeltaBuffer, adapter: LakeAdapter | DatabaseAdapter | null, deps: FlushCoordinatorDeps): Promise<Result<void, FlushError>>;
|
|
95
|
+
/**
|
|
96
|
+
* Flush a single table's deltas from the buffer.
|
|
97
|
+
*
|
|
98
|
+
* Drains only the specified table's deltas and flushes them,
|
|
99
|
+
* leaving other tables in the buffer.
|
|
100
|
+
*/
|
|
101
|
+
flushTable(table: string, buffer: DeltaBuffer, adapter: LakeAdapter | DatabaseAdapter | null, deps: FlushCoordinatorDeps): Promise<Result<void, FlushError>>;
|
|
102
|
+
/** Restore drained entries back to the buffer for retry. */
|
|
103
|
+
private restoreEntries;
|
|
104
|
+
}
|
|
105
|
+
|
|
159
106
|
/**
|
|
160
|
-
*
|
|
161
|
-
|
|
162
|
-
|
|
107
|
+
* Registry for named source adapters used in adapter-sourced pulls.
|
|
108
|
+
*
|
|
109
|
+
* Manages the mapping of source names to DatabaseAdapter instances,
|
|
110
|
+
* decoupling source adapter management from the gateway.
|
|
111
|
+
*/
|
|
112
|
+
declare class SourceRegistry {
|
|
113
|
+
private sources;
|
|
114
|
+
constructor(initial?: Record<string, DatabaseAdapter>);
|
|
115
|
+
/** Register a named source adapter. */
|
|
116
|
+
register(name: string, adapter: DatabaseAdapter): void;
|
|
117
|
+
/** Unregister a named source adapter. */
|
|
118
|
+
unregister(name: string): void;
|
|
119
|
+
/** Get a source adapter by name, or undefined if not registered. */
|
|
120
|
+
get(name: string): DatabaseAdapter | undefined;
|
|
121
|
+
/** List all registered source adapter names. */
|
|
122
|
+
list(): string[];
|
|
123
|
+
}
|
|
163
124
|
|
|
164
125
|
/** Validation error with HTTP status code. */
|
|
165
126
|
interface RequestError {
|
|
166
127
|
status: number;
|
|
167
128
|
message: string;
|
|
168
129
|
}
|
|
130
|
+
/** Parse a JSON string, returning Err on invalid JSON. */
|
|
131
|
+
declare function parseJson<T>(raw: string, reviver?: (key: string, value: unknown) => unknown): Result<T, RequestError>;
|
|
169
132
|
/**
|
|
170
133
|
* Validate and parse a push request body.
|
|
171
134
|
* Handles JSON parsing with bigint revival.
|
|
@@ -198,4 +161,4 @@ declare function pushErrorToStatus(code: string): number;
|
|
|
198
161
|
*/
|
|
199
162
|
declare function buildSyncRulesContext(rules: SyncRulesConfig | undefined, claims: ResolvedClaims): SyncRulesContext | undefined;
|
|
200
163
|
|
|
201
|
-
export { ActionPush,
|
|
164
|
+
export { ActionPush, DEFAULT_MAX_BUFFER_AGE_MS, DEFAULT_MAX_BUFFER_BYTES, DEFAULT_PULL_LIMIT, DeltaBuffer, type FlushConfig, FlushCoordinator, type FlushCoordinatorDeps, type FlushDeps, type FlushStrategy, MAX_DELTAS_PER_PUSH, MAX_PULL_LIMIT, MAX_PUSH_PAYLOAD_BYTES, type RequestError, SourceRegistry, SyncPull, SyncPush, VALID_COLUMN_TYPES, buildSyncRulesContext, commitToCatalogue, flushEntries, hlcRange, parseJson, parsePullParams, pushErrorToStatus, validateActionBody, validatePushBody, validateSchemaBody };
|
package/dist/gateway.js
CHANGED
|
@@ -4,11 +4,14 @@ import {
|
|
|
4
4
|
DEFAULT_MAX_BUFFER_BYTES,
|
|
5
5
|
DEFAULT_PULL_LIMIT,
|
|
6
6
|
DeltaBuffer,
|
|
7
|
+
FlushCoordinator,
|
|
7
8
|
MAX_DELTAS_PER_PUSH,
|
|
8
9
|
MAX_PULL_LIMIT,
|
|
9
10
|
MAX_PUSH_PAYLOAD_BYTES,
|
|
10
11
|
MemoryConfigStore,
|
|
12
|
+
MemoryIdempotencyCache,
|
|
11
13
|
SchemaManager,
|
|
14
|
+
SourceRegistry,
|
|
12
15
|
SyncGateway,
|
|
13
16
|
VALID_COLUMN_TYPES,
|
|
14
17
|
buildSyncRulesContext,
|
|
@@ -26,32 +29,35 @@ import {
|
|
|
26
29
|
handleSaveSyncRules,
|
|
27
30
|
handleUnregisterConnector,
|
|
28
31
|
hlcRange,
|
|
32
|
+
parseJson,
|
|
29
33
|
parsePullParams,
|
|
30
34
|
pushErrorToStatus,
|
|
31
35
|
validateActionBody,
|
|
32
36
|
validatePushBody,
|
|
33
37
|
validateSchemaBody
|
|
34
|
-
} from "./chunk-
|
|
35
|
-
import "./chunk-
|
|
36
|
-
import "./chunk-
|
|
37
|
-
import "./chunk-SF7Y6ZUA.js";
|
|
38
|
+
} from "./chunk-FIIHPQMQ.js";
|
|
39
|
+
import "./chunk-U2NV4DUX.js";
|
|
40
|
+
import "./chunk-ZU7RC7CT.js";
|
|
38
41
|
import {
|
|
39
42
|
bigintReplacer,
|
|
40
43
|
bigintReviver,
|
|
41
44
|
isIngestTarget
|
|
42
|
-
} from "./chunk-
|
|
43
|
-
import "./chunk-
|
|
45
|
+
} from "./chunk-4SG66H5K.js";
|
|
46
|
+
import "./chunk-DGUM43GV.js";
|
|
44
47
|
export {
|
|
45
48
|
ActionDispatcher,
|
|
46
49
|
DEFAULT_MAX_BUFFER_AGE_MS,
|
|
47
50
|
DEFAULT_MAX_BUFFER_BYTES,
|
|
48
51
|
DEFAULT_PULL_LIMIT,
|
|
49
52
|
DeltaBuffer,
|
|
53
|
+
FlushCoordinator,
|
|
50
54
|
MAX_DELTAS_PER_PUSH,
|
|
51
55
|
MAX_PULL_LIMIT,
|
|
52
56
|
MAX_PUSH_PAYLOAD_BYTES,
|
|
53
57
|
MemoryConfigStore,
|
|
58
|
+
MemoryIdempotencyCache,
|
|
54
59
|
SchemaManager,
|
|
60
|
+
SourceRegistry,
|
|
55
61
|
SyncGateway,
|
|
56
62
|
VALID_COLUMN_TYPES,
|
|
57
63
|
bigintReplacer,
|
|
@@ -72,6 +78,7 @@ export {
|
|
|
72
78
|
handleUnregisterConnector,
|
|
73
79
|
hlcRange,
|
|
74
80
|
isIngestTarget,
|
|
81
|
+
parseJson,
|
|
75
82
|
parsePullParams,
|
|
76
83
|
pushErrorToStatus,
|
|
77
84
|
validateActionBody,
|
package/dist/index.d.ts
CHANGED
|
@@ -2,15 +2,16 @@ import { A as Action, a as ActionValidationError, S as SyncRulesConfig, B as Buc
|
|
|
2
2
|
export { c as ActionDescriptor, d as ActionDiscovery, e as ActionErrorResult, f as ActionExecutionError, g as ActionHandler, h as ActionNotSupportedError, i as ActionPush, j as ActionResponse, k as ActionResult, l as AuthContext, m as SyncRuleFilter, n as SyncRuleOp, o as isActionError, p as isActionHandler } from './types-Bs-QyOe-.js';
|
|
3
3
|
import { H as HLCTimestamp, R as Result, C as ConflictError, L as LakeSyncError, S as SchemaError } from './result-CojzlFE2.js';
|
|
4
4
|
export { A as AdapterError, a as AdapterNotFoundError, B as BackpressureError, b as ClockDriftError, E as Err, F as FlushError, O as Ok, f as flatMapResult, c as fromPromise, m as mapResult, t as toError, u as unwrapOrThrow } from './result-CojzlFE2.js';
|
|
5
|
-
export {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
5
|
+
export { D as DatabaseAdapter, a as DatabaseAdapterConfig, L as LakeAdapter, M as Materialisable, O as ObjectInfo, i as isDatabaseAdapter, b as isMaterialisable } from './adapter-types-DwsQGQS4.js';
|
|
6
|
+
export { A as AuthClaims, a as AuthError, P as PollerFactory, b as PollerRegistry, c as createPoller, d as createPollerRegistry, v as verifyToken } from './create-poller-Cc2MGfhh.js';
|
|
7
|
+
import { P as PushTarget } from './base-poller-Y7ORYgUv.js';
|
|
8
|
+
export { B as BaseSourcePoller, C as ChunkedPusher, F as FlushableTarget, I as IngestTarget, a as PollerMemoryConfig, b as PressureManager, i as isIngestTarget } from './base-poller-Y7ORYgUv.js';
|
|
9
|
+
import { R as RowDelta, T as TableSchema, S as SyncPush } from './types-BdGBv2ba.js';
|
|
10
|
+
export { C as COLUMN_TYPES, a as ColumnDelta, b as ColumnReference, c as ColumnType, D as DeltaOp, d as RowKey, e as SyncPull, f as SyncResponse, r as rowKey } from './types-BdGBv2ba.js';
|
|
11
|
+
import { C as ConflictResolver } from './resolver-CXxmC0jR.js';
|
|
12
|
+
export { C as ConnectorCategory, a as ConnectorDescriptor, b as ConnectorRegistry, c as createConnectorRegistry, g as getConnectorDescriptor, l as listConnectorDescriptors, r as registerConnectorDescriptor, d as registerOutputSchemas } from './registry-Dd8JuW8T.js';
|
|
13
|
+
import { C as ConnectorConfig } from './types-D2C9jTbL.js';
|
|
14
|
+
export { B as BigQueryConnectorConfig, a as BigQueryConnectorConfigFull, b as CONNECTOR_TYPES, c as ConnectorConfigBase, d as ConnectorIngestConfig, e as ConnectorIngestTable, f as ConnectorType, J as JiraConnectorConfig, g as JiraConnectorConfigFull, M as MySQLConnectorConfig, h as MySQLConnectorConfigFull, P as PostgresConnectorConfig, i as PostgresConnectorConfigFull, S as SalesforceConnectorConfig, j as SalesforceConnectorConfigFull } from './types-D2C9jTbL.js';
|
|
14
15
|
export { H as HLC } from './hlc-DiD8QNG3.js';
|
|
15
16
|
export { b as bigintReplacer, a as bigintReviver } from './json-dYtqiL0F.js';
|
|
16
17
|
|
|
@@ -36,6 +37,42 @@ declare function generateActionId(params: {
|
|
|
36
37
|
*/
|
|
37
38
|
declare function validateAction(action: unknown): Result<Action, ActionValidationError>;
|
|
38
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Apply a delta to an existing row, returning the merged result.
|
|
42
|
+
*
|
|
43
|
+
* - DELETE → returns null
|
|
44
|
+
* - INSERT → creates a new row from delta columns
|
|
45
|
+
* - UPDATE → merges delta columns onto existing row (immutable — returns a new object)
|
|
46
|
+
*
|
|
47
|
+
* @param row - The current row state, or null if no row exists
|
|
48
|
+
* @param delta - The delta to apply
|
|
49
|
+
* @returns The merged row, or null for DELETE operations
|
|
50
|
+
*/
|
|
51
|
+
declare function applyDelta(row: Record<string, unknown> | null, delta: RowDelta): Record<string, unknown> | null;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Extract a column-level delta between two row states.
|
|
55
|
+
*
|
|
56
|
+
* - `before` null/undefined + `after` present -> INSERT (all columns)
|
|
57
|
+
* - `before` present + `after` null/undefined -> DELETE (empty columns)
|
|
58
|
+
* - Both present -> compare each column, emit only changed columns as UPDATE
|
|
59
|
+
* - No columns changed -> returns null (no-op)
|
|
60
|
+
*
|
|
61
|
+
* If `schema` is provided, only columns listed in the schema are considered.
|
|
62
|
+
*
|
|
63
|
+
* @param before - The previous row state, or null/undefined for a new row
|
|
64
|
+
* @param after - The current row state, or null/undefined for a deleted row
|
|
65
|
+
* @param opts - Table name, row ID, client ID, HLC timestamp, and optional schema
|
|
66
|
+
* @returns The extracted RowDelta, or null if nothing changed
|
|
67
|
+
*/
|
|
68
|
+
declare function extractDelta(before: Record<string, unknown> | null | undefined, after: Record<string, unknown> | null | undefined, opts: {
|
|
69
|
+
table: string;
|
|
70
|
+
rowId: string;
|
|
71
|
+
clientId: string;
|
|
72
|
+
hlc: HLCTimestamp;
|
|
73
|
+
schema?: TableSchema;
|
|
74
|
+
}): Promise<RowDelta | null>;
|
|
75
|
+
|
|
39
76
|
/**
|
|
40
77
|
* A simple PushTarget implementation that forwards every push to a
|
|
41
78
|
* user-supplied callback. Useful for testing, logging, or lightweight
|
|
@@ -102,56 +139,26 @@ declare class ConnectorValidationError extends LakeSyncError {
|
|
|
102
139
|
*/
|
|
103
140
|
declare function validateConnectorConfig(input: unknown): Result<ConnectorConfig, ConnectorValidationError>;
|
|
104
141
|
|
|
105
|
-
/** Factory function that creates a poller from a ConnectorConfig. */
|
|
106
|
-
type PollerFactory = (config: ConnectorConfig, gateway: PushTarget) => BaseSourcePoller;
|
|
107
|
-
/**
|
|
108
|
-
* Register a poller factory for a connector type.
|
|
109
|
-
* Connector packages call this at module load time so that
|
|
110
|
-
* `createPoller()` can instantiate the correct poller.
|
|
111
|
-
*/
|
|
112
|
-
declare function registerPollerFactory(type: string, factory: PollerFactory): void;
|
|
113
|
-
/**
|
|
114
|
-
* Create a poller from a {@link ConnectorConfig}.
|
|
115
|
-
*
|
|
116
|
-
* @throws If no factory has been registered for the config's `type`.
|
|
117
|
-
*/
|
|
118
|
-
declare function createPoller(config: ConnectorConfig, gateway: PushTarget): BaseSourcePoller;
|
|
119
|
-
|
|
120
142
|
/**
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
* - DELETE → returns null
|
|
124
|
-
* - INSERT → creates a new row from delta columns
|
|
125
|
-
* - UPDATE → merges delta columns onto existing row (immutable — returns a new object)
|
|
126
|
-
*
|
|
127
|
-
* @param row - The current row state, or null if no row exists
|
|
128
|
-
* @param delta - The delta to apply
|
|
129
|
-
* @returns The merged row, or null for DELETE operations
|
|
130
|
-
*/
|
|
131
|
-
declare function applyDelta(row: Record<string, unknown> | null, delta: RowDelta): Record<string, unknown> | null;
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Extract a column-level delta between two row states.
|
|
135
|
-
*
|
|
136
|
-
* - `before` null/undefined + `after` present -> INSERT (all columns)
|
|
137
|
-
* - `before` present + `after` null/undefined -> DELETE (empty columns)
|
|
138
|
-
* - Both present -> compare each column, emit only changed columns as UPDATE
|
|
139
|
-
* - No columns changed -> returns null (no-op)
|
|
140
|
-
*
|
|
141
|
-
* If `schema` is provided, only columns listed in the schema are considered.
|
|
142
|
-
*
|
|
143
|
-
* @param before - The previous row state, or null/undefined for a new row
|
|
144
|
-
* @param after - The current row state, or null/undefined for a deleted row
|
|
145
|
-
* @param opts - Table name, row ID, client ID, HLC timestamp, and optional schema
|
|
146
|
-
* @returns The extracted RowDelta, or null if nothing changed
|
|
143
|
+
* Manages the start/stop lifecycle and timer scheduling for a poll function.
|
|
144
|
+
* Has no knowledge of deltas, gateways, or sync protocol.
|
|
147
145
|
*/
|
|
148
|
-
declare
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
146
|
+
declare class PollingScheduler {
|
|
147
|
+
private readonly pollFn;
|
|
148
|
+
private readonly intervalMs;
|
|
149
|
+
private timer;
|
|
150
|
+
private running;
|
|
151
|
+
constructor(pollFn: () => Promise<void>, intervalMs: number);
|
|
152
|
+
/** Start the polling loop. No-op if already running. */
|
|
153
|
+
start(): void;
|
|
154
|
+
/** Stop the polling loop. */
|
|
155
|
+
stop(): void;
|
|
156
|
+
/** Whether the scheduler is currently running. */
|
|
157
|
+
get isRunning(): boolean;
|
|
158
|
+
/** Execute a single poll cycle without the timer loop. */
|
|
159
|
+
pollOnce(): Promise<void>;
|
|
160
|
+
private schedule;
|
|
161
|
+
}
|
|
155
162
|
|
|
156
163
|
/**
|
|
157
164
|
* Create a pass-all sync rules configuration.
|
|
@@ -270,4 +277,4 @@ declare function assertValidIdentifier(name: string): Result<void, SchemaError>;
|
|
|
270
277
|
*/
|
|
271
278
|
declare function quoteIdentifier(name: string): string;
|
|
272
279
|
|
|
273
|
-
export { Action, ActionValidationError,
|
|
280
|
+
export { Action, ActionValidationError, BucketDefinition, CallbackPushTarget, ConflictError, ConflictResolver, ConnectorConfig, ConnectorValidationError, HLCTimestamp, LWWResolver, LakeSyncError, PollingScheduler, PushTarget, ResolvedClaims, Result, RowDelta, SchemaError, SyncPush, SyncRuleError, SyncRulesConfig, SyncRulesContext, TableSchema, applyDelta, assertValidIdentifier, createPassAllRules, createUserScopedRules, deltaMatchesBucket, extractDelta, filterDeltas, generateActionId, isValidIdentifier, quoteIdentifier, resolveClientBuckets, resolveFilterValue, resolveLWW, validateAction, validateConnectorConfig, validateSyncRules };
|
package/dist/index.js
CHANGED
|
@@ -7,8 +7,10 @@ import {
|
|
|
7
7
|
AuthError,
|
|
8
8
|
BackpressureError,
|
|
9
9
|
BaseSourcePoller,
|
|
10
|
+
COLUMN_TYPES,
|
|
10
11
|
CONNECTOR_TYPES,
|
|
11
12
|
CallbackPushTarget,
|
|
13
|
+
ChunkedPusher,
|
|
12
14
|
ClockDriftError,
|
|
13
15
|
ConflictError,
|
|
14
16
|
ConnectorValidationError,
|
|
@@ -18,14 +20,18 @@ import {
|
|
|
18
20
|
LWWResolver,
|
|
19
21
|
LakeSyncError,
|
|
20
22
|
Ok,
|
|
23
|
+
PollingScheduler,
|
|
24
|
+
PressureManager,
|
|
21
25
|
SchemaError,
|
|
22
26
|
SyncRuleError,
|
|
23
27
|
applyDelta,
|
|
24
28
|
assertValidIdentifier,
|
|
25
29
|
bigintReplacer,
|
|
26
30
|
bigintReviver,
|
|
31
|
+
createConnectorRegistry,
|
|
27
32
|
createPassAllRules,
|
|
28
33
|
createPoller,
|
|
34
|
+
createPollerRegistry,
|
|
29
35
|
createUserScopedRules,
|
|
30
36
|
deltaMatchesBucket,
|
|
31
37
|
extractDelta,
|
|
@@ -36,14 +42,15 @@ import {
|
|
|
36
42
|
getConnectorDescriptor,
|
|
37
43
|
isActionError,
|
|
38
44
|
isActionHandler,
|
|
45
|
+
isDatabaseAdapter,
|
|
39
46
|
isIngestTarget,
|
|
47
|
+
isMaterialisable,
|
|
40
48
|
isValidIdentifier,
|
|
41
49
|
listConnectorDescriptors,
|
|
42
50
|
mapResult,
|
|
43
51
|
quoteIdentifier,
|
|
44
52
|
registerConnectorDescriptor,
|
|
45
53
|
registerOutputSchemas,
|
|
46
|
-
registerPollerFactory,
|
|
47
54
|
resolveClientBuckets,
|
|
48
55
|
resolveFilterValue,
|
|
49
56
|
resolveLWW,
|
|
@@ -54,8 +61,8 @@ import {
|
|
|
54
61
|
validateConnectorConfig,
|
|
55
62
|
validateSyncRules,
|
|
56
63
|
verifyToken
|
|
57
|
-
} from "./chunk-
|
|
58
|
-
import "./chunk-
|
|
64
|
+
} from "./chunk-4SG66H5K.js";
|
|
65
|
+
import "./chunk-DGUM43GV.js";
|
|
59
66
|
export {
|
|
60
67
|
ActionExecutionError,
|
|
61
68
|
ActionNotSupportedError,
|
|
@@ -65,8 +72,10 @@ export {
|
|
|
65
72
|
AuthError,
|
|
66
73
|
BackpressureError,
|
|
67
74
|
BaseSourcePoller,
|
|
75
|
+
COLUMN_TYPES,
|
|
68
76
|
CONNECTOR_TYPES,
|
|
69
77
|
CallbackPushTarget,
|
|
78
|
+
ChunkedPusher,
|
|
70
79
|
ClockDriftError,
|
|
71
80
|
ConflictError,
|
|
72
81
|
ConnectorValidationError,
|
|
@@ -76,14 +85,18 @@ export {
|
|
|
76
85
|
LWWResolver,
|
|
77
86
|
LakeSyncError,
|
|
78
87
|
Ok,
|
|
88
|
+
PollingScheduler,
|
|
89
|
+
PressureManager,
|
|
79
90
|
SchemaError,
|
|
80
91
|
SyncRuleError,
|
|
81
92
|
applyDelta,
|
|
82
93
|
assertValidIdentifier,
|
|
83
94
|
bigintReplacer,
|
|
84
95
|
bigintReviver,
|
|
96
|
+
createConnectorRegistry,
|
|
85
97
|
createPassAllRules,
|
|
86
98
|
createPoller,
|
|
99
|
+
createPollerRegistry,
|
|
87
100
|
createUserScopedRules,
|
|
88
101
|
deltaMatchesBucket,
|
|
89
102
|
extractDelta,
|
|
@@ -94,14 +107,15 @@ export {
|
|
|
94
107
|
getConnectorDescriptor,
|
|
95
108
|
isActionError,
|
|
96
109
|
isActionHandler,
|
|
110
|
+
isDatabaseAdapter,
|
|
97
111
|
isIngestTarget,
|
|
112
|
+
isMaterialisable,
|
|
98
113
|
isValidIdentifier,
|
|
99
114
|
listConnectorDescriptors,
|
|
100
115
|
mapResult,
|
|
101
116
|
quoteIdentifier,
|
|
102
117
|
registerConnectorDescriptor,
|
|
103
118
|
registerOutputSchemas,
|
|
104
|
-
registerPollerFactory,
|
|
105
119
|
resolveClientBuckets,
|
|
106
120
|
resolveFilterValue,
|
|
107
121
|
resolveLWW,
|
package/dist/parquet.d.ts
CHANGED
package/dist/parquet.js
CHANGED
|
@@ -3,9 +3,9 @@ import {
|
|
|
3
3
|
deltasToArrowTable,
|
|
4
4
|
readParquetToDeltas,
|
|
5
5
|
writeDeltasToParquet
|
|
6
|
-
} from "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
6
|
+
} from "./chunk-ZU7RC7CT.js";
|
|
7
|
+
import "./chunk-4SG66H5K.js";
|
|
8
|
+
import "./chunk-DGUM43GV.js";
|
|
9
9
|
export {
|
|
10
10
|
buildArrowSchema,
|
|
11
11
|
deltasToArrowTable,
|
package/dist/proto.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { H as HLCTimestamp, R as Result } from './result-CojzlFE2.js';
|
|
2
2
|
import { i as ActionPush$1, j as ActionResponse$1 } from './types-Bs-QyOe-.js';
|
|
3
|
-
import { R as RowDelta$1 } from './types-
|
|
3
|
+
import { R as RowDelta$1 } from './types-BdGBv2ba.js';
|
|
4
4
|
import { GenMessage, GenEnum } from '@bufbuild/protobuf/codegenv2';
|
|
5
5
|
import { Message } from '@bufbuild/protobuf';
|
|
6
6
|
|
package/dist/proto.js
CHANGED
|
@@ -29,9 +29,9 @@ import {
|
|
|
29
29
|
encodeSyncPull,
|
|
30
30
|
encodeSyncPush,
|
|
31
31
|
encodeSyncResponse
|
|
32
|
-
} from "./chunk-
|
|
33
|
-
import "./chunk-
|
|
34
|
-
import "./chunk-
|
|
32
|
+
} from "./chunk-YHYBLU6W.js";
|
|
33
|
+
import "./chunk-4SG66H5K.js";
|
|
34
|
+
import "./chunk-DGUM43GV.js";
|
|
35
35
|
export {
|
|
36
36
|
ActionPushSchema,
|
|
37
37
|
ActionResponseSchema,
|