lakesync 0.1.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/adapter-types-DwsQGQS4.d.ts +94 -0
  2. package/dist/adapter.d.ts +202 -63
  3. package/dist/adapter.js +20 -5
  4. package/dist/analyst.js +2 -2
  5. package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Y7ORYgUv.d.ts} +78 -19
  6. package/dist/catalogue.d.ts +1 -1
  7. package/dist/catalogue.js +3 -3
  8. package/dist/{chunk-P3FT7QCW.js → chunk-4SG66H5K.js} +395 -252
  9. package/dist/chunk-4SG66H5K.js.map +1 -0
  10. package/dist/{chunk-GUJWMK5P.js → chunk-C4KD6YKP.js} +419 -380
  11. package/dist/chunk-C4KD6YKP.js.map +1 -0
  12. package/dist/chunk-DGUM43GV.js +11 -0
  13. package/dist/{chunk-IRJ4QRWV.js → chunk-FIIHPQMQ.js} +396 -209
  14. package/dist/chunk-FIIHPQMQ.js.map +1 -0
  15. package/dist/{chunk-UAUQGP3B.js → chunk-U2NV4DUX.js} +2 -2
  16. package/dist/{chunk-NCZYFZ3B.js → chunk-XVP5DJJ7.js} +44 -18
  17. package/dist/{chunk-NCZYFZ3B.js.map → chunk-XVP5DJJ7.js.map} +1 -1
  18. package/dist/{chunk-FHVTUKXL.js → chunk-YHYBLU6W.js} +2 -2
  19. package/dist/{chunk-QMS7TGFL.js → chunk-ZNY4DSFU.js} +29 -15
  20. package/dist/{chunk-QMS7TGFL.js.map → chunk-ZNY4DSFU.js.map} +1 -1
  21. package/dist/{chunk-SF7Y6ZUA.js → chunk-ZU7RC7CT.js} +2 -2
  22. package/dist/client.d.ts +186 -17
  23. package/dist/client.js +456 -188
  24. package/dist/client.js.map +1 -1
  25. package/dist/compactor.d.ts +2 -2
  26. package/dist/compactor.js +4 -4
  27. package/dist/connector-jira.d.ts +13 -3
  28. package/dist/connector-jira.js +7 -3
  29. package/dist/connector-salesforce.d.ts +13 -3
  30. package/dist/connector-salesforce.js +7 -3
  31. package/dist/{coordinator-D32a5rNk.d.ts → coordinator-eGmZMnJ_.d.ts} +120 -30
  32. package/dist/create-poller-Cc2MGfhh.d.ts +55 -0
  33. package/dist/factory-DFfR-030.d.ts +33 -0
  34. package/dist/gateway-server.d.ts +516 -119
  35. package/dist/gateway-server.js +1201 -4035
  36. package/dist/gateway-server.js.map +1 -1
  37. package/dist/gateway.d.ts +69 -106
  38. package/dist/gateway.js +13 -6
  39. package/dist/index.d.ts +65 -58
  40. package/dist/index.js +18 -4
  41. package/dist/parquet.d.ts +1 -1
  42. package/dist/parquet.js +3 -3
  43. package/dist/proto.d.ts +1 -1
  44. package/dist/proto.js +3 -3
  45. package/dist/react.d.ts +47 -10
  46. package/dist/react.js +88 -40
  47. package/dist/react.js.map +1 -1
  48. package/dist/{registry-CPTgO9jv.d.ts → registry-Dd8JuW8T.d.ts} +19 -4
  49. package/dist/{gateway-Bpvatd9n.d.ts → request-handler-B1I5xDOx.d.ts} +193 -20
  50. package/dist/{resolver-CbuXm3nB.d.ts → resolver-CXxmC0jR.d.ts} +1 -1
  51. package/dist/{src-RHKJFQKR.js → src-WU7IBVC4.js} +19 -5
  52. package/dist/{types-CLlD4XOy.d.ts → types-BdGBv2ba.d.ts} +17 -2
  53. package/dist/{types-D-E0VrfS.d.ts → types-D2C9jTbL.d.ts} +39 -22
  54. package/package.json +1 -1
  55. package/dist/auth-CAVutXzx.d.ts +0 -30
  56. package/dist/chunk-7D4SUZUM.js +0 -38
  57. package/dist/chunk-GUJWMK5P.js.map +0 -1
  58. package/dist/chunk-IRJ4QRWV.js.map +0 -1
  59. package/dist/chunk-P3FT7QCW.js.map +0 -1
  60. package/dist/db-types-BlN-4KbQ.d.ts +0 -29
  61. package/dist/src-CLCALYDT.js +0 -25
  62. package/dist/src-FPJQYQNA.js +0 -27
  63. package/dist/src-FPJQYQNA.js.map +0 -1
  64. package/dist/src-RHKJFQKR.js.map +0 -1
  65. package/dist/types-DSC_EiwR.d.ts +0 -45
  66. /package/dist/{chunk-7D4SUZUM.js.map → chunk-DGUM43GV.js.map} +0 -0
  67. /package/dist/{chunk-UAUQGP3B.js.map → chunk-U2NV4DUX.js.map} +0 -0
  68. /package/dist/{chunk-FHVTUKXL.js.map → chunk-YHYBLU6W.js.map} +0 -0
  69. /package/dist/{chunk-SF7Y6ZUA.js.map → chunk-ZU7RC7CT.js.map} +0 -0
  70. /package/dist/{src-CLCALYDT.js.map → src-WU7IBVC4.js.map} +0 -0
package/dist/gateway.d.ts CHANGED
@@ -1,47 +1,16 @@
1
1
  import { S as SyncRulesConfig, R as ResolvedClaims, b as SyncRulesContext, i as ActionPush } from './types-Bs-QyOe-.js';
2
2
  export { d as ActionDiscovery, g as ActionHandler, j as ActionResponse, l as AuthContext } from './types-Bs-QyOe-.js';
3
- export { I as IngestTarget, i as isIngestTarget } from './base-poller-BpUyuG2R.js';
4
- import { T as TableSchema, R as RowDelta, b as SyncPull, S as SyncPush } from './types-CLlD4XOy.js';
5
- export { c as SyncResponse } from './types-CLlD4XOy.js';
3
+ import { T as TableSchema, R as RowDelta, e as SyncPull, S as SyncPush } from './types-BdGBv2ba.js';
4
+ export { f as SyncResponse } from './types-BdGBv2ba.js';
5
+ export { I as IngestTarget, i as isIngestTarget } from './base-poller-Y7ORYgUv.js';
6
6
  export { b as bigintReplacer, a as bigintReviver } from './json-dYtqiL0F.js';
7
- import { S as SyncGateway } from './gateway-Bpvatd9n.js';
8
- export { A as ActionDispatcher, B as BufferConfig, D as DeltaBuffer, F as FlushEnvelope, G as GatewayConfig, a as GatewayState, H as HandlePushResult, b as SchemaManager } from './gateway-Bpvatd9n.js';
9
- import { C as ConnectorConfig } from './types-D-E0VrfS.js';
10
- import { R as Result, F as FlushError, H as HLCTimestamp } from './result-CojzlFE2.js';
11
- import { D as DatabaseAdapter } from './db-types-BlN-4KbQ.js';
12
- import { L as LakeAdapter } from './types-DSC_EiwR.js';
7
+ import { D as DeltaBuffer } from './request-handler-B1I5xDOx.js';
8
+ export { A as ActionCacheConfig, a as ActionDispatcher, B as BufferConfig, C as CachedActionResult, b as ConfigStore, F as FlushEnvelope, G as GatewayConfig, c as GatewayState, H as HandlePushResult, d as HandlerResult, I as IdempotencyCache, e as IdempotencyCacheConfig, M as MemoryConfigStore, f as MemoryIdempotencyCache, S as SchemaManager, g as SyncGateway, h as handleActionRequest, i as handleFlushRequest, j as handleListConnectorTypes, k as handleListConnectors, l as handleMetrics, m as handlePullRequest, n as handlePushRequest, o as handleRegisterConnector, p as handleSaveSchema, q as handleSaveSyncRules, r as handleUnregisterConnector } from './request-handler-B1I5xDOx.js';
13
9
  import { N as NessieCatalogueClient } from './nessie-client-DrNikVXy.js';
10
+ import { R as Result, F as FlushError, H as HLCTimestamp } from './result-CojzlFE2.js';
11
+ import { L as LakeAdapter, D as DatabaseAdapter } from './adapter-types-DwsQGQS4.js';
14
12
  import './hlc-DiD8QNG3.js';
15
-
16
- /**
17
- * Platform-agnostic configuration storage interface.
18
- *
19
- * Implemented by MemoryConfigStore (tests, gateway-server) and
20
- * DurableStorageConfigStore (gateway-worker).
21
- */
22
- interface ConfigStore {
23
- getSchema(gatewayId: string): Promise<TableSchema | undefined>;
24
- setSchema(gatewayId: string, schema: TableSchema): Promise<void>;
25
- getSyncRules(gatewayId: string): Promise<SyncRulesConfig | undefined>;
26
- setSyncRules(gatewayId: string, rules: SyncRulesConfig): Promise<void>;
27
- getConnectors(): Promise<Record<string, ConnectorConfig>>;
28
- setConnectors(connectors: Record<string, ConnectorConfig>): Promise<void>;
29
- }
30
- /**
31
- * In-memory implementation of ConfigStore.
32
- * Used by tests and gateway-server.
33
- */
34
- declare class MemoryConfigStore implements ConfigStore {
35
- private schemas;
36
- private syncRules;
37
- private connectors;
38
- getSchema(gatewayId: string): Promise<TableSchema | undefined>;
39
- setSchema(gatewayId: string, schema: TableSchema): Promise<void>;
40
- getSyncRules(gatewayId: string): Promise<SyncRulesConfig | undefined>;
41
- setSyncRules(gatewayId: string, rules: SyncRulesConfig): Promise<void>;
42
- getConnectors(): Promise<Record<string, ConnectorConfig>>;
43
- setConnectors(connectors: Record<string, ConnectorConfig>): Promise<void>;
44
- }
13
+ import './types-D2C9jTbL.js';
45
14
 
46
15
  /** Maximum push payload size (1 MiB). */
47
16
  declare const MAX_PUSH_PAYLOAD_BYTES = 1048576;
@@ -51,8 +20,8 @@ declare const MAX_DELTAS_PER_PUSH = 10000;
51
20
  declare const MAX_PULL_LIMIT = 10000;
52
21
  /** Default number of deltas returned in a pull when no limit is specified. */
53
22
  declare const DEFAULT_PULL_LIMIT = 100;
54
- /** Allowed column types for schema validation. */
55
- declare const VALID_COLUMN_TYPES: Set<string>;
23
+ /** Allowed column types for schema validation. Derived from core COLUMN_TYPES. */
24
+ declare const VALID_COLUMN_TYPES: ReadonlySet<string>;
56
25
  /** Default maximum buffer size before triggering flush (4 MiB). */
57
26
  declare const DEFAULT_MAX_BUFFER_BYTES: number;
58
27
  /** Default maximum buffer age before triggering flush (30 seconds). */
@@ -64,6 +33,8 @@ interface FlushConfig {
64
33
  flushFormat?: "json" | "parquet";
65
34
  tableSchema?: TableSchema;
66
35
  catalogue?: NessieCatalogueClient;
36
+ /** Optional callback invoked when materialisation fails. Useful for metrics/alerting. */
37
+ onMaterialisationFailure?: (table: string, deltaCount: number, error: Error) => void;
67
38
  }
68
39
  /** Dependencies injected into flush operations. */
69
40
  interface FlushDeps {
@@ -77,6 +48,11 @@ declare function hlcRange(entries: RowDelta[]): {
77
48
  min: HLCTimestamp;
78
49
  max: HLCTimestamp;
79
50
  };
51
+ /** Strategy for flushing deltas to a specific adapter type. */
52
+ interface FlushStrategy {
53
+ /** Flush entries to the target adapter. */
54
+ flush(entries: RowDelta[], byteSize: number, deps: FlushDeps, keyPrefix?: string): Promise<Result<void, FlushError>>;
55
+ }
80
56
  /**
81
57
  * Flush a set of entries to the configured adapter.
82
58
  *
@@ -92,80 +68,67 @@ declare function flushEntries(entries: RowDelta[], byteSize: number, deps: Flush
92
68
  */
93
69
  declare function commitToCatalogue(objectKey: string, fileSizeInBytes: number, recordCount: number, catalogue: NessieCatalogueClient, schema: TableSchema): Promise<void>;
94
70
 
95
- /** Result from a request handler, ready for platform-specific serialisation. */
96
- interface HandlerResult {
97
- status: number;
98
- body: unknown;
71
+ /** Dependencies for flush operations. */
72
+ interface FlushCoordinatorDeps {
73
+ /** Gateway configuration for flush. */
74
+ config: FlushConfig;
75
+ /** Table schemas for materialisation after flush. */
76
+ schemas?: ReadonlyArray<TableSchema>;
99
77
  }
100
78
  /**
101
- * Handle a push request.
79
+ * Coordinates flush operations from the buffer to the adapter.
102
80
  *
103
- * @param gateway - The SyncGateway instance.
104
- * @param raw - The raw request body string.
105
- * @param headerClientId - Client ID from auth header (for mismatch check).
106
- * @param opts - Optional callbacks for persistence and broadcast.
107
- */
108
- declare function handlePushRequest(gateway: SyncGateway, raw: string, headerClientId?: string | null, opts?: {
109
- /** Persist deltas before processing (WAL-style). */
110
- persistBatch?: (deltas: RowDelta[]) => void;
111
- /** Clear persisted deltas after successful push. */
112
- clearPersistence?: () => void;
113
- /** Broadcast deltas to connected clients. */
114
- broadcastFn?: (deltas: RowDelta[], serverHlc: HLCTimestamp, excludeClientId: string) => void | Promise<void>;
115
- }): HandlerResult;
116
- /**
117
- * Handle a pull request.
118
- */
119
- declare function handlePullRequest(gateway: SyncGateway, params: {
120
- since: string | null;
121
- clientId: string | null;
122
- limit: string | null;
123
- source: string | null;
124
- }, claims?: ResolvedClaims, syncRules?: SyncRulesConfig): Promise<HandlerResult>;
125
- /**
126
- * Handle an action request.
127
- */
128
- declare function handleActionRequest(gateway: SyncGateway, raw: string, headerClientId?: string | null, claims?: ResolvedClaims): Promise<HandlerResult>;
129
- /**
130
- * Handle a flush request.
131
- */
132
- declare function handleFlushRequest(gateway: SyncGateway, opts?: {
133
- clearPersistence?: () => void;
134
- }): Promise<HandlerResult>;
135
- /**
136
- * Handle saving a table schema.
137
- */
138
- declare function handleSaveSchema(raw: string, store: ConfigStore, gatewayId: string): Promise<HandlerResult>;
139
- /**
140
- * Handle saving sync rules.
141
- */
142
- declare function handleSaveSyncRules(raw: string, store: ConfigStore, gatewayId: string): Promise<HandlerResult>;
143
- /**
144
- * Handle registering a connector.
145
- */
146
- declare function handleRegisterConnector(raw: string, store: ConfigStore): Promise<HandlerResult>;
147
- /**
148
- * Handle unregistering a connector.
149
- */
150
- declare function handleUnregisterConnector(name: string, store: ConfigStore): Promise<HandlerResult>;
151
- /**
152
- * Handle listing connectors.
153
- */
154
- declare function handleListConnectors(store: ConfigStore): Promise<HandlerResult>;
155
- /**
156
- * Handle listing available connector types (static metadata).
157
- */
158
- declare function handleListConnectorTypes(): HandlerResult;
81
+ * Owns the flushing state to prevent concurrent flushes and handles
82
+ * entry restoration on failure.
83
+ */
84
+ declare class FlushCoordinator {
85
+ private flushing;
86
+ /** Whether a flush is currently in progress. */
87
+ get isFlushing(): boolean;
88
+ /**
89
+ * Flush all entries from the buffer to the adapter.
90
+ *
91
+ * Drains the buffer first, then writes to the adapter. On failure,
92
+ * entries are restored to the buffer.
93
+ */
94
+ flush(buffer: DeltaBuffer, adapter: LakeAdapter | DatabaseAdapter | null, deps: FlushCoordinatorDeps): Promise<Result<void, FlushError>>;
95
+ /**
96
+ * Flush a single table's deltas from the buffer.
97
+ *
98
+ * Drains only the specified table's deltas and flushes them,
99
+ * leaving other tables in the buffer.
100
+ */
101
+ flushTable(table: string, buffer: DeltaBuffer, adapter: LakeAdapter | DatabaseAdapter | null, deps: FlushCoordinatorDeps): Promise<Result<void, FlushError>>;
102
+ /** Restore drained entries back to the buffer for retry. */
103
+ private restoreEntries;
104
+ }
105
+
159
106
  /**
160
- * Handle metrics request.
161
- */
162
- declare function handleMetrics(gateway: SyncGateway, extra?: Record<string, unknown>): HandlerResult;
107
+ * Registry for named source adapters used in adapter-sourced pulls.
108
+ *
109
+ * Manages the mapping of source names to DatabaseAdapter instances,
110
+ * decoupling source adapter management from the gateway.
111
+ */
112
+ declare class SourceRegistry {
113
+ private sources;
114
+ constructor(initial?: Record<string, DatabaseAdapter>);
115
+ /** Register a named source adapter. */
116
+ register(name: string, adapter: DatabaseAdapter): void;
117
+ /** Unregister a named source adapter. */
118
+ unregister(name: string): void;
119
+ /** Get a source adapter by name, or undefined if not registered. */
120
+ get(name: string): DatabaseAdapter | undefined;
121
+ /** List all registered source adapter names. */
122
+ list(): string[];
123
+ }
163
124
 
164
125
  /** Validation error with HTTP status code. */
165
126
  interface RequestError {
166
127
  status: number;
167
128
  message: string;
168
129
  }
130
+ /** Parse a JSON string, returning Err on invalid JSON. */
131
+ declare function parseJson<T>(raw: string, reviver?: (key: string, value: unknown) => unknown): Result<T, RequestError>;
169
132
  /**
170
133
  * Validate and parse a push request body.
171
134
  * Handles JSON parsing with bigint revival.
@@ -198,4 +161,4 @@ declare function pushErrorToStatus(code: string): number;
198
161
  */
199
162
  declare function buildSyncRulesContext(rules: SyncRulesConfig | undefined, claims: ResolvedClaims): SyncRulesContext | undefined;
200
163
 
201
- export { ActionPush, type ConfigStore, DEFAULT_MAX_BUFFER_AGE_MS, DEFAULT_MAX_BUFFER_BYTES, DEFAULT_PULL_LIMIT, type FlushConfig, type FlushDeps, type HandlerResult, MAX_DELTAS_PER_PUSH, MAX_PULL_LIMIT, MAX_PUSH_PAYLOAD_BYTES, MemoryConfigStore, type RequestError, SyncGateway, SyncPull, SyncPush, VALID_COLUMN_TYPES, buildSyncRulesContext, commitToCatalogue, flushEntries, handleActionRequest, handleFlushRequest, handleListConnectorTypes, handleListConnectors, handleMetrics, handlePullRequest, handlePushRequest, handleRegisterConnector, handleSaveSchema, handleSaveSyncRules, handleUnregisterConnector, hlcRange, parsePullParams, pushErrorToStatus, validateActionBody, validatePushBody, validateSchemaBody };
164
+ export { ActionPush, DEFAULT_MAX_BUFFER_AGE_MS, DEFAULT_MAX_BUFFER_BYTES, DEFAULT_PULL_LIMIT, DeltaBuffer, type FlushConfig, FlushCoordinator, type FlushCoordinatorDeps, type FlushDeps, type FlushStrategy, MAX_DELTAS_PER_PUSH, MAX_PULL_LIMIT, MAX_PUSH_PAYLOAD_BYTES, type RequestError, SourceRegistry, SyncPull, SyncPush, VALID_COLUMN_TYPES, buildSyncRulesContext, commitToCatalogue, flushEntries, hlcRange, parseJson, parsePullParams, pushErrorToStatus, validateActionBody, validatePushBody, validateSchemaBody };
package/dist/gateway.js CHANGED
@@ -4,11 +4,14 @@ import {
4
4
  DEFAULT_MAX_BUFFER_BYTES,
5
5
  DEFAULT_PULL_LIMIT,
6
6
  DeltaBuffer,
7
+ FlushCoordinator,
7
8
  MAX_DELTAS_PER_PUSH,
8
9
  MAX_PULL_LIMIT,
9
10
  MAX_PUSH_PAYLOAD_BYTES,
10
11
  MemoryConfigStore,
12
+ MemoryIdempotencyCache,
11
13
  SchemaManager,
14
+ SourceRegistry,
12
15
  SyncGateway,
13
16
  VALID_COLUMN_TYPES,
14
17
  buildSyncRulesContext,
@@ -26,32 +29,35 @@ import {
26
29
  handleSaveSyncRules,
27
30
  handleUnregisterConnector,
28
31
  hlcRange,
32
+ parseJson,
29
33
  parsePullParams,
30
34
  pushErrorToStatus,
31
35
  validateActionBody,
32
36
  validatePushBody,
33
37
  validateSchemaBody
34
- } from "./chunk-IRJ4QRWV.js";
35
- import "./chunk-GUJWMK5P.js";
36
- import "./chunk-UAUQGP3B.js";
37
- import "./chunk-SF7Y6ZUA.js";
38
+ } from "./chunk-FIIHPQMQ.js";
39
+ import "./chunk-U2NV4DUX.js";
40
+ import "./chunk-ZU7RC7CT.js";
38
41
  import {
39
42
  bigintReplacer,
40
43
  bigintReviver,
41
44
  isIngestTarget
42
- } from "./chunk-P3FT7QCW.js";
43
- import "./chunk-7D4SUZUM.js";
45
+ } from "./chunk-4SG66H5K.js";
46
+ import "./chunk-DGUM43GV.js";
44
47
  export {
45
48
  ActionDispatcher,
46
49
  DEFAULT_MAX_BUFFER_AGE_MS,
47
50
  DEFAULT_MAX_BUFFER_BYTES,
48
51
  DEFAULT_PULL_LIMIT,
49
52
  DeltaBuffer,
53
+ FlushCoordinator,
50
54
  MAX_DELTAS_PER_PUSH,
51
55
  MAX_PULL_LIMIT,
52
56
  MAX_PUSH_PAYLOAD_BYTES,
53
57
  MemoryConfigStore,
58
+ MemoryIdempotencyCache,
54
59
  SchemaManager,
60
+ SourceRegistry,
55
61
  SyncGateway,
56
62
  VALID_COLUMN_TYPES,
57
63
  bigintReplacer,
@@ -72,6 +78,7 @@ export {
72
78
  handleUnregisterConnector,
73
79
  hlcRange,
74
80
  isIngestTarget,
81
+ parseJson,
75
82
  parsePullParams,
76
83
  pushErrorToStatus,
77
84
  validateActionBody,
package/dist/index.d.ts CHANGED
@@ -2,15 +2,16 @@ import { A as Action, a as ActionValidationError, S as SyncRulesConfig, B as Buc
2
2
  export { c as ActionDescriptor, d as ActionDiscovery, e as ActionErrorResult, f as ActionExecutionError, g as ActionHandler, h as ActionNotSupportedError, i as ActionPush, j as ActionResponse, k as ActionResult, l as AuthContext, m as SyncRuleFilter, n as SyncRuleOp, o as isActionError, p as isActionHandler } from './types-Bs-QyOe-.js';
3
3
  import { H as HLCTimestamp, R as Result, C as ConflictError, L as LakeSyncError, S as SchemaError } from './result-CojzlFE2.js';
4
4
  export { A as AdapterError, a as AdapterNotFoundError, B as BackpressureError, b as ClockDriftError, E as Err, F as FlushError, O as Ok, f as flatMapResult, c as fromPromise, m as mapResult, t as toError, u as unwrapOrThrow } from './result-CojzlFE2.js';
5
- export { A as AuthClaims, a as AuthError, v as verifyToken } from './auth-CAVutXzx.js';
6
- import { P as PushTarget, B as BaseSourcePoller } from './base-poller-BpUyuG2R.js';
7
- export { I as IngestTarget, a as PollerMemoryConfig, i as isIngestTarget } from './base-poller-BpUyuG2R.js';
8
- import { S as SyncPush, R as RowDelta, T as TableSchema } from './types-CLlD4XOy.js';
9
- export { C as ColumnDelta, D as DeltaOp, a as RowKey, b as SyncPull, c as SyncResponse, r as rowKey } from './types-CLlD4XOy.js';
10
- import { C as ConflictResolver } from './resolver-CbuXm3nB.js';
11
- export { C as ConnectorCategory, a as ConnectorDescriptor, g as getConnectorDescriptor, l as listConnectorDescriptors, r as registerConnectorDescriptor, b as registerOutputSchemas } from './registry-CPTgO9jv.js';
12
- import { C as ConnectorConfig } from './types-D-E0VrfS.js';
13
- export { B as BigQueryConnectorConfig, a as CONNECTOR_TYPES, b as ConnectorIngestConfig, c as ConnectorIngestTable, d as ConnectorType, J as JiraConnectorConfig, M as MySQLConnectorConfig, P as PostgresConnectorConfig, S as SalesforceConnectorConfig } from './types-D-E0VrfS.js';
5
+ export { D as DatabaseAdapter, a as DatabaseAdapterConfig, L as LakeAdapter, M as Materialisable, O as ObjectInfo, i as isDatabaseAdapter, b as isMaterialisable } from './adapter-types-DwsQGQS4.js';
6
+ export { A as AuthClaims, a as AuthError, P as PollerFactory, b as PollerRegistry, c as createPoller, d as createPollerRegistry, v as verifyToken } from './create-poller-Cc2MGfhh.js';
7
+ import { P as PushTarget } from './base-poller-Y7ORYgUv.js';
8
+ export { B as BaseSourcePoller, C as ChunkedPusher, F as FlushableTarget, I as IngestTarget, a as PollerMemoryConfig, b as PressureManager, i as isIngestTarget } from './base-poller-Y7ORYgUv.js';
9
+ import { R as RowDelta, T as TableSchema, S as SyncPush } from './types-BdGBv2ba.js';
10
+ export { C as COLUMN_TYPES, a as ColumnDelta, b as ColumnReference, c as ColumnType, D as DeltaOp, d as RowKey, e as SyncPull, f as SyncResponse, r as rowKey } from './types-BdGBv2ba.js';
11
+ import { C as ConflictResolver } from './resolver-CXxmC0jR.js';
12
+ export { C as ConnectorCategory, a as ConnectorDescriptor, b as ConnectorRegistry, c as createConnectorRegistry, g as getConnectorDescriptor, l as listConnectorDescriptors, r as registerConnectorDescriptor, d as registerOutputSchemas } from './registry-Dd8JuW8T.js';
13
+ import { C as ConnectorConfig } from './types-D2C9jTbL.js';
14
+ export { B as BigQueryConnectorConfig, a as BigQueryConnectorConfigFull, b as CONNECTOR_TYPES, c as ConnectorConfigBase, d as ConnectorIngestConfig, e as ConnectorIngestTable, f as ConnectorType, J as JiraConnectorConfig, g as JiraConnectorConfigFull, M as MySQLConnectorConfig, h as MySQLConnectorConfigFull, P as PostgresConnectorConfig, i as PostgresConnectorConfigFull, S as SalesforceConnectorConfig, j as SalesforceConnectorConfigFull } from './types-D2C9jTbL.js';
14
15
  export { H as HLC } from './hlc-DiD8QNG3.js';
15
16
  export { b as bigintReplacer, a as bigintReviver } from './json-dYtqiL0F.js';
16
17
 
@@ -36,6 +37,42 @@ declare function generateActionId(params: {
36
37
  */
37
38
  declare function validateAction(action: unknown): Result<Action, ActionValidationError>;
38
39
 
40
+ /**
41
+ * Apply a delta to an existing row, returning the merged result.
42
+ *
43
+ * - DELETE → returns null
44
+ * - INSERT → creates a new row from delta columns
45
+ * - UPDATE → merges delta columns onto existing row (immutable — returns a new object)
46
+ *
47
+ * @param row - The current row state, or null if no row exists
48
+ * @param delta - The delta to apply
49
+ * @returns The merged row, or null for DELETE operations
50
+ */
51
+ declare function applyDelta(row: Record<string, unknown> | null, delta: RowDelta): Record<string, unknown> | null;
52
+
53
+ /**
54
+ * Extract a column-level delta between two row states.
55
+ *
56
+ * - `before` null/undefined + `after` present -> INSERT (all columns)
57
+ * - `before` present + `after` null/undefined -> DELETE (empty columns)
58
+ * - Both present -> compare each column, emit only changed columns as UPDATE
59
+ * - No columns changed -> returns null (no-op)
60
+ *
61
+ * If `schema` is provided, only columns listed in the schema are considered.
62
+ *
63
+ * @param before - The previous row state, or null/undefined for a new row
64
+ * @param after - The current row state, or null/undefined for a deleted row
65
+ * @param opts - Table name, row ID, client ID, HLC timestamp, and optional schema
66
+ * @returns The extracted RowDelta, or null if nothing changed
67
+ */
68
+ declare function extractDelta(before: Record<string, unknown> | null | undefined, after: Record<string, unknown> | null | undefined, opts: {
69
+ table: string;
70
+ rowId: string;
71
+ clientId: string;
72
+ hlc: HLCTimestamp;
73
+ schema?: TableSchema;
74
+ }): Promise<RowDelta | null>;
75
+
39
76
  /**
40
77
  * A simple PushTarget implementation that forwards every push to a
41
78
  * user-supplied callback. Useful for testing, logging, or lightweight
@@ -102,56 +139,26 @@ declare class ConnectorValidationError extends LakeSyncError {
102
139
  */
103
140
  declare function validateConnectorConfig(input: unknown): Result<ConnectorConfig, ConnectorValidationError>;
104
141
 
105
- /** Factory function that creates a poller from a ConnectorConfig. */
106
- type PollerFactory = (config: ConnectorConfig, gateway: PushTarget) => BaseSourcePoller;
107
- /**
108
- * Register a poller factory for a connector type.
109
- * Connector packages call this at module load time so that
110
- * `createPoller()` can instantiate the correct poller.
111
- */
112
- declare function registerPollerFactory(type: string, factory: PollerFactory): void;
113
- /**
114
- * Create a poller from a {@link ConnectorConfig}.
115
- *
116
- * @throws If no factory has been registered for the config's `type`.
117
- */
118
- declare function createPoller(config: ConnectorConfig, gateway: PushTarget): BaseSourcePoller;
119
-
120
142
  /**
121
- * Apply a delta to an existing row, returning the merged result.
122
- *
123
- * - DELETE → returns null
124
- * - INSERT → creates a new row from delta columns
125
- * - UPDATE → merges delta columns onto existing row (immutable — returns a new object)
126
- *
127
- * @param row - The current row state, or null if no row exists
128
- * @param delta - The delta to apply
129
- * @returns The merged row, or null for DELETE operations
130
- */
131
- declare function applyDelta(row: Record<string, unknown> | null, delta: RowDelta): Record<string, unknown> | null;
132
-
133
- /**
134
- * Extract a column-level delta between two row states.
135
- *
136
- * - `before` null/undefined + `after` present -> INSERT (all columns)
137
- * - `before` present + `after` null/undefined -> DELETE (empty columns)
138
- * - Both present -> compare each column, emit only changed columns as UPDATE
139
- * - No columns changed -> returns null (no-op)
140
- *
141
- * If `schema` is provided, only columns listed in the schema are considered.
142
- *
143
- * @param before - The previous row state, or null/undefined for a new row
144
- * @param after - The current row state, or null/undefined for a deleted row
145
- * @param opts - Table name, row ID, client ID, HLC timestamp, and optional schema
146
- * @returns The extracted RowDelta, or null if nothing changed
143
+ * Manages the start/stop lifecycle and timer scheduling for a poll function.
144
+ * Has no knowledge of deltas, gateways, or sync protocol.
147
145
  */
148
- declare function extractDelta(before: Record<string, unknown> | null | undefined, after: Record<string, unknown> | null | undefined, opts: {
149
- table: string;
150
- rowId: string;
151
- clientId: string;
152
- hlc: HLCTimestamp;
153
- schema?: TableSchema;
154
- }): Promise<RowDelta | null>;
146
+ declare class PollingScheduler {
147
+ private readonly pollFn;
148
+ private readonly intervalMs;
149
+ private timer;
150
+ private running;
151
+ constructor(pollFn: () => Promise<void>, intervalMs: number);
152
+ /** Start the polling loop. No-op if already running. */
153
+ start(): void;
154
+ /** Stop the polling loop. */
155
+ stop(): void;
156
+ /** Whether the scheduler is currently running. */
157
+ get isRunning(): boolean;
158
+ /** Execute a single poll cycle without the timer loop. */
159
+ pollOnce(): Promise<void>;
160
+ private schedule;
161
+ }
155
162
 
156
163
  /**
157
164
  * Create a pass-all sync rules configuration.
@@ -270,4 +277,4 @@ declare function assertValidIdentifier(name: string): Result<void, SchemaError>;
270
277
  */
271
278
  declare function quoteIdentifier(name: string): string;
272
279
 
273
- export { Action, ActionValidationError, BaseSourcePoller, BucketDefinition, CallbackPushTarget, ConflictError, ConflictResolver, ConnectorConfig, ConnectorValidationError, HLCTimestamp, LWWResolver, LakeSyncError, type PollerFactory, PushTarget, ResolvedClaims, Result, RowDelta, SchemaError, SyncPush, SyncRuleError, SyncRulesConfig, SyncRulesContext, TableSchema, applyDelta, assertValidIdentifier, createPassAllRules, createPoller, createUserScopedRules, deltaMatchesBucket, extractDelta, filterDeltas, generateActionId, isValidIdentifier, quoteIdentifier, registerPollerFactory, resolveClientBuckets, resolveFilterValue, resolveLWW, validateAction, validateConnectorConfig, validateSyncRules };
280
+ export { Action, ActionValidationError, BucketDefinition, CallbackPushTarget, ConflictError, ConflictResolver, ConnectorConfig, ConnectorValidationError, HLCTimestamp, LWWResolver, LakeSyncError, PollingScheduler, PushTarget, ResolvedClaims, Result, RowDelta, SchemaError, SyncPush, SyncRuleError, SyncRulesConfig, SyncRulesContext, TableSchema, applyDelta, assertValidIdentifier, createPassAllRules, createUserScopedRules, deltaMatchesBucket, extractDelta, filterDeltas, generateActionId, isValidIdentifier, quoteIdentifier, resolveClientBuckets, resolveFilterValue, resolveLWW, validateAction, validateConnectorConfig, validateSyncRules };
package/dist/index.js CHANGED
@@ -7,8 +7,10 @@ import {
7
7
  AuthError,
8
8
  BackpressureError,
9
9
  BaseSourcePoller,
10
+ COLUMN_TYPES,
10
11
  CONNECTOR_TYPES,
11
12
  CallbackPushTarget,
13
+ ChunkedPusher,
12
14
  ClockDriftError,
13
15
  ConflictError,
14
16
  ConnectorValidationError,
@@ -18,14 +20,18 @@ import {
18
20
  LWWResolver,
19
21
  LakeSyncError,
20
22
  Ok,
23
+ PollingScheduler,
24
+ PressureManager,
21
25
  SchemaError,
22
26
  SyncRuleError,
23
27
  applyDelta,
24
28
  assertValidIdentifier,
25
29
  bigintReplacer,
26
30
  bigintReviver,
31
+ createConnectorRegistry,
27
32
  createPassAllRules,
28
33
  createPoller,
34
+ createPollerRegistry,
29
35
  createUserScopedRules,
30
36
  deltaMatchesBucket,
31
37
  extractDelta,
@@ -36,14 +42,15 @@ import {
36
42
  getConnectorDescriptor,
37
43
  isActionError,
38
44
  isActionHandler,
45
+ isDatabaseAdapter,
39
46
  isIngestTarget,
47
+ isMaterialisable,
40
48
  isValidIdentifier,
41
49
  listConnectorDescriptors,
42
50
  mapResult,
43
51
  quoteIdentifier,
44
52
  registerConnectorDescriptor,
45
53
  registerOutputSchemas,
46
- registerPollerFactory,
47
54
  resolveClientBuckets,
48
55
  resolveFilterValue,
49
56
  resolveLWW,
@@ -54,8 +61,8 @@ import {
54
61
  validateConnectorConfig,
55
62
  validateSyncRules,
56
63
  verifyToken
57
- } from "./chunk-P3FT7QCW.js";
58
- import "./chunk-7D4SUZUM.js";
64
+ } from "./chunk-4SG66H5K.js";
65
+ import "./chunk-DGUM43GV.js";
59
66
  export {
60
67
  ActionExecutionError,
61
68
  ActionNotSupportedError,
@@ -65,8 +72,10 @@ export {
65
72
  AuthError,
66
73
  BackpressureError,
67
74
  BaseSourcePoller,
75
+ COLUMN_TYPES,
68
76
  CONNECTOR_TYPES,
69
77
  CallbackPushTarget,
78
+ ChunkedPusher,
70
79
  ClockDriftError,
71
80
  ConflictError,
72
81
  ConnectorValidationError,
@@ -76,14 +85,18 @@ export {
76
85
  LWWResolver,
77
86
  LakeSyncError,
78
87
  Ok,
88
+ PollingScheduler,
89
+ PressureManager,
79
90
  SchemaError,
80
91
  SyncRuleError,
81
92
  applyDelta,
82
93
  assertValidIdentifier,
83
94
  bigintReplacer,
84
95
  bigintReviver,
96
+ createConnectorRegistry,
85
97
  createPassAllRules,
86
98
  createPoller,
99
+ createPollerRegistry,
87
100
  createUserScopedRules,
88
101
  deltaMatchesBucket,
89
102
  extractDelta,
@@ -94,14 +107,15 @@ export {
94
107
  getConnectorDescriptor,
95
108
  isActionError,
96
109
  isActionHandler,
110
+ isDatabaseAdapter,
97
111
  isIngestTarget,
112
+ isMaterialisable,
98
113
  isValidIdentifier,
99
114
  listConnectorDescriptors,
100
115
  mapResult,
101
116
  quoteIdentifier,
102
117
  registerConnectorDescriptor,
103
118
  registerOutputSchemas,
104
- registerPollerFactory,
105
119
  resolveClientBuckets,
106
120
  resolveFilterValue,
107
121
  resolveLWW,
package/dist/parquet.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { T as TableSchema, R as RowDelta } from './types-CLlD4XOy.js';
1
+ import { T as TableSchema, R as RowDelta } from './types-BdGBv2ba.js';
2
2
  import * as arrow from 'apache-arrow';
3
3
  import { R as Result, F as FlushError } from './result-CojzlFE2.js';
4
4
 
package/dist/parquet.js CHANGED
@@ -3,9 +3,9 @@ import {
3
3
  deltasToArrowTable,
4
4
  readParquetToDeltas,
5
5
  writeDeltasToParquet
6
- } from "./chunk-SF7Y6ZUA.js";
7
- import "./chunk-P3FT7QCW.js";
8
- import "./chunk-7D4SUZUM.js";
6
+ } from "./chunk-ZU7RC7CT.js";
7
+ import "./chunk-4SG66H5K.js";
8
+ import "./chunk-DGUM43GV.js";
9
9
  export {
10
10
  buildArrowSchema,
11
11
  deltasToArrowTable,
package/dist/proto.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { H as HLCTimestamp, R as Result } from './result-CojzlFE2.js';
2
2
  import { i as ActionPush$1, j as ActionResponse$1 } from './types-Bs-QyOe-.js';
3
- import { R as RowDelta$1 } from './types-CLlD4XOy.js';
3
+ import { R as RowDelta$1 } from './types-BdGBv2ba.js';
4
4
  import { GenMessage, GenEnum } from '@bufbuild/protobuf/codegenv2';
5
5
  import { Message } from '@bufbuild/protobuf';
6
6
 
package/dist/proto.js CHANGED
@@ -29,9 +29,9 @@ import {
29
29
  encodeSyncPull,
30
30
  encodeSyncPush,
31
31
  encodeSyncResponse
32
- } from "./chunk-FHVTUKXL.js";
33
- import "./chunk-P3FT7QCW.js";
34
- import "./chunk-7D4SUZUM.js";
32
+ } from "./chunk-YHYBLU6W.js";
33
+ import "./chunk-4SG66H5K.js";
34
+ import "./chunk-DGUM43GV.js";
35
35
  export {
36
36
  ActionPushSchema,
37
37
  ActionResponseSchema,