lakesync 0.1.8 → 0.2.0

package/dist/adapter-types-DwsQGQS4.d.ts

@@ -0,0 +1,94 @@
+import { R as RowDelta, T as TableSchema } from './types-BdGBv2ba.js';
+import { R as Result, A as AdapterError, H as HLCTimestamp } from './result-CojzlFE2.js';
+
+/** Information about an object in the lake store. */
+interface ObjectInfo {
+    /** S3 object key */
+    key: string;
+    /** Object size in bytes */
+    size: number;
+    /** Last modification date */
+    lastModified: Date;
+}
+/** Abstract interface for lake storage operations. */
+interface LakeAdapter {
+    /** Store an object in the lake. */
+    putObject(path: string, data: Uint8Array, contentType?: string): Promise<Result<void, AdapterError>>;
+    /** Retrieve an object from the lake. */
+    getObject(path: string): Promise<Result<Uint8Array, AdapterError>>;
+    /** Get object metadata without retrieving the body. */
+    headObject(path: string): Promise<Result<{
+        size: number;
+        lastModified: Date;
+    }, AdapterError>>;
+    /** List objects matching a given prefix. */
+    listObjects(prefix: string): Promise<Result<ObjectInfo[], AdapterError>>;
+    /** Delete a single object from the lake. */
+    deleteObject(path: string): Promise<Result<void, AdapterError>>;
+    /** Delete multiple objects from the lake in a single batch operation. */
+    deleteObjects(paths: string[]): Promise<Result<void, AdapterError>>;
+}
+/** Configuration for a database adapter connection. */
+interface DatabaseAdapterConfig {
+    /** Connection string (e.g. postgres://user:pass@host/db) */
+    connectionString: string;
+    /** Maximum number of connections in the pool (default: 10). */
+    poolMax?: number;
+    /** Connection idle timeout in milliseconds (default: 10000). */
+    idleTimeoutMs?: number;
+    /** Connection acquisition timeout in milliseconds (default: 30000). */
+    connectionTimeoutMs?: number;
+    /** Statement timeout in milliseconds (default: 30000). */
+    statementTimeoutMs?: number;
+}
+/**
+ * Abstract interface for SQL database storage operations.
+ * Alternative to LakeAdapter for small-data backends (Postgres, MySQL, etc).
+ */
+interface DatabaseAdapter {
+    /** Insert deltas into the database in a single batch. Idempotent via deltaId uniqueness. */
+    insertDeltas(deltas: RowDelta[]): Promise<Result<void, AdapterError>>;
+    /** Query deltas with HLC greater than the given timestamp, optionally filtered by table. */
+    queryDeltasSince(hlc: HLCTimestamp, tables?: string[]): Promise<Result<RowDelta[], AdapterError>>;
+    /** Get the latest merged state for a specific row. Returns null if the row doesn't exist. */
+    getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
+    /** Ensure the database schema matches the given TableSchema. Creates/alters tables as needed. */
+    ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Close the database connection and release resources. */
+    close(): Promise<void>;
+}
+/** Type guard to distinguish DatabaseAdapter from LakeAdapter at runtime. */
+declare function isDatabaseAdapter(adapter: unknown): adapter is DatabaseAdapter;
+/**
+ * Opt-in capability for adapters that can materialise deltas into destination tables.
+ *
+ * Materialisation is a separate concern from delta storage — adapters that store
+ * deltas (via `DatabaseAdapter.insertDeltas`) may also materialise them into
+ * queryable destination tables by implementing this interface.
+ *
+ * Destination tables follow the hybrid column model:
+ * - Synced columns (written by materialiser, derived from `TableSchema.columns`)
+ * - `props JSONB DEFAULT '{}'` — consumer-extensible, never touched by materialiser
+ * - `synced_at` — updated on every materialise cycle
+ */
+interface Materialisable {
+    /**
+     * Materialise deltas into destination tables.
+     *
+     * For each table with a matching schema, merges delta history into the
+     * latest row state and upserts into the destination table. Tombstoned
+     * rows are deleted. The `props` column is never touched.
+     *
+     * @param deltas - The deltas that were just flushed.
+     * @param schemas - Table schemas defining destination tables and column mappings.
+     */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
+}
+/**
+ * Type guard to check if an adapter supports materialisation.
+ *
+ * Uses duck-typing (same pattern as `isDatabaseAdapter`).
+ */
+declare function isMaterialisable(adapter: unknown): adapter is Materialisable;
+
+export { type DatabaseAdapter as D, type LakeAdapter as L, type Materialisable as M, type ObjectInfo as O, type DatabaseAdapterConfig as a, isMaterialisable as b, isDatabaseAdapter as i };

package/dist/adapter.d.ts

@@ -1,44 +1,15 @@
 import { BigQuery } from '@google-cloud/bigquery';
 import { R as Result, A as AdapterError, H as HLCTimestamp } from './result-CojzlFE2.js';
-import { R as RowDelta, T as TableSchema, a as ColumnDelta } from './types-BdGBv2ba.js';
-import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-CfLMUBfW.js';
-export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-CfLMUBfW.js';
-import { C as ConnectorConfig } from './types-BrcD1oJg.js';
-import { L as LakeAdapter, A as AdapterConfig, O as ObjectInfo } from './types-DSC_EiwR.js';
+import { D as DatabaseAdapter, M as Materialisable, L as LakeAdapter, O as ObjectInfo, a as DatabaseAdapterConfig } from './adapter-types-DwsQGQS4.js';
+export { i as isDatabaseAdapter, b as isMaterialisable } from './adapter-types-DwsQGQS4.js';
+import { T as TableSchema, a as ColumnDelta, R as RowDelta } from './types-BdGBv2ba.js';
+export { A as AdapterFactory, a as AdapterFactoryRegistry, c as createAdapterFactoryRegistry, b as createDatabaseAdapter, d as defaultAdapterFactoryRegistry } from './factory-DFfR-030.js';
 import mysql from 'mysql2/promise';
 import { Pool } from 'pg';
+import { C as ConnectorConfig } from './types-D2C9jTbL.js';
+
+declare function lakeSyncTypeToBigQuery(type: TableSchema["columns"][number]["type"]): string;
 
-/**
- * Opt-in capability for adapters that can materialise deltas into destination tables.
- *
- * Materialisation is a separate concern from delta storage — adapters that store
- * deltas (via `DatabaseAdapter.insertDeltas`) may also materialise them into
- * queryable destination tables by implementing this interface.
- *
- * Destination tables follow the hybrid column model:
- * - Synced columns (written by materialiser, derived from `TableSchema.columns`)
- * - `props JSONB DEFAULT '{}'` — consumer-extensible, never touched by materialiser
- * - `synced_at` — updated on every materialise cycle
- */
-interface Materialisable {
-    /**
-     * Materialise deltas into destination tables.
-     *
-     * For each table with a matching schema, merges delta history into the
-     * latest row state and upserts into the destination table. Tombstoned
-     * rows are deleted. The `props` column is never touched.
-     *
-     * @param deltas - The deltas that were just flushed.
-     * @param schemas - Table schemas defining destination tables and column mappings.
-     */
-    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
-}
-/**
- * Type guard to check if an adapter supports materialisation.
- *
- * Uses duck-typing (same pattern as `isDatabaseAdapter`).
- */
-declare function isMaterialisable(adapter: unknown): adapter is Materialisable;
 /**
  * Resolve the primary key columns for a table schema.
  * Defaults to `["row_id"]` when not explicitly set.
@@ -258,18 +229,6 @@ declare class CompositeAdapter implements DatabaseAdapter {
     close(): Promise<void>;
 }
 
-/**
- * Instantiate a {@link DatabaseAdapter} from a {@link ConnectorConfig}.
- *
- * Switches on `config.type` and creates the matching adapter using
- * the type-specific connection configuration. Returns an {@link AdapterError}
- * if the type is unsupported or the adapter constructor throws.
- *
- * @param config - Validated connector configuration.
- * @returns The instantiated adapter or an error.
- */
-declare function createDatabaseAdapter(config: ConnectorConfig): Result<DatabaseAdapter, AdapterError>;
-
 /** Configuration for the FanOutAdapter. */
 interface FanOutAdapterConfig {
     /** The primary adapter that handles all reads and authoritative writes. */
@@ -413,6 +372,21 @@ interface MigrateResult {
  */
 declare function migrateAdapter(opts: MigrateOptions): Promise<Result<MigrateResult, AdapterError>>;
 
+/** Configuration for connecting to the lake store */
+interface AdapterConfig {
+    /** Endpoint URL (e.g. http://localhost:9000) */
+    endpoint: string;
+    /** Bucket name */
+    bucket: string;
+    /** AWS region (defaults to us-east-1) */
+    region?: string;
+    /** Access credentials */
+    credentials: {
+        accessKeyId: string;
+        secretAccessKey: string;
+    };
+}
+
 /**
  * MinIO/S3-compatible lake adapter.
  *
@@ -624,4 +598,4 @@ declare function mergeLatestState(rows: Array<{
     op: string;
 }>): Record<string, unknown> | null;
 
-export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, BigQuerySqlDialect, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, MySqlDialect, ObjectInfo, PostgresAdapter, PostgresSqlDialect, type QueryExecutor, type QueryFn, type SqlDialect, buildSchemaIndex, createDatabaseAdapter, createQueryFn, executeMaterialise, groupAndMerge, groupDeltasByTable, isMaterialisable, isSoftDelete, mergeLatestState, migrateAdapter, migrateToTier, resolveConflictColumns, resolvePrimaryKey, toCause, wrapAsync };
+export { type AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, BigQuerySqlDialect, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, MySqlDialect, ObjectInfo, PostgresAdapter, PostgresSqlDialect, type QueryExecutor, type QueryFn, type SqlDialect, buildSchemaIndex, createQueryFn, executeMaterialise, groupAndMerge, groupDeltasByTable, isSoftDelete, lakeSyncTypeToBigQuery, mergeLatestState, migrateAdapter, migrateToTier, resolveConflictColumns, resolvePrimaryKey, toCause, wrapAsync };

package/dist/adapter.js

@@ -10,13 +10,13 @@ import {
   PostgresAdapter,
   PostgresSqlDialect,
   buildSchemaIndex,
+  createAdapterFactoryRegistry,
   createDatabaseAdapter,
   createQueryFn,
+  defaultAdapterFactoryRegistry,
   executeMaterialise,
   groupAndMerge,
   groupDeltasByTable,
-  isDatabaseAdapter,
-  isMaterialisable,
   isSoftDelete,
   lakeSyncTypeToBigQuery,
   mergeLatestState,
@@ -26,8 +26,11 @@ import {
   resolvePrimaryKey,
   toCause,
   wrapAsync
-} from "./chunk-LPWXOYNS.js";
-import "./chunk-LDFFCG2K.js";
+} from "./chunk-C4KD6YKP.js";
+import {
+  isDatabaseAdapter,
+  isMaterialisable
+} from "./chunk-4SG66H5K.js";
 import "./chunk-DGUM43GV.js";
 export {
   BigQueryAdapter,
@@ -41,8 +44,10 @@ export {
   PostgresAdapter,
   PostgresSqlDialect,
   buildSchemaIndex,
+  createAdapterFactoryRegistry,
   createDatabaseAdapter,
   createQueryFn,
+  defaultAdapterFactoryRegistry,
   executeMaterialise,
   groupAndMerge,
   groupDeltasByTable,

package/dist/analyst.js

@@ -2,7 +2,7 @@ import {
   Err,
   LakeSyncError,
   Ok
-} from "./chunk-LDFFCG2K.js";
+} from "./chunk-4SG66H5K.js";
 import "./chunk-DGUM43GV.js";
 
 // ../analyst/src/duckdb.ts

package/dist/{base-poller-Bj9kX9dv.d.ts → base-poller-Y7ORYgUv.d.ts}

@@ -111,6 +111,8 @@ declare abstract class BaseSourcePoller {
     protected readonly clientId: string;
     private readonly scheduler;
     private readonly pusher;
+    /** Optional callback invoked after each poll with the current cursor state. */
+    onCursorUpdate?: (state: Record<string, unknown>) => void;
     constructor(config: {
         name: string;
         intervalMs: number;

package/dist/catalogue.js

@@ -4,8 +4,8 @@ import {
   buildPartitionSpec,
   lakeSyncTableName,
   tableSchemaToIceberg
-} from "./chunk-TMLG32QV.js";
-import "./chunk-LDFFCG2K.js";
+} from "./chunk-U2NV4DUX.js";
+import "./chunk-4SG66H5K.js";
 import "./chunk-DGUM43GV.js";
 export {
   CatalogueError,

package/dist/{chunk-LDFFCG2K.js → chunk-4SG66H5K.js}

@@ -156,6 +156,14 @@ function validateAction(action) {
   return Ok(action);
 }
 
+// ../core/src/adapter-types.ts
+function isDatabaseAdapter(adapter) {
+  return adapter !== null && typeof adapter === "object" && "insertDeltas" in adapter && "queryDeltasSince" in adapter && typeof adapter.insertDeltas === "function";
+}
+function isMaterialisable(adapter) {
+  return adapter !== null && typeof adapter === "object" && "materialise" in adapter && typeof adapter.materialise === "function";
+}
+
 // ../core/src/auth.ts
 var AuthError = class extends Error {
   constructor(message) {
@@ -536,6 +544,8 @@ var BaseSourcePoller = class {
   clientId;
   scheduler;
   pusher;
+  /** Optional callback invoked after each poll with the current cursor state. */
+  onCursorUpdate;
   constructor(config) {
     this.gateway = config.gateway;
     this.hlc = new HLC();
@@ -551,7 +561,12 @@ var BaseSourcePoller = class {
       chunkSize: config.memory?.chunkSize ?? DEFAULT_CHUNK_SIZE,
       pressure
     });
-    this.scheduler = new PollingScheduler(() => this.poll(), config.intervalMs);
+    this.scheduler = new PollingScheduler(async () => {
+      await this.poll();
+      if (this.onCursorUpdate) {
+        this.onCursorUpdate(this.getCursorState());
+      }
+    }, config.intervalMs);
   }
   /** Start the polling loop. */
   start() {
@@ -909,31 +924,32 @@ function validateConnectorConfig(input) {
   if (typeof obj.name !== "string" || obj.name.length === 0) {
     return Err(new ConnectorValidationError("Connector name must be a non-empty string"));
   }
-  if (typeof obj.type !== "string" || !CONNECTOR_TYPES.includes(obj.type)) {
-    return Err(
-      new ConnectorValidationError(`Connector type must be one of: ${CONNECTOR_TYPES.join(", ")}`)
-    );
+  if (typeof obj.type !== "string" || obj.type.length === 0) {
+    return Err(new ConnectorValidationError("Connector type must be a non-empty string"));
   }
   const connectorType = obj.type;
-  let typeResult;
-  switch (connectorType) {
-    case "postgres":
-      typeResult = validatePostgresConfig(obj);
-      break;
-    case "mysql":
-      typeResult = validateMySQLConfig(obj);
-      break;
-    case "bigquery":
-      typeResult = validateBigQueryConfig(obj);
-      break;
-    case "jira":
-      typeResult = validateJiraConfig(obj);
-      break;
-    case "salesforce":
-      typeResult = validateSalesforceConfig(obj);
-      break;
-  }
-  if (!typeResult.ok) return typeResult;
+  if (CONNECTOR_TYPES.includes(connectorType)) {
+    const knownType = connectorType;
+    let typeResult;
+    switch (knownType) {
+      case "postgres":
+        typeResult = validatePostgresConfig(obj);
+        break;
+      case "mysql":
+        typeResult = validateMySQLConfig(obj);
+        break;
+      case "bigquery":
+        typeResult = validateBigQueryConfig(obj);
+        break;
+      case "jira":
+        typeResult = validateJiraConfig(obj);
+        break;
+      case "salesforce":
+        typeResult = validateSalesforceConfig(obj);
+        break;
+    }
+    if (!typeResult.ok) return typeResult;
+  }
   const ingestResult = validateIngestConfig(obj, connectorType);
   if (!ingestResult.ok) return ingestResult;
   return Ok(input);
@@ -1228,12 +1244,8 @@ function buildPollerRegistry(map) {
     }
   };
 }
-var pollerFactories = /* @__PURE__ */ new Map();
-function registerPollerFactory(type, factory) {
-  pollerFactories.set(type, factory);
-}
 function createPoller(config, gateway, registry) {
-  const factory = registry ? registry.get(config.type) : pollerFactories.get(config.type);
+  const factory = registry.get(config.type);
   if (!factory) {
     throw new Error(
       `No poller factory registered for connector type "${config.type}". Did you import the connector package (e.g. "@lakesync/connector-${config.type}")?`
@@ -1596,6 +1608,8 @@ export {
   unwrapOrThrow,
   fromPromise,
   validateAction,
+  isDatabaseAdapter,
+  isMaterialisable,
   AuthError,
   verifyToken,
   HLC,
@@ -1617,7 +1631,6 @@ export {
   CONNECTOR_TYPES,
   validateConnectorConfig,
   createPollerRegistry,
-  registerPollerFactory,
   createPoller,
   applyDelta,
   extractDelta,
@@ -1637,4 +1650,4 @@ export {
   assertValidIdentifier,
   quoteIdentifier
 };
-//# sourceMappingURL=chunk-LDFFCG2K.js.map
+//# sourceMappingURL=chunk-4SG66H5K.js.map