lakesync 0.1.6 → 0.1.8

package/dist/adapter.d.ts

@@ -1,9 +1,9 @@
 import { BigQuery } from '@google-cloud/bigquery';
 import { R as Result, A as AdapterError, H as HLCTimestamp } from './result-CojzlFE2.js';
-import { R as RowDelta, T as TableSchema, C as ColumnDelta } from './types-CLlD4XOy.js';
-import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-BlN-4KbQ.js';
-export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-BlN-4KbQ.js';
-import { C as ConnectorConfig } from './types-D-E0VrfS.js';
+import { R as RowDelta, T as TableSchema, a as ColumnDelta } from './types-BdGBv2ba.js';
+import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-CfLMUBfW.js';
+export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-CfLMUBfW.js';
+import { C as ConnectorConfig } from './types-BrcD1oJg.js';
 import { L as LakeAdapter, A as AdapterConfig, O as ObjectInfo } from './types-DSC_EiwR.js';
 import mysql from 'mysql2/promise';
 import { Pool } from 'pg';
@@ -70,6 +70,56 @@ declare function groupDeltasByTable(deltas: ReadonlyArray<RowDelta>): Map<string
  * @returns A map from source table name to schema.
  */
 declare function buildSchemaIndex(schemas: ReadonlyArray<TableSchema>): Map<string, TableSchema>;
+/** Minimal query interface for executing SQL against a database. */
+interface QueryExecutor {
+    query(sql: string, params: unknown[]): Promise<void>;
+    queryRows(sql: string, params: unknown[]): Promise<Array<{
+        row_id: string;
+        columns: string | ColumnDelta[];
+        op: string;
+    }>>;
+}
+/**
+ * SQL dialect interface — encapsulates the syntactic differences between
+ * Postgres, MySQL, and BigQuery for the materialise algorithm.
+ */
+interface SqlDialect {
+    /** Generate CREATE TABLE IF NOT EXISTS for the destination table. */
+    createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+    /** Generate a query to fetch delta history for a set of affected row IDs. */
+    queryDeltaHistory(sourceTable: string, rowIds: string[]): {
+        sql: string;
+        params: unknown[];
+    };
+    /** Generate an upsert statement for the merged row states. */
+    buildUpsert(dest: string, schema: TableSchema, conflictCols: string[], softDelete: boolean, upserts: Array<{
+        rowId: string;
+        state: Record<string, unknown>;
+    }>): {
+        sql: string;
+        params: unknown[];
+    };
+    /** Generate a delete (hard or soft) statement for tombstoned row IDs. */
+    buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+}
+/**
+ * Execute the shared materialise algorithm using the provided dialect and executor.
+ *
+ * Algorithm: group by table -> build schema index -> for each table:
+ * create dest table -> query history -> merge -> upsert -> delete.
+ *
+ * @param executor - Executes SQL statements against the database.
+ * @param dialect - Generates dialect-specific SQL.
+ * @param deltas - The deltas that were just flushed.
+ * @param schemas - Table schemas defining destination tables and column mappings.
+ */
+declare function executeMaterialise(executor: QueryExecutor, dialect: SqlDialect, deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
 
 /**
  * Configuration for the BigQuery adapter.
@@ -85,6 +135,40 @@ interface BigQueryAdapterConfig {
     /** Dataset location (default: "US"). */
     location?: string;
 }
+/**
+ * BigQuery SQL dialect for the shared materialise algorithm.
+ *
+ * Uses `@name` named parameters, `MERGE` for upserts, `JSON` type,
+ * `TIMESTAMP` types, and `CLUSTER BY` for table creation.
+ *
+ * BigQuery's parameterised query API uses named params in a flat object,
+ * so this dialect encodes params as named keys and returns them as an
+ * array of `[key, value]` pairs that the executor unpacks into a
+ * `Record<string, unknown>`.
+ */
+declare class BigQuerySqlDialect implements SqlDialect {
+    private readonly dataset;
+    constructor(dataset: string);
+    createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+    queryDeltaHistory(sourceTable: string, rowIds: string[]): {
+        sql: string;
+        params: unknown[];
+    };
+    buildUpsert(dest: string, schema: TableSchema, conflictCols: string[], softDelete: boolean, upserts: Array<{
+        rowId: string;
+        state: Record<string, unknown>;
+    }>): {
+        sql: string;
+        params: unknown[];
+    };
+    buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+}
 /**
  * BigQuery database adapter for LakeSync.
  *
@@ -103,6 +187,8 @@ declare class BigQueryAdapter implements DatabaseAdapter, Materialisable {
     readonly dataset: string;
     /** @internal */
     readonly location: string;
+    private readonly dialect;
+    private readonly executor;
     constructor(config: BigQueryAdapterConfig);
     /**
      * Insert deltas into the database in a single batch.
@@ -127,10 +213,8 @@ declare class BigQueryAdapter implements DatabaseAdapter, Materialisable {
     /**
      * Materialise deltas into destination tables.
      *
-     * For each affected table, queries the full delta history for touched rows,
-     * merges to latest state via column-level LWW, then upserts live rows and
-     * deletes tombstoned rows. The consumer-owned `props` column is never
-     * touched on UPDATE.
+     * Delegates to the shared `executeMaterialise` algorithm with the
+     * BigQuery SQL dialect.
      */
     materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /**
@@ -179,7 +263,7 @@ declare class CompositeAdapter implements DatabaseAdapter {
  *
  * Switches on `config.type` and creates the matching adapter using
  * the type-specific connection configuration. Returns an {@link AdapterError}
- * if the type-specific config is missing or the adapter constructor throws.
+ * if the type is unsupported or the adapter constructor throws.
  *
  * @param config - Validated connector configuration.
  * @returns The instantiated adapter or an error.
@@ -199,8 +283,14 @@ interface FanOutAdapterConfig {
  *
  * Secondary failures are silently caught and never affect the return value.
  * Use case: write to Postgres (fast, operational), replicate to BigQuery (analytics).
+ *
+ * **Materialisation:** This adapter exposes a `materialise()` method for
+ * duck-type compatibility with `isMaterialisable()`. When the primary adapter
+ * is itself materialisable, materialisation is delegated to it; otherwise
+ * the method is a graceful no-op returning `Ok`. Materialisable secondaries
+ * receive fire-and-forget replication.
  */
-declare class FanOutAdapter implements DatabaseAdapter, Materialisable {
+declare class FanOutAdapter implements DatabaseAdapter {
     private readonly primary;
     private readonly secondaries;
     constructor(config: FanOutAdapterConfig);
@@ -241,8 +331,14 @@ interface LifecycleAdapterConfig {
  *
  * Use {@link migrateToTier} as a background job to copy aged-out deltas
  * from hot to cold.
+ *
+ * **Materialisation:** This adapter exposes a `materialise()` method for
+ * duck-type compatibility with `isMaterialisable()`. Materialisation is
+ * delegated to the hot tier only — cold tier stores archived deltas, not
+ * destination tables. When the hot adapter is not materialisable, the
+ * method is a graceful no-op returning `Ok`.
  */
-declare class LifecycleAdapter implements DatabaseAdapter, Materialisable {
+declare class LifecycleAdapter implements DatabaseAdapter {
     private readonly hot;
     private readonly cold;
     private readonly maxAgeMs;
@@ -345,6 +441,33 @@ declare class MinIOAdapter implements LakeAdapter {
     deleteObjects(paths: string[]): Promise<Result<void, AdapterError>>;
 }
 
+/**
+ * MySQL SQL dialect for the shared materialise algorithm.
+ *
+ * Uses `?` positional parameters, `ON DUPLICATE KEY UPDATE`,
+ * `JSON` type, and `TIMESTAMP` types.
+ */
+declare class MySqlDialect implements SqlDialect {
+    createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+    queryDeltaHistory(sourceTable: string, rowIds: string[]): {
+        sql: string;
+        params: unknown[];
+    };
+    buildUpsert(dest: string, schema: TableSchema, _conflictCols: string[], softDelete: boolean, upserts: Array<{
+        rowId: string;
+        state: Record<string, unknown>;
+    }>): {
+        sql: string;
+        params: unknown[];
+    };
+    buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+}
 /**
  * MySQL database adapter for LakeSync.
  *
@@ -355,7 +478,9 @@ declare class MinIOAdapter implements LakeAdapter {
 declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
     /** @internal */
     readonly pool: mysql.Pool;
+    private readonly dialect;
     constructor(config: DatabaseAdapterConfig);
+    private get executor();
     /**
      * Insert deltas into the database in a single batch.
      * Uses INSERT IGNORE for idempotent writes — duplicate deltaIds are silently skipped.
@@ -379,16 +504,41 @@ declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
     /**
      * Materialise deltas into destination tables.
      *
-     * For each table with a matching schema, merges delta history into the
-     * latest row state and upserts into the destination table. Tombstoned
-     * rows are soft-deleted (default) or hard-deleted. The `props` column
-     * is never touched.
+     * Delegates to the shared `executeMaterialise` algorithm with the
+     * MySQL SQL dialect.
      */
     materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /** Close the database connection pool and release resources. */
     close(): Promise<void>;
 }
 
+/**
+ * Postgres SQL dialect for the shared materialise algorithm.
+ *
+ * Uses `$N` positional parameters, `ON CONFLICT DO UPDATE`, `JSONB`,
+ * and `TIMESTAMPTZ` types.
+ */
+declare class PostgresSqlDialect implements SqlDialect {
+    createDestinationTable(dest: string, schema: TableSchema, pk: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+    queryDeltaHistory(sourceTable: string, rowIds: string[]): {
+        sql: string;
+        params: unknown[];
+    };
+    buildUpsert(dest: string, schema: TableSchema, conflictCols: string[], softDelete: boolean, upserts: Array<{
+        rowId: string;
+        state: Record<string, unknown>;
+    }>): {
+        sql: string;
+        params: unknown[];
+    };
+    buildDelete(dest: string, deleteIds: string[], softDelete: boolean): {
+        sql: string;
+        params: unknown[];
+    };
+}
 /**
  * PostgreSQL database adapter for LakeSync.
  *
@@ -398,7 +548,9 @@ declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
 declare class PostgresAdapter implements DatabaseAdapter, Materialisable {
     /** @internal */
     readonly pool: Pool;
+    private readonly dialect;
     constructor(config: DatabaseAdapterConfig);
+    private get executor();
     /**
      * Insert deltas into the database in a single batch.
      * Idempotent via `ON CONFLICT (delta_id) DO NOTHING`.
@@ -422,9 +574,8 @@ declare class PostgresAdapter implements DatabaseAdapter, Materialisable {
     /**
      * Materialise deltas into destination tables.
      *
-     * For each table with a matching schema, merges delta history into the
-     * latest row state and upserts into the destination table. Tombstoned
-     * rows are deleted. The `props` column is never touched.
+     * Delegates to the shared `executeMaterialise` algorithm with the
+     * Postgres SQL dialect.
      */
     materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /** Close the database connection pool and release resources. */
@@ -438,13 +589,27 @@ type QueryFn = (sql: string, params?: unknown[]) => Promise<Record<string, unkno
  *
  * Uses dynamic imports so the database drivers (pg, mysql2) are only
  * loaded when actually needed. Returns `null` for connector types that
- * do not support the standard SQL polling model (e.g. BigQuery).
+ * do not support the standard SQL polling model (e.g. BigQuery, Jira, Salesforce).
  *
  * @param config - Validated connector configuration.
  * @returns A query function or `null` if the connector type is unsupported.
  */
 declare function createQueryFn(config: ConnectorConfig): Promise<QueryFn | null>;
 
+/**
+ * Group raw delta rows by row_id, merge to latest state, and partition into upserts and deletes.
+ */
+declare function groupAndMerge(rows: Array<{
+    row_id: string;
+    columns: string | ColumnDelta[];
+    op: string;
+}>): {
+    upserts: Array<{
+        rowId: string;
+        state: Record<string, unknown>;
+    }>;
+    deleteIds: string[];
+};
 /** Normalise a caught value into an Error or undefined. */
 declare function toCause(error: unknown): Error | undefined;
 /** Execute an async operation and wrap errors into an AdapterError Result. */
@@ -459,4 +624,4 @@ declare function mergeLatestState(rows: Array<{
     op: string;
 }>): Record<string, unknown> | null;
 
-export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, ObjectInfo, PostgresAdapter, type QueryFn, buildSchemaIndex, createDatabaseAdapter, createQueryFn, groupDeltasByTable, isMaterialisable, isSoftDelete, mergeLatestState, migrateAdapter, migrateToTier, resolveConflictColumns, resolvePrimaryKey, toCause, wrapAsync };
+export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, BigQuerySqlDialect, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, MySqlDialect, ObjectInfo, PostgresAdapter, PostgresSqlDialect, type QueryExecutor, type QueryFn, type SqlDialect, buildSchemaIndex, createDatabaseAdapter, createQueryFn, executeMaterialise, groupAndMerge, groupDeltasByTable, isMaterialisable, isSoftDelete, mergeLatestState, migrateAdapter, migrateToTier, resolveConflictColumns, resolvePrimaryKey, toCause, wrapAsync };

package/dist/adapter.js

@@ -1,14 +1,19 @@
 import {
   BigQueryAdapter,
+  BigQuerySqlDialect,
   CompositeAdapter,
   FanOutAdapter,
   LifecycleAdapter,
   MinIOAdapter,
   MySQLAdapter,
+  MySqlDialect,
   PostgresAdapter,
+  PostgresSqlDialect,
   buildSchemaIndex,
   createDatabaseAdapter,
   createQueryFn,
+  executeMaterialise,
+  groupAndMerge,
   groupDeltasByTable,
   isDatabaseAdapter,
   isMaterialisable,
@@ -21,20 +26,25 @@ import {
   resolvePrimaryKey,
   toCause,
   wrapAsync
-} from "./chunk-GUJWMK5P.js";
-import "./chunk-P3FT7QCW.js";
-import "./chunk-7D4SUZUM.js";
+} from "./chunk-LPWXOYNS.js";
+import "./chunk-LDFFCG2K.js";
+import "./chunk-DGUM43GV.js";
 export {
   BigQueryAdapter,
+  BigQuerySqlDialect,
   CompositeAdapter,
   FanOutAdapter,
   LifecycleAdapter,
   MinIOAdapter,
   MySQLAdapter,
+  MySqlDialect,
   PostgresAdapter,
+  PostgresSqlDialect,
   buildSchemaIndex,
   createDatabaseAdapter,
   createQueryFn,
+  executeMaterialise,
+  groupAndMerge,
   groupDeltasByTable,
   isDatabaseAdapter,
   isMaterialisable,

package/dist/analyst.js

@@ -2,8 +2,8 @@ import {
   Err,
   LakeSyncError,
   Ok
-} from "./chunk-P3FT7QCW.js";
-import "./chunk-7D4SUZUM.js";
+} from "./chunk-LDFFCG2K.js";
+import "./chunk-DGUM43GV.js";
 
 // ../analyst/src/duckdb.ts
 var DuckDBClient = class {

package/dist/{base-poller-BpUyuG2R.d.ts → base-poller-Bj9kX9dv.d.ts}

@@ -1,11 +1,78 @@
-import { S as SyncPush, R as RowDelta } from './types-CLlD4XOy.js';
+import { S as SyncPush, R as RowDelta } from './types-BdGBv2ba.js';
 import { H as HLC } from './hlc-DiD8QNG3.js';
 import { R as Result, F as FlushError } from './result-CojzlFE2.js';
 
+/**
+ * Target that supports flush and buffer inspection.
+ * Implemented by SyncGateway so pollers can trigger flushes to relieve memory pressure.
+ */
+interface FlushableTarget {
+    flush(): Promise<Result<void, FlushError>>;
+    shouldFlush(): boolean;
+    readonly bufferStats: {
+        logSize: number;
+        indexSize: number;
+        byteSize: number;
+    };
+}
+/**
+ * Monitors buffer pressure on a {@link FlushableTarget} and triggers
+ * flushes when thresholds are exceeded. Created at construction time
+ * only when the target supports flushing — no runtime type checks needed.
+ */
+declare class PressureManager {
+    private readonly target;
+    private readonly memoryBudgetBytes;
+    private readonly flushThreshold;
+    constructor(config: {
+        target: FlushableTarget;
+        memoryBudgetBytes?: number;
+        flushThreshold?: number;
+    });
+    /** Check buffer pressure and flush if thresholds are exceeded. */
+    checkAndFlush(): Promise<void>;
+    /** Force a flush regardless of current pressure. */
+    forceFlush(): Promise<void>;
+    private shouldFlush;
+}
+
 /** Minimal interface for a push target (avoids depending on @lakesync/gateway). */
 interface PushTarget {
     handlePush(push: SyncPush): unknown;
 }
+/**
+ * Manages chunked pushing of deltas to a {@link PushTarget}.
+ * Handles backpressure retry (flush + retry once) when a
+ * {@link PressureManager} is present.
+ *
+ * No runtime type checks — the PressureManager is either provided
+ * at construction (target supports flush) or null.
+ */
+declare class ChunkedPusher {
+    private readonly target;
+    private readonly clientId;
+    private readonly chunkSize;
+    private readonly pressure;
+    private pendingDeltas;
+    constructor(config: {
+        target: PushTarget;
+        clientId: string;
+        chunkSize: number;
+        pressure: PressureManager | null;
+    });
+    /**
+     * Accumulate a single delta. When `chunkSize` is reached, the pending
+     * deltas are automatically pushed (and flushed if needed).
+     */
+    accumulate(delta: RowDelta): Promise<void>;
+    /** Flush any remaining accumulated deltas. */
+    flush(): Promise<void>;
+    /** Push deltas directly (single-shot, backward compat). */
+    pushImmediate(deltas: RowDelta[]): void;
+    private pushPendingChunk;
+    private pushChunkWithFlush;
+}
+
 /**
  * Extended push target that supports flush and buffer inspection.
  * Implemented by SyncGateway so pollers can trigger flushes to relieve memory pressure.
@@ -32,19 +99,18 @@ interface PollerMemoryConfig {
 }
 /**
  * Base class for source pollers that poll an external API and push deltas
- * to a SyncGateway. Handles lifecycle (start/stop/schedule), and push.
+ * to a SyncGateway.
+ *
+ * Composes {@link PollingScheduler} (lifecycle), {@link ChunkedPusher}
+ * (chunked push with backpressure), and {@link PressureManager}
+ * (memory-budget flush decisions).
  */
 declare abstract class BaseSourcePoller {
     protected readonly gateway: PushTarget;
     protected readonly hlc: HLC;
     protected readonly clientId: string;
-    private readonly intervalMs;
-    private timer;
-    private running;
-    private readonly chunkSize;
-    private readonly memoryBudgetBytes;
-    private readonly flushThreshold;
-    private pendingDeltas;
+    private readonly scheduler;
+    private readonly pusher;
     constructor(config: {
         name: string;
         intervalMs: number;
@@ -77,15 +143,6 @@ declare abstract class BaseSourcePoller {
     protected accumulateDelta(delta: RowDelta): Promise<void>;
     /** Flush any remaining accumulated deltas. Call at the end of `poll()`. */
     protected flushAccumulator(): Promise<void>;
-    /**
-     * Push a chunk of pending deltas. If the gateway is an IngestTarget,
-     * checks memory pressure and flushes before/after push when needed.
-     * On backpressure, flushes once and retries.
-     */
-    private pushPendingChunk;
-    private pushChunkWithFlush;
-    private shouldFlushTarget;
-    private schedulePoll;
 }
 
-export { BaseSourcePoller as B, type IngestTarget as I, type PushTarget as P, type PollerMemoryConfig as a, isIngestTarget as i };
+export { BaseSourcePoller as B, ChunkedPusher as C, type FlushableTarget as F, type IngestTarget as I, type PushTarget as P, type PollerMemoryConfig as a, PressureManager as b, isIngestTarget as i };

package/dist/catalogue.d.ts

@@ -1,6 +1,6 @@
 import { I as IcebergSchema, P as PartitionSpec } from './nessie-client-DrNikVXy.js';
 export { C as CatalogueConfig, a as CatalogueError, D as DataFile, b as IcebergField, N as NessieCatalogueClient, S as Snapshot, T as TableMetadata } from './nessie-client-DrNikVXy.js';
-import { T as TableSchema } from './types-CLlD4XOy.js';
+import { T as TableSchema } from './types-BdGBv2ba.js';
 import './result-CojzlFE2.js';
 
 /**

package/dist/catalogue.js

@@ -4,9 +4,9 @@ import {
   buildPartitionSpec,
   lakeSyncTableName,
   tableSchemaToIceberg
-} from "./chunk-UAUQGP3B.js";
-import "./chunk-P3FT7QCW.js";
-import "./chunk-7D4SUZUM.js";
+} from "./chunk-TMLG32QV.js";
+import "./chunk-LDFFCG2K.js";
+import "./chunk-DGUM43GV.js";
 export {
   CatalogueError,
   NessieCatalogueClient,

package/dist/chunk-DGUM43GV.js

@@ -0,0 +1,11 @@
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+
+export {
+  __require
+};
+//# sourceMappingURL=chunk-DGUM43GV.js.map