lakesync 0.1.3 → 0.1.5

package/dist/adapter.d.ts

@@ -1,13 +1,62 @@
 import { BigQuery } from '@google-cloud/bigquery';
 import { R as Result, A as AdapterError, H as HLCTimestamp } from './result-CojzlFE2.js';
-import { R as RowDelta, T as TableSchema, C as ColumnDelta } from './types-BUzzVRD6.js';
-import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-CPAPw8Ws.js';
-export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-CPAPw8Ws.js';
+import { R as RowDelta, T as TableSchema, C as ColumnDelta } from './types-GGBfZBKQ.js';
+import { D as DatabaseAdapter, a as DatabaseAdapterConfig } from './db-types-B6_JKQWK.js';
+export { i as isDatabaseAdapter, l as lakeSyncTypeToBigQuery } from './db-types-B6_JKQWK.js';
 import { C as ConnectorConfig } from './types-D-E0VrfS.js';
 import { L as LakeAdapter, A as AdapterConfig, O as ObjectInfo } from './types-DSC_EiwR.js';
 import mysql from 'mysql2/promise';
 import { Pool } from 'pg';
 
+/**
+ * Opt-in capability for adapters that can materialise deltas into destination tables.
+ *
+ * Materialisation is a separate concern from delta storage — adapters that store
+ * deltas (via `DatabaseAdapter.insertDeltas`) may also materialise them into
+ * queryable destination tables by implementing this interface.
+ *
+ * Destination tables follow the hybrid column model:
+ * - Synced columns (written by materialiser, derived from `TableSchema.columns`)
+ * - `props JSONB DEFAULT '{}'` — consumer-extensible, never touched by materialiser
+ * - `synced_at` — updated on every materialise cycle
+ */
+interface Materialisable {
+    /**
+     * Materialise deltas into destination tables.
+     *
+     * For each table with a matching schema, merges delta history into the
+     * latest row state and upserts into the destination table. Tombstoned
+     * rows are deleted. The `props` column is never touched.
+     *
+     * @param deltas - The deltas that were just flushed.
+     * @param schemas - Table schemas defining destination tables and column mappings.
+     */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
+}
+/**
+ * Type guard to check if an adapter supports materialisation.
+ *
+ * Uses duck-typing (same pattern as `isDatabaseAdapter`).
+ */
+declare function isMaterialisable(adapter: unknown): adapter is Materialisable;
+/**
+ * Group deltas by their table name, collecting the set of affected row IDs per table.
+ *
+ * @param deltas - The deltas to group.
+ * @returns A map from table name to the set of affected row IDs.
+ */
+declare function groupDeltasByTable(deltas: ReadonlyArray<RowDelta>): Map<string, Set<string>>;
+/**
+ * Build an index from source table name to schema.
+ *
+ * Keys are `schema.sourceTable ?? schema.table`, so deltas can be matched
+ * by their `table` field to find the correct destination schema.
+ *
+ * @param schemas - The table schemas to index.
+ * @returns A map from source table name to schema.
+ */
+declare function buildSchemaIndex(schemas: ReadonlyArray<TableSchema>): Map<string, TableSchema>;
+
 /**
  * Configuration for the BigQuery adapter.
  * Unlike SQL adapters, BigQuery is HTTP-based — no connection string needed.
@@ -33,7 +82,7 @@ interface BigQueryAdapterConfig {
  * on standard (non-partitioned) tables. Query latency is seconds, not
  * milliseconds — this adapter is designed for the analytics tier.
  */
-declare class BigQueryAdapter implements DatabaseAdapter {
+declare class BigQueryAdapter implements DatabaseAdapter, Materialisable {
     /** @internal */
     readonly client: BigQuery;
     /** @internal */
@@ -61,6 +110,15 @@ declare class BigQueryAdapter implements DatabaseAdapter {
      * internal table structure is fixed (deltas store column data as JSON).
      */
     ensureSchema(_schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /**
+     * Materialise deltas into destination tables.
+     *
+     * For each affected table, queries the full delta history for touched rows,
+     * merges to latest state via column-level LWW, then upserts live rows and
+     * deletes tombstoned rows. The consumer-owned `props` column is never
+     * touched on UPDATE.
+     */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /**
      * No-op — BigQuery client is HTTP-based with no persistent connections.
      */
@@ -128,7 +186,7 @@ interface FanOutAdapterConfig {
  * Secondary failures are silently caught and never affect the return value.
  * Use case: write to Postgres (fast, operational), replicate to BigQuery (analytics).
  */
-declare class FanOutAdapter implements DatabaseAdapter {
+declare class FanOutAdapter implements DatabaseAdapter, Materialisable {
     private readonly primary;
     private readonly secondaries;
     constructor(config: FanOutAdapterConfig);
@@ -140,6 +198,8 @@ declare class FanOutAdapter implements DatabaseAdapter {
     getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
     /** Ensure schema on the primary first, then best-effort on secondaries. */
     ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Materialise via primary, then replicate to materialisable secondaries (fire-and-forget). */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /** Close primary and all secondary adapters. */
     close(): Promise<void>;
 }
@@ -168,7 +228,7 @@ interface LifecycleAdapterConfig {
  * Use {@link migrateToTier} as a background job to copy aged-out deltas
  * from hot to cold.
  */
-declare class LifecycleAdapter implements DatabaseAdapter {
+declare class LifecycleAdapter implements DatabaseAdapter, Materialisable {
     private readonly hot;
     private readonly cold;
     private readonly maxAgeMs;
@@ -186,6 +246,8 @@ declare class LifecycleAdapter implements DatabaseAdapter {
     getLatestState(table: string, rowId: string): Promise<Result<Record<string, unknown> | null, AdapterError>>;
     /** Ensure schema exists on both hot and cold adapters. */
     ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /** Materialise via hot tier only — cold tier stores archived deltas, not destination tables. */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /** Close both hot and cold adapters. */
     close(): Promise<void>;
 }
@@ -276,7 +338,7 @@ declare class MinIOAdapter implements LakeAdapter {
  * idempotent writes. All public methods return `Result` and never throw.
  * Uses mysql2/promise connection pool for async operations.
  */
-declare class MySQLAdapter implements DatabaseAdapter {
+declare class MySQLAdapter implements DatabaseAdapter, Materialisable {
     /** @internal */
     readonly pool: mysql.Pool;
     constructor(config: DatabaseAdapterConfig);
@@ -300,6 +362,14 @@ declare class MySQLAdapter implements DatabaseAdapter {
      * and a user table matching the given TableSchema definition.
      */
     ensureSchema(schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /**
+     * Materialise deltas into destination tables.
+     *
+     * For each table with a matching schema, merges delta history into the
+     * latest row state and upserts into the destination table. Tombstoned
+     * rows are deleted. The `props` column is never touched.
+     */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /** Close the database connection pool and release resources. */
     close(): Promise<void>;
 }
@@ -310,7 +380,7 @@ declare class MySQLAdapter implements DatabaseAdapter {
  * Stores deltas in a `lakesync_deltas` table using pg Pool.
  * All public methods return `Result` and never throw.
  */
-declare class PostgresAdapter implements DatabaseAdapter {
+declare class PostgresAdapter implements DatabaseAdapter, Materialisable {
     /** @internal */
     readonly pool: Pool;
     constructor(config: DatabaseAdapterConfig);
@@ -334,6 +404,14 @@ declare class PostgresAdapter implements DatabaseAdapter {
      * internal table structure is fixed (deltas store column data as JSONB).
      */
     ensureSchema(_schema: TableSchema): Promise<Result<void, AdapterError>>;
+    /**
+     * Materialise deltas into destination tables.
+     *
+     * For each table with a matching schema, merges delta history into the
+     * latest row state and upserts into the destination table. Tombstoned
+     * rows are deleted. The `props` column is never touched.
+     */
+    materialise(deltas: RowDelta[], schemas: ReadonlyArray<TableSchema>): Promise<Result<void, AdapterError>>;
     /** Close the database connection pool and release resources. */
     close(): Promise<void>;
 }
@@ -366,4 +444,4 @@ declare function mergeLatestState(rows: Array<{
     op: string;
 }>): Record<string, unknown> | null;
 
-export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, ObjectInfo, PostgresAdapter, type QueryFn, createDatabaseAdapter, createQueryFn, mergeLatestState, migrateAdapter, migrateToTier, toCause, wrapAsync };
+export { AdapterConfig, BigQueryAdapter, type BigQueryAdapterConfig, CompositeAdapter, type CompositeAdapterConfig, type CompositeRoute, DatabaseAdapter, DatabaseAdapterConfig, FanOutAdapter, type FanOutAdapterConfig, LakeAdapter, LifecycleAdapter, type LifecycleAdapterConfig, type Materialisable, type MigrateOptions, type MigrateProgress, type MigrateResult, MinIOAdapter, MySQLAdapter, ObjectInfo, PostgresAdapter, type QueryFn, buildSchemaIndex, createDatabaseAdapter, createQueryFn, groupDeltasByTable, isMaterialisable, mergeLatestState, migrateAdapter, migrateToTier, toCause, wrapAsync };

package/dist/adapter.js

@@ -6,17 +6,20 @@ import {
   MinIOAdapter,
   MySQLAdapter,
   PostgresAdapter,
+  buildSchemaIndex,
   createDatabaseAdapter,
   createQueryFn,
+  groupDeltasByTable,
   isDatabaseAdapter,
+  isMaterialisable,
   lakeSyncTypeToBigQuery,
   mergeLatestState,
   migrateAdapter,
   migrateToTier,
   toCause,
   wrapAsync
-} from "./chunk-NCMXLWEW.js";
-import "./chunk-C7ECMNQ4.js";
+} from "./chunk-Z7FGLEQU.js";
+import "./chunk-7UBS6MFH.js";
 import "./chunk-7D4SUZUM.js";
 export {
   BigQueryAdapter,
@@ -26,9 +29,12 @@ export {
   MinIOAdapter,
   MySQLAdapter,
   PostgresAdapter,
+  buildSchemaIndex,
   createDatabaseAdapter,
   createQueryFn,
+  groupDeltasByTable,
   isDatabaseAdapter,
+  isMaterialisable,
   lakeSyncTypeToBigQuery,
   mergeLatestState,
   migrateAdapter,

package/dist/analyst.js

@@ -2,7 +2,7 @@ import {
   Err,
   LakeSyncError,
   Ok
-} from "./chunk-C7ECMNQ4.js";
+} from "./chunk-7UBS6MFH.js";
 import "./chunk-7D4SUZUM.js";
 
 // ../analyst/src/duckdb.ts

package/dist/{base-poller-Dfaj05py.d.ts → base-poller-CBvhdvcj.d.ts}

@@ -1,4 +1,4 @@
-import { S as SyncPush, R as RowDelta } from './types-BUzzVRD6.js';
+import { S as SyncPush, R as RowDelta } from './types-GGBfZBKQ.js';
 import { H as HLC } from './hlc-DiD8QNG3.js';
 import { R as Result, F as FlushError } from './result-CojzlFE2.js';
 

package/dist/catalogue.d.ts

@@ -1,6 +1,6 @@
 import { I as IcebergSchema, P as PartitionSpec } from './nessie-client-DrNikVXy.js';
 export { C as CatalogueConfig, a as CatalogueError, D as DataFile, b as IcebergField, N as NessieCatalogueClient, S as Snapshot, T as TableMetadata } from './nessie-client-DrNikVXy.js';
-import { T as TableSchema } from './types-BUzzVRD6.js';
+import { T as TableSchema } from './types-GGBfZBKQ.js';
 import './result-CojzlFE2.js';
 
 /**

package/dist/catalogue.js

@@ -4,8 +4,8 @@ import {
   buildPartitionSpec,
   lakeSyncTableName,
   tableSchemaToIceberg
-} from "./chunk-TIPMVLIG.js";
-import "./chunk-C7ECMNQ4.js";
+} from "./chunk-B3QEUG6E.js";
+import "./chunk-7UBS6MFH.js";
 import "./chunk-7D4SUZUM.js";
 export {
   CatalogueError,

package/dist/{chunk-265CMYJH.js → chunk-46CKACNC.js}

@@ -2,7 +2,7 @@ import {
   Err,
   FlushError,
   Ok
-} from "./chunk-C7ECMNQ4.js";
+} from "./chunk-7UBS6MFH.js";
 
 // ../parquet/src/arrow-schema.ts
 import * as arrow from "apache-arrow";
@@ -254,4 +254,4 @@ export {
   readParquetToDeltas,
   writeDeltasToParquet
 };
-//# sourceMappingURL=chunk-265CMYJH.js.map
+//# sourceMappingURL=chunk-46CKACNC.js.map

package/dist/{chunk-C7ECMNQ4.js → chunk-7UBS6MFH.js}

@@ -620,6 +620,23 @@ var ConnectorValidationError = class extends LakeSyncError {
   }
 };
 
+// ../core/src/connector/registry.ts
+var descriptors = /* @__PURE__ */ new Map();
+function registerConnectorDescriptor(descriptor) {
+  descriptors.set(descriptor.type, descriptor);
+}
+function registerOutputSchemas(type, schemas) {
+  const existing = descriptors.get(type);
+  if (!existing) return;
+  descriptors.set(type, { ...existing, outputTables: schemas });
+}
+function getConnectorDescriptor(type) {
+  return descriptors.get(type);
+}
+function listConnectorDescriptors() {
+  return [...descriptors.values()].sort((a, b) => a.type.localeCompare(b.type));
+}
+
 // ../core/src/connector/types.ts
 var CONNECTOR_TYPES = ["postgres", "mysql", "bigquery", "jira", "salesforce"];
 
@@ -814,6 +831,279 @@ function validateConnectorConfig(input) {
   return Ok(input);
 }
 
+// ../core/src/connector/schemas/bigquery.ts
+var BIGQUERY_CONFIG_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    projectId: {
+      type: "string",
+      description: "GCP project ID."
+    },
+    dataset: {
+      type: "string",
+      description: "BigQuery dataset name."
+    },
+    keyFilename: {
+      type: "string",
+      description: "Path to service account JSON key file. Falls back to ADC when omitted."
+    },
+    location: {
+      type: "string",
+      description: 'Dataset location (default "US").'
+    }
+  },
+  required: ["projectId", "dataset"],
+  additionalProperties: false
+};
+
+// ../core/src/connector/schemas/ingest.ts
+var DATABASE_INGEST_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    tables: {
+      type: "array",
+      description: "Tables to poll for changes.",
+      items: {
+        type: "object",
+        properties: {
+          table: { type: "string", description: "Target table name in LakeSync." },
+          query: {
+            type: "string",
+            description: "SQL query to poll (must return rowId + data columns)."
+          },
+          rowIdColumn: {
+            type: "string",
+            description: 'Primary key column name (default "id").'
+          },
+          strategy: {
+            oneOf: [
+              {
+                type: "object",
+                properties: {
+                  type: { const: "cursor" },
+                  cursorColumn: { type: "string" },
+                  lookbackMs: { type: "number" }
+                },
+                required: ["type", "cursorColumn"]
+              },
+              {
+                type: "object",
+                properties: {
+                  type: { const: "diff" }
+                },
+                required: ["type"]
+              }
+            ]
+          }
+        },
+        required: ["table", "query", "strategy"]
+      }
+    },
+    intervalMs: {
+      type: "number",
+      description: "Poll interval in milliseconds (default 10 000)."
+    },
+    chunkSize: {
+      type: "number",
+      description: "Deltas per push chunk (default 500)."
+    },
+    memoryBudgetBytes: {
+      type: "number",
+      description: "Approximate memory budget in bytes \u2014 triggers flush at 70%."
+    }
+  },
+  required: ["tables"],
+  additionalProperties: false
+};
+var API_INGEST_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    intervalMs: {
+      type: "number",
+      description: "Poll interval in milliseconds (default 10 000)."
+    },
+    chunkSize: {
+      type: "number",
+      description: "Deltas per push chunk (default 500)."
+    },
+    memoryBudgetBytes: {
+      type: "number",
+      description: "Approximate memory budget in bytes \u2014 triggers flush at 70%."
+    }
+  },
+  additionalProperties: false
+};
+
+// ../core/src/connector/schemas/jira.ts
+var JIRA_CONFIG_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    domain: {
+      type: "string",
+      description: "Jira Cloud domain (e.g. mycompany for mycompany.atlassian.net)."
+    },
+    email: {
+      type: "string",
+      description: "Email address for Basic auth."
+    },
+    apiToken: {
+      type: "string",
+      description: "API token paired with the email."
+    },
+    jql: {
+      type: "string",
+      description: "Optional JQL filter to scope issue polling."
+    },
+    includeComments: {
+      type: "boolean",
+      description: "Whether to include comments (default true)."
+    },
+    includeProjects: {
+      type: "boolean",
+      description: "Whether to include projects (default true)."
+    }
+  },
+  required: ["domain", "email", "apiToken"],
+  additionalProperties: false
+};
+
+// ../core/src/connector/schemas/mysql.ts
+var MYSQL_CONFIG_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    connectionString: {
+      type: "string",
+      description: "MySQL connection string (e.g. mysql://user:pass@host/db)."
+    }
+  },
+  required: ["connectionString"],
+  additionalProperties: false
+};
+
+// ../core/src/connector/schemas/postgres.ts
+var POSTGRES_CONFIG_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    connectionString: {
+      type: "string",
+      description: "PostgreSQL connection string (e.g. postgres://user:pass@host/db)."
+    }
+  },
+  required: ["connectionString"],
+  additionalProperties: false
+};
+
+// ../core/src/connector/schemas/salesforce.ts
+var SALESFORCE_CONFIG_SCHEMA = {
+  $schema: "http://json-schema.org/draft-07/schema#",
+  type: "object",
+  properties: {
+    instanceUrl: {
+      type: "string",
+      description: "Salesforce instance URL (e.g. https://mycompany.salesforce.com)."
+    },
+    clientId: {
+      type: "string",
+      description: "Connected App consumer key."
+    },
+    clientSecret: {
+      type: "string",
+      description: "Connected App consumer secret."
+    },
+    username: {
+      type: "string",
+      description: "Salesforce username."
+    },
+    password: {
+      type: "string",
+      description: "Salesforce password + security token concatenated."
+    },
+    apiVersion: {
+      type: "string",
+      description: 'REST API version (default "v62.0").'
+    },
+    isSandbox: {
+      type: "boolean",
+      description: "Use test.salesforce.com for auth (default false)."
+    },
+    soqlFilter: {
+      type: "string",
+      description: "Optional WHERE clause fragment appended to all SOQL queries."
+    },
+    includeAccounts: {
+      type: "boolean",
+      description: "Whether to include Account objects (default true)."
+    },
+    includeContacts: {
+      type: "boolean",
+      description: "Whether to include Contact objects (default true)."
+    },
+    includeOpportunities: {
+      type: "boolean",
+      description: "Whether to include Opportunity objects (default true)."
+    },
+    includeLeads: {
+      type: "boolean",
+      description: "Whether to include Lead objects (default true)."
+    }
+  },
+  required: ["instanceUrl", "clientId", "clientSecret", "username", "password"],
+  additionalProperties: false
+};
+
+// ../core/src/connector/register-builtin.ts
+registerConnectorDescriptor({
+  type: "bigquery",
+  displayName: "BigQuery",
+  description: "Google BigQuery data warehouse connector.",
+  category: "database",
+  configSchema: BIGQUERY_CONFIG_SCHEMA,
+  ingestSchema: DATABASE_INGEST_SCHEMA,
+  outputTables: null
+});
+registerConnectorDescriptor({
+  type: "jira",
+  displayName: "Jira",
+  description: "Atlassian Jira Cloud issue tracker connector.",
+  category: "api",
+  configSchema: JIRA_CONFIG_SCHEMA,
+  ingestSchema: API_INGEST_SCHEMA,
+  outputTables: null
+});
+registerConnectorDescriptor({
+  type: "mysql",
+  displayName: "MySQL",
+  description: "MySQL relational database connector.",
+  category: "database",
+  configSchema: MYSQL_CONFIG_SCHEMA,
+  ingestSchema: DATABASE_INGEST_SCHEMA,
+  outputTables: null
+});
+registerConnectorDescriptor({
+  type: "postgres",
+  displayName: "PostgreSQL",
+  description: "PostgreSQL relational database connector.",
+  category: "database",
+  configSchema: POSTGRES_CONFIG_SCHEMA,
+  ingestSchema: DATABASE_INGEST_SCHEMA,
+  outputTables: null
+});
+registerConnectorDescriptor({
+  type: "salesforce",
+  displayName: "Salesforce",
+  description: "Salesforce CRM connector for accounts, contacts, opportunities, and leads.",
+  category: "api",
+  configSchema: SALESFORCE_CONFIG_SCHEMA,
+  ingestSchema: API_INGEST_SCHEMA,
+  outputTables: null
+});
+
 // ../core/src/create-poller.ts
 var pollerFactories = /* @__PURE__ */ new Map();
 function registerPollerFactory(type, factory) {
@@ -1192,6 +1482,10 @@ export {
   resolveLWW,
   isActionHandler,
   ConnectorValidationError,
+  registerConnectorDescriptor,
+  registerOutputSchemas,
+  getConnectorDescriptor,
+  listConnectorDescriptors,
   CONNECTOR_TYPES,
   validateConnectorConfig,
   registerPollerFactory,
@@ -1213,4 +1507,4 @@ export {
   assertValidIdentifier,
   quoteIdentifier
 };
-//# sourceMappingURL=chunk-C7ECMNQ4.js.map
+//# sourceMappingURL=chunk-7UBS6MFH.js.map