lakesync 0.1.8 → 0.2.0

package/dist/{chunk-JI4C4R5H.js → chunk-FIIHPQMQ.js}

@@ -1,15 +1,11 @@
-import {
-  isDatabaseAdapter,
-  isMaterialisable
-} from "./chunk-LPWXOYNS.js";
 import {
   buildPartitionSpec,
   lakeSyncTableName,
   tableSchemaToIceberg
-} from "./chunk-TMLG32QV.js";
+} from "./chunk-U2NV4DUX.js";
 import {
   writeDeltasToParquet
-} from "./chunk-SSICS5KI.js";
+} from "./chunk-ZU7RC7CT.js";
 import {
   AdapterNotFoundError,
   BackpressureError,
@@ -22,6 +18,8 @@ import {
   bigintReplacer,
   bigintReviver,
   filterDeltas,
+  isDatabaseAdapter,
+  isMaterialisable,
   listConnectorDescriptors,
   resolveLWW,
   rowKey,
@@ -29,25 +27,84 @@ import {
   validateAction,
   validateConnectorConfig,
   validateSyncRules
-} from "./chunk-LDFFCG2K.js";
+} from "./chunk-4SG66H5K.js";
 
-// ../gateway/src/action-dispatcher.ts
+// ../gateway/src/idempotency-cache.ts
 var DEFAULT_MAX_CACHE_SIZE = 1e4;
 var DEFAULT_CACHE_TTL_MS = 5 * 60 * 1e3;
+var MemoryIdempotencyCache = class {
+  entries = /* @__PURE__ */ new Map();
+  maxSize;
+  ttlMs;
+  constructor(config) {
+    this.maxSize = config?.maxSize ?? DEFAULT_MAX_CACHE_SIZE;
+    this.ttlMs = config?.ttlMs ?? DEFAULT_CACHE_TTL_MS;
+  }
+  /** {@inheritDoc IdempotencyCache.has} */
+  has(actionId) {
+    const entry = this.entries.get(actionId);
+    if (!entry) return false;
+    if (Date.now() - entry.cachedAt > this.ttlMs) {
+      this.entries.delete(actionId);
+      return false;
+    }
+    return true;
+  }
+  /** {@inheritDoc IdempotencyCache.get} */
+  get(key) {
+    const entry = this.entries.get(key);
+    if (!entry) return void 0;
+    if (Date.now() - entry.cachedAt > this.ttlMs) {
+      this.entries.delete(key);
+      return void 0;
+    }
+    return entry.value;
+  }
+  /** {@inheritDoc IdempotencyCache.set} */
+  set(actionId, result, idempotencyKey) {
+    this.evictStaleEntries();
+    const entry = { value: result, cachedAt: Date.now() };
+    this.entries.set(actionId, entry);
+    if (idempotencyKey) {
+      this.entries.set(`idem:${idempotencyKey}`, entry);
+    }
+  }
+  /** Evict expired entries and trim to max size (counting only non-idem entries). */
+  evictStaleEntries() {
+    const now = Date.now();
+    for (const [key, entry] of this.entries) {
+      if (now - entry.cachedAt > this.ttlMs) {
+        this.entries.delete(key);
+      }
+    }
+    const actionKeys = [...this.entries.keys()].filter((k) => !k.startsWith("idem:"));
+    if (actionKeys.length > this.maxSize) {
+      const excess = actionKeys.length - this.maxSize;
+      for (let i = 0; i < excess; i++) {
+        this.entries.delete(actionKeys[i]);
+      }
+    }
+  }
+};
+
+// ../gateway/src/action-dispatcher.ts
 var ActionDispatcher = class {
   actionHandlers = /* @__PURE__ */ new Map();
-  executedActions = /* @__PURE__ */ new Set();
-  idempotencyMap = /* @__PURE__ */ new Map();
-  maxCacheSize;
-  cacheTtlMs;
-  constructor(handlers, cacheConfig) {
+  cache;
+  /**
+   * Create an ActionDispatcher.
+   *
+   * @param handlers - Optional map of connector name to action handler.
+   * @param cacheConfig - Optional cache configuration (used when no `cache` is provided).
+   * @param cache - Optional pre-built idempotency cache; defaults to a {@link MemoryIdempotencyCache}.
+   */
+  constructor(handlers, cacheConfig, cache) {
     if (handlers) {
       for (const [name, handler] of Object.entries(handlers)) {
         this.actionHandlers.set(name, handler);
       }
     }
-    this.maxCacheSize = cacheConfig?.maxSize ?? DEFAULT_MAX_CACHE_SIZE;
-    this.cacheTtlMs = cacheConfig?.ttlMs ?? DEFAULT_CACHE_TTL_MS;
+    this.cache = cache ?? new MemoryIdempotencyCache(cacheConfig);
   }
   /**
    * Dispatch an action push to registered handlers.
@@ -62,15 +119,14 @@ var ActionDispatcher = class {
    * @returns A `Result` containing results for each action.
    */
   async dispatch(msg, hlcNow, context) {
-    this.evictStaleEntries();
     const results = [];
     for (const action of msg.actions) {
       const validation = validateAction(action);
       if (!validation.ok) {
         return Err(validation.error);
       }
-      if (this.executedActions.has(action.actionId)) {
-        const cached = this.getCachedResult(action.actionId);
+      if (this.cache.has(action.actionId)) {
+        const cached = this.cache.get(action.actionId);
         if (cached) {
           results.push(cached);
           continue;
@@ -78,7 +134,7 @@ var ActionDispatcher = class {
         continue;
       }
       if (action.idempotencyKey) {
-        const cached = this.getCachedResult(`idem:${action.idempotencyKey}`);
+        const cached = this.cache.get(`idem:${action.idempotencyKey}`);
         if (cached) {
           results.push(cached);
           continue;
@@ -93,7 +149,7 @@ var ActionDispatcher = class {
           retryable: false
         };
         results.push(errorResult);
-        this.cacheActionResult(action, errorResult);
+        this.cache.set(action.actionId, errorResult, action.idempotencyKey);
         continue;
       }
       const supported = handler.supportedActions.some((d) => d.actionType === action.actionType);
@@ -105,13 +161,13 @@ var ActionDispatcher = class {
           retryable: false
         };
         results.push(errorResult);
-        this.cacheActionResult(action, errorResult);
+        this.cache.set(action.actionId, errorResult, action.idempotencyKey);
         continue;
       }
       const execResult = await handler.executeAction(action, context);
       if (execResult.ok) {
         results.push(execResult.value);
-        this.cacheActionResult(action, execResult.value);
+        this.cache.set(action.actionId, execResult.value, action.idempotencyKey);
       } else {
         const err = execResult.error;
         const errorResult = {
@@ -122,7 +178,7 @@ var ActionDispatcher = class {
         };
         results.push(errorResult);
         if (!errorResult.retryable) {
-          this.cacheActionResult(action, errorResult);
+          this.cache.set(action.actionId, errorResult, action.idempotencyKey);
         }
       }
     }
@@ -169,47 +225,6 @@ var ActionDispatcher = class {
     }
     return { connectors };
   }
-  /** Cache an action result for idempotency deduplication. */
-  cacheActionResult(action, result) {
-    const entry = { value: result, cachedAt: Date.now() };
-    this.executedActions.add(action.actionId);
-    this.idempotencyMap.set(action.actionId, entry);
-    if (action.idempotencyKey) {
-      this.idempotencyMap.set(`idem:${action.idempotencyKey}`, entry);
-    }
-  }
-  /** Get a cached result if it exists and hasn't expired. */
-  getCachedResult(key) {
-    const entry = this.idempotencyMap.get(key);
-    if (!entry) return void 0;
-    if (Date.now() - entry.cachedAt > this.cacheTtlMs) {
-      this.idempotencyMap.delete(key);
-      return void 0;
-    }
-    return entry.value;
-  }
-  /** Evict expired entries and trim to max size. */
-  evictStaleEntries() {
-    const now = Date.now();
-    for (const [key, entry] of this.idempotencyMap) {
-      if (now - entry.cachedAt > this.cacheTtlMs) {
-        this.idempotencyMap.delete(key);
-        if (!key.startsWith("idem:")) {
-          this.executedActions.delete(key);
-        }
-      }
-    }
-    if (this.executedActions.size > this.maxCacheSize) {
-      const excess = this.executedActions.size - this.maxCacheSize;
-      let removed = 0;
-      for (const actionId of this.executedActions) {
-        if (removed >= excess) break;
-        this.executedActions.delete(actionId);
-        this.idempotencyMap.delete(actionId);
-        removed++;
-      }
-    }
-  }
 };
 
 // ../gateway/src/buffer.ts
@@ -414,26 +429,39 @@ function hlcRange(entries) {
   }
   return { min, max };
 }
-async function flushEntries(entries, byteSize, deps, keyPrefix) {
-  if (isDatabaseAdapter(deps.adapter)) {
+function notifyMaterialisationFailure(entries, error, config) {
+  if (!config.onMaterialisationFailure) return;
+  const tables = new Set(entries.map((e) => e.table));
+  for (const table of tables) {
+    const count = entries.filter((e) => e.table === table).length;
+    config.onMaterialisationFailure(table, count, error);
+  }
+}
+var DatabaseFlushStrategy = class {
+  async flush(entries, _byteSize, deps) {
+    const adapter = deps.adapter;
     try {
-      const result = await deps.adapter.insertDeltas(entries);
+      const result = await adapter.insertDeltas(entries);
       if (!result.ok) {
         deps.restoreEntries(entries);
         return Err(new FlushError(`Database flush failed: ${result.error.message}`));
       }
-      if (deps.schemas && deps.schemas.length > 0 && isMaterialisable(deps.adapter)) {
+      if (deps.schemas && deps.schemas.length > 0 && isMaterialisable(adapter)) {
         try {
-          const matResult = await deps.adapter.materialise(entries, deps.schemas);
+          const matResult = await adapter.materialise(entries, deps.schemas);
           if (!matResult.ok) {
+            const error = new Error(matResult.error.message);
             console.warn(
               `[lakesync] Materialisation failed (${entries.length} deltas): ${matResult.error.message}`
             );
+            notifyMaterialisationFailure(entries, error, deps.config);
           }
         } catch (error) {
+          const err = error instanceof Error ? error : new Error(String(error));
           console.warn(
-            `[lakesync] Materialisation error (${entries.length} deltas): ${error instanceof Error ? error.message : String(error)}`
+            `[lakesync] Materialisation error (${entries.length} deltas): ${err.message}`
           );
+          notifyMaterialisationFailure(entries, err, deps.config);
         }
       }
       return Ok(void 0);
@@ -442,14 +470,14 @@ async function flushEntries(entries, byteSize, deps, keyPrefix) {
       return Err(new FlushError(`Unexpected database flush failure: ${toError(error).message}`));
     }
   }
-  try {
-    const { min, max } = hlcRange(entries);
-    const date = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
-    const prefix = keyPrefix ? `${keyPrefix}-` : "";
-    let objectKey;
-    let data;
-    let contentType;
-    if (deps.config.flushFormat === "json") {
+};
+var LakeJsonFlushStrategy = class {
+  async flush(entries, byteSize, deps, keyPrefix) {
+    const adapter = deps.adapter;
+    try {
+      const { min, max } = hlcRange(entries);
+      const date = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
+      const prefix = keyPrefix ? `${keyPrefix}-` : "";
       const envelope = {
         version: 1,
         gatewayId: deps.config.gatewayId,
@@ -459,10 +487,33 @@ async function flushEntries(entries, byteSize, deps, keyPrefix) {
         byteSize,
         deltas: entries
       };
-      objectKey = `deltas/${date}/${deps.config.gatewayId}/${prefix}${min.toString()}-${max.toString()}.json`;
-      data = new TextEncoder().encode(JSON.stringify(envelope, bigintReplacer));
-      contentType = "application/json";
-    } else {
+      const objectKey = `deltas/${date}/${deps.config.gatewayId}/${prefix}${min.toString()}-${max.toString()}.json`;
+      const data = new TextEncoder().encode(JSON.stringify(envelope, bigintReplacer));
+      const result = await adapter.putObject(objectKey, data, "application/json");
+      if (!result.ok) {
+        deps.restoreEntries(entries);
+        return Err(new FlushError(`Failed to write flush envelope: ${result.error.message}`));
+      }
+      if (deps.config.catalogue && deps.config.tableSchema) {
+        await commitToCatalogue(
+          objectKey,
+          data.byteLength,
+          entries.length,
+          deps.config.catalogue,
+          deps.config.tableSchema
+        );
+      }
+      return Ok(void 0);
+    } catch (error) {
+      deps.restoreEntries(entries);
+      return Err(new FlushError(`Unexpected flush failure: ${toError(error).message}`));
+    }
+  }
+};
+var LakeParquetFlushStrategy = class {
+  async flush(entries, _byteSize, deps, keyPrefix) {
+    const adapter = deps.adapter;
+    try {
       if (!deps.config.tableSchema) {
         deps.restoreEntries(entries);
         return Err(new FlushError("tableSchema required for Parquet flush"));
@@ -472,29 +523,43 @@ async function flushEntries(entries, byteSize, deps, keyPrefix) {
         deps.restoreEntries(entries);
         return Err(parquetResult.error);
       }
-      objectKey = `deltas/${date}/${deps.config.gatewayId}/${prefix}${min.toString()}-${max.toString()}.parquet`;
-      data = parquetResult.value;
-      contentType = "application/vnd.apache.parquet";
-    }
-    const result = await deps.adapter.putObject(objectKey, data, contentType);
-    if (!result.ok) {
+      const { min, max } = hlcRange(entries);
+      const date = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
+      const prefix = keyPrefix ? `${keyPrefix}-` : "";
+      const objectKey = `deltas/${date}/${deps.config.gatewayId}/${prefix}${min.toString()}-${max.toString()}.parquet`;
+      const data = parquetResult.value;
+      const result = await adapter.putObject(objectKey, data, "application/vnd.apache.parquet");
+      if (!result.ok) {
+        deps.restoreEntries(entries);
+        return Err(new FlushError(`Failed to write flush envelope: ${result.error.message}`));
+      }
+      if (deps.config.catalogue && deps.config.tableSchema) {
+        await commitToCatalogue(
+          objectKey,
+          data.byteLength,
+          entries.length,
+          deps.config.catalogue,
+          deps.config.tableSchema
+        );
+      }
+      return Ok(void 0);
+    } catch (error) {
       deps.restoreEntries(entries);
-      return Err(new FlushError(`Failed to write flush envelope: ${result.error.message}`));
-    }
-    if (deps.config.catalogue && deps.config.tableSchema) {
-      await commitToCatalogue(
-        objectKey,
-        data.byteLength,
-        entries.length,
-        deps.config.catalogue,
-        deps.config.tableSchema
-      );
+      return Err(new FlushError(`Unexpected flush failure: ${toError(error).message}`));
     }
-    return Ok(void 0);
-  } catch (error) {
-    deps.restoreEntries(entries);
-    return Err(new FlushError(`Unexpected flush failure: ${toError(error).message}`));
   }
+};
+var databaseStrategy = new DatabaseFlushStrategy();
+var lakeJsonStrategy = new LakeJsonFlushStrategy();
+var lakeParquetStrategy = new LakeParquetFlushStrategy();
+function selectFlushStrategy(adapter, format) {
+  if (isDatabaseAdapter(adapter)) return databaseStrategy;
+  if (format === "json") return lakeJsonStrategy;
+  return lakeParquetStrategy;
+}
+async function flushEntries(entries, byteSize, deps, keyPrefix) {
+  const strategy = selectFlushStrategy(deps.adapter, deps.config.flushFormat);
+  return strategy.flush(entries, byteSize, deps, keyPrefix);
 }
 async function commitToCatalogue(objectKey, fileSizeInBytes, recordCount, catalogue, schema) {
   const { namespace, name } = lakeSyncTableName(schema.table);
@@ -862,6 +927,15 @@ var SyncGateway = class {
     return this.sources.list();
   }
   // -----------------------------------------------------------------------
+  // Rehydration — restore persisted deltas without push validation
+  // -----------------------------------------------------------------------
+  /** Rehydrate the buffer with persisted deltas (bypasses push validation). */
+  rehydrate(deltas) {
+    for (const delta of deltas) {
+      this.buffer.append(delta);
+    }
+  }
+  // -----------------------------------------------------------------------
   // Buffer queries
   // -----------------------------------------------------------------------
   /** Get per-table buffer statistics. */
@@ -1167,17 +1241,17 @@ function handleMetrics(gateway, extra) {
 
 // ../gateway/src/schema-manager.ts
 var SchemaManager = class {
-  currentSchema;
-  version;
-  allowedColumns;
+  state;
   constructor(schema, version) {
-    this.currentSchema = schema;
-    this.version = version ?? 1;
-    this.allowedColumns = new Set(schema.columns.map((c) => c.name));
+    this.state = {
+      schema,
+      version: version ?? 1,
+      allowedColumns: new Set(schema.columns.map((c) => c.name))
+    };
   }
   /** Get the current schema and version. */
   getSchema() {
-    return { schema: this.currentSchema, version: this.version };
+    return { schema: this.state.schema, version: this.state.version };
   }
   /**
    * Validate that a delta's columns are compatible with the current schema.
@@ -1190,10 +1264,10 @@ var SchemaManager = class {
       return Ok(void 0);
     }
     for (const col of delta.columns) {
-      if (!this.allowedColumns.has(col.column)) {
+      if (!this.state.allowedColumns.has(col.column)) {
         return Err(
           new SchemaError(
-            `Unknown column "${col.column}" in delta for table "${delta.table}". Schema version ${this.version} does not include this column.`
+            `Unknown column "${col.column}" in delta for table "${delta.table}". Schema version ${this.state.version} does not include this column.`
           )
         );
       }
@@ -1207,10 +1281,10 @@ var SchemaManager = class {
    * returns a SchemaError.
    */
   evolveSchema(newSchema) {
-    if (newSchema.table !== this.currentSchema.table) {
+    if (newSchema.table !== this.state.schema.table) {
       return Err(new SchemaError("Cannot evolve schema: table name mismatch"));
     }
-    const oldColumnMap = new Map(this.currentSchema.columns.map((c) => [c.name, c.type]));
+    const oldColumnMap = new Map(this.state.schema.columns.map((c) => [c.name, c.type]));
     const newColumnMap = new Map(newSchema.columns.map((c) => [c.name, c.type]));
     for (const [name] of oldColumnMap) {
       if (!newColumnMap.has(name)) {
@@ -1231,14 +1305,18 @@ var SchemaManager = class {
         );
       }
     }
-    this.currentSchema = newSchema;
-    this.version++;
-    this.allowedColumns = new Set(newSchema.columns.map((c) => c.name));
-    return Ok({ version: this.version });
+    const newVersion = this.state.version + 1;
+    this.state = {
+      schema: newSchema,
+      version: newVersion,
+      allowedColumns: new Set(newSchema.columns.map((c) => c.name))
+    };
+    return Ok({ version: newVersion });
   }
 };
 
 export {
+  MemoryIdempotencyCache,
   ActionDispatcher,
   DeltaBuffer,
   MemoryConfigStore,
@@ -1275,4 +1353,4 @@ export {
   handleMetrics,
   SchemaManager
 };
-//# sourceMappingURL=chunk-JI4C4R5H.js.map
+//# sourceMappingURL=chunk-FIIHPQMQ.js.map