lakesync 0.1.6 → 0.1.8

package/dist/{chunk-IRJ4QRWV.js → chunk-JI4C4R5H.js}

@@ -1,18 +1,19 @@
 import {
   isDatabaseAdapter,
   isMaterialisable
-} from "./chunk-GUJWMK5P.js";
+} from "./chunk-LPWXOYNS.js";
 import {
   buildPartitionSpec,
   lakeSyncTableName,
   tableSchemaToIceberg
-} from "./chunk-UAUQGP3B.js";
+} from "./chunk-TMLG32QV.js";
 import {
   writeDeltasToParquet
-} from "./chunk-SF7Y6ZUA.js";
+} from "./chunk-SSICS5KI.js";
 import {
   AdapterNotFoundError,
   BackpressureError,
+  COLUMN_TYPES,
   Err,
   FlushError,
   HLC,
@@ -28,19 +29,25 @@ import {
   validateAction,
   validateConnectorConfig,
   validateSyncRules
-} from "./chunk-P3FT7QCW.js";
+} from "./chunk-LDFFCG2K.js";
 
 // ../gateway/src/action-dispatcher.ts
+var DEFAULT_MAX_CACHE_SIZE = 1e4;
+var DEFAULT_CACHE_TTL_MS = 5 * 60 * 1e3;
 var ActionDispatcher = class {
   actionHandlers = /* @__PURE__ */ new Map();
   executedActions = /* @__PURE__ */ new Set();
   idempotencyMap = /* @__PURE__ */ new Map();
-  constructor(handlers) {
+  maxCacheSize;
+  cacheTtlMs;
+  constructor(handlers, cacheConfig) {
     if (handlers) {
       for (const [name, handler] of Object.entries(handlers)) {
         this.actionHandlers.set(name, handler);
       }
     }
+    this.maxCacheSize = cacheConfig?.maxSize ?? DEFAULT_MAX_CACHE_SIZE;
+    this.cacheTtlMs = cacheConfig?.ttlMs ?? DEFAULT_CACHE_TTL_MS;
   }
   /**
    * Dispatch an action push to registered handlers.
@@ -55,6 +62,7 @@ var ActionDispatcher = class {
    * @returns A `Result` containing results for each action.
    */
   async dispatch(msg, hlcNow, context) {
+    this.evictStaleEntries();
     const results = [];
     for (const action of msg.actions) {
       const validation = validateAction(action);
@@ -62,7 +70,7 @@ var ActionDispatcher = class {
         return Err(validation.error);
       }
       if (this.executedActions.has(action.actionId)) {
-        const cached = this.idempotencyMap.get(action.actionId);
+        const cached = this.getCachedResult(action.actionId);
         if (cached) {
           results.push(cached);
           continue;
@@ -70,7 +78,7 @@ var ActionDispatcher = class {
         continue;
       }
       if (action.idempotencyKey) {
-        const cached = this.idempotencyMap.get(`idem:${action.idempotencyKey}`);
+        const cached = this.getCachedResult(`idem:${action.idempotencyKey}`);
         if (cached) {
           results.push(cached);
           continue;
@@ -163,10 +171,43 @@ var ActionDispatcher = class {
   }
   /** Cache an action result for idempotency deduplication. */
   cacheActionResult(action, result) {
+    const entry = { value: result, cachedAt: Date.now() };
     this.executedActions.add(action.actionId);
-    this.idempotencyMap.set(action.actionId, result);
+    this.idempotencyMap.set(action.actionId, entry);
     if (action.idempotencyKey) {
-      this.idempotencyMap.set(`idem:${action.idempotencyKey}`, result);
+      this.idempotencyMap.set(`idem:${action.idempotencyKey}`, entry);
+    }
+  }
+  /** Get a cached result if it exists and hasn't expired. */
+  getCachedResult(key) {
+    const entry = this.idempotencyMap.get(key);
+    if (!entry) return void 0;
+    if (Date.now() - entry.cachedAt > this.cacheTtlMs) {
+      this.idempotencyMap.delete(key);
+      return void 0;
+    }
+    return entry.value;
+  }
+  /** Evict expired entries and trim to max size. */
+  evictStaleEntries() {
+    const now = Date.now();
+    for (const [key, entry] of this.idempotencyMap) {
+      if (now - entry.cachedAt > this.cacheTtlMs) {
+        this.idempotencyMap.delete(key);
+        if (!key.startsWith("idem:")) {
+          this.executedActions.delete(key);
+        }
+      }
+    }
+    if (this.executedActions.size > this.maxCacheSize) {
+      const excess = this.executedActions.size - this.maxCacheSize;
+      let removed = 0;
+      for (const actionId of this.executedActions) {
+        if (removed >= excess) break;
+        this.executedActions.delete(actionId);
+        this.idempotencyMap.delete(actionId);
+        removed++;
+      }
     }
   }
 };
@@ -284,9 +325,18 @@ var DeltaBuffer = class {
     this.tableLog.delete(table);
     return tableDeltas;
   }
-  /** Drain the log for flush. Returns log entries and clears both structures. */
-  drain() {
-    const entries = [...this.log];
+  /**
+   * Snapshot the current buffer state without clearing it.
+   *
+   * Useful for inspecting the buffer contents without draining.
+   * Use {@link clear} separately after a successful flush for
+   * transactional semantics.
+   */
+  snapshot() {
+    return { entries: [...this.log], byteSize: this.estimatedBytes };
+  }
+  /** Clear all buffer state. */
+  clear() {
     this.log = [];
     this.index.clear();
     this.deltaIds.clear();
@@ -294,6 +344,11 @@ var DeltaBuffer = class {
     this.createdAt = Date.now();
     this.tableBytes.clear();
     this.tableLog.clear();
+  }
+  /** Drain the log for flush. Returns log entries and clears both structures. */
+  drain() {
+    const { entries } = this.snapshot();
+    this.clear();
     return entries;
   }
   /** Number of log entries */
@@ -344,7 +399,7 @@ var MAX_PUSH_PAYLOAD_BYTES = 1048576;
 var MAX_DELTAS_PER_PUSH = 1e4;
 var MAX_PULL_LIMIT = 1e4;
 var DEFAULT_PULL_LIMIT = 100;
-var VALID_COLUMN_TYPES = /* @__PURE__ */ new Set(["string", "number", "boolean", "json", "null"]);
+var VALID_COLUMN_TYPES = new Set(COLUMN_TYPES);
 var DEFAULT_MAX_BUFFER_BYTES = 4 * 1024 * 1024;
 var DEFAULT_MAX_BUFFER_AGE_MS = 3e4;
 
@@ -463,6 +518,117 @@ async function commitToCatalogue(objectKey, fileSizeInBytes, recordCount, catalo
   }
 }
 
+// ../gateway/src/flush-coordinator.ts
+var FlushCoordinator = class {
+  flushing = false;
+  /** Whether a flush is currently in progress. */
+  get isFlushing() {
+    return this.flushing;
+  }
+  /**
+   * Flush all entries from the buffer to the adapter.
+   *
+   * Drains the buffer first, then writes to the adapter. On failure,
+   * entries are restored to the buffer.
+   */
+  async flush(buffer, adapter, deps) {
+    if (this.flushing) {
+      return Err(new FlushError("Flush already in progress"));
+    }
+    if (buffer.logSize === 0) {
+      return Ok(void 0);
+    }
+    if (!adapter) {
+      return Err(new FlushError("No adapter configured"));
+    }
+    this.flushing = true;
+    const byteSize = isDatabaseAdapter(adapter) ? 0 : buffer.byteSize;
+    const entries = buffer.drain();
+    if (entries.length === 0) {
+      this.flushing = false;
+      return Ok(void 0);
+    }
+    try {
+      return await flushEntries(entries, byteSize, {
+        adapter,
+        config: deps.config,
+        restoreEntries: (e) => this.restoreEntries(buffer, e),
+        schemas: deps.schemas
+      });
+    } finally {
+      this.flushing = false;
+    }
+  }
+  /**
+   * Flush a single table's deltas from the buffer.
+   *
+   * Drains only the specified table's deltas and flushes them,
+   * leaving other tables in the buffer.
+   */
+  async flushTable(table, buffer, adapter, deps) {
+    if (this.flushing) {
+      return Err(new FlushError("Flush already in progress"));
+    }
+    if (!adapter) {
+      return Err(new FlushError("No adapter configured"));
+    }
+    const entries = buffer.drainTable(table);
+    if (entries.length === 0) {
+      return Ok(void 0);
+    }
+    this.flushing = true;
+    try {
+      return await flushEntries(
+        entries,
+        0,
+        {
+          adapter,
+          config: deps.config,
+          restoreEntries: (e) => this.restoreEntries(buffer, e),
+          schemas: deps.schemas
+        },
+        table
+      );
+    } finally {
+      this.flushing = false;
+    }
+  }
+  /** Restore drained entries back to the buffer for retry. */
+  restoreEntries(buffer, entries) {
+    for (const entry of entries) {
+      buffer.append(entry);
+    }
+  }
+};
+
+// ../gateway/src/source-registry.ts
+var SourceRegistry = class {
+  sources = /* @__PURE__ */ new Map();
+  constructor(initial) {
+    if (initial) {
+      for (const [name, adapter] of Object.entries(initial)) {
+        this.sources.set(name, adapter);
+      }
+    }
+  }
+  /** Register a named source adapter. */
+  register(name, adapter) {
+    this.sources.set(name, adapter);
+  }
+  /** Unregister a named source adapter. */
+  unregister(name) {
+    this.sources.delete(name);
+  }
+  /** Get a source adapter by name, or undefined if not registered. */
+  get(name) {
+    return this.sources.get(name);
+  }
+  /** List all registered source adapter names. */
+  list() {
+    return [...this.sources.keys()];
+  }
+};
+
 // ../gateway/src/gateway.ts
 var SyncGateway = class {
   hlc;
@@ -470,20 +636,20 @@ var SyncGateway = class {
   actions;
   config;
   adapter;
-  flushing = false;
+  sources;
+  flushCoordinator;
   constructor(config, adapter) {
     this.config = { sourceAdapters: {}, ...config };
     this.hlc = new HLC();
     this.buffer = new DeltaBuffer();
     this.adapter = this.config.adapter ?? adapter ?? null;
     this.actions = new ActionDispatcher(config.actionHandlers);
+    this.sources = new SourceRegistry(this.config.sourceAdapters);
+    this.flushCoordinator = new FlushCoordinator();
   }
-  /** Restore drained entries back to the buffer for retry. */
-  restoreEntries(entries) {
-    for (const entry of entries) {
-      this.buffer.append(entry);
-    }
-  }
+  // -----------------------------------------------------------------------
+  // Push — pipeline of validation steps then buffer append
+  // -----------------------------------------------------------------------
   /**
    * Handle an incoming push from a client.
    *
@@ -494,14 +660,8 @@ var SyncGateway = class {
    *          or a `ClockDriftError` if the client clock is too far ahead.
    */
   handlePush(msg) {
-    const backpressureLimit = this.config.maxBackpressureBytes ?? this.config.maxBufferBytes * 2;
-    if (this.buffer.byteSize >= backpressureLimit) {
-      return Err(
-        new BackpressureError(
-          `Buffer backpressure exceeded (${this.buffer.byteSize} >= ${backpressureLimit} bytes)`
-        )
-      );
-    }
+    const bpResult = this.checkBackpressure();
+    if (!bpResult.ok) return bpResult;
     let accepted = 0;
     const ingested = [];
     for (const delta of msg.deltas) {
@@ -536,6 +696,18 @@ var SyncGateway = class {
     const serverHlc = this.hlc.now();
     return Ok({ serverHlc, accepted, deltas: ingested });
   }
+  /** Check buffer backpressure. */
+  checkBackpressure() {
+    const backpressureLimit = this.config.maxBackpressureBytes ?? this.config.maxBufferBytes * 2;
+    if (this.buffer.byteSize >= backpressureLimit) {
+      return Err(
+        new BackpressureError(
+          `Buffer backpressure exceeded (${this.buffer.byteSize} >= ${backpressureLimit} bytes)`
+        )
+      );
+    }
+    return Ok(void 0);
+  }
   handlePull(msg, context) {
     if (msg.source) {
       return this.handleAdapterPull(msg, context);
@@ -580,7 +752,7 @@ var SyncGateway = class {
   }
   /** Pull from a named source adapter. */
   async handleAdapterPull(msg, context) {
-    const adapter = this.config.sourceAdapters?.[msg.source];
+    const adapter = this.sources.get(msg.source);
     if (!adapter) {
       return Err(new AdapterNotFoundError(`Source adapter "${msg.source}" not found`));
     }
@@ -598,7 +770,7 @@ var SyncGateway = class {
     return Ok({ deltas: sliced, serverHlc, hasMore });
   }
   // -----------------------------------------------------------------------
-  // Flush — delegates to flush module
+  // Flush — delegates to FlushCoordinator
   // -----------------------------------------------------------------------
   /**
    * Flush the buffer to the configured adapter.
@@ -611,55 +783,15 @@ var SyncGateway = class {
    * @returns A `Result` indicating success or a `FlushError`.
    */
   async flush() {
-    if (this.flushing) {
-      return Err(new FlushError("Flush already in progress"));
-    }
-    if (this.buffer.logSize === 0) {
-      return Ok(void 0);
-    }
-    if (!this.adapter) {
-      return Err(new FlushError("No adapter configured"));
-    }
-    this.flushing = true;
-    if (isDatabaseAdapter(this.adapter)) {
-      const entries2 = this.buffer.drain();
-      if (entries2.length === 0) {
-        this.flushing = false;
-        return Ok(void 0);
-      }
-      try {
-        return await flushEntries(entries2, 0, {
-          adapter: this.adapter,
-          config: {
-            gatewayId: this.config.gatewayId,
-            flushFormat: this.config.flushFormat,
-            tableSchema: this.config.tableSchema,
-            catalogue: this.config.catalogue
-          },
-          restoreEntries: (e) => this.restoreEntries(e),
-          schemas: this.config.schemas
-        });
-      } finally {
-        this.flushing = false;
-      }
-    }
-    const byteSize = this.buffer.byteSize;
-    const entries = this.buffer.drain();
-    try {
-      return await flushEntries(entries, byteSize, {
-        adapter: this.adapter,
-        config: {
-          gatewayId: this.config.gatewayId,
-          flushFormat: this.config.flushFormat,
-          tableSchema: this.config.tableSchema,
-          catalogue: this.config.catalogue
-        },
-        restoreEntries: (e) => this.restoreEntries(e),
-        schemas: this.config.schemas
-      });
-    } finally {
-      this.flushing = false;
-    }
+    return this.flushCoordinator.flush(this.buffer, this.adapter, {
+      config: {
+        gatewayId: this.config.gatewayId,
+        flushFormat: this.config.flushFormat,
+        tableSchema: this.config.tableSchema,
+        catalogue: this.config.catalogue
+      },
+      schemas: this.config.schemas
+    });
   }
   /**
    * Flush a single table's deltas from the buffer.
@@ -668,37 +800,15 @@ var SyncGateway = class {
    * leaving other tables in the buffer.
    */
   async flushTable(table) {
-    if (this.flushing) {
-      return Err(new FlushError("Flush already in progress"));
-    }
-    if (!this.adapter) {
-      return Err(new FlushError("No adapter configured"));
-    }
-    const entries = this.buffer.drainTable(table);
-    if (entries.length === 0) {
-      return Ok(void 0);
-    }
-    this.flushing = true;
-    try {
-      return await flushEntries(
-        entries,
-        0,
-        {
-          adapter: this.adapter,
-          config: {
-            gatewayId: this.config.gatewayId,
-            flushFormat: this.config.flushFormat,
-            tableSchema: this.config.tableSchema,
-            catalogue: this.config.catalogue
-          },
-          restoreEntries: (e) => this.restoreEntries(e),
-          schemas: this.config.schemas
-        },
-        table
-      );
-    } finally {
-      this.flushing = false;
-    }
+    return this.flushCoordinator.flushTable(table, this.buffer, this.adapter, {
+      config: {
+        gatewayId: this.config.gatewayId,
+        flushFormat: this.config.flushFormat,
+        tableSchema: this.config.tableSchema,
+        catalogue: this.config.catalogue
+      },
+      schemas: this.config.schemas
+    });
   }
   // -----------------------------------------------------------------------
   // Actions — delegates to ActionDispatcher
@@ -724,7 +834,7 @@ var SyncGateway = class {
     return this.actions.describe();
   }
   // -----------------------------------------------------------------------
-  // Source adapters
+  // Source adapters — delegates to SourceRegistry
   // -----------------------------------------------------------------------
   /**
    * Register a named source adapter for adapter-sourced pulls.
@@ -733,7 +843,7 @@ var SyncGateway = class {
    * @param adapter - The database adapter to register.
    */
   registerSource(name, adapter) {
-    this.config.sourceAdapters[name] = adapter;
+    this.sources.register(name, adapter);
   }
   /**
    * Unregister a named source adapter.
@@ -741,7 +851,7 @@ var SyncGateway = class {
    * @param name - The source name to remove.
    */
   unregisterSource(name) {
-    delete this.config.sourceAdapters[name];
+    this.sources.unregister(name);
   }
   /**
    * List all registered source adapter names.
@@ -749,7 +859,7 @@ var SyncGateway = class {
    * @returns Array of registered source adapter names.
    */
   listSources() {
-    return Object.keys(this.config.sourceAdapters);
+    return this.sources.list();
   }
   // -----------------------------------------------------------------------
   // Buffer queries
@@ -789,13 +899,17 @@ var SyncGateway = class {
 };
 
 // ../gateway/src/validation.ts
-function validatePushBody(raw, headerClientId) {
-  let body;
+function parseJson(raw, reviver) {
   try {
-    body = JSON.parse(raw, bigintReviver);
+    return Ok(JSON.parse(raw, reviver));
   } catch {
     return Err({ status: 400, message: "Invalid JSON body" });
   }
+}
+function validatePushBody(raw, headerClientId) {
+  const parsed = parseJson(raw, bigintReviver);
+  if (!parsed.ok) return parsed;
+  const body = parsed.value;
   if (!body.clientId || !Array.isArray(body.deltas)) {
     return Err({ status: 400, message: "Missing required fields: clientId, deltas" });
   }
@@ -840,12 +954,9 @@ function parsePullParams(params) {
   return Ok(msg);
 }
 function validateActionBody(raw, headerClientId) {
-  let body;
-  try {
-    body = JSON.parse(raw, bigintReviver);
-  } catch {
-    return Err({ status: 400, message: "Invalid JSON body" });
-  }
+  const parsed = parseJson(raw, bigintReviver);
+  if (!parsed.ok) return parsed;
+  const body = parsed.value;
   if (!body.clientId || !Array.isArray(body.actions)) {
     return Err({ status: 400, message: "Missing required fields: clientId, actions" });
   }
@@ -858,12 +969,9 @@ function validateActionBody(raw, headerClientId) {
   return Ok(body);
 }
 function validateSchemaBody(raw) {
-  let schema;
-  try {
-    schema = JSON.parse(raw);
-  } catch {
-    return Err({ status: 400, message: "Invalid JSON body" });
-  }
+  const parsed = parseJson(raw);
+  if (!parsed.ok) return parsed;
+  const schema = parsed.value;
   if (!schema.table || !Array.isArray(schema.columns)) {
     return Err({ status: 400, message: "Missing required fields: table, columns" });
   }
@@ -1000,12 +1108,11 @@ async function handleSaveSchema(raw, store, gatewayId) {
   return { status: 200, body: { saved: true } };
 }
 async function handleSaveSyncRules(raw, store, gatewayId) {
-  let config;
-  try {
-    config = JSON.parse(raw);
-  } catch {
-    return { status: 400, body: { error: "Invalid JSON body" } };
+  const parsed = parseJson(raw);
+  if (!parsed.ok) {
+    return { status: parsed.error.status, body: { error: parsed.error.message } };
   }
+  const config = parsed.value;
   const validation = validateSyncRules(config);
   if (!validation.ok) {
     return { status: 400, body: { error: validation.error.message } };
@@ -1014,12 +1121,11 @@ async function handleSaveSyncRules(raw, store, gatewayId) {
   return { status: 200, body: { saved: true } };
 }
 async function handleRegisterConnector(raw, store) {
-  let body;
-  try {
-    body = JSON.parse(raw);
-  } catch {
-    return { status: 400, body: { error: "Invalid JSON body" } };
+  const parsed = parseJson(raw);
+  if (!parsed.ok) {
+    return { status: parsed.error.status, body: { error: parsed.error.message } };
   }
+  const body = parsed.value;
   const validation = validateConnectorConfig(body);
   if (!validation.ok) {
     return { status: 400, body: { error: validation.error.message } };
@@ -1146,7 +1252,10 @@ export {
   hlcRange,
   flushEntries,
   commitToCatalogue,
+  FlushCoordinator,
+  SourceRegistry,
   SyncGateway,
+  parseJson,
   validatePushBody,
   parsePullParams,
   validateActionBody,
@@ -1166,4 +1275,4 @@ export {
   handleMetrics,
   SchemaManager
 };
-//# sourceMappingURL=chunk-IRJ4QRWV.js.map
+//# sourceMappingURL=chunk-JI4C4R5H.js.map