@xeplr/actions 1.1.1

package/lib/runner.js

@@ -0,0 +1,81 @@
+var { applySchema } = require('@xeplr/schema-handler');
+var { get } = require('./registry');
+var { ActionNotRegisteredError } = require('./errors');
+
+/**
+ * Safely run an action. Two call forms supported:
+ *
+ *   runAction({ name, input, system?, timeoutMs? })
+ *       — look up a registered action by name (the Jobs / config path).
+ *
+ *   runAction(nameOrAction, input, opts?)
+ *       — shorthand for one-off programmatic use. First arg can be a name
+ *         (string) OR an action module `{ name, inputSchema, execute, ... }`.
+ *         `opts` = { system, timeoutMs, metaStore }.
+ *
+ * Returns { status: 'success' | 'failed', output, error, durationMs }.
+ * Never throws — every failure (unregistered action, validation, executor
+ * exception, timeout) is captured in `error`.
+ */
+async function runAction(first, second, third) {
+  var start = Date.now();
+  var opts = normalizeArgs(first, second, third);
+
+  var def = opts.action || get(opts.name);
+  if (!def) return finish(start, 'failed', null, new ActionNotRegisteredError(opts.name));
+
+  var input;
+  try {
+    input = applySchema(def.inputSchema, opts.input || {}, 'input');
+  } catch (err) {
+    return finish(start, 'failed', null, err);
+  }
+
+  try {
+    var call = def.execute({ input: input, system: opts.system || {}, metaStore: opts.metaStore });
+    var settled = (opts.timeoutMs && call && typeof call.then === 'function')
+      ? withTimeout(call, opts.timeoutMs)
+      : call;
+    var output = await settled;
+    return finish(start, 'success', output, null);
+  } catch (err) {
+    return finish(start, 'failed', null, err);
+  }
+}
+
+// Accept either the options-object form or the shorthand (action, input, opts).
+function normalizeArgs(first, second, third) {
+  // Shorthand form: first arg is a string name or an action module.
+  if (typeof first === 'string' || (first && typeof first.execute === 'function')) {
+    var extra = third || {};
+    var result = { input: second || {}, system: extra.system, timeoutMs: extra.timeoutMs, metaStore: extra.metaStore };
+    if (typeof first === 'string') result.name = first;
+    else                            result.action = first;
+    return result;
+  }
+  // Options-object form.
+  return first || {};
+}
+
+function finish(start, status, output, err) {
+  var e = null;
+  if (err) {
+    e = { name: err.name || 'Error', message: err.message };
+    if (err.stack) e.stack = err.stack;
+    if (err.details) e.details = err.details;
+    if (err.actionName) e.actionName = err.actionName;
+  }
+  return { status: status, output: output, error: e, durationMs: Date.now() - start };
+}
+
+function withTimeout(promise, ms) {
+  return new Promise(function(resolve, reject) {
+    var t = setTimeout(function() { reject(new Error('Action timed out after ' + ms + 'ms')); }, ms);
+    promise.then(
+      function(v) { clearTimeout(t); resolve(v); },
+      function(e) { clearTimeout(t); reject(e); }
+    );
+  });
+}
+
+module.exports = { runAction: runAction };

package/lib/streaming/checkpoint.js

@@ -0,0 +1,46 @@
+// Per-run checkpoint file. Written atomically (write .tmp, rename) after every
+// batch completes its onBatch hook. Enables resume() to skip already-processed
+// batches on retry.
+//
+// Shape:
+//   { runId, batchDir, batchSize, lastCompletedBatch, updatedAt }
+
+var fs = require('fs');
+var fsp = require('fs/promises');
+var path = require('path');
+
+function checkpointPath(dir, runId) {
+  return path.join(dir, runId + '.json');
+}
+
+async function ensureDir(dir) {
+  await fsp.mkdir(dir, { recursive: true });
+}
+
+async function read(dir, runId) {
+  if (!dir) return null;
+  try {
+    var raw = await fsp.readFile(checkpointPath(dir, runId), 'utf8');
+    return JSON.parse(raw);
+  } catch (err) {
+    if (err.code === 'ENOENT') return null;
+    throw err;
+  }
+}
+
+async function write(dir, checkpoint) {
+  if (!dir) return;
+  await ensureDir(dir);
+  var target = checkpointPath(dir, checkpoint.runId);
+  var tmp = target + '.tmp';
+  var payload = JSON.stringify(Object.assign({}, checkpoint, { updatedAt: new Date().toISOString() }));
+  await fsp.writeFile(tmp, payload, { encoding: 'utf8' });
+  await fsp.rename(tmp, target);
+}
+
+async function clear(dir, runId) {
+  if (!dir) return;
+  await fsp.unlink(checkpointPath(dir, runId)).catch(function() { /* ignore */ });
+}
+
+module.exports = { read: read, write: write, clear: clear, checkpointPath: checkpointPath };

package/lib/streaming/index.js

@@ -0,0 +1,10 @@
+var { spool, readBatchFile } = require('./spool');
+var { resume } = require('./resume');
+var checkpoint = require('./checkpoint');
+
+module.exports = {
+  spool:         spool,
+  resume:        resume,
+  readBatchFile: readBatchFile,
+  checkpoint:    checkpoint
+};

package/lib/streaming/resume.js

@@ -0,0 +1,35 @@
+// resume(runId, source, opts)
+//
+// Look up the checkpoint for runId (in opts.checkpointDir), figure out how
+// many batches were already completed, then re-invoke spool() telling it to
+// skip forward by that much. The caller's `source` must be replayable / at
+// least positioned so the first row it yields corresponds to the first row
+// of batch `lastCompletedBatch + 1`.
+//
+// Concretely: if `source` is `for-await SELECT * FROM foo ORDER BY id LIMIT ...
+// OFFSET (lastCompletedBatch * batchSize)`, resume() will "just work."
+
+var checkpoint = require('./checkpoint');
+var { spool } = require('./spool');
+
+async function resume(runId, source, opts) {
+  opts = opts || {};
+  if (!runId) throw new Error('resume: runId is required');
+  if (!opts.checkpointDir) throw new Error('resume: opts.checkpointDir is required');
+
+  var cp = await checkpoint.read(opts.checkpointDir, runId);
+  if (!cp) {
+    // No checkpoint → cold start with this runId.
+    return spool(source, Object.assign({}, opts, { runId: runId }));
+  }
+
+  var startFromBatch = (cp.lastCompletedBatch != null ? cp.lastCompletedBatch : -1) + 1;
+
+  return spool(source, Object.assign({}, opts, {
+    runId:          runId,
+    startFromBatch: startFromBatch,
+    batchSize:      cp.batchSize || opts.batchSize
+  }));
+}
+
+module.exports = { resume: resume };

package/lib/streaming/spool.js

@@ -0,0 +1,253 @@
+// spool(source, opts)
+//
+// Consume an async iterable of rows, write them as rotating NDJSON batch
+// files, and drive `target` hooks in order. Never throws mid-stream —
+// hook errors reject the whole spool.
+//
+// See project_xeplr_actions_streaming.md for the locked design.
+
+var fs = require('fs');
+var fsp = require('fs/promises');
+var path = require('path');
+
+var { semaphore, rateLimiter, memoryMonitor, toNDJSON } = require('./utils');
+var checkpoint = require('./checkpoint');
+
+var DEFAULTS = {
+  batchSize:     5000,
+  maxInFlight:   1,
+  ratePerSecond: null,
+  maxMemoryMB:   256
+};
+
+/**
+ * @param {AsyncIterable<object>} source
+ * @param {object} opts
+ * @param {string} opts.runId            stable id — used for the sub-dir + checkpoint file
+ * @param {string} opts.batchDir         where the run dir goes (final dir = batchDir/<runId>)
+ * @param {number} [opts.batchSize=5000]
+ * @param {number} [opts.startFromBatch=0]  skip N batches (drives resume)
+ * @param {number} [opts.maxInFlight=1]
+ * @param {number} [opts.ratePerSecond]
+ * @param {number} [opts.maxMemoryMB=256]
+ * @param {string} [opts.checkpointDir]  enables checkpoint writes
+ * @param {object} [opts.target]
+ * @param {function} [opts.target.beforeAll]  (ctx) => void          fires ONCE, after 1st batch is buffered to disk
+ * @param {function} [opts.target.onBatch]    (rows, meta) => void   fires per completed batch, in order
+ * @param {function} [opts.target.afterAll]   (summary) => void      fires ONCE, after all batches settle
+ * @returns {Promise<{runId, batchDir, totalRows, totalBatches, files, durationMs}>}
+ */
+async function spool(source, opts) {
+  opts = Object.assign({}, DEFAULTS, opts || {});
+
+  if (!opts.runId)    throw new Error('spool: opts.runId is required');
+  if (!opts.batchDir) throw new Error('spool: opts.batchDir is required');
+  if (!source || typeof source[Symbol.asyncIterator] !== 'function') {
+    throw new Error('spool: source must be an async iterable');
+  }
+
+  var target = opts.target || {};
+  var runDir = path.join(opts.batchDir, opts.runId);
+  await fsp.mkdir(runDir, { recursive: true });
+
+  var started = Date.now();
+  var sem = semaphore(opts.maxInFlight);
+  var rl  = rateLimiter(opts.ratePerSecond);
+  var mem = memoryMonitor(opts.maxMemoryMB);
+
+  // Ordered checkpoint tracking: batches may complete out of order when
+  // maxInFlight > 1. `lastContiguous` = highest N where every batch 0..N has
+  // reported done. Only checkpoint that highwater — safer for resume.
+  var completedSet = new Set();
+  var lastContiguous = (opts.startFromBatch || 0) - 1;
+
+  // First-error latch. Since spool guarantees "all hooks awaited in order,
+  // any hook error rejects the whole thing," we surface the first error
+  // deterministically even if newer batches also fail.
+  var firstError = null;
+
+  var fileIndex = opts.startFromBatch || 0;
+  var currentFile = null;         // { stream, path, byteCount, rowCount, buffered }
+  var beforeAllFired = false;
+  var pendingChain = Promise.resolve();      // sequential-arrival ordering into the semaphore
+  var filesCreated = [];
+  var totalRows = 0;
+
+  async function openNextFile() {
+    var name = String(fileIndex).padStart(6, '0') + '.ndjson';
+    var filePath = path.join(runDir, name);
+    var stream = fs.createWriteStream(filePath);
+    currentFile = {
+      stream:      stream,
+      path:        filePath,
+      byteCount:   0,
+      rowCount:    0,
+      // Buffer rows in memory too — we need them to invoke onBatch(rows, meta).
+      // Same footprint as reading the file back, but avoids the round trip.
+      buffered:    []
+    };
+  }
+
+  async function closeCurrentFile() {
+    if (!currentFile) return null;
+    var cf = currentFile;
+    currentFile = null;
+    await new Promise(function(resolve) { cf.stream.end(resolve); });
+    return cf;
+  }
+
+  // Drop the current file if it's empty (avoids leaving 000042.ndjson behind
+  // when the source ended right on a rotation boundary).
+  async function dropEmptyFile(cf) {
+    if (!cf || cf.rowCount > 0) return false;
+    await fsp.unlink(cf.path).catch(function() { /* ignore */ });
+    return true;
+  }
+
+  // Schedule a completed file for hook processing.
+  // Adds to the pendingChain to preserve arrival order into the semaphore,
+  // then the semaphore itself gates concurrency (maxInFlight).
+  function schedule(cf, batchIndex, isLast) {
+    pendingChain = pendingChain.then(async function() {
+      if (firstError) return;              // short-circuit on failure
+
+      var release = await sem.acquire();
+
+      try {
+        // First-batch bootstrap — fire beforeAll AFTER the first batch is on
+        // disk (so consumers can peek at it), BEFORE its onBatch runs.
+        if (!beforeAllFired && batchIndex === 0 && target.beforeAll) {
+          beforeAllFired = true;
+          await target.beforeAll({
+            runId:          opts.runId,
+            batchDir:       runDir,
+            firstBatchPath: cf.path
+          });
+        }
+
+        if (target.onBatch) {
+          await target.onBatch(cf.buffered, {
+            batchIndex: batchIndex,
+            filePath:   cf.path,
+            rowCount:   cf.rowCount,
+            byteCount:  cf.byteCount,
+            isLast:     isLast
+          });
+        }
+
+        completedSet.add(batchIndex);
+        // Advance the contiguous highwater.
+        while (completedSet.has(lastContiguous + 1)) {
+          lastContiguous++;
+          completedSet.delete(lastContiguous);
+        }
+
+        // Release the row bytes for GC + memory accounting.
+        mem.sub(cf.byteCount);
+        cf.buffered = null;
+
+        if (opts.checkpointDir) {
+          await checkpoint.write(opts.checkpointDir, {
+            runId:              opts.runId,
+            batchDir:           runDir,
+            batchSize:          opts.batchSize,
+            lastCompletedBatch: lastContiguous
+          });
+        }
+      } catch (err) {
+        if (!firstError) firstError = err;
+      } finally {
+        release();
+      }
+    });
+  }
+
+  // ─── main pump ──────────────────────────────────────────────────────────
+  try {
+    await openNextFile();
+
+    for await (var row of source) {
+      if (firstError) break;
+
+      await rl.take(1);
+      var line = toNDJSON(row);
+      mem.add(line.length);
+
+      currentFile.stream.write(line);
+      currentFile.byteCount += line.length;
+      currentFile.rowCount++;
+      currentFile.buffered.push(row);
+      totalRows++;
+
+      if (currentFile.rowCount >= opts.batchSize) {
+        var idx = fileIndex;
+        var cf = await closeCurrentFile();
+        filesCreated.push(cf.path);
+        schedule(cf, idx, false);
+        fileIndex++;
+        await openNextFile();
+      }
+    }
+
+    // Close the trailing file. If it's empty, drop it. If it had rows,
+    // it counts as the last batch.
+    var tail = await closeCurrentFile();
+    if (tail) {
+      if (await dropEmptyFile(tail)) {
+        // Nothing to schedule.
+      } else {
+        var tailIdx = fileIndex;
+        filesCreated.push(tail.path);
+        schedule(tail, tailIdx, true);
+      }
+    }
+
+    // Wait for every scheduled hook to settle.
+    await pendingChain;
+    if (firstError) throw firstError;
+
+    if (target.afterAll) {
+      await target.afterAll({
+        runId:        opts.runId,
+        batchDir:     runDir,
+        totalRows:    totalRows,
+        totalBatches: filesCreated.length,
+        files:        filesCreated,
+        durationMs:   Date.now() - started
+      });
+    }
+
+    // Success — clear the checkpoint so a future resume() with the same runId
+    // treats the run as fresh (rather than "already done at batch N").
+    if (opts.checkpointDir) {
+      await checkpoint.clear(opts.checkpointDir, opts.runId).catch(function() {});
+    }
+
+    return {
+      runId:        opts.runId,
+      batchDir:     runDir,
+      totalRows:    totalRows,
+      totalBatches: filesCreated.length,
+      files:        filesCreated,
+      durationMs:   Date.now() - started
+    };
+  } catch (err) {
+    // Best-effort teardown of the currently open file (nothing to schedule).
+    if (currentFile) {
+      try { await new Promise(function(r) { currentFile.stream.end(r); }); } catch (_) {}
+      currentFile = null;
+    }
+    throw err;
+  }
+}
+
+/**
+ * Read one NDJSON batch file back into an array of rows. Consumer counterpart
+ * to spool's writer — one JSON object per non-blank line.
+ */
+async function readBatchFile(filePath) {
+  var raw = await fsp.readFile(filePath, 'utf8');
+  return raw.split(/\r?\n/).filter(function(l) { return l.trim().length > 0; }).map(function(l) { return JSON.parse(l); });
+}
+
+module.exports = { spool: spool, readBatchFile: readBatchFile };

package/lib/streaming/utils.js

@@ -0,0 +1,82 @@
+// Small building blocks used by spool.js. No external deps.
+
+// Semaphore that gates concurrency to `n` (default 1).
+// acquire() returns a release() function that MUST be called (finally block).
+function semaphore(n) {
+  var slots = n;
+  var queue = [];
+  return {
+    async acquire() {
+      if (slots > 0) { slots--; return release; }
+      await new Promise(function(resolve) { queue.push(resolve); });
+      slots--;
+      return release;
+    },
+    inFlight() { return n - slots; }
+  };
+  function release() {
+    slots++;
+    var next = queue.shift();
+    if (next) next();
+  }
+}
+
+// Simple leaky-bucket rate limiter. `perSecond=null` disables (no-op).
+function rateLimiter(perSecond) {
+  if (!perSecond || perSecond <= 0) {
+    return { async take(_n) { /* no-op */ } };
+  }
+  var tokens = perSecond;
+  var lastRefill = Date.now();
+  return {
+    async take(n) {
+      n = n || 1;
+      while (true) {
+        // Refill based on elapsed time.
+        var now = Date.now();
+        var elapsed = (now - lastRefill) / 1000;
+        tokens = Math.min(perSecond, tokens + elapsed * perSecond);
+        lastRefill = now;
+        if (tokens >= n) { tokens -= n; return; }
+        // Wait for enough tokens.
+        var need = (n - tokens) / perSecond * 1000;
+        await new Promise(function(r) { setTimeout(r, Math.max(1, Math.ceil(need))); });
+      }
+    }
+  };
+}
+
+// Coarse memory tracker. Bytes added via .add(n); throws with a clear message
+// when the running total crosses the ceiling. Consumer decrements via .sub()
+// when a batch is released (file flushed + onBatch settled).
+function memoryMonitor(maxMB) {
+  if (!maxMB || maxMB <= 0) {
+    return { add() {}, sub() {}, current() { return 0; } };
+  }
+  var ceiling = maxMB * 1024 * 1024;
+  var used = 0;
+  return {
+    add(n) {
+      used += n;
+      if (used > ceiling) {
+        var mb = Math.round(used / 1024 / 1024);
+        throw new Error('Streaming memory ceiling exceeded: ' + mb + 'MB > ' + maxMB + 'MB (raise maxMemoryMB or lower batchSize)');
+      }
+    },
+    sub(n) { used = Math.max(0, used - n); },
+    current() { return used; }
+  };
+}
+
+// Serialize a row to a NDJSON line. Replaces embedded newlines so each row is
+// exactly one line — the invariant `readNDJSONLines` relies on.
+function toNDJSON(row) {
+  return JSON.stringify(row).replace(/\r?\n/g, ' ') + '\n';
+}
+
+module.exports = {
+  semaphore: semaphore,
+  rateLimiter: rateLimiter,
+  memoryMonitor: memoryMonitor,
+  toNDJSON: toNDJSON
+};

package/lib/uploader/attach.js

@@ -0,0 +1,69 @@
+// attachConfig — connect an app to the shared xeplr_configs control-plane DB
+// and get back a ready-to-use metaStore + ImportMeta model, in one call.
+// Mirrors @xeplr/auth's attach() shape exactly (ready()/model-style getters,
+// required env owned by the library, lazy + memoized) so every consuming app
+// writes the SAME few lines instead of each hand-rolling its own
+// bootstrapConfigDb()/bindKnex()/meta-store-knex() dance:
+//
+//   const { attachConfig } = require('@xeplr/actions');
+//   const xcfg = attachConfig({ service: 'xeplr-bi' });   // service = this app's name
+//   await xcfg.ready();
+//   xcfg.metaStore     // pass into uploads/loadMapping, uploader.rollback
+//   xcfg.ImportMeta    // BaseModel bound to xeplr_configs — reads/soft-deletes
+//   xcfg.SERVICE_NAME  // same string, for `.where('service', ...)` filters
+//
+// import_meta's SCHEMA is identical for every app (see xeplr-db/config/
+// migrations/0001_import_meta.js + 0002_...) — there is nothing app-specific
+// about the ImportMeta model, so it lives here once instead of being
+// copy-pasted into every app's models/ directory.
+var { BaseModel, bootstrapConfigDb } = require('@xeplr/db');
+var { decrypt } = require('@xeplr/utils/isomorphic/crypto');
+var makeMetaStore = require('./meta-store-knex');
+
+var requiredEnv = [
+  'XCFG_DB_NAME',                        // xeplr_configs db name
+  'XCFG_DB_CONNECTION_INFO_ENCRYPTED',   // xeplr_configs server login
+];
+
+class ImportMeta extends BaseModel {
+  static get tableName() { return 'import_meta'; }
+  static get idColumn() { return 'id'; }
+}
+
+function attachConfig(options) {
+  options = options || {};
+  var service = options.service;
+  if (!service) throw new Error('attachConfig: options.service is required');
+
+  var _ready = null;
+  var _xcfg = null;
+  var _boundImportMeta = null;
+  var _metaStore = null;
+
+  function ready() {
+    if (!_ready) {
+      _ready = (async function() {
+        var login = JSON.parse(await decrypt(process.env.XCFG_DB_CONNECTION_INFO_ENCRYPTED, process.env.ENCRYPTION_KEY));
+        _xcfg = await bootstrapConfigDb({ connection: login, database: process.env.XCFG_DB_NAME });
+        _boundImportMeta = ImportMeta.bindKnex(_xcfg.db);
+        _metaStore = makeMetaStore(_xcfg.db, { table: 'import_meta', service: service });
+        return _xcfg;
+      })();
+    }
+    return _ready;
+  }
+
+  function requireReady(name) {
+    if (!_xcfg) throw new Error('attachConfig("' + service + '"): ' + name + ' accessed before ready() resolved');
+  }
+
+  return {
+    ready: ready,
+    SERVICE_NAME: service,
+    get metaStore() { requireReady('metaStore'); return _metaStore; },
+    get ImportMeta() { requireReady('ImportMeta'); return _boundImportMeta; },
+    get db() { requireReady('db'); return _xcfg.db; }
+  };
+}
+
+module.exports = { attachConfig: attachConfig, requiredEnv: requiredEnv, ImportMeta: ImportMeta };