@xeplr/actions 1.4.6 → 1.4.7

package/README.md

@@ -125,7 +125,10 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
 | `emailDelete` | `email-delete` | Flag `\Deleted` **and expunge** — permanent, not a move to Trash. |
 | `emailDownloadEmail` | `email-download-email` | Stream the raw message to an `.eml` file, named by `fileName`. `{ count, saved[], path, existing[] }`. |
 | `emailDownloadAttachments` | `email-download-attachments` | Save attachments, optionally filtered by `extension` / `filenameContains`, named by `fileName` (see [Saving files](#saving-files-names-and-a-name-already-taken)). `{ count, saved[], path, skipped, existing[] }`. |
-| `httpRequest`, `spawnProgram`, `dbQuery`, `fileMove` | — | **Placeholders.** Empty modules with no `name` or `execute`; `register()` rejects them. |
+| `fileList` | `file-list` | The files in a folder, optionally matching `pattern` (`*.csv`, `Bank-2026-??.pdf`), sorted by name or date. `{ files[{ name, path, size, modifiedAt }], count }`. "Is it there?" is the exact name and `count`. See [File steps](#file-steps-and-the-folders-they-may-use). |
+| `fileRead` | `file-read` | A CSV, text, Excel, JSON or NDJSON file as rows (format from the extension). `{ rows, count, columns, truncated }`, at most `maxRows` (50,000). |
+| `fileMove` | `file-move` | Move or copy a file into a folder (made if missing) or to a new name; `ifExists` as every saving step. `{ path, name, moved, skipped }`. |
+| `httpRequest`, `spawnProgram`, `dbQuery` | — | **Placeholders.** Empty modules with no `name` or `execute`; `register()` rejects them. |
 
 ### Connections
 
@@ -245,6 +248,19 @@ A job or a workflow step stores `connectionInfoId`, never a password. `resolveCo
 - **`resolver(id, { field, inputs })`** is the host's: a login, or `{ dbType, connection }`. It sees all the inputs, for a companion id such as BI's `dbInfoId`.
 - **An id that resolves to nothing is refused** (`CONNECTION_NOT_FOUND`), never replaced by a literal `connection`. An id with no resolver configured: `NO_CONNECTION_RESOLVER`.
 
+## File steps, and the folders they may use
+
+`file-list`, `file-read` and `file-move` take a path from whoever wrote the workflow, so they work **only inside the folders the install names** (`lib/fileRoots.js`):
+
+| | |
+|---|---|
+| `XEPLR_FILE_ROOTS` | Absolute folders, comma-separated: `/data/bank,/data/erp`. |
+| always allowed | `XEPLR_ACTIONS_TMP_DIR` (where downloads and exports land by default), so the next step can read what the last one saved. |
+
+- **Paths are absolute.** A relative one is refused: it would mean different files depending on where the server was started.
+- **No way out:** `../` and symbolic links are followed to the real path before it is checked; a target that does not exist yet is checked by its nearest existing folder.
+- **Not defaulted:** with no `XEPLR_FILE_ROOTS`, a path outside the temp folder is refused (`PATH_NOT_ALLOWED`), naming the setting. Why: without it, a workflow could read any file the server can, its own settings included.
+
 ## Saving files: names, and a name already taken
 
 Every action that saves a file follows one rule (`lib/fileNames.js`): `email-download-attachments`, `email-download-email` and `export-table`.
@@ -461,6 +477,7 @@ await actions.runAction(actions.builtins.dbPush, input, { metaStore: xcfg.metaSt
 | name | required? | meaning |
 |---|---|---|
 | `XEPLR_ACTIONS_TMP_DIR` | no | Where `db-fetch` and `email-read` spool NDJSON, email downloads go (unless `outDir`), and the uploader spools batches. Default `<os.tmpdir()>/xeplr-actions`. Files are not cleaned up by `db-fetch` or `email-read`. |
+| `XEPLR_FILE_ROOTS` | for `file-list`, `file-read`, `file-move` outside the temp folder | Absolute folders, comma-separated, that file steps may use. See [File steps](#file-steps-and-the-folders-they-may-use). |
 | `XCFG_DB_NAME` | for `attachConfig` | The `xeplr_configs` database name. In `configRequiredEnv`. |
 | `XCFG_DB_CONNECTION_INFO_ENCRYPTED` | no | Encrypted server login for `attachConfig`; overrides `XEPLR_DB_CONNECTION`. |
 | `XEPLR_DB_CONNECTION` | for `attachConfig`, unless the above is set | Shared encrypted server login. |

package/lib/builtins/file/list.js

@@ -0,0 +1,60 @@
+// file-list action — the files in a folder, optionally matching a pattern.
+//
+// Also the way to ask "is it there yet?": list with the exact name and branch
+// on output.count. Only inside the folders file steps may use (lib/fileRoots.js).
+
+var fs = require('fs');
+var path = require('path');
+var fileRoots = require('../../fileRoots');
+
+/** `*.csv`, `Bank-2026-??.pdf` → a test on a name. Case-insensitive. */
+function matcher(pattern) {
+  if (!pattern) return function() { return true; };
+  var re = new RegExp('^' + String(pattern).split('').map(function(ch) {
+    if (ch === '*') return '.*';
+    if (ch === '?') return '.';
+    return ch.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+  }).join('') + '$', 'i');
+  return function(name) { return re.test(name); };
+}
+
+module.exports = {
+  name: 'file-list',
+  description: 'The files in a folder, optionally matching a pattern such as *.csv. Returns { files[], count }. ' +
+               'Ask "is it there?" with the exact name and branch on count.',
+  requires: [],
+  inputSchema: [
+    { name: 'folder', type: 'string', required: true, order: 1,
+      description: 'An absolute folder inside XEPLR_FILE_ROOTS, e.g. /data/bank.' },
+    { name: 'pattern', type: 'string', order: 2,
+      description: 'Only names matching this: * any characters, ? one. E.g. Bank-{trigger.window.from|date:YYYY-MM}*.csv' },
+    { name: 'sort', type: 'string', default: 'name', options: ['name', 'newest', 'oldest'], order: 3,
+      description: 'name: A to Z. newest / oldest: by when the file last changed.' }
+  ],
+  outputSchema: [
+    { name: 'files', type: 'array', description: '[{ name, path, size, modifiedAt }]' },
+    { name: 'count', type: 'number', description: 'How many, so the next step can branch on none.' }
+  ],
+
+  execute: async function(ctx) {
+    var input = ctx.input || {};
+    var dir = fileRoots.allowed(input.folder, 'The folder');
+    if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
+      var missing = new Error('"' + input.folder + '" is not a folder that exists.');
+      missing.code = 'NOT_A_FOLDER';
+      throw missing;
+    }
+    var test = matcher(input.pattern);
+    var files = fs.readdirSync(dir, { withFileTypes: true })
+      .filter(function(e) { return e.isFile() && test(e.name); })
+      .map(function(e) {
+        var full = path.join(dir, e.name);
+        var st = fs.statSync(full);
+        return { name: e.name, path: full, size: st.size, modifiedAt: st.mtime.toISOString() };
+      });
+    if (input.sort === 'newest') files.sort(function(a, b) { return b.modifiedAt.localeCompare(a.modifiedAt); });
+    else if (input.sort === 'oldest') files.sort(function(a, b) { return a.modifiedAt.localeCompare(b.modifiedAt); });
+    else files.sort(function(a, b) { return a.name.localeCompare(b.name); });
+    return { files: files, count: files.length };
+  }
+};

package/lib/builtins/file/move.js

@@ -1,8 +1,63 @@
-// file-move action — copies/moves files or entire directories from one
-// location to another. Source + target can each be local/sharepoint/google
-// (i.e. it uses getSource(sourceType) on both ends).
+// file-move action — move or copy a file, into a folder or to a new name.
 //
-// Streaming: always streamed when files are large. Input flag:
-//   { operation: 'copy' | 'move', overwrite: boolean }
-//
-// Implementation pending user spec.
+// "Processed" folders, an archive by month, a copy for another team. Only
+// inside the folders file steps may use (lib/fileRoots.js), at both ends. A
+// name already taken at the target follows the rule every file-writing step
+// follows (lib/fileNames.js): fail by default.
+
+var fs = require('fs');
+var path = require('path');
+var fileRoots = require('../../fileRoots');
+var fileNames = require('../../fileNames');
+
+module.exports = {
+  name: 'file-move',
+  description: 'Move or copy a file into a folder, or to a new name. Returns { path, name, moved, skipped }.',
+  requires: [],
+  inputSchema: [
+    { name: 'from', type: 'string', required: true, order: 1,
+      description: 'The file, by absolute path, e.g. {steps.save.output.path}.' },
+    { name: 'to', type: 'string', required: true, order: 2,
+      description: 'A folder (it keeps its name), or a full path to rename it, e.g. /data/archive/{trigger.window.from|date:YYYY-MM}/' },
+    { name: 'operation', type: 'string', default: 'move', options: ['move', 'copy'], order: 3,
+      description: 'move: it is no longer where it was. copy: it is in both places.' },
+    { name: 'ifExists', type: 'string', default: 'fail', options: fileNames.IF_EXISTS, order: 4,
+      description: 'When the target name is taken. fail: the step fails and nothing moves. skip: left where it is. ' +
+                   'overwrite: replaced. addSuffix: "name (2).ext".' }
+  ],
+  outputSchema: [
+    { name: 'path', type: 'string' }, { name: 'name', type: 'string' },
+    { name: 'moved', type: 'boolean' }, { name: 'skipped', type: 'boolean' }
+  ],
+
+  execute: async function(ctx) {
+    var input = ctx.input || {};
+    var from = fileRoots.allowed(input.from, 'The file to move');
+    if (!fs.existsSync(from) || !fs.statSync(from).isFile()) {
+      var missing = new Error('"' + input.from + '" is not a file that exists.');
+      missing.code = 'FILE_NOT_FOUND';
+      throw missing;
+    }
+    // A folder: ends with a separator, or is one already.
+    var toIsFolder = /[\/\\]$/.test(input.to) || (fs.existsSync(input.to) && fs.statSync(input.to).isDirectory());
+    var dir = fileRoots.allowed(toIsFolder ? input.to : path.dirname(input.to), 'The target folder');
+    fs.mkdirSync(dir, { recursive: true });
+    var wanted = toIsFolder ? path.basename(from) : fileNames.safeName(path.basename(input.to));
+    var place = fileNames.plan(dir, [wanted], input.ifExists)[0];
+    if (!place.write) return { path: place.path, name: place.name, moved: false, skipped: true };
+
+    if (input.operation === 'copy') {
+      fs.copyFileSync(from, place.path);
+    } else {
+      try {
+        fs.renameSync(from, place.path);
+      } catch (err) {
+        // Another disk: copy, then remove.
+        if (err.code !== 'EXDEV') throw err;
+        fs.copyFileSync(from, place.path);
+        fs.unlinkSync(from);
+      }
+    }
+    return { path: place.path, name: place.name, moved: input.operation !== 'copy', skipped: false };
+  }
+};

package/lib/builtins/file/read.js

@@ -0,0 +1,77 @@
+// file-read action — a CSV, Excel, JSON or NDJSON file as rows.
+//
+// What the next step works with: the data step filters them, export-table
+// writes them, a matcher compares them. Only inside the folders file steps may
+// use (lib/fileRoots.js). Bounded by maxRows, because a step's output is
+// stored with the run.
+
+var path = require('path');
+var fs = require('fs');
+var readline = require('readline');
+var fileRoots = require('../../fileRoots');
+var formats = require('../../formats');
+
+var BY_EXTENSION = { csv: 'csv', txt: 'txt', tsv: 'txt', xlsx: 'excel', xls: 'excel', json: 'json', ndjson: 'ndjson', jsonl: 'ndjson' };
+
+async function* ndjsonRows(file) {
+  var lines = readline.createInterface({ input: fs.createReadStream(file), crlfDelay: Infinity });
+  for await (var line of lines) if (line.trim()) yield JSON.parse(line);
+}
+
+module.exports = {
+  name: 'file-read',
+  description: 'Read a CSV, Excel (.xlsx), JSON or NDJSON file into rows. Returns { rows, count, columns, truncated }.',
+  requires: [],
+  inputSchema: [
+    { name: 'path', type: 'string', required: true, order: 1,
+      description: 'An absolute file path inside XEPLR_FILE_ROOTS, e.g. {steps.save.output.path}.' },
+    { name: 'format', type: 'string', order: 2, options: ['csv', 'txt', 'excel', 'json', 'ndjson'],
+      description: 'Empty: from the file\'s extension.' },
+    { name: 'sheet', type: 'string', order: 3, group: 'Excel',
+      description: 'Which sheet. Empty: the first.' },
+    { name: 'maxRows', type: 'number', default: 50000, order: 4,
+      description: 'Rows to read before stopping (truncated: true). A step\'s rows are stored with the run.' }
+  ],
+  outputSchema: [
+    { name: 'rows', type: 'array' }, { name: 'count', type: 'number' },
+    { name: 'columns', type: 'array', description: 'The file\'s columns, in order: [{ key, header }].' },
+    { name: 'truncated', type: 'boolean' }
+  ],
+
+  execute: async function(ctx) {
+    var input = ctx.input || {};
+    var file = fileRoots.allowed(input.path, 'The file');
+    if (!fs.existsSync(file) || !fs.statSync(file).isFile()) {
+      var missing = new Error('"' + input.path + '" is not a file that exists.');
+      missing.code = 'FILE_NOT_FOUND';
+      throw missing;
+    }
+    var ext = path.extname(file).slice(1).toLowerCase();
+    var kind = input.format || BY_EXTENSION[ext];
+    if (!kind) {
+      var unknown = new Error('Cannot tell how to read ".' + ext + '". Set format: csv, txt, excel, json or ndjson.');
+      unknown.code = 'FORMAT_UNKNOWN';
+      throw unknown;
+    }
+    var source;
+    if (kind === 'ndjson') {
+      source = ndjsonRows(file);
+    } else {
+      var reader = formats.getFormat(kind);
+      formats.checkFormatRequires('file-read', kind, reader);
+      var opts = input.sheet ? { sheet: input.sheet } : {};
+      source = reader.inputMode === 'path' ? reader.parsePath(file, opts) : reader.parseStream(fs.createReadStream(file), opts);
+    }
+    var max = Math.max(1, Number(input.maxRows) || 50000);
+    var rows = [];
+    var truncated = false;
+    for await (var row of source) {
+      if (rows.length >= max) { truncated = true; break; }
+      rows.push(row);
+    }
+    // The order of the file's own columns, said out loud: a step's output is
+    // stored as JSONB, which reorders a row's keys.
+    var columns = rows.length ? Object.keys(rows[0]).map(function(k) { return { key: k, header: k }; }) : [];
+    return { rows: rows, count: rows.length, columns: columns, truncated: truncated };
+  }
+};

package/lib/builtins/index.js

@@ -48,6 +48,8 @@ module.exports = {
   // File actions — factory-shaped for the same reason (meta store binding).
   fileUpload:   require('./file/upload'),
   fileMove:     require('./file/move'),
+  fileList:     require('./file/list'),
+  fileRead:     require('./file/read'),
 
   // Email actions. Inbound routes to getInboundProvider(input.emailType)
   // (imap today), outbound to getOutboundProvider (smtp) — see

package/lib/fileRoots.js

@@ -0,0 +1,71 @@
+// WHERE A FILE STEP MAY LOOK.
+//
+// file-list, file-read and file-move take a path from whoever wrote the
+// workflow. Unbounded, that is every file the server can read: its own
+// settings, other installs' data. So they work only inside the folders the
+// install names:
+//
+//   XEPLR_FILE_ROOTS   absolute folders, comma-separated, e.g. /data/bank,/data/erp
+//
+// plus XEPLR_ACTIONS_TMP_DIR (or the system temp folder's xeplr-actions),
+// where downloads and exports land by default, so the file a step just saved
+// can be read by the next one.
+//
+// NOT DEFAULTED: no roots configured and a path outside the temp folder is
+// refused, naming the variable. "Where data lives is never defaulted."
+
+var fs = require('fs');
+var os = require('os');
+var path = require('path');
+
+function rootsError(message) {
+  var err = new Error(message);
+  err.code = 'PATH_NOT_ALLOWED';
+  err.status = 400;
+  return err;
+}
+
+function tmpRoot() {
+  return path.resolve(process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions'));
+}
+
+/** The folders a file step may use, resolved and real (symlinks followed). */
+function roots() {
+  var listed = String(process.env.XEPLR_FILE_ROOTS || '').split(',')
+    .map(function(r) { return r.trim(); })
+    .filter(Boolean);
+  listed.forEach(function(r) {
+    if (!path.isAbsolute(r)) throw rootsError('XEPLR_FILE_ROOTS must list absolute folders, and "' + r + '" is not one.');
+  });
+  return listed.concat([tmpRoot()]).map(function(r) {
+    try { return fs.realpathSync(r); } catch (e) { return path.resolve(r); }
+  });
+}
+
+/**
+ * The path, absolute and real, when it is inside an allowed folder; refused
+ * otherwise. A path that does not exist yet (a move's target) is checked by
+ * its nearest existing parent, so "../" and symlinks cannot step outside.
+ */
+function allowed(p, what) {
+  if (!p || typeof p !== 'string') throw rootsError((what || 'A path') + ' is required.');
+  if (!path.isAbsolute(p)) throw rootsError((what || 'The path') + ' "' + p + '" must be absolute, e.g. /data/bank/statement.csv.');
+  var resolved = path.resolve(p);
+  var probe = resolved;
+  var rest = '';
+  while (!fs.existsSync(probe) && path.dirname(probe) !== probe) {
+    rest = path.join(path.basename(probe), rest);
+    probe = path.dirname(probe);
+  }
+  var real = path.join(fs.realpathSync(probe), rest);
+  var ok = roots().some(function(r) { return real === r || real.indexOf(r + path.sep) === 0; });
+  if (!ok) {
+    throw rootsError((what || 'The path') + ' "' + p + '" is outside the folders file steps may use. ' +
+      (process.env.XEPLR_FILE_ROOTS
+        ? 'Allowed: ' + process.env.XEPLR_FILE_ROOTS + ' and the temp folder.'
+        : 'Set XEPLR_FILE_ROOTS to the folders they may use (absolute, comma-separated).'));
+  }
+  return real;
+}
+
+module.exports = { allowed: allowed, roots: roots, tmpRoot: tmpRoot };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.4.6",
+  "version": "1.4.7",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "scripts": {