@xeplr/actions 1.4.6 → 1.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -1
- package/lib/builtins/file/list.js +60 -0
- package/lib/builtins/file/move.js +62 -7
- package/lib/builtins/file/read.js +77 -0
- package/lib/builtins/http-request.js +271 -3
- package/lib/builtins/index.js +2 -0
- package/lib/fileRoots.js +71 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -125,7 +125,11 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
|
|
|
125
125
|
| `emailDelete` | `email-delete` | Flag `\Deleted` **and expunge** — permanent, not a move to Trash. |
|
|
126
126
|
| `emailDownloadEmail` | `email-download-email` | Stream the raw message to an `.eml` file, named by `fileName`. `{ count, saved[], path, existing[] }`. |
|
|
127
127
|
| `emailDownloadAttachments` | `email-download-attachments` | Save attachments, optionally filtered by `extension` / `filenameContains`, named by `fileName` (see [Saving files](#saving-files-names-and-a-name-already-taken)). `{ count, saved[], path, skipped, existing[] }`. |
|
|
128
|
-
| `
|
|
128
|
+
| `fileList` | `file-list` | The files in a folder, optionally matching `pattern` (`*.csv`, `Bank-2026-??.pdf`), sorted by name or date. `{ files[{ name, path, size, modifiedAt }], count }`. "Is it there?" is the exact name and `count`. See [File steps](#file-steps-and-the-folders-they-may-use). |
|
|
129
|
+
| `fileRead` | `file-read` | A CSV, text, Excel, JSON or NDJSON file as rows (format from the extension). `{ rows, count, columns, truncated }`, at most `maxRows` (50,000). |
|
|
130
|
+
| `fileMove` | `file-move` | Move or copy a file into a folder (made if missing) or to a new name; `ifExists` as every saving step. `{ path, name, moved, skipped }`. |
|
|
131
|
+
| `httpRequest` | `http-request` | Call another system's API: `method`, `path`, `query`, `body`, `headers`, signing in with a saved connection. `{ status, ok, body, headers }`. See [Calling an API](#calling-an-api-http-request). |
|
|
132
|
+
| `spawnProgram`, `dbQuery` | — | **Placeholders.** Empty modules with no `name` or `execute`; `register()` rejects them. |
|
|
129
133
|
|
|
130
134
|
### Connections
|
|
131
135
|
|
|
@@ -245,6 +249,52 @@ A job or a workflow step stores `connectionInfoId`, never a password. `resolveCo
|
|
|
245
249
|
- **`resolver(id, { field, inputs })`** is the host's: a login, or `{ dbType, connection }`. It sees all the inputs, for a companion id such as BI's `dbInfoId`.
|
|
246
250
|
- **An id that resolves to nothing is refused** (`CONNECTION_NOT_FOUND`), never replaced by a literal `connection`. An id with no resolver configured: `NO_CONNECTION_RESOLVER`.
|
|
247
251
|
|
|
252
|
+
## Calling an API: `http-request`
|
|
253
|
+
|
|
254
|
+
```js
|
|
255
|
+
await runAction({ name: 'http-request', input: {
|
|
256
|
+
connectionInfoId: 'bank-api', // resolved by the host to `connection`
|
|
257
|
+
method: 'GET', path: '/statements',
|
|
258
|
+
query: { from: '2026-09-01', to: '2026-09-30' }
|
|
259
|
+
} })
|
|
260
|
+
// → { status: 200, ok: true, body: { rows: [...] }, headers: { 'content-type': … } }
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
**The connection** (what a saved API connection resolves to, so no key is ever in a workflow):
|
|
264
|
+
|
|
265
|
+
```js
|
|
266
|
+
{ baseUrl: 'https://api.bank.example/v2', headers?: { 'X-Tenant': 'acme' }, auth }
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
| `auth.type` | fields | sends |
|
|
270
|
+
|---|---|---|
|
|
271
|
+
| `none` | | nothing |
|
|
272
|
+
| `apiKey` | `in` (`header` / `query`), `name`, `value` | `X-API-Key: …`, or `?key=…` |
|
|
273
|
+
| `bearer` | `token` | `Authorization: Bearer …` |
|
|
274
|
+
| `basic` | `username`, `password` | `Authorization: Basic …` |
|
|
275
|
+
| `login` | `url` (path or address), `method` (POST), `body`, `tokenPath` (`data.token`), `expiresInPath`, `header`, `prefix` | calls the login first, then `Authorization: Bearer <token>` |
|
|
276
|
+
| `oauth2` | `tokenUrl`, `clientId`, `clientSecret`, `scope`, `audience` | client credentials; then `Authorization: Bearer <access_token>` |
|
|
277
|
+
|
|
278
|
+
- **A token from `login` or `oauth2` is kept in this process** until it runs out (`expires_in` / `expiresInPath`, a minute early; 10 minutes when none is given). A 401 throws it away and the call is tried once more with a fresh one.
|
|
279
|
+
- **Retries:** a network failure, 429 or 5xx is tried again (`retries`, 2), waiting `retryAfterMs` (1 s) and doubling, or what `Retry-After` says. A 4xx other than 401 / 429 is not retried.
|
|
280
|
+
- **A status that is not 2xx fails the step** with `HTTP_<status>` and the start of the answer. `failOnError: false`: it succeeds, so the arrows can branch on `output.status`.
|
|
281
|
+
- **A saved connection signs in to its own address only:** a whole URL on another host is refused (`URL_NOT_ON_CONNECTION`), so the key cannot be sent elsewhere. With no connection, `path` may be a whole `https://` address.
|
|
282
|
+
- **Bodies:** JSON by default; `bodyType: 'form'` or `'text'`. The answer is parsed as JSON when it is JSON, else text.
|
|
283
|
+
- **Not yet:** paging through many pages of results.
|
|
284
|
+
|
|
285
|
+
## File steps, and the folders they may use
|
|
286
|
+
|
|
287
|
+
`file-list`, `file-read` and `file-move` take a path from whoever wrote the workflow, so they work **only inside the folders the install names** (`lib/fileRoots.js`):
|
|
288
|
+
|
|
289
|
+
| | |
|
|
290
|
+
|---|---|
|
|
291
|
+
| `XEPLR_FILE_ROOTS` | Absolute folders, comma-separated: `/data/bank,/data/erp`. |
|
|
292
|
+
| always allowed | `XEPLR_ACTIONS_TMP_DIR` (where downloads and exports land by default), so the next step can read what the last one saved. |
|
|
293
|
+
|
|
294
|
+
- **Paths are absolute.** A relative one is refused: it would mean different files depending on where the server was started.
|
|
295
|
+
- **No way out:** `../` and symbolic links are followed to the real path before it is checked; a target that does not exist yet is checked by its nearest existing folder.
|
|
296
|
+
- **Not defaulted:** with no `XEPLR_FILE_ROOTS`, a path outside the temp folder is refused (`PATH_NOT_ALLOWED`), naming the setting. Why: without it, a workflow could read any file the server can, its own settings included.
|
|
297
|
+
|
|
248
298
|
## Saving files: names, and a name already taken
|
|
249
299
|
|
|
250
300
|
Every action that saves a file follows one rule (`lib/fileNames.js`): `email-download-attachments`, `email-download-email` and `export-table`.
|
|
@@ -461,6 +511,7 @@ await actions.runAction(actions.builtins.dbPush, input, { metaStore: xcfg.metaSt
|
|
|
461
511
|
| name | required? | meaning |
|
|
462
512
|
|---|---|---|
|
|
463
513
|
| `XEPLR_ACTIONS_TMP_DIR` | no | Where `db-fetch` and `email-read` spool NDJSON, email downloads go (unless `outDir`), and the uploader spools batches. Default `<os.tmpdir()>/xeplr-actions`. Files are not cleaned up by `db-fetch` or `email-read`. |
|
|
514
|
+
| `XEPLR_FILE_ROOTS` | for `file-list`, `file-read`, `file-move` outside the temp folder | Absolute folders, comma-separated, that file steps may use. See [File steps](#file-steps-and-the-folders-they-may-use). |
|
|
464
515
|
| `XCFG_DB_NAME` | for `attachConfig` | The `xeplr_configs` database name. In `configRequiredEnv`. |
|
|
465
516
|
| `XCFG_DB_CONNECTION_INFO_ENCRYPTED` | no | Encrypted server login for `attachConfig`; overrides `XEPLR_DB_CONNECTION`. |
|
|
466
517
|
| `XEPLR_DB_CONNECTION` | for `attachConfig`, unless the above is set | Shared encrypted server login. |
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// file-list action — the files in a folder, optionally matching a pattern.
|
|
2
|
+
//
|
|
3
|
+
// Also the way to ask "is it there yet?": list with the exact name and branch
|
|
4
|
+
// on output.count. Only inside the folders file steps may use (lib/fileRoots.js).
|
|
5
|
+
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var fileRoots = require('../../fileRoots');
|
|
9
|
+
|
|
10
|
+
/** `*.csv`, `Bank-2026-??.pdf` → a test on a name. Case-insensitive. */
|
|
11
|
+
function matcher(pattern) {
|
|
12
|
+
if (!pattern) return function() { return true; };
|
|
13
|
+
var re = new RegExp('^' + String(pattern).split('').map(function(ch) {
|
|
14
|
+
if (ch === '*') return '.*';
|
|
15
|
+
if (ch === '?') return '.';
|
|
16
|
+
return ch.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
|
17
|
+
}).join('') + '$', 'i');
|
|
18
|
+
return function(name) { return re.test(name); };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
module.exports = {
|
|
22
|
+
name: 'file-list',
|
|
23
|
+
description: 'The files in a folder, optionally matching a pattern such as *.csv. Returns { files[], count }. ' +
|
|
24
|
+
'Ask "is it there?" with the exact name and branch on count.',
|
|
25
|
+
requires: [],
|
|
26
|
+
inputSchema: [
|
|
27
|
+
{ name: 'folder', type: 'string', required: true, order: 1,
|
|
28
|
+
description: 'An absolute folder inside XEPLR_FILE_ROOTS, e.g. /data/bank.' },
|
|
29
|
+
{ name: 'pattern', type: 'string', order: 2,
|
|
30
|
+
description: 'Only names matching this: * any characters, ? one. E.g. Bank-{trigger.window.from|date:YYYY-MM}*.csv' },
|
|
31
|
+
{ name: 'sort', type: 'string', default: 'name', options: ['name', 'newest', 'oldest'], order: 3,
|
|
32
|
+
description: 'name: A to Z. newest / oldest: by when the file last changed.' }
|
|
33
|
+
],
|
|
34
|
+
outputSchema: [
|
|
35
|
+
{ name: 'files', type: 'array', description: '[{ name, path, size, modifiedAt }]' },
|
|
36
|
+
{ name: 'count', type: 'number', description: 'How many, so the next step can branch on none.' }
|
|
37
|
+
],
|
|
38
|
+
|
|
39
|
+
execute: async function(ctx) {
|
|
40
|
+
var input = ctx.input || {};
|
|
41
|
+
var dir = fileRoots.allowed(input.folder, 'The folder');
|
|
42
|
+
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
|
|
43
|
+
var missing = new Error('"' + input.folder + '" is not a folder that exists.');
|
|
44
|
+
missing.code = 'NOT_A_FOLDER';
|
|
45
|
+
throw missing;
|
|
46
|
+
}
|
|
47
|
+
var test = matcher(input.pattern);
|
|
48
|
+
var files = fs.readdirSync(dir, { withFileTypes: true })
|
|
49
|
+
.filter(function(e) { return e.isFile() && test(e.name); })
|
|
50
|
+
.map(function(e) {
|
|
51
|
+
var full = path.join(dir, e.name);
|
|
52
|
+
var st = fs.statSync(full);
|
|
53
|
+
return { name: e.name, path: full, size: st.size, modifiedAt: st.mtime.toISOString() };
|
|
54
|
+
});
|
|
55
|
+
if (input.sort === 'newest') files.sort(function(a, b) { return b.modifiedAt.localeCompare(a.modifiedAt); });
|
|
56
|
+
else if (input.sort === 'oldest') files.sort(function(a, b) { return a.modifiedAt.localeCompare(b.modifiedAt); });
|
|
57
|
+
else files.sort(function(a, b) { return a.name.localeCompare(b.name); });
|
|
58
|
+
return { files: files, count: files.length };
|
|
59
|
+
}
|
|
60
|
+
};
|
|
@@ -1,8 +1,63 @@
|
|
|
1
|
-
// file-move action —
|
|
2
|
-
// location to another. Source + target can each be local/sharepoint/google
|
|
3
|
-
// (i.e. it uses getSource(sourceType) on both ends).
|
|
1
|
+
// file-move action — move or copy a file, into a folder or to a new name.
|
|
4
2
|
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
3
|
+
// "Processed" folders, an archive by month, a copy for another team. Only
|
|
4
|
+
// inside the folders file steps may use (lib/fileRoots.js), at both ends. A
|
|
5
|
+
// name already taken at the target follows the rule every file-writing step
|
|
6
|
+
// follows (lib/fileNames.js): fail by default.
|
|
7
|
+
|
|
8
|
+
var fs = require('fs');
|
|
9
|
+
var path = require('path');
|
|
10
|
+
var fileRoots = require('../../fileRoots');
|
|
11
|
+
var fileNames = require('../../fileNames');
|
|
12
|
+
|
|
13
|
+
module.exports = {
|
|
14
|
+
name: 'file-move',
|
|
15
|
+
description: 'Move or copy a file into a folder, or to a new name. Returns { path, name, moved, skipped }.',
|
|
16
|
+
requires: [],
|
|
17
|
+
inputSchema: [
|
|
18
|
+
{ name: 'from', type: 'string', required: true, order: 1,
|
|
19
|
+
description: 'The file, by absolute path, e.g. {steps.save.output.path}.' },
|
|
20
|
+
{ name: 'to', type: 'string', required: true, order: 2,
|
|
21
|
+
description: 'A folder (it keeps its name), or a full path to rename it, e.g. /data/archive/{trigger.window.from|date:YYYY-MM}/' },
|
|
22
|
+
{ name: 'operation', type: 'string', default: 'move', options: ['move', 'copy'], order: 3,
|
|
23
|
+
description: 'move: it is no longer where it was. copy: it is in both places.' },
|
|
24
|
+
{ name: 'ifExists', type: 'string', default: 'fail', options: fileNames.IF_EXISTS, order: 4,
|
|
25
|
+
description: 'When the target name is taken. fail: the step fails and nothing moves. skip: left where it is. ' +
|
|
26
|
+
'overwrite: replaced. addSuffix: "name (2).ext".' }
|
|
27
|
+
],
|
|
28
|
+
outputSchema: [
|
|
29
|
+
{ name: 'path', type: 'string' }, { name: 'name', type: 'string' },
|
|
30
|
+
{ name: 'moved', type: 'boolean' }, { name: 'skipped', type: 'boolean' }
|
|
31
|
+
],
|
|
32
|
+
|
|
33
|
+
execute: async function(ctx) {
|
|
34
|
+
var input = ctx.input || {};
|
|
35
|
+
var from = fileRoots.allowed(input.from, 'The file to move');
|
|
36
|
+
if (!fs.existsSync(from) || !fs.statSync(from).isFile()) {
|
|
37
|
+
var missing = new Error('"' + input.from + '" is not a file that exists.');
|
|
38
|
+
missing.code = 'FILE_NOT_FOUND';
|
|
39
|
+
throw missing;
|
|
40
|
+
}
|
|
41
|
+
// A folder: ends with a separator, or is one already.
|
|
42
|
+
var toIsFolder = /[\/\\]$/.test(input.to) || (fs.existsSync(input.to) && fs.statSync(input.to).isDirectory());
|
|
43
|
+
var dir = fileRoots.allowed(toIsFolder ? input.to : path.dirname(input.to), 'The target folder');
|
|
44
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
45
|
+
var wanted = toIsFolder ? path.basename(from) : fileNames.safeName(path.basename(input.to));
|
|
46
|
+
var place = fileNames.plan(dir, [wanted], input.ifExists)[0];
|
|
47
|
+
if (!place.write) return { path: place.path, name: place.name, moved: false, skipped: true };
|
|
48
|
+
|
|
49
|
+
if (input.operation === 'copy') {
|
|
50
|
+
fs.copyFileSync(from, place.path);
|
|
51
|
+
} else {
|
|
52
|
+
try {
|
|
53
|
+
fs.renameSync(from, place.path);
|
|
54
|
+
} catch (err) {
|
|
55
|
+
// Another disk: copy, then remove.
|
|
56
|
+
if (err.code !== 'EXDEV') throw err;
|
|
57
|
+
fs.copyFileSync(from, place.path);
|
|
58
|
+
fs.unlinkSync(from);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return { path: place.path, name: place.name, moved: input.operation !== 'copy', skipped: false };
|
|
62
|
+
}
|
|
63
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// file-read action — a CSV, Excel, JSON or NDJSON file as rows.
|
|
2
|
+
//
|
|
3
|
+
// What the next step works with: the data step filters them, export-table
|
|
4
|
+
// writes them, a matcher compares them. Only inside the folders file steps may
|
|
5
|
+
// use (lib/fileRoots.js). Bounded by maxRows, because a step's output is
|
|
6
|
+
// stored with the run.
|
|
7
|
+
|
|
8
|
+
var path = require('path');
|
|
9
|
+
var fs = require('fs');
|
|
10
|
+
var readline = require('readline');
|
|
11
|
+
var fileRoots = require('../../fileRoots');
|
|
12
|
+
var formats = require('../../formats');
|
|
13
|
+
|
|
14
|
+
var BY_EXTENSION = { csv: 'csv', txt: 'txt', tsv: 'txt', xlsx: 'excel', xls: 'excel', json: 'json', ndjson: 'ndjson', jsonl: 'ndjson' };
|
|
15
|
+
|
|
16
|
+
async function* ndjsonRows(file) {
|
|
17
|
+
var lines = readline.createInterface({ input: fs.createReadStream(file), crlfDelay: Infinity });
|
|
18
|
+
for await (var line of lines) if (line.trim()) yield JSON.parse(line);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
module.exports = {
|
|
22
|
+
name: 'file-read',
|
|
23
|
+
description: 'Read a CSV, Excel (.xlsx), JSON or NDJSON file into rows. Returns { rows, count, columns, truncated }.',
|
|
24
|
+
requires: [],
|
|
25
|
+
inputSchema: [
|
|
26
|
+
{ name: 'path', type: 'string', required: true, order: 1,
|
|
27
|
+
description: 'An absolute file path inside XEPLR_FILE_ROOTS, e.g. {steps.save.output.path}.' },
|
|
28
|
+
{ name: 'format', type: 'string', order: 2, options: ['csv', 'txt', 'excel', 'json', 'ndjson'],
|
|
29
|
+
description: 'Empty: from the file\'s extension.' },
|
|
30
|
+
{ name: 'sheet', type: 'string', order: 3, group: 'Excel',
|
|
31
|
+
description: 'Which sheet. Empty: the first.' },
|
|
32
|
+
{ name: 'maxRows', type: 'number', default: 50000, order: 4,
|
|
33
|
+
description: 'Rows to read before stopping (truncated: true). A step\'s rows are stored with the run.' }
|
|
34
|
+
],
|
|
35
|
+
outputSchema: [
|
|
36
|
+
{ name: 'rows', type: 'array' }, { name: 'count', type: 'number' },
|
|
37
|
+
{ name: 'columns', type: 'array', description: 'The file\'s columns, in order: [{ key, header }].' },
|
|
38
|
+
{ name: 'truncated', type: 'boolean' }
|
|
39
|
+
],
|
|
40
|
+
|
|
41
|
+
execute: async function(ctx) {
|
|
42
|
+
var input = ctx.input || {};
|
|
43
|
+
var file = fileRoots.allowed(input.path, 'The file');
|
|
44
|
+
if (!fs.existsSync(file) || !fs.statSync(file).isFile()) {
|
|
45
|
+
var missing = new Error('"' + input.path + '" is not a file that exists.');
|
|
46
|
+
missing.code = 'FILE_NOT_FOUND';
|
|
47
|
+
throw missing;
|
|
48
|
+
}
|
|
49
|
+
var ext = path.extname(file).slice(1).toLowerCase();
|
|
50
|
+
var kind = input.format || BY_EXTENSION[ext];
|
|
51
|
+
if (!kind) {
|
|
52
|
+
var unknown = new Error('Cannot tell how to read ".' + ext + '". Set format: csv, txt, excel, json or ndjson.');
|
|
53
|
+
unknown.code = 'FORMAT_UNKNOWN';
|
|
54
|
+
throw unknown;
|
|
55
|
+
}
|
|
56
|
+
var source;
|
|
57
|
+
if (kind === 'ndjson') {
|
|
58
|
+
source = ndjsonRows(file);
|
|
59
|
+
} else {
|
|
60
|
+
var reader = formats.getFormat(kind);
|
|
61
|
+
formats.checkFormatRequires('file-read', kind, reader);
|
|
62
|
+
var opts = input.sheet ? { sheet: input.sheet } : {};
|
|
63
|
+
source = reader.inputMode === 'path' ? reader.parsePath(file, opts) : reader.parseStream(fs.createReadStream(file), opts);
|
|
64
|
+
}
|
|
65
|
+
var max = Math.max(1, Number(input.maxRows) || 50000);
|
|
66
|
+
var rows = [];
|
|
67
|
+
var truncated = false;
|
|
68
|
+
for await (var row of source) {
|
|
69
|
+
if (rows.length >= max) { truncated = true; break; }
|
|
70
|
+
rows.push(row);
|
|
71
|
+
}
|
|
72
|
+
// The order of the file's own columns, said out loud: a step's output is
|
|
73
|
+
// stored as JSONB, which reorders a row's keys.
|
|
74
|
+
var columns = rows.length ? Object.keys(rows[0]).map(function(k) { return { key: k, header: k }; }) : [];
|
|
75
|
+
return { rows: rows, count: rows.length, columns: columns, truncated: truncated };
|
|
76
|
+
}
|
|
77
|
+
};
|
|
@@ -1,3 +1,271 @@
|
|
|
1
|
-
// http-request action —
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// http-request action — call another system's API.
|
|
2
|
+
//
|
|
3
|
+
// One call: a method, a path on the API, query values, headers and a body,
|
|
4
|
+
// and the answer back as { status, ok, body, headers }. The body is parsed
|
|
5
|
+
// JSON when the answer says it is JSON, else text.
|
|
6
|
+
//
|
|
7
|
+
// WHERE AND AS WHOM come from a saved connection (`connectionInfoId`), which
|
|
8
|
+
// the host turns into `connection` just before the call (resolveConnections):
|
|
9
|
+
//
|
|
10
|
+
// { baseUrl: 'https://api.bank.example/v2',
|
|
11
|
+
// headers?: { 'X-Tenant': 'acme' },
|
|
12
|
+
// auth: { type: 'none' }
|
|
13
|
+
// | { type: 'apiKey', in: 'header' | 'query', name: 'X-API-Key', value }
|
|
14
|
+
// | { type: 'bearer', token }
|
|
15
|
+
// | { type: 'basic', username, password }
|
|
16
|
+
// | { type: 'login', url, method?, body?, tokenPath, header?, prefix?, expiresInPath? }
|
|
17
|
+
// | { type: 'oauth2', tokenUrl, clientId, clientSecret, scope?, audience? } }
|
|
18
|
+
//
|
|
19
|
+
// so a workflow never holds a key or a password, and a step's recorded input
|
|
20
|
+
// holds only the id. Without a saved connection, `path` may be a whole
|
|
21
|
+
// address (a public API that needs no sign-in).
|
|
22
|
+
//
|
|
23
|
+
// A TOKEN FROM A LOGIN CALL OR OAUTH is fetched once and kept in this process
|
|
24
|
+
// until it runs out (or, with no expiry given, for 10 minutes). A 401 throws
|
|
25
|
+
// it away and the call is tried once more with a fresh one.
|
|
26
|
+
//
|
|
27
|
+
// RETRIES: a network failure, 429 or 5xx is tried again (`retries`, default
|
|
28
|
+
// 2), waiting longer each time, or as long as Retry-After asks. A 4xx other
|
|
29
|
+
// than 401/429 is the API refusing the request and is not retried.
|
|
30
|
+
//
|
|
31
|
+
// A STATUS THAT IS NOT 2xx fails the step (`failOnError`, default true),
|
|
32
|
+
// saying the status and the start of the answer. Off: the step succeeds and
|
|
33
|
+
// the arrows can branch on output.status.
|
|
34
|
+
|
|
35
|
+
var crypto = require('crypto');
|
|
36
|
+
|
|
37
|
+
var TOKEN_DEFAULT_MS = 10 * 60 * 1000;
|
|
38
|
+
var tokens = {}; // key → { value, until }
|
|
39
|
+
|
|
40
|
+
function httpError(message, code, extra) {
|
|
41
|
+
var err = new Error(message);
|
|
42
|
+
err.code = code;
|
|
43
|
+
Object.assign(err, extra || {});
|
|
44
|
+
return err;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function sleep(ms) { return new Promise(function(r) { setTimeout(r, ms); }); }
|
|
48
|
+
|
|
49
|
+
/** A value at a dotted path in an object: 'data.token' → obj.data.token. */
|
|
50
|
+
function at(obj, path) {
|
|
51
|
+
return String(path || '').split('.').filter(Boolean).reduce(function(o, k) { return o == null ? undefined : o[k]; }, obj);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The full address: a path is joined to the connection's base; a whole URL must stay on it. */
|
|
55
|
+
function addressOf(conn, input) {
|
|
56
|
+
var raw = String(input.path || '').trim();
|
|
57
|
+
var base = conn && conn.baseUrl ? String(conn.baseUrl).replace(/\/+$/, '') : '';
|
|
58
|
+
var url;
|
|
59
|
+
if (/^https?:\/\//i.test(raw)) {
|
|
60
|
+
url = new URL(raw);
|
|
61
|
+
// A saved connection signs in to ITS address only: a whole URL elsewhere
|
|
62
|
+
// would send the key to whoever is there.
|
|
63
|
+
if (base && url.origin !== new URL(base).origin) {
|
|
64
|
+
throw httpError('"' + raw + '" is not on this connection\'s address (' + base + '). Give a path, e.g. /statements.', 'URL_NOT_ON_CONNECTION');
|
|
65
|
+
}
|
|
66
|
+
} else {
|
|
67
|
+
if (!base) throw httpError('Give a whole address (https://…), or pick a saved connection and give a path.', 'URL_REQUIRED');
|
|
68
|
+
url = new URL(base + (raw ? '/' + raw.replace(/^\/+/, '') : ''));
|
|
69
|
+
}
|
|
70
|
+
if (url.protocol !== 'https:' && url.protocol !== 'http:') throw httpError('Only http and https addresses.', 'URL_REQUIRED');
|
|
71
|
+
Object.keys(input.query || {}).forEach(function(k) {
|
|
72
|
+
var v = input.query[k];
|
|
73
|
+
if (v !== undefined && v !== null && v !== '') url.searchParams.set(k, typeof v === 'object' ? JSON.stringify(v) : String(v));
|
|
74
|
+
});
|
|
75
|
+
return url;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** A body as fetch sends it, and its content type. */
|
|
79
|
+
function bodyOf(input) {
|
|
80
|
+
if (input.body === undefined || input.body === null || input.body === '') return { body: undefined };
|
|
81
|
+
var kind = input.bodyType || (typeof input.body === 'string' ? 'text' : 'json');
|
|
82
|
+
if (kind === 'json') return { body: typeof input.body === 'string' ? input.body : JSON.stringify(input.body), type: 'application/json' };
|
|
83
|
+
if (kind === 'form') {
|
|
84
|
+
var form = new URLSearchParams();
|
|
85
|
+
Object.keys(input.body || {}).forEach(function(k) { form.set(k, String(input.body[k])); });
|
|
86
|
+
return { body: form.toString(), type: 'application/x-www-form-urlencoded' };
|
|
87
|
+
}
|
|
88
|
+
return { body: String(input.body), type: 'text/plain' };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function readBody(res) {
|
|
92
|
+
var text = await res.text();
|
|
93
|
+
var type = res.headers.get('content-type') || '';
|
|
94
|
+
if (/json/i.test(type) || /^\s*[\[{]/.test(text)) {
|
|
95
|
+
try { return JSON.parse(text); } catch (e) { /* not JSON after all */ }
|
|
96
|
+
}
|
|
97
|
+
return text;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function cacheKey(conn) {
|
|
101
|
+
return crypto.createHash('sha256').update(JSON.stringify([conn.baseUrl, conn.auth])).digest('hex');
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** A token from the connection's login call or OAuth, kept until it runs out. */
|
|
105
|
+
async function tokenFor(conn, timeoutMs) {
|
|
106
|
+
var key = cacheKey(conn);
|
|
107
|
+
var kept = tokens[key];
|
|
108
|
+
if (kept && kept.until > Date.now()) return kept.value;
|
|
109
|
+
var auth = conn.auth;
|
|
110
|
+
var res, answer, token, expiresIn;
|
|
111
|
+
if (auth.type === 'oauth2') {
|
|
112
|
+
var form = new URLSearchParams({ grant_type: 'client_credentials', client_id: auth.clientId || '', client_secret: auth.clientSecret || '' });
|
|
113
|
+
if (auth.scope) form.set('scope', auth.scope);
|
|
114
|
+
if (auth.audience) form.set('audience', auth.audience);
|
|
115
|
+
res = await timed(auth.tokenUrl, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', accept: 'application/json' }, body: form.toString() }, timeoutMs);
|
|
116
|
+
answer = await readBody(res);
|
|
117
|
+
token = answer && answer.access_token;
|
|
118
|
+
expiresIn = answer && answer.expires_in;
|
|
119
|
+
} else {
|
|
120
|
+
var loginUrl = /^https?:\/\//i.test(auth.url || '') ? auth.url : String(conn.baseUrl || '').replace(/\/+$/, '') + '/' + String(auth.url || '').replace(/^\/+/, '');
|
|
121
|
+
var b = auth.body === undefined ? undefined : (typeof auth.body === 'string' ? auth.body : JSON.stringify(auth.body));
|
|
122
|
+
res = await timed(loginUrl, { method: auth.method || 'POST', headers: { 'content-type': 'application/json', accept: 'application/json' }, body: b }, timeoutMs);
|
|
123
|
+
answer = await readBody(res);
|
|
124
|
+
token = at(answer, auth.tokenPath || 'token');
|
|
125
|
+
expiresIn = auth.expiresInPath ? at(answer, auth.expiresInPath) : undefined;
|
|
126
|
+
}
|
|
127
|
+
if (!res.ok || !token) {
|
|
128
|
+
tokens[key] = undefined;
|
|
129
|
+
throw httpError('Signing in to ' + (conn.baseUrl || 'the API') + ' failed: ' + res.status +
|
|
130
|
+
(token ? '' : ', and no token in the answer' + (auth.type === 'login' ? ' at "' + (auth.tokenPath || 'token') + '"' : '')) + '.', 'SIGN_IN_FAILED', { status: res.status });
|
|
131
|
+
}
|
|
132
|
+
// A minute early, so a token never runs out mid-call.
|
|
133
|
+
var lifeMs = Number(expiresIn) > 0 ? Math.max(0, Number(expiresIn) * 1000 - 60000) : TOKEN_DEFAULT_MS;
|
|
134
|
+
tokens[key] = { value: String(token), until: Date.now() + lifeMs };
|
|
135
|
+
return String(token);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function timed(url, init, timeoutMs) {
|
|
139
|
+
var controller = new AbortController();
|
|
140
|
+
var timer = setTimeout(function() { controller.abort(); }, timeoutMs);
|
|
141
|
+
if (timer.unref) timer.unref();
|
|
142
|
+
try {
|
|
143
|
+
return await fetch(url, Object.assign({}, init, { signal: controller.signal }));
|
|
144
|
+
} catch (err) {
|
|
145
|
+
var why = err.name === 'AbortError' ? 'no answer within ' + timeoutMs + ' ms' : (err.cause && err.cause.code) || err.message;
|
|
146
|
+
throw httpError('Could not reach ' + String(url).split('?')[0] + ': ' + why + '.', 'UNREACHABLE', { transient: true });
|
|
147
|
+
} finally {
|
|
148
|
+
clearTimeout(timer);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Headers for one call: the connection's, the sign-in's, then the step's own. */
|
|
153
|
+
async function headersFor(conn, input, url, timeoutMs) {
|
|
154
|
+
var h = { accept: 'application/json, text/plain;q=0.9, */*;q=0.8' };
|
|
155
|
+
Object.assign(h, (conn && conn.headers) || {});
|
|
156
|
+
var auth = (conn && conn.auth) || { type: 'none' };
|
|
157
|
+
if (auth.type === 'apiKey') {
|
|
158
|
+
if (auth.in === 'query') url.searchParams.set(auth.name || 'api_key', auth.value || '');
|
|
159
|
+
else h[auth.name || 'X-API-Key'] = auth.value || '';
|
|
160
|
+
} else if (auth.type === 'bearer') {
|
|
161
|
+
h.authorization = 'Bearer ' + (auth.token || '');
|
|
162
|
+
} else if (auth.type === 'basic') {
|
|
163
|
+
h.authorization = 'Basic ' + Buffer.from((auth.username || '') + ':' + (auth.password || '')).toString('base64');
|
|
164
|
+
} else if (auth.type === 'login' || auth.type === 'oauth2') {
|
|
165
|
+
var token = await tokenFor(conn, timeoutMs);
|
|
166
|
+
var name = auth.header || 'authorization';
|
|
167
|
+
h[name] = (auth.prefix === undefined ? 'Bearer ' : auth.prefix) + token;
|
|
168
|
+
} else if (auth.type && auth.type !== 'none') {
|
|
169
|
+
throw httpError('"' + auth.type + '" is not a way of signing in. Kinds: none, apiKey, bearer, basic, login, oauth2.', 'AUTH_UNKNOWN');
|
|
170
|
+
}
|
|
171
|
+
Object.assign(h, input.headers || {});
|
|
172
|
+
return h;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** How long to wait before try `n`: Retry-After when the API says, else 1s, 2s, 4s… */
|
|
176
|
+
function waitFor(res, n, baseMs) {
|
|
177
|
+
var after = res && res.headers && res.headers.get('retry-after');
|
|
178
|
+
if (after && !isNaN(Number(after))) return Math.min(60000, Number(after) * 1000);
|
|
179
|
+
return baseMs * Math.pow(2, n - 1);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
module.exports = {
|
|
183
|
+
name: 'http-request',
|
|
184
|
+
description: 'Call another system\'s API: a method, a path, query values, headers and a body. ' +
|
|
185
|
+
'Signs in with a saved connection. Returns { status, ok, body, headers }.',
|
|
186
|
+
requires: [],
|
|
187
|
+
inputSchema: [
|
|
188
|
+
{ name: 'connectionInfoId', type: 'string', order: 1, optionsFrom: 'apis',
|
|
189
|
+
description: 'A saved API connection: its address and how to sign in. Resolved on the server.' },
|
|
190
|
+
{ name: 'connection', type: 'object', order: 2, system: true,
|
|
191
|
+
description: 'Filled from the saved connection: { baseUrl, headers?, auth }.' },
|
|
192
|
+
{ name: 'method', type: 'string', default: 'GET', order: 3, options: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'],
|
|
193
|
+
description: 'GET reads; POST sends something new; PUT / PATCH change; DELETE removes.' },
|
|
194
|
+
{ name: 'path', type: 'string', order: 4,
|
|
195
|
+
description: 'Where on the API, e.g. /statements. Or a whole address (https://…) with no saved connection.' },
|
|
196
|
+
{ name: 'query', type: 'object', order: 5,
|
|
197
|
+
description: 'Values for the address, e.g. { "from": "{trigger.window.from|date:YYYY-MM-DD}" }.' },
|
|
198
|
+
{ name: 'body', type: 'object', order: 6,
|
|
199
|
+
description: 'What to send with POST / PUT / PATCH, e.g. { "rows": "{steps.match.output.rows}" }.' },
|
|
200
|
+
{ name: 'bodyType', type: 'string', default: 'json', order: 7, group: 'Advanced', options: ['json', 'form', 'text'],
|
|
201
|
+
description: 'How the body is sent: json, a form, or plain text.' },
|
|
202
|
+
{ name: 'headers', type: 'object', order: 8, group: 'Advanced',
|
|
203
|
+
description: 'Extra headers for this call. The sign-in is added from the connection.' },
|
|
204
|
+
{ name: 'failOnError', type: 'boolean', default: true, order: 9, group: 'Advanced',
|
|
205
|
+
description: 'On: a status that is not 2xx fails the step. Off: the step succeeds and the arrows branch on output.status.' },
|
|
206
|
+
{ name: 'retries', type: 'number', default: 2, order: 10, group: 'Advanced',
|
|
207
|
+
description: 'Tries again on a network failure, 429 or 5xx, waiting longer each time.' },
|
|
208
|
+
{ name: 'retryAfterMs', type: 'number', default: 1000, order: 11, group: 'Advanced',
|
|
209
|
+
description: 'The first wait between tries; it doubles each time, unless the API says Retry-After.' },
|
|
210
|
+
{ name: 'timeoutMs', type: 'number', default: 30000, order: 12, group: 'Advanced',
|
|
211
|
+
description: 'How long to wait for an answer.' }
|
|
212
|
+
],
|
|
213
|
+
outputSchema: [
|
|
214
|
+
{ name: 'status', type: 'number', description: 'The HTTP status, e.g. 200.' },
|
|
215
|
+
{ name: 'ok', type: 'boolean', description: 'Whether it was 2xx.' },
|
|
216
|
+
{ name: 'body', type: 'object', description: 'The answer: parsed JSON, or text.' },
|
|
217
|
+
{ name: 'headers', type: 'object', description: 'The answer\'s content-type, and any rate-limit or paging headers.' }
|
|
218
|
+
],
|
|
219
|
+
|
|
220
|
+
execute: async function(ctx) {
|
|
221
|
+
var input = ctx.input || {};
|
|
222
|
+
var conn = input.connection || null;
|
|
223
|
+
if (input.connectionInfoId && !conn) throw httpError('The saved connection was not resolved.', 'CONNECTION_NOT_FOUND');
|
|
224
|
+
var method = String(input.method || 'GET').toUpperCase();
|
|
225
|
+
var timeoutMs = Math.max(1000, Number(input.timeoutMs) || 30000);
|
|
226
|
+
var retries = Math.max(0, Math.min(10, input.retries == null ? 2 : Number(input.retries)));
|
|
227
|
+
var baseWait = Math.max(0, input.retryAfterMs == null ? 1000 : Number(input.retryAfterMs));
|
|
228
|
+
var sent = method === 'GET' || method === 'DELETE' ? { body: undefined } : bodyOf(input);
|
|
229
|
+
|
|
230
|
+
var refreshed = false;
|
|
231
|
+
var res;
|
|
232
|
+
for (var attempt = 0; ; attempt++) {
|
|
233
|
+
var url = addressOf(conn, input);
|
|
234
|
+
var headers = await headersFor(conn, input, url, timeoutMs);
|
|
235
|
+
if (sent.type && !Object.keys(headers).some(function(k) { return k.toLowerCase() === 'content-type'; })) headers['content-type'] = sent.type;
|
|
236
|
+
try {
|
|
237
|
+
res = await timed(url.toString(), { method: method, headers: headers, body: sent.body }, timeoutMs);
|
|
238
|
+
} catch (err) {
|
|
239
|
+
if (err.transient && attempt < retries) { await sleep(baseWait * Math.pow(2, attempt)); continue; }
|
|
240
|
+
throw err;
|
|
241
|
+
}
|
|
242
|
+
// A kept token the API no longer takes: throw it away and sign in again, once.
|
|
243
|
+
if (res.status === 401 && conn && conn.auth && (conn.auth.type === 'login' || conn.auth.type === 'oauth2') && !refreshed) {
|
|
244
|
+
tokens[cacheKey(conn)] = undefined;
|
|
245
|
+
refreshed = true;
|
|
246
|
+
attempt--;
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if ((res.status === 429 || res.status >= 500) && attempt < retries) {
|
|
250
|
+
await res.text().catch(function() {});
|
|
251
|
+
await sleep(waitFor(res, attempt + 1, baseWait));
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
break;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
var body = await readBody(res);
|
|
258
|
+
var kept = {};
|
|
259
|
+
res.headers.forEach(function(v, k) {
|
|
260
|
+
if (k === 'content-type' || k === 'link' || /^x-(rate|ratelimit|total|page|next)/i.test(k) || /^ratelimit/i.test(k)) kept[k] = v;
|
|
261
|
+
});
|
|
262
|
+
if (!res.ok && input.failOnError !== false) {
|
|
263
|
+
var said = typeof body === 'string' ? body : JSON.stringify(body);
|
|
264
|
+
throw httpError(method + ' ' + url.pathname + ' answered ' + res.status + (said ? ': ' + said.slice(0, 300) : '') + '.', 'HTTP_' + res.status, { status: res.status });
|
|
265
|
+
}
|
|
266
|
+
return { status: res.status, ok: res.ok, body: body, headers: kept };
|
|
267
|
+
},
|
|
268
|
+
|
|
269
|
+
// For tests: forget every kept token.
|
|
270
|
+
_forgetTokens: function() { tokens = {}; }
|
|
271
|
+
};
|
package/lib/builtins/index.js
CHANGED
|
@@ -48,6 +48,8 @@ module.exports = {
|
|
|
48
48
|
// File actions — factory-shaped for the same reason (meta store binding).
|
|
49
49
|
fileUpload: require('./file/upload'),
|
|
50
50
|
fileMove: require('./file/move'),
|
|
51
|
+
fileList: require('./file/list'),
|
|
52
|
+
fileRead: require('./file/read'),
|
|
51
53
|
|
|
52
54
|
// Email actions. Inbound routes to getInboundProvider(input.emailType)
|
|
53
55
|
// (imap today), outbound to getOutboundProvider (smtp) — see
|
package/lib/fileRoots.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// WHERE A FILE STEP MAY LOOK.
|
|
2
|
+
//
|
|
3
|
+
// file-list, file-read and file-move take a path from whoever wrote the
|
|
4
|
+
// workflow. Unbounded, that is every file the server can read: its own
|
|
5
|
+
// settings, other installs' data. So they work only inside the folders the
|
|
6
|
+
// install names:
|
|
7
|
+
//
|
|
8
|
+
// XEPLR_FILE_ROOTS absolute folders, comma-separated, e.g. /data/bank,/data/erp
|
|
9
|
+
//
|
|
10
|
+
// plus XEPLR_ACTIONS_TMP_DIR (or the system temp folder's xeplr-actions),
|
|
11
|
+
// where downloads and exports land by default, so the file a step just saved
|
|
12
|
+
// can be read by the next one.
|
|
13
|
+
//
|
|
14
|
+
// NOT DEFAULTED: no roots configured and a path outside the temp folder is
|
|
15
|
+
// refused, naming the variable. "Where data lives is never defaulted."
|
|
16
|
+
|
|
17
|
+
var fs = require('fs');
|
|
18
|
+
var os = require('os');
|
|
19
|
+
var path = require('path');
|
|
20
|
+
|
|
21
|
+
function rootsError(message) {
|
|
22
|
+
var err = new Error(message);
|
|
23
|
+
err.code = 'PATH_NOT_ALLOWED';
|
|
24
|
+
err.status = 400;
|
|
25
|
+
return err;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function tmpRoot() {
|
|
29
|
+
return path.resolve(process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions'));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The folders a file step may use, resolved and real (symlinks followed). */
|
|
33
|
+
function roots() {
|
|
34
|
+
var listed = String(process.env.XEPLR_FILE_ROOTS || '').split(',')
|
|
35
|
+
.map(function(r) { return r.trim(); })
|
|
36
|
+
.filter(Boolean);
|
|
37
|
+
listed.forEach(function(r) {
|
|
38
|
+
if (!path.isAbsolute(r)) throw rootsError('XEPLR_FILE_ROOTS must list absolute folders, and "' + r + '" is not one.');
|
|
39
|
+
});
|
|
40
|
+
return listed.concat([tmpRoot()]).map(function(r) {
|
|
41
|
+
try { return fs.realpathSync(r); } catch (e) { return path.resolve(r); }
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The path, absolute and real, when it is inside an allowed folder; refused
|
|
47
|
+
* otherwise. A path that does not exist yet (a move's target) is checked by
|
|
48
|
+
* its nearest existing parent, so "../" and symlinks cannot step outside.
|
|
49
|
+
*/
|
|
50
|
+
function allowed(p, what) {
|
|
51
|
+
if (!p || typeof p !== 'string') throw rootsError((what || 'A path') + ' is required.');
|
|
52
|
+
if (!path.isAbsolute(p)) throw rootsError((what || 'The path') + ' "' + p + '" must be absolute, e.g. /data/bank/statement.csv.');
|
|
53
|
+
var resolved = path.resolve(p);
|
|
54
|
+
var probe = resolved;
|
|
55
|
+
var rest = '';
|
|
56
|
+
while (!fs.existsSync(probe) && path.dirname(probe) !== probe) {
|
|
57
|
+
rest = path.join(path.basename(probe), rest);
|
|
58
|
+
probe = path.dirname(probe);
|
|
59
|
+
}
|
|
60
|
+
var real = path.join(fs.realpathSync(probe), rest);
|
|
61
|
+
var ok = roots().some(function(r) { return real === r || real.indexOf(r + path.sep) === 0; });
|
|
62
|
+
if (!ok) {
|
|
63
|
+
throw rootsError((what || 'The path') + ' "' + p + '" is outside the folders file steps may use. ' +
|
|
64
|
+
(process.env.XEPLR_FILE_ROOTS
|
|
65
|
+
? 'Allowed: ' + process.env.XEPLR_FILE_ROOTS + ' and the temp folder.'
|
|
66
|
+
: 'Set XEPLR_FILE_ROOTS to the folders they may use (absolute, comma-separated).'));
|
|
67
|
+
}
|
|
68
|
+
return real;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
module.exports = { allowed: allowed, roots: roots, tmpRoot: tmpRoot };
|
package/package.json
CHANGED