@xeplr/actions 1.5.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -13
- package/index.js +8 -1
- package/lib/builtins/db/fetch.js +3 -4
- package/lib/builtins/db/push.js +1 -1
- package/lib/builtins/email/download-attachments.js +1 -1
- package/lib/builtins/email/download-email.js +1 -1
- package/lib/builtins/email/read.js +3 -4
- package/lib/builtins/export/pages.js +5 -4
- package/lib/builtins/export/table.js +6 -9
- package/lib/builtins/file/delete.js +5 -5
- package/lib/builtins/file/list.js +4 -4
- package/lib/builtins/file/move.js +8 -7
- package/lib/builtins/file/read.js +5 -5
- package/lib/builtins/index.js +1 -1
- package/lib/builtins/send-email.js +41 -5
- package/lib/dataDir.js +135 -0
- package/lib/drivers/email/files.js +6 -5
- package/lib/drivers/file/local.js +16 -10
- package/lib/uploader/index.js +3 -4
- package/package.json +2 -2
- package/lib/fileRoots.js +0 -71
package/README.md
CHANGED
|
@@ -125,7 +125,7 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
|
|
|
125
125
|
| `emailDelete` | `email-delete` | Flag `\Deleted` **and expunge** — permanent, not a move to Trash. |
|
|
126
126
|
| `emailDownloadEmail` | `email-download-email` | Stream the raw message to an `.eml` file, named by `fileName`. `{ count, saved[], path, existing[] }`. |
|
|
127
127
|
| `emailDownloadAttachments` | `email-download-attachments` | Save attachments, optionally filtered by `extension` / `filenameContains`, named by `fileName` (see [Saving files](#saving-files-names-and-a-name-already-taken)). `{ count, saved[], path, skipped, existing[] }`. |
|
|
128
|
-
| `fileList` | `file-list` | The files in a folder, optionally matching `pattern` (`*.csv`, `Bank-2026-??.pdf`), sorted by name or date. `{ files[{ name, path, size, modifiedAt }], count }`. "Is it there?" is the exact name and `count`. See [
|
|
128
|
+
| `fileList` | `file-list` | The files in a folder, optionally matching `pattern` (`*.csv`, `Bank-2026-??.pdf`), sorted by name or date. `{ files[{ name, path, size, modifiedAt }], count }`. "Is it there?" is the exact name and `count`. See [Where files are](#where-files-are-one-data-root-a-folder-per-tenant). |
|
|
129
129
|
| `fileRead` | `file-read` | A CSV, text, Excel, JSON or NDJSON file as rows (format from the extension). `{ rows, count, columns, truncated }`, at most `maxRows` (50,000). |
|
|
130
130
|
| `fileMove` | `file-move` | Move or copy a file into a folder (made if missing) or to a new name; `ifExists` as every saving step. `{ path, name, moved, skipped }`. |
|
|
131
131
|
| `fileDelete` | `file-delete` | Delete one file, **permanently** (no trash; `file-move` to a "processed" folder is the reversible one). Never a folder. `ifMissing`: `fail` (default) or `skip`. `{ path, deleted, skipped }`. |
|
|
@@ -283,18 +283,28 @@ await runAction({ name: 'http-request', input: {
|
|
|
283
283
|
- **Bodies:** JSON by default; `bodyType: 'form'` or `'text'`. The answer is parsed as JSON when it is JSON, else text.
|
|
284
284
|
- **Not yet:** paging through many pages of results.
|
|
285
285
|
|
|
286
|
-
##
|
|
286
|
+
## Where files are: one data root, a folder per tenant
|
|
287
287
|
|
|
288
|
-
`
|
|
288
|
+
**`XEPLR_DATA_DIR` is required** (`requiredEnv` — spread it into your `env.required.js`). Every file any step reads or writes is inside the folder of the tenant the work runs as, and nowhere else (`lib/dataDir.js`):
|
|
289
289
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
290
|
+
```
|
|
291
|
+
<XEPLR_DATA_DIR>/<applicationId>/<mtId1>/<mtId2>/…
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
- **The application** is the one registered with `@xeplr/db`'s `registerApplication()`, which a host calls at boot. A company id is minted per app, so company 42 of BI and company 42 of ERP never share a folder. No application registered → refused.
|
|
295
|
+
- **The tenant** comes from the context every query runs in: a request after the membership gate, a job, or a workflow run (`runWithMt`). **Never from a step's input.** An app with no tenancy registered is one tenant: `<root>/<app>`.
|
|
296
|
+
- **A step names files inside its folder:** `bank/inbox/sept.csv`. A full path is accepted only when it is already inside, such as what an earlier step returned (`file-list`'s `path`). Anything else is refused (`PATH_NOT_ALLOWED`): a path elsewhere, `../`, a symbolic link pointing out, or an address (`http://…`, `file://…`). `../` and links are followed to the real path before the check; a path that does not exist yet is checked by its nearest existing folder.
|
|
297
|
+
- **`*` at a level** (a job for every company) is a folder of its own, `_all`, which no company's step reaches. **No tenant at all is refused** (`NO_TENANT`) — never the root — as a query with no tenant finds nothing.
|
|
298
|
+
- **Temporary files** (db-fetch and email-read spools, uploader batches, an export or email download nobody named) go in the tenant's `tmp/`.
|
|
299
|
+
- **Not defaulted:** a missing or relative `XEPLR_DATA_DIR`, or one that does not exist, is refused by name (`DATA_DIR_UNSET`). A wrong root that boots would read as empty data rather than as an error.
|
|
300
|
+
|
|
301
|
+
Which inputs are checked: `file-list` `folder`; `file-read` / `file-delete` `path`; `file-move` `from` and `to`; `file-upload`'s local source (`sourcePath`, and every local-source operation); `db-push` `filePath`; `export-table` `rowsFile` and `outputPath`; `export-pages` `outputPath`; the email download steps' folder; and `email-send` attachments.
|
|
302
|
+
|
|
303
|
+
**`email-send` attachments** are `{ filename, path }` (inside the folder) or `{ filename, content }`. `href` and `raw` are refused, and so is a body that is not text: the mail library reads `path`, `href` and `html: { path }` from disk or from the network, so an unchecked one would mail the server's own settings or fetch an internal address.
|
|
304
|
+
|
|
305
|
+
`dataDir` is exported (`resolve(name)`, `tenantFolder()`, `scratch()`, `nameOf(path)`) for a host's own code that keeps files per tenant.
|
|
294
306
|
|
|
295
|
-
|
|
296
|
-
- **No way out:** `../` and symbolic links are followed to the real path before it is checked; a target that does not exist yet is checked by its nearest existing folder.
|
|
297
|
-
- **Not defaulted:** with no `XEPLR_FILE_ROOTS`, a path outside the temp folder is refused (`PATH_NOT_ALLOWED`), naming the setting. Why: without it, a workflow could read any file the server can, its own settings included.
|
|
307
|
+
**From 1.x:** `XEPLR_FILE_ROOTS` and `XEPLR_ACTIONS_TMP_DIR` are gone. Set `XEPLR_DATA_DIR`, call `registerApplication()`, and run actions inside a tenant context (jobs and the workflow engine already do). A saved path outside the tenant's folder is refused until its files are moved in.
|
|
298
308
|
|
|
299
309
|
## Saving files: names, and a name already taken
|
|
300
310
|
|
|
@@ -358,7 +368,7 @@ await runAction(builtins.exportTable, {
|
|
|
358
368
|
|
|
359
369
|
- **Streaming.** CSV and Excel stream rows to disk, from `rows` or from `rowsFile` (NDJSON, one row per line, as `streaming.spool` writes it); 200,000 rows to xlsx take about 2 s in flat memory. Our own xlsx writer runs on `fflate`. `exceljs` was rejected: it pulls a `uuid` with a published advisory and has had no release since December 2024.
|
|
360
370
|
- **Fonts.** pdfkit's built-in fonts cannot draw `₹` or non-Latin text; a missing glyph draws as nothing. The PDF embeds **DejaVu Sans Condensed** (the `dejavu-fonts-ttf` peer). Pass `document.font` / `boldFont` (TTF/OTF paths) for scripts it lacks. Only the glyphs used are embedded.
|
|
361
|
-
- **Output** goes to `outputPath
|
|
371
|
+
- **Output** goes to `outputPath` (a name inside the tenant's folder), or to a new directory in its `tmp/` with a file named after the title. A failed export leaves no half-written file.
|
|
362
372
|
|
|
363
373
|
`export-pages` lays out `pages: [{ blocks }]`, each block positioned by **fractions** of the page's content box (A4 landscape by default):
|
|
364
374
|
- `text` (`text`, `size`, `bold`, `color`, `align`)
|
|
@@ -511,8 +521,7 @@ await actions.runAction(actions.builtins.dbPush, input, { metaStore: xcfg.metaSt
|
|
|
511
521
|
|
|
512
522
|
| name | required? | meaning |
|
|
513
523
|
|---|---|---|
|
|
514
|
-
| `
|
|
515
|
-
| `XEPLR_FILE_ROOTS` | for `file-list`, `file-read`, `file-move`, `file-delete` outside the temp folder | Absolute folders, comma-separated, that file steps may use. See [File steps](#file-steps-and-the-folders-they-may-use). |
|
|
524
|
+
| `XEPLR_DATA_DIR` | **yes** (`requiredEnv`) | The root of all data: absolute, existing. Each tenant's files are in `<root>/<app>/<mtId1>/…`, temporary ones in its `tmp/`. See [Where files are](#where-files-are-one-data-root-a-folder-per-tenant). Files are not cleaned up by `db-fetch` or `email-read`. |
|
|
516
525
|
| `XCFG_DB_NAME` | for `attachConfig` | The `xeplr_configs` database name. In `configRequiredEnv`. |
|
|
517
526
|
| `XCFG_DB_CONNECTION_INFO_ENCRYPTED` | no | Encrypted server login for `attachConfig`; overrides `XEPLR_DB_CONNECTION`. |
|
|
518
527
|
| `XEPLR_DB_CONNECTION` | for `attachConfig`, unless the above is set | Shared encrypted server login. |
|
|
@@ -599,6 +608,7 @@ node --test test/uploader/reconcile.test.js
|
|
|
599
608
|
|
|
600
609
|
- **No `*_HOST`:** that database's tests are **skipped**, and each says which settings run it.
|
|
601
610
|
- **`*_HOST` set:** they run, and a login the database refuses **fails** them. A configured run never passes by not connecting.
|
|
611
|
+
- **A data root and an application per test process** (`--require ./test/setup-data-dir.cjs`, in `npm test`, CI, and the gate through `@flags` on `test/data-dir.test.js`), as an app has at boot. `test/data-dir.test.js` registers tenants: two companies with the same names, each refused the other's files by full path, `../`, a link and a move; `*` and no tenant; email attachments, db-push and a local upload.
|
|
602
612
|
- **One file at a time** (`--test-concurrency=1`): the database suites share the `xeplr_actions_test` database and race to create it when run side by side.
|
|
603
613
|
- **Why:** these files used to carry a fixed host, port and password each (Postgres on `:5435`). On any other machine they all failed, so the pre-commit gate and CI ran only `test/*.test.js` and never this package's sub-folders. That is how `data-operation` shipped returning no rows (fixed in 1.4.2).
|
|
604
614
|
|
package/index.js
CHANGED
|
@@ -56,5 +56,12 @@ module.exports = {
|
|
|
56
56
|
// lib/uploader/attach.js. configRequiredEnv spreads into the app's own
|
|
57
57
|
// env.required.js, same convention as @xeplr/auth's requiredEnv.
|
|
58
58
|
attachConfig: configDb.attachConfig,
|
|
59
|
-
configRequiredEnv: configDb.requiredEnv
|
|
59
|
+
configRequiredEnv: configDb.requiredEnv,
|
|
60
|
+
|
|
61
|
+
// WHERE FILES ARE: XEPLR_DATA_DIR, one folder per tenant under it
|
|
62
|
+
// (<root>/<app>/<mtId1>/…). requiredEnv spreads into the app's
|
|
63
|
+
// env.required.js — the root is never defaulted. dataDir.resolve(name)
|
|
64
|
+
// is the one check every file path goes through (lib/dataDir.js).
|
|
65
|
+
requiredEnv: require('./lib/dataDir').requiredEnv,
|
|
66
|
+
dataDir: require('./lib/dataDir')
|
|
60
67
|
};
|
package/lib/builtins/db/fetch.js
CHANGED
|
@@ -14,13 +14,12 @@
|
|
|
14
14
|
// modes:
|
|
15
15
|
// 'table' → SELECT * FROM <table> (input.table required)
|
|
16
16
|
// 'query' → raw input.sql (+ input.params) (input.sql required)
|
|
17
|
-
// Temp file:
|
|
17
|
+
// Temp file: the tenant's tmp/ under XEPLR_DATA_DIR (lib/dataDir.js),
|
|
18
18
|
// named <occurrenceId>_db-fetch_<seq>.jsonl. Cleanup: none in-action — rely
|
|
19
19
|
// on the sweep-temp-files action (or db-push after it consumes the file).
|
|
20
20
|
|
|
21
21
|
var dbDrivers = require('../../drivers/db');
|
|
22
22
|
var { generateId } = require('@xeplr/utils/lib/helpers');
|
|
23
|
-
var os = require('os');
|
|
24
23
|
var path = require('path');
|
|
25
24
|
var fs = require('fs');
|
|
26
25
|
var fsp = require('fs/promises');
|
|
@@ -137,8 +136,8 @@ function buildFetchSql(driver, input) {
|
|
|
137
136
|
|
|
138
137
|
|
|
139
138
|
async function resolveOutputPath(system) {
|
|
140
|
-
|
|
141
|
-
|
|
139
|
+
// The tenant's own temporary folder (lib/dataDir.js).
|
|
140
|
+
var dir = require('../../dataDir').scratch();
|
|
142
141
|
var base = (system.occurrenceId || generateId()) + '_db-fetch_' + generateId() + '.jsonl';
|
|
143
142
|
return path.join(dir, base);
|
|
144
143
|
}
|
package/lib/builtins/db/push.js
CHANGED
|
@@ -99,7 +99,7 @@ module.exports = {
|
|
|
99
99
|
// Return an async iterable for whichever input shape the caller supplied.
|
|
100
100
|
function resolveSource(input) {
|
|
101
101
|
if (Array.isArray(input.rows) && input.rows.length > 0) return asyncFromArray(input.rows);
|
|
102
|
-
if (input.filePath) return asyncFromNDJSONFile(input.filePath);
|
|
102
|
+
if (input.filePath) return asyncFromNDJSONFile(require('../../dataDir').resolve(input.filePath, 'The rows file'));
|
|
103
103
|
throw new Error('db-push: either input.rows (non-empty) or input.filePath is required');
|
|
104
104
|
}
|
|
105
105
|
|
|
@@ -34,7 +34,7 @@ module.exports = {
|
|
|
34
34
|
{ name: 'folder', type: 'string', default: 'INBOX', order: 4,
|
|
35
35
|
description: 'Folder the message is currently in — where its id is valid.' },
|
|
36
36
|
{ name: 'outDir', type: 'string', order: 5,
|
|
37
|
-
description: '
|
|
37
|
+
description: 'A folder in this company\'s data folder, e.g. mail/attachments. Default: its tmp/ folder.' },
|
|
38
38
|
|
|
39
39
|
{ name: 'extension', type: 'string', order: 6, group: 'Filters',
|
|
40
40
|
description: 'Only attachments with this extension, e.g. "pdf" or ".pdf".' },
|
|
@@ -32,7 +32,7 @@ module.exports = {
|
|
|
32
32
|
{ name: 'folder', type: 'string', default: 'INBOX', order: 4,
|
|
33
33
|
description: 'Folder the message is currently in — where its id is valid.' },
|
|
34
34
|
{ name: 'outDir', type: 'string', order: 5,
|
|
35
|
-
description: '
|
|
35
|
+
description: 'A folder in this company\'s data folder, e.g. mail/attachments. Default: its tmp/ folder.' }
|
|
36
36
|
].concat(fileNames.nameInputs(6, '[subject], [id] (the message), [ext] (eml)')),
|
|
37
37
|
|
|
38
38
|
execute: async function(ctx) {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// the normal case is a handful of messages a workflow branches on immediately.
|
|
6
6
|
// Set streaming_mode:true when pulling bodies in bulk — HTML mail with inline
|
|
7
7
|
// base64 images is large, and spooling keeps it off the heap and out of the
|
|
8
|
-
// run record. Temp file:
|
|
8
|
+
// run record. Temp file: the tenant's tmp/ (lib/dataDir.js), same as db-fetch.
|
|
9
9
|
//
|
|
10
10
|
// COST CONTROL is `body`, not `limit`. body:'none' skips downloading message
|
|
11
11
|
// sources entirely and fetches envelopes only — the right setting for a
|
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
var emailDrivers = require('../../drivers/email');
|
|
15
15
|
var emailConfig = require('../../drivers/email/config');
|
|
16
16
|
var crypto = require('crypto');
|
|
17
|
-
var os = require('os');
|
|
18
17
|
var path = require('path');
|
|
19
18
|
var fs = require('fs');
|
|
20
19
|
var fsp = require('fs/promises');
|
|
@@ -92,8 +91,8 @@ module.exports = {
|
|
|
92
91
|
// ─── helpers ────────────────────────────────────────────────────────────
|
|
93
92
|
|
|
94
93
|
async function resolveOutputPath(system) {
|
|
95
|
-
|
|
96
|
-
|
|
94
|
+
// The tenant's own temporary folder (lib/dataDir.js).
|
|
95
|
+
var dir = require('../../dataDir').scratch();
|
|
97
96
|
var base = (system.occurrenceId || crypto.randomBytes(6).toString('hex'))
|
|
98
97
|
+ '_email-read_' + crypto.randomBytes(6).toString('hex') + '.jsonl';
|
|
99
98
|
return path.join(dir, base);
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
// of a raster type — no fetching, no markup.
|
|
12
12
|
|
|
13
13
|
var fs = require('fs');
|
|
14
|
-
var os = require('os');
|
|
15
14
|
var path = require('path');
|
|
16
15
|
var pdf = require('../../formats/writers/pdf');
|
|
17
16
|
var { checkFormatRequires } = require('../../formats');
|
|
@@ -91,9 +90,11 @@ module.exports = {
|
|
|
91
90
|
checkFormatRequires('export-pages', 'pdf', pdf);
|
|
92
91
|
validate(input);
|
|
93
92
|
var name = (String(input.title || 'dashboard').replace(/[\\/:*?"<>|\u0000-\u001F]+/g, ' ').trim().slice(0, 80) || 'dashboard') + '.pdf';
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
var
|
|
93
|
+
// Inside the tenant's own folder (lib/dataDir.js): a name the step gives,
|
|
94
|
+
// or a new file in its tmp/.
|
|
95
|
+
var dataDir = require('../../dataDir');
|
|
96
|
+
var filePath = input.outputPath ? dataDir.resolve(input.outputPath, 'Where to save it')
|
|
97
|
+
: path.join(fs.mkdtempSync(path.join(dataDir.scratch(), 'export-')), name);
|
|
97
98
|
var done = await pdf.renderPages(filePath, {
|
|
98
99
|
pages: input.pages, title: input.title, footer: input.footer, landscape: input.landscape !== false,
|
|
99
100
|
page: input.page || null
|
|
@@ -17,18 +17,13 @@
|
|
|
17
17
|
// default) because a page is laid out from them.
|
|
18
18
|
|
|
19
19
|
var fs = require('fs');
|
|
20
|
-
var os = require('os');
|
|
21
20
|
var path = require('path');
|
|
22
21
|
var readline = require('readline');
|
|
23
22
|
var writers = require('../../formats/writers');
|
|
24
23
|
var { checkFormatRequires } = require('../../formats');
|
|
25
24
|
var fileNames = require('../../fileNames');
|
|
26
25
|
|
|
27
|
-
|
|
28
|
-
var dir = process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions');
|
|
29
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
30
|
-
return dir;
|
|
31
|
-
}
|
|
26
|
+
var dataDir = require('../../dataDir');
|
|
32
27
|
|
|
33
28
|
/** A file name from a title: what a person would recognise in their downloads. */
|
|
34
29
|
function fileNameFor(title, extension) {
|
|
@@ -50,7 +45,7 @@ function asRow(row, columns) {
|
|
|
50
45
|
|
|
51
46
|
async function* rowsFrom(input, columns) {
|
|
52
47
|
if (Array.isArray(input.rows)) { for (var i = 0; i < input.rows.length; i++) yield asRow(input.rows[i], columns); return; }
|
|
53
|
-
var lines = readline.createInterface({ input: fs.createReadStream(input.rowsFile), crlfDelay: Infinity });
|
|
48
|
+
var lines = readline.createInterface({ input: fs.createReadStream(dataDir.resolve(input.rowsFile, 'The rows file')), crlfDelay: Infinity });
|
|
54
49
|
for await (var line of lines) if (line.trim()) yield asRow(JSON.parse(line), columns);
|
|
55
50
|
}
|
|
56
51
|
|
|
@@ -80,7 +75,7 @@ module.exports = {
|
|
|
80
75
|
description: 'An NDJSON file with one row per line, instead of rows.' },
|
|
81
76
|
{ name: 'outputPath', type: 'string', order: 5,
|
|
82
77
|
description: 'Where to write, e.g. /data/recon/Recon-{trigger.window.from|date:YYYY-MM}.xlsx. ' +
|
|
83
|
-
'Default: a new file in
|
|
78
|
+
'In this company\'s data folder, e.g. exports/sales.xlsx. Default: a new file in its tmp/ folder named after the title.' },
|
|
84
79
|
{ name: 'ifExists', type: 'string', order: 7, default: 'fail', options: fileNames.IF_EXISTS,
|
|
85
80
|
description: 'When outputPath is already there. fail: the step fails and nothing is written. skip: left alone ' +
|
|
86
81
|
'(skipped: true). overwrite: replaced. addSuffix: "name (2).xlsx".' },
|
|
@@ -115,7 +110,9 @@ module.exports = {
|
|
|
115
110
|
|
|
116
111
|
var info = writers.INFO[format];
|
|
117
112
|
var fileName = fileNameFor(doc.title, info.extension);
|
|
118
|
-
|
|
113
|
+
// A name the step gives is inside the tenant's folder; none, its tmp/.
|
|
114
|
+
var filePath = input.outputPath ? dataDir.resolve(input.outputPath, 'Where to save it')
|
|
115
|
+
: path.join(fs.mkdtempSync(path.join(dataDir.scratch(), 'export-')), fileName);
|
|
119
116
|
if (input.outputPath) {
|
|
120
117
|
// The same rule as every action that saves a file (lib/fileNames.js):
|
|
121
118
|
// a name already taken is never silently replaced.
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
// A file a workflow has finished with: the bank file read and posted, so the
|
|
4
4
|
// next check of the folder does not find it again. PERMANENT — there is no
|
|
5
5
|
// trash to take it back from; file-move into a "processed" folder is the one
|
|
6
|
-
// that can be undone. Only inside the
|
|
7
|
-
//
|
|
6
|
+
// that can be undone. Only inside the tenant's own folder (lib/dataDir.js),
|
|
7
|
+
// and only a file, never a folder.
|
|
8
8
|
|
|
9
9
|
var fs = require('fs');
|
|
10
|
-
var
|
|
10
|
+
var dataDir = require('../../dataDir');
|
|
11
11
|
|
|
12
12
|
module.exports = {
|
|
13
13
|
name: 'file-delete',
|
|
@@ -15,7 +15,7 @@ module.exports = {
|
|
|
15
15
|
requires: [],
|
|
16
16
|
inputSchema: [
|
|
17
17
|
{ name: 'path', type: 'string', required: true, order: 1,
|
|
18
|
-
description: 'The file,
|
|
18
|
+
description: 'The file, in this company\'s data folder, e.g. {item.path}.' },
|
|
19
19
|
{ name: 'ifMissing', type: 'string', default: 'fail', options: ['fail', 'skip'], order: 2,
|
|
20
20
|
description: 'When there is no such file. fail: the step fails. skip: it goes on, deleted false.' }
|
|
21
21
|
],
|
|
@@ -25,7 +25,7 @@ module.exports = {
|
|
|
25
25
|
|
|
26
26
|
execute: async function(ctx) {
|
|
27
27
|
var input = ctx.input || {};
|
|
28
|
-
var file =
|
|
28
|
+
var file = dataDir.resolve(input.path, 'The file to delete');
|
|
29
29
|
if (!fs.existsSync(file)) {
|
|
30
30
|
if (input.ifMissing === 'skip') return { path: file, deleted: false, skipped: true };
|
|
31
31
|
var missing = new Error('"' + input.path + '" is not a file that exists.');
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// file-list action — the files in a folder, optionally matching a pattern.
|
|
2
2
|
//
|
|
3
3
|
// Also the way to ask "is it there yet?": list with the exact name and branch
|
|
4
|
-
// on output.count. Only inside the
|
|
4
|
+
// on output.count. Only inside the tenant's own folder (lib/dataDir.js).
|
|
5
5
|
|
|
6
6
|
var fs = require('fs');
|
|
7
7
|
var path = require('path');
|
|
8
|
-
var
|
|
8
|
+
var dataDir = require('../../dataDir');
|
|
9
9
|
|
|
10
10
|
/** `*.csv`, `Bank-2026-??.pdf` → a test on a name. Case-insensitive. */
|
|
11
11
|
function matcher(pattern) {
|
|
@@ -25,7 +25,7 @@ module.exports = {
|
|
|
25
25
|
requires: [],
|
|
26
26
|
inputSchema: [
|
|
27
27
|
{ name: 'folder', type: 'string', required: true, order: 1,
|
|
28
|
-
description: '
|
|
28
|
+
description: 'A folder in this company\'s data folder, e.g. bank/inbox.' },
|
|
29
29
|
{ name: 'pattern', type: 'string', order: 2,
|
|
30
30
|
description: 'Only names matching this: * any characters, ? one. E.g. Bank-{trigger.window.from|date:YYYY-MM}*.csv' },
|
|
31
31
|
{ name: 'sort', type: 'string', default: 'name', options: ['name', 'newest', 'oldest'], order: 3,
|
|
@@ -38,7 +38,7 @@ module.exports = {
|
|
|
38
38
|
|
|
39
39
|
execute: async function(ctx) {
|
|
40
40
|
var input = ctx.input || {};
|
|
41
|
-
var dir =
|
|
41
|
+
var dir = dataDir.resolve(input.folder, 'The folder');
|
|
42
42
|
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
|
|
43
43
|
var missing = new Error('"' + input.folder + '" is not a folder that exists.');
|
|
44
44
|
missing.code = 'NOT_A_FOLDER';
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
// file-move action — move or copy a file, into a folder or to a new name.
|
|
2
2
|
//
|
|
3
3
|
// "Processed" folders, an archive by month, a copy for another team. Only
|
|
4
|
-
// inside the
|
|
4
|
+
// inside the tenant's own folder (lib/dataDir.js), at both ends. A
|
|
5
5
|
// name already taken at the target follows the rule every file-writing step
|
|
6
6
|
// follows (lib/fileNames.js): fail by default.
|
|
7
7
|
|
|
8
8
|
var fs = require('fs');
|
|
9
9
|
var path = require('path');
|
|
10
|
-
var
|
|
10
|
+
var dataDir = require('../../dataDir');
|
|
11
11
|
var fileNames = require('../../fileNames');
|
|
12
12
|
|
|
13
13
|
module.exports = {
|
|
@@ -16,9 +16,9 @@ module.exports = {
|
|
|
16
16
|
requires: [],
|
|
17
17
|
inputSchema: [
|
|
18
18
|
{ name: 'from', type: 'string', required: true, order: 1,
|
|
19
|
-
description: 'The file,
|
|
19
|
+
description: 'The file, in this company\'s data folder, e.g. {item.path}.' },
|
|
20
20
|
{ name: 'to', type: 'string', required: true, order: 2,
|
|
21
|
-
description: 'A folder (it keeps its name), or a
|
|
21
|
+
description: 'A folder (it keeps its name; end it with /), or a new name for it, e.g. archive/{trigger.window.from|date:YYYY-MM}/' },
|
|
22
22
|
{ name: 'operation', type: 'string', default: 'move', options: ['move', 'copy'], order: 3,
|
|
23
23
|
description: 'move: it is no longer where it was. copy: it is in both places.' },
|
|
24
24
|
{ name: 'ifExists', type: 'string', default: 'fail', options: fileNames.IF_EXISTS, order: 4,
|
|
@@ -32,15 +32,16 @@ module.exports = {
|
|
|
32
32
|
|
|
33
33
|
execute: async function(ctx) {
|
|
34
34
|
var input = ctx.input || {};
|
|
35
|
-
var from =
|
|
35
|
+
var from = dataDir.resolve(input.from, 'The file to move');
|
|
36
36
|
if (!fs.existsSync(from) || !fs.statSync(from).isFile()) {
|
|
37
37
|
var missing = new Error('"' + input.from + '" is not a file that exists.');
|
|
38
38
|
missing.code = 'FILE_NOT_FOUND';
|
|
39
39
|
throw missing;
|
|
40
40
|
}
|
|
41
41
|
// A folder: ends with a separator, or is one already.
|
|
42
|
-
var
|
|
43
|
-
var
|
|
42
|
+
var target = dataDir.resolve(input.to, 'Where it goes');
|
|
43
|
+
var toIsFolder = /[\/\\]$/.test(input.to) || (fs.existsSync(target) && fs.statSync(target).isDirectory());
|
|
44
|
+
var dir = toIsFolder ? target : path.dirname(target);
|
|
44
45
|
fs.mkdirSync(dir, { recursive: true });
|
|
45
46
|
var wanted = toIsFolder ? path.basename(from) : fileNames.safeName(path.basename(input.to));
|
|
46
47
|
var place = fileNames.plan(dir, [wanted], input.ifExists)[0];
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// file-read action — a CSV, Excel, JSON or NDJSON file as rows.
|
|
2
2
|
//
|
|
3
3
|
// What the next step works with: the data step filters them, export-table
|
|
4
|
-
// writes them, a matcher compares them. Only inside the
|
|
5
|
-
//
|
|
4
|
+
// writes them, a matcher compares them. Only inside the tenant's own folder
|
|
5
|
+
// (lib/dataDir.js). Bounded by maxRows, because a step's output is
|
|
6
6
|
// stored with the run.
|
|
7
7
|
|
|
8
8
|
var path = require('path');
|
|
9
9
|
var fs = require('fs');
|
|
10
10
|
var readline = require('readline');
|
|
11
|
-
var
|
|
11
|
+
var dataDir = require('../../dataDir');
|
|
12
12
|
var formats = require('../../formats');
|
|
13
13
|
|
|
14
14
|
var BY_EXTENSION = { csv: 'csv', txt: 'txt', tsv: 'txt', xlsx: 'excel', xls: 'excel', json: 'json', ndjson: 'ndjson', jsonl: 'ndjson' };
|
|
@@ -24,7 +24,7 @@ module.exports = {
|
|
|
24
24
|
requires: [],
|
|
25
25
|
inputSchema: [
|
|
26
26
|
{ name: 'path', type: 'string', required: true, order: 1,
|
|
27
|
-
description: '
|
|
27
|
+
description: 'A file in this company\'s data folder, e.g. bank/inbox/sept.csv or {item.path}.' },
|
|
28
28
|
{ name: 'format', type: 'string', order: 2, options: ['csv', 'txt', 'excel', 'json', 'ndjson'],
|
|
29
29
|
description: 'Empty: from the file\'s extension.' },
|
|
30
30
|
{ name: 'sheet', type: 'string', order: 3, group: 'Excel',
|
|
@@ -40,7 +40,7 @@ module.exports = {
|
|
|
40
40
|
|
|
41
41
|
execute: async function(ctx) {
|
|
42
42
|
var input = ctx.input || {};
|
|
43
|
-
var file =
|
|
43
|
+
var file = dataDir.resolve(input.path, 'The file');
|
|
44
44
|
if (!fs.existsSync(file) || !fs.statSync(file).isFile()) {
|
|
45
45
|
var missing = new Error('"' + input.path + '" is not a file that exists.');
|
|
46
46
|
missing.code = 'FILE_NOT_FOUND';
|
package/lib/builtins/index.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// in the output.
|
|
10
10
|
// - When false, data stays inline in output.rows / output.content.
|
|
11
11
|
// - Writes may accept EITHER `rows: []` OR a spooled `filePath`/`dirPath`.
|
|
12
|
-
// -
|
|
12
|
+
// - Files: inside the tenant's own folder under XEPLR_DATA_DIR (lib/dataDir.js); temp in its tmp/.
|
|
13
13
|
//
|
|
14
14
|
// Uploader primitive (lib/uploader/) handles DDL, type inference, TZ, batch
|
|
15
15
|
// bisection, and error-table routing. The DB actions here compose it with
|
|
@@ -3,16 +3,47 @@
|
|
|
3
3
|
// Peer dep: `nodemailer`, lazy-required inside the driver so a read-only
|
|
4
4
|
// deployment never has to install it.
|
|
5
5
|
//
|
|
6
|
-
// ATTACHMENTS
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
6
|
+
// ATTACHMENTS are { filename, path } or { filename, content } — which is why
|
|
7
|
+
// the output of email-download-attachments (`saved[]`, each with filename +
|
|
8
|
+
// path) can be passed straight back in to forward what was just downloaded.
|
|
9
|
+
//
|
|
10
|
+
// A PATH IS A FILE IN THE TENANT'S OWN FOLDER (lib/dataDir.js), never anywhere
|
|
11
|
+
// on the server and never an address: nodemailer reads whatever `path` names,
|
|
12
|
+
// a file or a URL, so an unchecked one would mail the server's own settings to
|
|
13
|
+
// anyone, or fetch an internal address. `href` is refused for the same reason,
|
|
14
|
+
// and the body must be text — nodemailer reads `html: { path }` from disk too.
|
|
10
15
|
//
|
|
11
16
|
// `accepted` / `rejected` are both returned. SMTP accepts per-recipient, so a
|
|
12
17
|
// send can succeed at the protocol level while dropping half its recipients;
|
|
13
18
|
// reporting only success would hide that.
|
|
14
19
|
|
|
15
20
|
var emailDrivers = require('../drivers/email');
|
|
21
|
+
var dataDir = require('../dataDir');
|
|
22
|
+
|
|
23
|
+
function bad(message) {
|
|
24
|
+
var err = new Error(message);
|
|
25
|
+
err.code = 'PATH_NOT_ALLOWED';
|
|
26
|
+
return err;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Each attachment, with its path checked; anything that would read elsewhere refused. */
|
|
30
|
+
function safeAttachments(list) {
|
|
31
|
+
if (list === undefined || list === null) return list;
|
|
32
|
+
if (!Array.isArray(list)) throw bad('Attachments must be a list of { filename, path } or { filename, content }.');
|
|
33
|
+
return list.map(function(a, i) {
|
|
34
|
+
if (!a || typeof a !== 'object') throw bad('Attachment ' + (i + 1) + ' must be { filename, path } or { filename, content }.');
|
|
35
|
+
if (a.href !== undefined || a.raw !== undefined) throw bad('Attachment ' + (i + 1) + ' names an address; attach a file from this company\'s folder instead.');
|
|
36
|
+
var out = {};
|
|
37
|
+
['filename', 'content', 'contentType', 'encoding', 'cid', 'contentDisposition'].forEach(function(k) { if (a[k] !== undefined) out[k] = a[k]; });
|
|
38
|
+
if (a.path !== undefined) out.path = dataDir.resolve(String(a.path), 'Attachment ' + (i + 1));
|
|
39
|
+
if (out.path === undefined && out.content === undefined) throw bad('Attachment ' + (i + 1) + ' has neither a path nor content.');
|
|
40
|
+
return out;
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function textOnly(value, what) {
|
|
45
|
+
if (value !== undefined && value !== null && typeof value !== 'string') throw bad('The ' + what + ' must be text.');
|
|
46
|
+
}
|
|
16
47
|
var emailConfig = require('../drivers/email/config');
|
|
17
48
|
|
|
18
49
|
module.exports = {
|
|
@@ -68,7 +99,7 @@ module.exports = {
|
|
|
68
99
|
showWhen: { field: 'templateName', isSet: false },
|
|
69
100
|
description: 'Plain-text body, as an alternative to html.' },
|
|
70
101
|
{ name: 'attachments', type: 'array', order: 9, group: 'Message options',
|
|
71
|
-
description: '
|
|
102
|
+
description: 'Files to attach, e.g. [{ filename, path }] — path in this company\'s data folder, such as {steps.save.output.filePath}.' },
|
|
72
103
|
|
|
73
104
|
{ name: 'cc', type: 'array', order: 10, group: 'More addresses',
|
|
74
105
|
description: 'CC addresses.' },
|
|
@@ -82,6 +113,11 @@ module.exports = {
|
|
|
82
113
|
|
|
83
114
|
execute: async function(ctx) {
|
|
84
115
|
var input = ctx.input || {};
|
|
116
|
+
// Before anything is opened or rendered: a body that is not text, or an
|
|
117
|
+
// attachment outside this company's folder, is refused here.
|
|
118
|
+
textOnly(input.html, 'body (html)');
|
|
119
|
+
textOnly(input.text, 'body (text)');
|
|
120
|
+
if (input.attachments !== undefined) input = Object.assign({}, input, { attachments: safeAttachments(input.attachments) });
|
|
85
121
|
|
|
86
122
|
// A template supplies subject/html/text; anything typed on the step is
|
|
87
123
|
// OVERRIDDEN rather than merged, so "which subject actually went out" has
|
package/lib/dataDir.js
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// WHERE A STEP'S FILES ARE — one data root, one folder per tenant.
|
|
2
|
+
//
|
|
3
|
+
// XEPLR_DATA_DIR the root of all data, absolute, required (requiredEnv)
|
|
4
|
+
//
|
|
5
|
+
// Every file a step reads or writes is inside the folder of the tenant the
|
|
6
|
+
// work runs as, and nowhere else:
|
|
7
|
+
//
|
|
8
|
+
// <XEPLR_DATA_DIR>/<applicationId>/<mtId1>/<mtId2>/…
|
|
9
|
+
//
|
|
10
|
+
// The application comes from @xeplr/db's registerApplication (a company id is
|
|
11
|
+
// minted per app, so company 42 of BI and company 42 of ERP are two tenants);
|
|
12
|
+
// the tenant from the context every query already runs in — a request after
|
|
13
|
+
// the membership gate, a job or a workflow run (runWithMt). Never from a
|
|
14
|
+
// step's input: a step names `bank/inbox/sept.csv`, and cannot name anything
|
|
15
|
+
// outside its own folder.
|
|
16
|
+
//
|
|
17
|
+
// a name bank/inbox/sept.csv → inside the folder
|
|
18
|
+
// a full path accepted only when it is inside the folder already (what
|
|
19
|
+
// an earlier step returned: file-list's `path`)
|
|
20
|
+
// '*' at a level `_all` — a job for every company has a folder of its own,
|
|
21
|
+
// and no company's step can reach it
|
|
22
|
+
// no tenant refused, as a query with no tenant finds nothing; never
|
|
23
|
+
// the root
|
|
24
|
+
//
|
|
25
|
+
// `..` and symlinks are resolved before the check, so neither steps out.
|
|
26
|
+
// NOT DEFAULTED: "Where data lives is never defaulted." A missing or relative
|
|
27
|
+
// XEPLR_DATA_DIR is refused by name, and so is one that does not exist — a
|
|
28
|
+
// wrong root that boots reads as empty data, not as an error.
|
|
29
|
+
|
|
30
|
+
var fs = require('fs');
|
|
31
|
+
var path = require('path');
|
|
32
|
+
|
|
33
|
+
var VAR = 'XEPLR_DATA_DIR';
|
|
34
|
+
var requiredEnv = [VAR];
|
|
35
|
+
var ALL = '_all';
|
|
36
|
+
var SCRATCH = 'tmp';
|
|
37
|
+
|
|
38
|
+
function refused(message, code) {
|
|
39
|
+
var err = new Error(message);
|
|
40
|
+
err.code = code || 'PATH_NOT_ALLOWED';
|
|
41
|
+
err.status = 400;
|
|
42
|
+
return err;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function db() { return require('@xeplr/db'); }
|
|
46
|
+
|
|
47
|
+
/** The data root, absolute and real. */
|
|
48
|
+
function root() {
|
|
49
|
+
var dir = process.env[VAR];
|
|
50
|
+
if (!dir || !String(dir).trim()) {
|
|
51
|
+
throw refused(VAR + ' is not set: it is the folder all data lives under (absolute), e.g. /srv/xeplr-data.', 'DATA_DIR_UNSET');
|
|
52
|
+
}
|
|
53
|
+
if (!path.isAbsolute(dir)) throw refused(VAR + ' must be an absolute folder, and "' + dir + '" is not one.', 'DATA_DIR_UNSET');
|
|
54
|
+
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
|
|
55
|
+
throw refused(VAR + ' is "' + dir + '", which is not a folder that exists. Create it, or point ' + VAR + ' at the one that holds the data.', 'DATA_DIR_UNSET');
|
|
56
|
+
}
|
|
57
|
+
return fs.realpathSync(dir);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// An id becomes a folder name as it is. One that could not be one — or could
|
|
61
|
+
// climb out (`..`) — is refused rather than cleaned up into someone else's.
|
|
62
|
+
function segment(value, what) {
|
|
63
|
+
if (value === '*') return ALL;
|
|
64
|
+
var s = value === null || value === undefined ? '' : String(value);
|
|
65
|
+
if (!/^[A-Za-z0-9_.-]{1,100}$/.test(s) || s === '.' || s === '..') {
|
|
66
|
+
throw refused('The ' + what + ' "' + s + '" cannot name a folder.', 'NO_TENANT');
|
|
67
|
+
}
|
|
68
|
+
return s;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The folder the current work may use: <root>/<app>/<mtId1>/…, made if it is
|
|
73
|
+
* not there yet. An app with no tenancy registered is one tenant: <root>/<app>.
|
|
74
|
+
*/
|
|
75
|
+
function tenantFolder() {
|
|
76
|
+
var x = db();
|
|
77
|
+
var app = x.getApplicationId();
|
|
78
|
+
if (!app) throw refused('No application is registered (registerApplication), so there is no folder for its files.', 'NO_TENANT');
|
|
79
|
+
var parts = [root(), segment(app, 'application')];
|
|
80
|
+
var levels = x.getMtConfig().levels || 0;
|
|
81
|
+
var ctx = x.getMtContext() || {};
|
|
82
|
+
for (var i = 1; i <= levels; i++) {
|
|
83
|
+
var v = ctx['mtId' + i];
|
|
84
|
+
if (v === undefined || v === null || v === '') {
|
|
85
|
+
throw refused('This work runs as no tenant, so it has no folder: files are only ever inside the folder of the company it runs for.', 'NO_TENANT');
|
|
86
|
+
}
|
|
87
|
+
parts.push(segment(v, 'tenant id'));
|
|
88
|
+
}
|
|
89
|
+
var dir = path.join.apply(path, parts);
|
|
90
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
91
|
+
return fs.realpathSync(dir);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** The tenant's own temporary folder: spools, downloads, exports nobody named. */
|
|
95
|
+
function scratch() {
|
|
96
|
+
var dir = path.join(tenantFolder(), SCRATCH);
|
|
97
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
98
|
+
return dir;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function inside(real, folder) {
|
|
102
|
+
return real === folder || real.indexOf(folder + path.sep) === 0;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* A name (or a full path already inside the folder) → the real absolute path,
|
|
107
|
+
* inside the tenant's folder; refused otherwise. A path that does not exist
|
|
108
|
+
* yet (a move's target, an export) is checked by its nearest existing parent,
|
|
109
|
+
* so `../` and symlinks cannot step out.
|
|
110
|
+
*/
|
|
111
|
+
function resolve(p, what) {
|
|
112
|
+
if (!p || typeof p !== 'string' || !p.trim()) throw refused((what || 'A path') + ' is required.');
|
|
113
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) throw refused((what || 'The path') + ' "' + p + '" is an address, not a file in this company\'s folder.');
|
|
114
|
+
var folder = tenantFolder();
|
|
115
|
+
var resolved = path.isAbsolute(p) ? path.resolve(p) : path.resolve(folder, p);
|
|
116
|
+
var probe = resolved;
|
|
117
|
+
var rest = '';
|
|
118
|
+
while (!fs.existsSync(probe) && path.dirname(probe) !== probe) {
|
|
119
|
+
rest = path.join(path.basename(probe), rest);
|
|
120
|
+
probe = path.dirname(probe);
|
|
121
|
+
}
|
|
122
|
+
var real = path.join(fs.realpathSync(probe), rest);
|
|
123
|
+
if (!inside(real, folder)) {
|
|
124
|
+
throw refused((what || 'The path') + ' "' + p + '" is outside this company\'s folder. Name it inside, e.g. bank/inbox/statement.csv.');
|
|
125
|
+
}
|
|
126
|
+
return real;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The name a path has inside the tenant's folder — what a screen shows. */
|
|
130
|
+
function nameOf(real) {
|
|
131
|
+
var folder = tenantFolder();
|
|
132
|
+
return inside(real, folder) ? path.relative(folder, real) : null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
module.exports = { VAR: VAR, requiredEnv: requiredEnv, ALL: ALL, root: root, tenantFolder: tenantFolder, scratch: scratch, resolve: resolve, nameOf: nameOf };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Shared on-disk helpers for the email actions that write files
|
|
2
2
|
// (email-download-email, email-download-attachments).
|
|
3
3
|
//
|
|
4
|
-
// Output goes to
|
|
4
|
+
// Output goes to the tenant's tmp/ (lib/dataDir.js) — the SAME directory db-fetch spools
|
|
5
5
|
// NDJSON into — so every artifact an action leaves behind is swept by one
|
|
6
6
|
// mechanism rather than one per action family. An explicit `outDir` input
|
|
7
7
|
// still wins for the case where a later step (or another service) needs the
|
|
@@ -11,7 +11,6 @@ var crypto = require('crypto');
|
|
|
11
11
|
var path = require('path');
|
|
12
12
|
var fs = require('fs');
|
|
13
13
|
var fsp = require('fs/promises');
|
|
14
|
-
var os = require('os');
|
|
15
14
|
var fileNames = require('../../fileNames');
|
|
16
15
|
|
|
17
16
|
// Short random prefix. Attachment names collide constantly in mail —
|
|
@@ -34,10 +33,12 @@ function sanitizeFilename(name) {
|
|
|
34
33
|
return out;
|
|
35
34
|
}
|
|
36
35
|
|
|
37
|
-
//
|
|
38
|
-
//
|
|
36
|
+
// A folder named by the step, inside the tenant's own folder; otherwise the
|
|
37
|
+
// tenant's temporary folder (lib/dataDir.js). Created recursively — a first
|
|
38
|
+
// run has no directory yet.
|
|
39
39
|
async function resolveOutDir(outDir) {
|
|
40
|
-
var
|
|
40
|
+
var dataDir = require('../../dataDir');
|
|
41
|
+
var dir = outDir ? dataDir.resolve(outDir, 'The folder to save into') : dataDir.scratch();
|
|
41
42
|
await fsp.mkdir(dir, { recursive: true });
|
|
42
43
|
return dir;
|
|
43
44
|
}
|
|
@@ -7,18 +7,22 @@
|
|
|
7
7
|
// deleteFile(config) → void
|
|
8
8
|
//
|
|
9
9
|
// `config.path` is the file/dir path — local source config is intentionally
|
|
10
|
-
// minimal since fs handles most concerns natively.
|
|
10
|
+
// minimal since fs handles most concerns natively. Every path is inside the
|
|
11
|
+
// tenant's own folder (lib/dataDir.js): a name, or a full path already in it.
|
|
11
12
|
|
|
12
13
|
var fs = require('fs');
|
|
13
14
|
var fsp = require('fs/promises');
|
|
14
15
|
var path = require('path');
|
|
16
|
+
var dataDir = require('../../dataDir');
|
|
17
|
+
|
|
18
|
+
function at(p, what) { return dataDir.resolve(p, what || 'The file'); }
|
|
15
19
|
|
|
16
20
|
module.exports = {
|
|
17
21
|
requires: [],
|
|
18
22
|
|
|
19
23
|
async openStream(config) {
|
|
20
24
|
if (!config || !config.path) throw new Error('local source: config.path is required');
|
|
21
|
-
return fs.createReadStream(config.path, config.encoding ? { encoding: config.encoding } : {});
|
|
25
|
+
return fs.createReadStream(at(config.path), config.encoding ? { encoding: config.encoding } : {});
|
|
22
26
|
},
|
|
23
27
|
|
|
24
28
|
// Formats that need random access (e.g. .xlsx via unzipper — the zip central
|
|
@@ -27,29 +31,31 @@ module.exports = {
|
|
|
27
31
|
// must implement this by downloading to a caller-provided tmp path.
|
|
28
32
|
async openPath(config) {
|
|
29
33
|
if (!config || !config.path) throw new Error('local source: config.path is required');
|
|
30
|
-
return config.path;
|
|
34
|
+
return at(config.path);
|
|
31
35
|
},
|
|
32
36
|
|
|
33
37
|
async writeFile(config, opts) {
|
|
34
38
|
if (!config.path) throw new Error('local source: config.path is required for writeFile');
|
|
35
39
|
if (opts.filePath) {
|
|
36
|
-
|
|
37
|
-
|
|
40
|
+
var to = at(config.path);
|
|
41
|
+
await fsp.copyFile(at(opts.filePath, 'The file to copy'), to);
|
|
42
|
+
return { bytes: (await fsp.stat(to)).size };
|
|
38
43
|
}
|
|
39
44
|
if (opts.content != null) {
|
|
40
|
-
await fsp.writeFile(config.path, opts.content);
|
|
45
|
+
await fsp.writeFile(at(config.path), opts.content);
|
|
41
46
|
return { bytes: Buffer.byteLength(opts.content) };
|
|
42
47
|
}
|
|
43
48
|
throw new Error('local source: writeFile requires opts.content or opts.filePath');
|
|
44
49
|
},
|
|
45
50
|
|
|
46
51
|
async listFiles(config) {
|
|
47
|
-
var
|
|
52
|
+
var dir = at(config.path, 'The folder');
|
|
53
|
+
var entries = await fsp.readdir(dir, { withFileTypes: true });
|
|
48
54
|
var out = [];
|
|
49
55
|
for (var i = 0; i < entries.length; i++) {
|
|
50
56
|
var e = entries[i];
|
|
51
57
|
if (!e.isFile()) continue;
|
|
52
|
-
var full = path.join(
|
|
58
|
+
var full = path.join(dir, e.name);
|
|
53
59
|
var stat = await fsp.stat(full);
|
|
54
60
|
out.push({ name: e.name, path: full, size: stat.size, mtime: stat.mtime.toISOString() });
|
|
55
61
|
}
|
|
@@ -58,11 +64,11 @@ module.exports = {
|
|
|
58
64
|
|
|
59
65
|
async moveFile(_config, opts) {
|
|
60
66
|
if (!opts || !opts.from || !opts.to) throw new Error('local source: moveFile requires from + to');
|
|
61
|
-
await fsp.rename(opts.from, opts.to);
|
|
67
|
+
await fsp.rename(at(opts.from, 'The file to move'), at(opts.to, 'Where it goes'));
|
|
62
68
|
},
|
|
63
69
|
|
|
64
70
|
async deleteFile(config) {
|
|
65
71
|
if (!config.path) throw new Error('local source: config.path is required for deleteFile');
|
|
66
|
-
await fsp.unlink(config.path);
|
|
72
|
+
await fsp.unlink(at(config.path));
|
|
67
73
|
}
|
|
68
74
|
};
|
package/lib/uploader/index.js
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
// // (filename, connectionId, who ran it, ...) the engine itself
|
|
27
27
|
// // has no business knowing about
|
|
28
28
|
// errorTable, // defaults to `<targetTable>_import_errors`
|
|
29
|
-
// batchDir, // where NDJSON files spool to. Defaults to
|
|
29
|
+
// batchDir, // where NDJSON files spool to. Defaults to the tenant's tmp/ (lib/dataDir.js)
|
|
30
30
|
// keepBatchFiles, // default false — clean up NDJSON files after success
|
|
31
31
|
// replaceFrom, // optional, staged only: { column } — the move deletes target
|
|
32
32
|
// // rows from the earliest staged value on first (see mergeStaged)
|
|
@@ -59,7 +59,6 @@
|
|
|
59
59
|
//
|
|
60
60
|
// See project_xeplr_actions_streaming.md for the full contract.
|
|
61
61
|
|
|
62
|
-
var os = require('os');
|
|
63
62
|
var path = require('path');
|
|
64
63
|
var fsp = require('fs/promises');
|
|
65
64
|
|
|
@@ -468,8 +467,8 @@ function normalizeOptions(opts) {
|
|
|
468
467
|
opts.metaStore = opts.metaStore || require('./meta-store-noop');
|
|
469
468
|
opts.errorTable = opts.errorTable || (opts.targetTable + '_import_errors');
|
|
470
469
|
opts.connectionName = opts.connectionName || opts.targetTable;
|
|
471
|
-
|
|
472
|
-
|
|
470
|
+
// The tenant's own temporary folder (lib/dataDir.js) unless the host names one.
|
|
471
|
+
opts.batchDir = opts.batchDir || require('../dataDir').scratch();
|
|
473
472
|
opts.keepBatchFiles = !!opts.keepBatchFiles;
|
|
474
473
|
opts.staged = !!opts.staged;
|
|
475
474
|
// A driver that matches upsert keys only in a staged move (DuckDB: its
|
package/package.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xeplr/actions",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
7
|
-
"test": "node --test --test-concurrency=1 \"test/**/*.test.js\""
|
|
7
|
+
"test": "node --test --test-concurrency=1 --require ./test/setup-data-dir.cjs \"test/**/*.test.js\""
|
|
8
8
|
},
|
|
9
9
|
"files": [
|
|
10
10
|
"index.js",
|
package/lib/fileRoots.js
DELETED
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
// WHERE A FILE STEP MAY LOOK.
|
|
2
|
-
//
|
|
3
|
-
// file-list, file-read and file-move take a path from whoever wrote the
|
|
4
|
-
// workflow. Unbounded, that is every file the server can read: its own
|
|
5
|
-
// settings, other installs' data. So they work only inside the folders the
|
|
6
|
-
// install names:
|
|
7
|
-
//
|
|
8
|
-
// XEPLR_FILE_ROOTS absolute folders, comma-separated, e.g. /data/bank,/data/erp
|
|
9
|
-
//
|
|
10
|
-
// plus XEPLR_ACTIONS_TMP_DIR (or the system temp folder's xeplr-actions),
|
|
11
|
-
// where downloads and exports land by default, so the file a step just saved
|
|
12
|
-
// can be read by the next one.
|
|
13
|
-
//
|
|
14
|
-
// NOT DEFAULTED: no roots configured and a path outside the temp folder is
|
|
15
|
-
// refused, naming the variable. "Where data lives is never defaulted."
|
|
16
|
-
|
|
17
|
-
var fs = require('fs');
|
|
18
|
-
var os = require('os');
|
|
19
|
-
var path = require('path');
|
|
20
|
-
|
|
21
|
-
function rootsError(message) {
|
|
22
|
-
var err = new Error(message);
|
|
23
|
-
err.code = 'PATH_NOT_ALLOWED';
|
|
24
|
-
err.status = 400;
|
|
25
|
-
return err;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
function tmpRoot() {
|
|
29
|
-
return path.resolve(process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions'));
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/** The folders a file step may use, resolved and real (symlinks followed). */
|
|
33
|
-
function roots() {
|
|
34
|
-
var listed = String(process.env.XEPLR_FILE_ROOTS || '').split(',')
|
|
35
|
-
.map(function(r) { return r.trim(); })
|
|
36
|
-
.filter(Boolean);
|
|
37
|
-
listed.forEach(function(r) {
|
|
38
|
-
if (!path.isAbsolute(r)) throw rootsError('XEPLR_FILE_ROOTS must list absolute folders, and "' + r + '" is not one.');
|
|
39
|
-
});
|
|
40
|
-
return listed.concat([tmpRoot()]).map(function(r) {
|
|
41
|
-
try { return fs.realpathSync(r); } catch (e) { return path.resolve(r); }
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* The path, absolute and real, when it is inside an allowed folder; refused
|
|
47
|
-
* otherwise. A path that does not exist yet (a move's target) is checked by
|
|
48
|
-
* its nearest existing parent, so "../" and symlinks cannot step outside.
|
|
49
|
-
*/
|
|
50
|
-
function allowed(p, what) {
|
|
51
|
-
if (!p || typeof p !== 'string') throw rootsError((what || 'A path') + ' is required.');
|
|
52
|
-
if (!path.isAbsolute(p)) throw rootsError((what || 'The path') + ' "' + p + '" must be absolute, e.g. /data/bank/statement.csv.');
|
|
53
|
-
var resolved = path.resolve(p);
|
|
54
|
-
var probe = resolved;
|
|
55
|
-
var rest = '';
|
|
56
|
-
while (!fs.existsSync(probe) && path.dirname(probe) !== probe) {
|
|
57
|
-
rest = path.join(path.basename(probe), rest);
|
|
58
|
-
probe = path.dirname(probe);
|
|
59
|
-
}
|
|
60
|
-
var real = path.join(fs.realpathSync(probe), rest);
|
|
61
|
-
var ok = roots().some(function(r) { return real === r || real.indexOf(r + path.sep) === 0; });
|
|
62
|
-
if (!ok) {
|
|
63
|
-
throw rootsError((what || 'The path') + ' "' + p + '" is outside the folders file steps may use. ' +
|
|
64
|
-
(process.env.XEPLR_FILE_ROOTS
|
|
65
|
-
? 'Allowed: ' + process.env.XEPLR_FILE_ROOTS + ' and the temp folder.'
|
|
66
|
-
: 'Set XEPLR_FILE_ROOTS to the folders they may use (absolute, comma-separated).'));
|
|
67
|
-
}
|
|
68
|
-
return real;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
module.exports = { allowed: allowed, roots: roots, tmpRoot: tmpRoot };
|