@xeplr/actions 2.2.3 → 2.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -6
- package/lib/builtins/db/list-foreign-keys.js +41 -0
- package/lib/builtins/file/archive.js +151 -0
- package/lib/builtins/index.js +5 -0
- package/lib/drivers/db/duckdb.js +24 -0
- package/lib/drivers/db/foreignKeys.js +26 -0
- package/lib/drivers/db/mssql.js +21 -0
- package/lib/drivers/db/mysql.js +17 -0
- package/lib/drivers/db/postgres.js +21 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -122,7 +122,7 @@ The input is checked exactly as a run checks it.
|
|
|
122
122
|
|
|
123
123
|
| previewed (reads, up to 5) | example only (writes or sends; never run) |
|
|
124
124
|
|---|---|
|
|
125
|
-
| `db-fetch` (through the run's own cursor, then stops; nothing spooled), `db-list-tables` / `-views` / `-procedures` / `-columns`, `file-list`, `file-read`, `data-operation` (the run itself: it only computes), `email-read` (the mailbox opened **read-only**, IMAP `EXAMINE`, so nothing is marked read) | `file-move`, `file-delete`, `email-send`, `push-send`, `email-move`, `email-delete`, `email-download-email`, `email-download-attachments`, `export-table`, `export-pages`, `db-procedure` (a procedure may write) |
|
|
125
|
+
| `db-fetch` (through the run's own cursor, then stops; nothing spooled), `db-list-tables` / `-views` / `-procedures` / `-columns` / `-foreign-keys`, `file-list`, `file-read`, `data-operation` (the run itself: it only computes), `email-read` (the mailbox opened **read-only**, IMAP `EXAMINE`, so nothing is marked read) | `file-move`, `file-delete`, `email-send`, `push-send`, `email-move`, `email-delete`, `email-download-email`, `email-download-attachments`, `export-table`, `export-pages`, `db-procedure` (a procedure may write) |
|
|
126
126
|
|
|
127
127
|
Neither: `db-push`, `db-move`, `http-request` (an outside system, which may change something), `spawn-program`, `file-upload`; a designer asks for an example by hand. `test/previews.test.js` checks it on a real folder, file and Postgres: five rows read and no file spooled, the real count kept, a deleting step's file still there.
|
|
128
128
|
|
|
@@ -138,12 +138,13 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
|
|
|
138
138
|
| `dbFetch` | `db-fetch` | Stream rows from a table (`mode: 'table'`) or SQL (`mode: 'query'`, `params`). Default `streaming_mode: true` → NDJSON file `{ filePath, format: 'jsonl', bytes, rows }`; `false` → `{ rows, rowCount }`. |
|
|
139
139
|
| `dbPush` | `db-push` | Write `rows` or an NDJSON `filePath` into `targetTable` through the uploader. `primaryKeys` → upsert. |
|
|
140
140
|
| `dbMove` | `db-move` | One window of rows from a source table, query or procedure into a target table in one streamed pass. See [db-move](#db-move). |
|
|
141
|
-
| `dbDump` | `db-dump` | A whole database to one backup file with its own dump program: `pg_dump -Fc` (Postgres, `.dump`) or `mysqldump` (MySQL, `.sql.gz`). See [Backups](#backups-db-dump-and-s3-upload). |
|
|
141
|
+
| `dbDump` | `db-dump` | A whole database to one backup file with its own dump program: `pg_dump -Fc` (Postgres, `.dump`) or `mysqldump` (MySQL, `.sql.gz`). See [Backups](#backups-db-dump-folder-archive-and-s3-upload). |
|
|
142
142
|
| `dbProcedure` | `db-procedure` | Call a stored procedure; returns `{ rows, rowCount, procedure, sql }`. See [db-procedure](#db-procedure). |
|
|
143
143
|
| `dbListTables` | `db-list-tables` | `{ tables }` — base tables in a schema/database. |
|
|
144
144
|
| `dbListViews` | `db-list-views` | `{ views }`. |
|
|
145
145
|
| `dbListProcedures` | `db-list-procedures` | `{ procedures }` (functions excluded). |
|
|
146
146
|
| `dbListColumns` | `db-list-columns` | `{ columns }` — `name`, `dataType`, `udtName` and `maxLength` (characters; `-1` is SQL Server `(max)`, `null` when it has none) for one table. |
|
|
147
|
+
| `dbListForeignKeys` | `db-list-foreign-keys` | `{ foreignKeys }` — each foreign key between the tables of a schema/database: `{ name, table, columns, refTable, refColumns }`, a composite key's columns in key order. Only keys whose **both** tables are in the schema `db-list-tables` lists, so every name is one it returns. Postgres (`public`), MySQL (the connection's database), SQL Server (every schema unless `schema`), DuckDB (`main`). What the BI dataset canvas joins two tables on without asking. |
|
|
147
148
|
| `fileUpload` | `file-upload` | Read a file (`sourceType`, `sourcePath`), parse it (`format`), load it into `targetTable` (`dbType`, `dbConnection`) as a New table, Append or Replace (`mode`, see [Import modes](#import-modes)). Passes `system.onProgress` (`{ rowsRead, batches, rowsWritten }`, per batch) and `system.signal` (stop) to the uploader, as `db-move` does. |
|
|
148
149
|
| `sendEmail` | `email-send` | Send one message. See [Email](#email). |
|
|
149
150
|
| `pushSend` | `push-send` | A push notification on people's phones and computers, even with the app closed, by `emails` and/or `userIds`: `title` (required), `body`, `url` (a page of the app a tap opens), `tag`. Returns `{ devices, sent, removed, failed }`. Through [`@xeplr/push`](https://www.npmjs.com/package/@xeplr/push), which the host sets up. |
|
|
@@ -153,7 +154,8 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
|
|
|
153
154
|
| `emailDownloadEmail` | `email-download-email` | Stream the raw message to an `.eml` file, named by `fileName`. `{ count, saved[], path, existing[] }`. |
|
|
154
155
|
| `emailDownloadAttachments` | `email-download-attachments` | Save attachments, optionally filtered by `extension` / `filenameContains`, named by `fileName` (see [Saving files](#saving-files-names-and-a-name-already-taken)). `{ count, saved[], path, skipped, existing[] }`. |
|
|
155
156
|
| `fileList` | `file-list` | The files in a folder, optionally matching `pattern` (`*.csv`, `Bank-2026-??.pdf`), sorted by name or date. `{ files[{ name, path, size, modifiedAt }], count }`. "Is it there?" is the exact name and `count`. See [Where files are](#where-files-are-one-data-root-a-folder-per-tenant). |
|
|
156
|
-
| `s3Upload` | `s3-upload` | One file to an S3 bucket (or an S3-compatible `endpoint`), in parts, with a SHA-256 checksum. Only writes. See [Backups](#backups-db-dump-and-s3-upload). |
|
|
157
|
+
| `s3Upload` | `s3-upload` | One file to an S3 bucket (or an S3-compatible `endpoint`), in parts, with a SHA-256 checksum. Only writes. See [Backups](#backups-db-dump-folder-archive-and-s3-upload). |
|
|
158
|
+
| `folderArchive` | `folder-archive` | A folder of the tenant's own data folder to one `.tar.gz` (`tar`, no shell); its `backups/` and `tmp/` always left out. See [Backups](#backups-db-dump-folder-archive-and-s3-upload). |
|
|
157
159
|
| `fileRead` | `file-read` | A CSV, text, Excel, JSON or NDJSON file as rows (format from the extension). `{ rows, count, columns, truncated }`, at most `maxRows` (50,000). |
|
|
158
160
|
| `fileMove` | `file-move` | Move or copy a file into a folder (made if missing) or to a new name; `ifExists` as every saving step. `{ path, name, moved, skipped }`. |
|
|
159
161
|
| `fileDelete` | `file-delete` | Delete one file, **permanently** (no trash; `file-move` to a "processed" folder is the reversible one). Never a folder. `ifMissing`: `fail` (default) or `skip`. `{ path, deleted, skipped }`. |
|
|
@@ -208,9 +210,9 @@ For `mode: 'table'` the source's declared column types (`getTableSchema`) are pa
|
|
|
208
210
|
|
|
209
211
|
Message ids are IMAP UIDs **scoped to `folder`**, and a move changes the id — download and delete before moving.
|
|
210
212
|
|
|
211
|
-
## Backups: `db-dump` and `s3-upload`
|
|
213
|
+
## Backups: `db-dump`, `folder-archive` and `s3-upload`
|
|
212
214
|
|
|
213
|
-
A
|
|
215
|
+
A backup is two steps: make one file — a database's dump, or a folder's archive — then send the file away.
|
|
214
216
|
|
|
215
217
|
```js
|
|
216
218
|
var dumped = await runAction({ name: 'db-dump', input: {
|
|
@@ -242,6 +244,20 @@ await runAction({ name: 's3-upload', input: {
|
|
|
242
244
|
- **Written under `<name>.partial`, renamed only when the program ends cleanly.** A failure removes it and fails the step with the end of what the program said, the password taken out. A program that is not installed is named ("pg_dump is not installed on this server").
|
|
243
245
|
- `system.signal` stops it (the program is killed); `system.onProgress({ bytes })` every second.
|
|
244
246
|
|
|
247
|
+
**folder-archive**
|
|
248
|
+
|
|
249
|
+
| input | |
|
|
250
|
+
|---|---|
|
|
251
|
+
| `folder` | A folder in the tenant's own folder, checked like every file step's (outside, `../`, a link pointing out: refused). Blank or `.`: all of it. |
|
|
252
|
+
| `exclude` | Patterns left out, comma separated (`node_modules, *.log`) — tar's `--exclude`. One starting with `-` is refused. |
|
|
253
|
+
| `fileName` | Default `<folder>-<UTC time>.tar.gz` (`all-…` for the whole folder), in the tenant's `backups/`. A name, not a path. |
|
|
254
|
+
|
|
255
|
+
→ `{ filePath, fileName, bytes, sha256, folder, warning }`
|
|
256
|
+
|
|
257
|
+
- **An app backs up only its own files.** The folder is inside the tenant's folder like every file step's, so an app (or the monitor) embedding this can never archive anything else on the server.
|
|
258
|
+
- **Never its own `backups/` and `tmp/`:** the whole folder is archived entry by entry without them, and naming either (or a folder in them) is refused — a backup must not hold the backups before it.
|
|
259
|
+
- `tar -czf` with an argument list, no shell; written as `<name>.partial`, renamed when tar ends. Exit 1 — GNU tar's "a file changed while it was read", the archive still whole — succeeds with `warning`; anything else removes the file and fails with the end of what tar said. `system.signal` kills tar.
|
|
260
|
+
|
|
245
261
|
**s3-upload**
|
|
246
262
|
|
|
247
263
|
| input | |
|
|
@@ -260,7 +276,7 @@ await runAction({ name: 's3-upload', input: {
|
|
|
260
276
|
```
|
|
261
277
|
- Peers, optional: `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` (`^3`). Missing → the step says to install them.
|
|
262
278
|
|
|
263
|
-
`test/backup/` checks
|
|
279
|
+
`test/backup/` checks all three: `folder-archive` with a real `tar` (patterns left out, `backups/` and `tmp/` never in it, outside paths and links refused); the dump programs with a stand-in on `PATH` (arguments, password only in the environment, a failure leaving no file), a real `pg_dump` restored with `pg_restore --list` when Postgres is here, and the upload against an S3 stand-in (one PUT, then parts, a SHA-256 checksum, never a read).
|
|
264
280
|
|
|
265
281
|
## Data operation
|
|
266
282
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// db-list-foreign-keys action — the foreign keys between the tables of one
|
|
2
|
+
// schema/database, each with its columns in key order: { name, table,
|
|
3
|
+
// columns, refTable, refColumns }. What an app joins two tables on without
|
|
4
|
+
// asking (the BI dataset canvas does). Only keys whose both tables are in the
|
|
5
|
+
// schema db-list-tables lists. A small catalogue read, like db-list-tables.
|
|
6
|
+
|
|
7
|
+
var dbDrivers = require('../../drivers/db');
|
|
8
|
+
|
|
9
|
+
module.exports = {
|
|
10
|
+
name: 'db-list-foreign-keys',
|
|
11
|
+
description: 'List foreign keys between tables of a schema/database. Routes to the driver named by input.dbType.',
|
|
12
|
+
requires: [], // per-driver requires checked at runtime once dbType is known
|
|
13
|
+
|
|
14
|
+
inputSchema: [
|
|
15
|
+
{ name: 'dbType', type: 'string', required: true, default: 'postgres', order: 1,
|
|
16
|
+
description: 'postgres | mysql | mssql | duckdb' },
|
|
17
|
+
{ name: 'connection', type: 'object', required: true, order: 2,
|
|
18
|
+
description: 'Connection config: { host, port, user, password, database } (duckdb: { file })' },
|
|
19
|
+
{ name: 'schema', type: 'string', order: 3,
|
|
20
|
+
description: 'Schema to read (postgres public, duckdb main, mssql every schema, mysql the connection database by default)' }
|
|
21
|
+
],
|
|
22
|
+
|
|
23
|
+
// Only reads the database's own catalogue, so the preview is the run.
|
|
24
|
+
preview: async function(ctx) { return { output: await module.exports.execute(ctx) }; },
|
|
25
|
+
|
|
26
|
+
execute: async function(ctx) {
|
|
27
|
+
var input = ctx.input || {};
|
|
28
|
+
var driver = dbDrivers.getDriver(input.dbType);
|
|
29
|
+
dbDrivers.checkDriverRequires('db-list-foreign-keys', input.dbType, driver);
|
|
30
|
+
if (typeof driver.listForeignKeys !== 'function') {
|
|
31
|
+
throw new Error('db-list-foreign-keys: driver "' + input.dbType + '" does not implement listForeignKeys');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
var pool = await driver.connect(input.connection);
|
|
35
|
+
try {
|
|
36
|
+
return { foreignKeys: await driver.listForeignKeys(pool, input.schema) };
|
|
37
|
+
} finally {
|
|
38
|
+
await driver.close(pool);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
};
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
// folder-archive action — a folder of this company's data folder, packed into
|
|
2
|
+
// one .tar.gz backup file (then s3-upload sends it away).
|
|
3
|
+
//
|
|
4
|
+
// folder inside the tenant's own folder (lib/dataDir.js), checked like
|
|
5
|
+
// every file step's: a path outside, ../ or a link pointing out is
|
|
6
|
+
// refused. '' or '.' is the whole tenant folder
|
|
7
|
+
// never the tenant's own backups/ and tmp/ — a backup must not hold the
|
|
8
|
+
// backups before it, nor half-written spools. Archiving the whole
|
|
9
|
+
// folder names every other entry; naming backups/ or tmp/ (or a
|
|
10
|
+
// folder inside them) is refused
|
|
11
|
+
// exclude patterns left out ("node_modules, *.log"), tar's --exclude; none
|
|
12
|
+
// may start with "-", which tar would read as an option
|
|
13
|
+
//
|
|
14
|
+
// `tar -czf` is RUN WITH AN ARGUMENT LIST, never a shell. The file is written
|
|
15
|
+
// as <name>.partial and renamed only once tar ends: exit 0, or 1 — GNU tar's
|
|
16
|
+
// "a file changed while it was read" (a log being written), where the archive
|
|
17
|
+
// is whole — which comes back as `warning`. Anything else removes the file and
|
|
18
|
+
// fails the step with the end of what tar said. A stop (system.signal) kills
|
|
19
|
+
// tar. Written to the tenant's backups/: <slug>-<UTC time>.tar.gz.
|
|
20
|
+
|
|
21
|
+
var fs = require('fs');
|
|
22
|
+
var path = require('path');
|
|
23
|
+
var crypto = require('crypto');
|
|
24
|
+
var { spawn } = require('child_process');
|
|
25
|
+
var dataDir = require('../../dataDir');
|
|
26
|
+
|
|
27
|
+
var OUT_FOLDER = 'backups';
|
|
28
|
+
var NEVER = ['backups', 'tmp'];
|
|
29
|
+
var STDERR_KEEP = 2048;
|
|
30
|
+
|
|
31
|
+
function refuse(message, code) {
|
|
32
|
+
var err = new Error(message);
|
|
33
|
+
err.code = code;
|
|
34
|
+
return err;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function stamp(date) {
|
|
38
|
+
return date.toISOString().slice(0, 19).replace(/:/g, '-');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** "uploads/2026" → "uploads-2026"; the whole folder → "all". */
|
|
42
|
+
function slugOf(rel) {
|
|
43
|
+
return String(rel).replace(/^[./]+|\/+$/g, '').replace(/[^A-Za-z0-9._-]+/g, '-') || 'all';
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** "node_modules, *.log" → the patterns; one starting with "-" is refused. */
|
|
47
|
+
function excludesOf(text) {
|
|
48
|
+
var list = (Array.isArray(text) ? text : String(text || '').split(/[\n,]/)).map(function(p) { return String(p).trim(); }).filter(Boolean);
|
|
49
|
+
list.forEach(function(p) {
|
|
50
|
+
if (p[0] === '-') throw refuse('folder-archive: the pattern "' + p + '" cannot start with "-".', 'BAD_ARGUMENT');
|
|
51
|
+
});
|
|
52
|
+
return list;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** What tar is asked to pack: from the tenant folder, its entries except backups/ and tmp/; else the folder itself. */
|
|
56
|
+
function targetOf(input) {
|
|
57
|
+
var root = fs.realpathSync(dataDir.tenantFolder());
|
|
58
|
+
var asked = String(input.folder === undefined || input.folder === null ? '' : input.folder).trim();
|
|
59
|
+
var dir = asked === '' || asked === '.' ? root : dataDir.resolve(asked, 'The folder');
|
|
60
|
+
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) throw refuse('"' + asked + '" is not a folder that exists.', 'NOT_A_FOLDER');
|
|
61
|
+
var rel = path.relative(root, dir);
|
|
62
|
+
var top = rel.split(path.sep)[0];
|
|
63
|
+
if (rel && NEVER.indexOf(top) >= 0) {
|
|
64
|
+
throw refuse('folder-archive: ' + top + '/ holds the backups and temporary files themselves — name a folder beside it.', 'PATH_NOT_ALLOWED');
|
|
65
|
+
}
|
|
66
|
+
if (!rel) {
|
|
67
|
+
var entries = fs.readdirSync(root).filter(function(n) { return NEVER.indexOf(n) < 0; }).sort();
|
|
68
|
+
if (!entries.length) throw refuse('folder-archive: there is nothing in this company\'s folder to back up.', 'NOTHING_TO_ARCHIVE');
|
|
69
|
+
return { dir: root, rel: '', cwd: root, members: entries };
|
|
70
|
+
}
|
|
71
|
+
return { dir: dir, rel: rel, cwd: path.dirname(dir), members: [path.basename(dir)] };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
module.exports = {
|
|
75
|
+
name: 'folder-archive',
|
|
76
|
+
example: { filePath: '/data/app/backups/uploads-2026-11-13T02-00-05.tar.gz', fileName: 'uploads-2026-11-13T02-00-05.tar.gz', bytes: 10485760,
|
|
77
|
+
sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', folder: 'uploads', warning: null },
|
|
78
|
+
description: 'Pack a folder of this company\'s data folder into one .tar.gz backup file (its backups/ and tmp/ always left out). Returns { filePath, fileName, bytes, sha256, folder, warning }.',
|
|
79
|
+
requires: [], // tar is checked when it runs
|
|
80
|
+
|
|
81
|
+
inputSchema: [
|
|
82
|
+
{ name: 'folder', type: 'string', order: 1,
|
|
83
|
+
description: 'A folder in this company\'s data folder, e.g. uploads. Blank or "." for all of it.' },
|
|
84
|
+
{ name: 'exclude', type: 'string', order: 2,
|
|
85
|
+
description: 'Names or patterns to leave out, comma separated — e.g. node_modules, *.log.' },
|
|
86
|
+
{ name: 'fileName', type: 'string', order: 3, group: 'File',
|
|
87
|
+
description: 'The archive\'s name. Default: <folder>-<time>.tar.gz.' }
|
|
88
|
+
],
|
|
89
|
+
outputSchema: [
|
|
90
|
+
{ name: 'filePath', type: 'string' }, { name: 'fileName', type: 'string' }, { name: 'bytes', type: 'number' },
|
|
91
|
+
{ name: 'sha256', type: 'string' }, { name: 'folder', type: 'string' }, { name: 'warning', type: 'string' }
|
|
92
|
+
],
|
|
93
|
+
|
|
94
|
+
execute: async function(ctx) {
|
|
95
|
+
var input = ctx.input || {};
|
|
96
|
+
var system = ctx.system || {};
|
|
97
|
+
var target = targetOf(input);
|
|
98
|
+
var excludes = excludesOf(input.exclude);
|
|
99
|
+
var outDir = dataDir.resolve(OUT_FOLDER, 'The backup folder');
|
|
100
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
101
|
+
var name = input.fileName ? String(input.fileName) : slugOf(target.rel) + '-' + stamp(new Date()) + '.tar.gz';
|
|
102
|
+
var filePath = dataDir.resolve(path.join(outDir, name), 'The backup file');
|
|
103
|
+
if (path.dirname(filePath) !== fs.realpathSync(outDir)) throw refuse('folder-archive: fileName "' + name + '" must be a name, not a path.', 'PATH_NOT_ALLOWED');
|
|
104
|
+
var partial = filePath + '.partial';
|
|
105
|
+
|
|
106
|
+
var args = ['-czf', partial].concat(excludes.map(function(p) { return '--exclude=' + p; }), ['-C', target.cwd, '--'], target.members);
|
|
107
|
+
var stderr = '';
|
|
108
|
+
var child;
|
|
109
|
+
var code = await new Promise(function(resolve, reject) {
|
|
110
|
+
try {
|
|
111
|
+
child = spawn('tar', args, { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
112
|
+
} catch (e) {
|
|
113
|
+
reject(e);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
child.on('error', function(e) { reject(e.code === 'ENOENT' ? refuse('tar is not installed on this server.', 'PROGRAM_NOT_FOUND') : e); });
|
|
117
|
+
child.stderr.on('data', function(d) { stderr = (stderr + d.toString()).slice(-STDERR_KEEP); });
|
|
118
|
+
var onAbort = function() { if (child.exitCode === null) child.kill('SIGTERM'); };
|
|
119
|
+
if (system.signal) {
|
|
120
|
+
if (system.signal.aborted) onAbort();
|
|
121
|
+
else system.signal.addEventListener('abort', onAbort, { once: true });
|
|
122
|
+
}
|
|
123
|
+
child.on('close', function(c, signal) {
|
|
124
|
+
if (system.signal) system.signal.removeEventListener('abort', onAbort);
|
|
125
|
+
if (signal) return reject(refuse('tar was stopped (' + signal + ').', 'ARCHIVE_FAILED'));
|
|
126
|
+
resolve(c);
|
|
127
|
+
});
|
|
128
|
+
}).catch(function(err) {
|
|
129
|
+
try { fs.unlinkSync(partial); } catch (_) {}
|
|
130
|
+
throw err;
|
|
131
|
+
});
|
|
132
|
+
var said = stderr.trim().split('\n').slice(-3).join(' ');
|
|
133
|
+
if (code !== 0 && code !== 1) {
|
|
134
|
+
try { fs.unlinkSync(partial); } catch (_) {}
|
|
135
|
+
throw refuse('tar ended with code ' + code + (said ? ': ' + said : ''), 'ARCHIVE_FAILED');
|
|
136
|
+
}
|
|
137
|
+
fs.renameSync(partial, filePath);
|
|
138
|
+
var hash = crypto.createHash('sha256');
|
|
139
|
+
await new Promise(function(resolve, reject) {
|
|
140
|
+
fs.createReadStream(filePath).on('data', function(d) { hash.update(d); }).on('end', resolve).on('error', reject);
|
|
141
|
+
});
|
|
142
|
+
return {
|
|
143
|
+
filePath: filePath, fileName: path.basename(filePath), bytes: fs.statSync(filePath).size, sha256: hash.digest('hex'),
|
|
144
|
+
folder: target.rel || '.', warning: code === 1 ? (said || 'some files changed while they were read') : null
|
|
145
|
+
};
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
// For tests.
|
|
149
|
+
_targetOf: targetOf,
|
|
150
|
+
_excludesOf: excludesOf
|
|
151
|
+
};
|
package/lib/builtins/index.js
CHANGED
|
@@ -49,6 +49,8 @@ module.exports = {
|
|
|
49
49
|
dbListViews: require('./db/list-views'),
|
|
50
50
|
dbListProcedures: require('./db/list-procedures'),
|
|
51
51
|
dbListColumns: require('./db/list-columns'),
|
|
52
|
+
// The foreign keys between a schema's tables, composite ones in key order.
|
|
53
|
+
dbListForeignKeys: require('./db/list-foreign-keys'),
|
|
52
54
|
|
|
53
55
|
// File actions — factory-shaped for the same reason (meta store binding).
|
|
54
56
|
fileUpload: require('./file/upload'),
|
|
@@ -59,6 +61,9 @@ module.exports = {
|
|
|
59
61
|
// One file to an S3 bucket, in parts, write-only. Peers: @aws-sdk/client-s3
|
|
60
62
|
// and @aws-sdk/lib-storage — optional, lazy-required.
|
|
61
63
|
s3Upload: require('./file/s3-upload'),
|
|
64
|
+
// A folder of the tenant's own data folder to one .tar.gz backup file (tar,
|
|
65
|
+
// no shell; its backups/ and tmp/ always left out).
|
|
66
|
+
folderArchive: require('./file/archive'),
|
|
62
67
|
|
|
63
68
|
// Email actions. Inbound routes to getInboundProvider(input.emailType)
|
|
64
69
|
// (imap today), outbound to getOutboundProvider (smtp) — see
|
package/lib/drivers/db/duckdb.js
CHANGED
|
@@ -354,6 +354,7 @@ async function acquire(entry) {
|
|
|
354
354
|
// own connection, and the time limit is a timer that does the same. A query
|
|
355
355
|
// still waiting for a free connection is let go of before it ever runs.
|
|
356
356
|
var stop = require('./stop');
|
|
357
|
+
var foreignKeys = require('./foreignKeys');
|
|
357
358
|
|
|
358
359
|
/** acquire(), unless the stop comes first; a connection that arrives too late goes straight back. */
|
|
359
360
|
function acquireUnlessStopped(entry, signal) {
|
|
@@ -658,6 +659,28 @@ async function listTables(pool, schema) {
|
|
|
658
659
|
return r.rows.map(function(row) { return row.table_name; });
|
|
659
660
|
}
|
|
660
661
|
|
|
662
|
+
// Foreign keys between this schema's tables — see foreignKeys.js.
|
|
663
|
+
async function listForeignKeys(pool, schema) {
|
|
664
|
+
var r = await query(pool,
|
|
665
|
+
"SELECT constraint_index, table_name, constraint_column_names, referenced_table, referenced_column_names " +
|
|
666
|
+
"FROM duckdb_constraints() WHERE constraint_type = 'FOREIGN KEY' AND schema_name = $1 " +
|
|
667
|
+
'ORDER BY table_name, constraint_index',
|
|
668
|
+
[schema || 'main']);
|
|
669
|
+
var rows = [];
|
|
670
|
+
r.rows.forEach(function(row) {
|
|
671
|
+
// LIST columns: this driver hands them back as arrays.
|
|
672
|
+
var cols = row.constraint_column_names || [];
|
|
673
|
+
var refs = row.referenced_column_names || [];
|
|
674
|
+
cols.forEach(function(col, i) {
|
|
675
|
+
rows.push({ name: row.table_name + '_fk_' + row.constraint_index, table: row.table_name,
|
|
676
|
+
refTable: row.referenced_table, column: col, refColumn: refs[i] });
|
|
677
|
+
});
|
|
678
|
+
});
|
|
679
|
+
return foreignKeys.group(rows);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
|
|
661
684
|
async function listViews(pool, schema) {
|
|
662
685
|
var r = await query(pool,
|
|
663
686
|
'SELECT table_name FROM information_schema.views WHERE table_schema = $1 ORDER BY table_name',
|
|
@@ -1039,6 +1062,7 @@ module.exports = {
|
|
|
1039
1062
|
|
|
1040
1063
|
getTableSchema: getTableSchema,
|
|
1041
1064
|
listTables: listTables,
|
|
1065
|
+
listForeignKeys: listForeignKeys,
|
|
1042
1066
|
listViews: listViews,
|
|
1043
1067
|
listProcedures: listProcedures,
|
|
1044
1068
|
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Foreign keys between the tables of one schema, as every SQL driver's
|
|
2
|
+
// listForeignKeys returns them — what the BI dataset canvas joins two tables
|
|
3
|
+
// on without asking. One entry per constraint, its columns in key order (a
|
|
4
|
+
// composite key keeps every pair): { name, table, columns, refTable,
|
|
5
|
+
// refColumns }. Each driver reads its own catalogue one row per column pair
|
|
6
|
+
// and hands the rows here; only keys whose BOTH tables are in the schema
|
|
7
|
+
// listTables lists, so every name is one listTables also returns.
|
|
8
|
+
|
|
9
|
+
/** Rows of { name, table, refTable, column, refColumn }, in key order → one entry per key. */
|
|
10
|
+
function group(rows) {
|
|
11
|
+
var byKey = {};
|
|
12
|
+
var out = [];
|
|
13
|
+
(rows || []).forEach(function(r) {
|
|
14
|
+
var key = r.table + '\u0000' + r.name;
|
|
15
|
+
var fk = byKey[key];
|
|
16
|
+
if (!fk) {
|
|
17
|
+
fk = byKey[key] = { name: r.name, table: r.table, columns: [], refTable: r.refTable, refColumns: [] };
|
|
18
|
+
out.push(fk);
|
|
19
|
+
}
|
|
20
|
+
fk.columns.push(r.column);
|
|
21
|
+
fk.refColumns.push(r.refColumn);
|
|
22
|
+
});
|
|
23
|
+
return out;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
module.exports = { group: group };
|
package/lib/drivers/db/mssql.js
CHANGED
|
@@ -57,6 +57,7 @@ var NATIVE_TO_LOGICAL = {
|
|
|
57
57
|
|
|
58
58
|
// Dates and times: one shared rule for every driver (dateValues.js).
|
|
59
59
|
var dateValues = require('./dateValues');
|
|
60
|
+
var foreignKeys = require('./foreignKeys');
|
|
60
61
|
|
|
61
62
|
var INTERNAL_ID_COL = '__xeplr_id__';
|
|
62
63
|
var INTERNAL_MOVEMENT_COL = '__xeplr_movement_id__';
|
|
@@ -348,6 +349,25 @@ async function listTables(pool, schema) {
|
|
|
348
349
|
return r.rows.map(function(row) { return row.name; });
|
|
349
350
|
}
|
|
350
351
|
|
|
352
|
+
// Foreign keys between this schema's tables — see foreignKeys.js.
|
|
353
|
+
async function listForeignKeys(pool, schema) {
|
|
354
|
+
var r = await query(pool,
|
|
355
|
+
'SELECT fk.name AS name, t.name AS [table], rt.name AS refTable, ' +
|
|
356
|
+
'c.name AS [column], rc.name AS refColumn ' +
|
|
357
|
+
'FROM sys.foreign_keys fk ' +
|
|
358
|
+
'JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id ' +
|
|
359
|
+
'JOIN sys.tables t ON t.object_id = fk.parent_object_id ' +
|
|
360
|
+
'JOIN sys.tables rt ON rt.object_id = fk.referenced_object_id ' +
|
|
361
|
+
'JOIN sys.columns c ON c.object_id = fkc.parent_object_id AND c.column_id = fkc.parent_column_id ' +
|
|
362
|
+
'JOIN sys.columns rc ON rc.object_id = fkc.referenced_object_id AND rc.column_id = fkc.referenced_column_id ' +
|
|
363
|
+
'WHERE (@p0 IS NULL OR (SCHEMA_NAME(t.schema_id) = @p0 AND SCHEMA_NAME(rt.schema_id) = @p0)) ' +
|
|
364
|
+
'ORDER BY t.name, fk.name, fkc.constraint_column_id',
|
|
365
|
+
[schema || null]
|
|
366
|
+
);
|
|
367
|
+
return foreignKeys.group(r.rows);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
|
|
351
371
|
async function listViews(pool, schema) {
|
|
352
372
|
var r = await query(pool,
|
|
353
373
|
'SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES ' +
|
|
@@ -545,6 +565,7 @@ module.exports = {
|
|
|
545
565
|
// Introspection (used by uploader.reconcile + the db-list-* actions) + rollback
|
|
546
566
|
getTableSchema: getTableSchema,
|
|
547
567
|
listTables: listTables,
|
|
568
|
+
listForeignKeys: listForeignKeys,
|
|
548
569
|
listViews: listViews,
|
|
549
570
|
listProcedures: listProcedures,
|
|
550
571
|
rollbackMovement: rollbackMovement,
|
package/lib/drivers/db/mysql.js
CHANGED
|
@@ -55,6 +55,7 @@ var NATIVE_TO_LOGICAL = {
|
|
|
55
55
|
|
|
56
56
|
// Dates and times: one shared rule for every driver (dateValues.js).
|
|
57
57
|
var dateValues = require('./dateValues');
|
|
58
|
+
var foreignKeys = require('./foreignKeys');
|
|
58
59
|
|
|
59
60
|
var INTERNAL_ID_COL = '__xeplr_id__';
|
|
60
61
|
var INTERNAL_MOVEMENT_COL = '__xeplr_movement_id__';
|
|
@@ -383,6 +384,21 @@ async function listTables(pool, schema) {
|
|
|
383
384
|
return r.rows.map(function(row) { return row.name; });
|
|
384
385
|
}
|
|
385
386
|
|
|
387
|
+
// Foreign keys between this schema's tables — see foreignKeys.js.
|
|
388
|
+
async function listForeignKeys(pool, schema) {
|
|
389
|
+
var r = await query(pool,
|
|
390
|
+
'SELECT CONSTRAINT_NAME AS name, TABLE_NAME AS `table`, REFERENCED_TABLE_NAME AS refTable, ' +
|
|
391
|
+
'COLUMN_NAME AS `column`, REFERENCED_COLUMN_NAME AS refColumn ' +
|
|
392
|
+
'FROM information_schema.KEY_COLUMN_USAGE ' +
|
|
393
|
+
'WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND REFERENCED_TABLE_NAME IS NOT NULL ' +
|
|
394
|
+
'AND REFERENCED_TABLE_SCHEMA = TABLE_SCHEMA ' +
|
|
395
|
+
'ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION',
|
|
396
|
+
[schema || null]
|
|
397
|
+
);
|
|
398
|
+
return foreignKeys.group(r.rows);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
|
|
386
402
|
async function listViews(pool, schema) {
|
|
387
403
|
var r = await query(pool,
|
|
388
404
|
"SELECT TABLE_NAME AS name FROM information_schema.TABLES " +
|
|
@@ -577,6 +593,7 @@ module.exports = {
|
|
|
577
593
|
// Introspection (used by uploader.reconcile + the db-list-* actions) + rollback
|
|
578
594
|
getTableSchema: getTableSchema,
|
|
579
595
|
listTables: listTables,
|
|
596
|
+
listForeignKeys: listForeignKeys,
|
|
580
597
|
listViews: listViews,
|
|
581
598
|
listProcedures: listProcedures,
|
|
582
599
|
rollbackMovement: rollbackMovement,
|
|
@@ -89,6 +89,7 @@ async function close(pool) {
|
|
|
89
89
|
// time limit is statement_timeout, so Postgres enforces it even if this
|
|
90
90
|
// process dies; our own timer, a little later, is only the backstop.
|
|
91
91
|
var stop = require('./stop');
|
|
92
|
+
var foreignKeys = require('./foreignKeys');
|
|
92
93
|
var QUERY_CANCELED = '57014';
|
|
93
94
|
var BACKSTOP_MS = 2000;
|
|
94
95
|
|
|
@@ -381,6 +382,25 @@ async function listTables(pool, schema) {
|
|
|
381
382
|
return r.rows.map(function(row) { return row.table_name; });
|
|
382
383
|
}
|
|
383
384
|
|
|
385
|
+
// Foreign keys between this schema's tables — see foreignKeys.js.
|
|
386
|
+
async function listForeignKeys(pool, schema) {
|
|
387
|
+
var r = await pool.query(
|
|
388
|
+
'SELECT c.conname AS name, t.relname AS "table", rt.relname AS "refTable", ' +
|
|
389
|
+
'a.attname AS "column", ra.attname AS "refColumn" ' +
|
|
390
|
+
'FROM pg_constraint c ' +
|
|
391
|
+
'JOIN pg_class t ON t.oid = c.conrelid JOIN pg_namespace n ON n.oid = t.relnamespace ' +
|
|
392
|
+
'JOIN pg_class rt ON rt.oid = c.confrelid JOIN pg_namespace rn ON rn.oid = rt.relnamespace ' +
|
|
393
|
+
'CROSS JOIN LATERAL unnest(c.conkey, c.confkey) WITH ORDINALITY AS k(col, refcol, ord) ' +
|
|
394
|
+
'JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.col ' +
|
|
395
|
+
'JOIN pg_attribute ra ON ra.attrelid = c.confrelid AND ra.attnum = k.refcol ' +
|
|
396
|
+
"WHERE c.contype = 'f' AND n.nspname = $1 AND rn.nspname = $1 " +
|
|
397
|
+
'ORDER BY t.relname, c.conname, k.ord',
|
|
398
|
+
[schema || 'public']
|
|
399
|
+
);
|
|
400
|
+
return foreignKeys.group(r.rows);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
|
|
384
404
|
async function listViews(pool, schema) {
|
|
385
405
|
var r = await pool.query(
|
|
386
406
|
"SELECT table_name FROM information_schema.views " +
|
|
@@ -546,6 +566,7 @@ module.exports = {
|
|
|
546
566
|
// Introspection (used by uploader.reconcile + the db-list-* actions)
|
|
547
567
|
getTableSchema: getTableSchema,
|
|
548
568
|
listTables: listTables,
|
|
569
|
+
listForeignKeys: listForeignKeys,
|
|
549
570
|
listViews: listViews,
|
|
550
571
|
listProcedures: listProcedures,
|
|
551
572
|
|
package/package.json
CHANGED