@xeplr/actions 2.2.2 → 2.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/lib/builtins/db/dump.js +199 -0
- package/lib/builtins/file/s3-upload.js +131 -0
- package/lib/builtins/index.js +6 -0
- package/package.json +11 -1
package/README.md
CHANGED
|
@@ -138,6 +138,7 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
|
|
|
138
138
|
| `dbFetch` | `db-fetch` | Stream rows from a table (`mode: 'table'`) or SQL (`mode: 'query'`, `params`). Default `streaming_mode: true` → NDJSON file `{ filePath, format: 'jsonl', bytes, rows }`; `false` → `{ rows, rowCount }`. |
|
|
139
139
|
| `dbPush` | `db-push` | Write `rows` or an NDJSON `filePath` into `targetTable` through the uploader. `primaryKeys` → upsert. |
|
|
140
140
|
| `dbMove` | `db-move` | One window of rows from a source table, query or procedure into a target table in one streamed pass. See [db-move](#db-move). |
|
|
141
|
+
| `dbDump` | `db-dump` | A whole database to one backup file with its own dump program: `pg_dump -Fc` (Postgres, `.dump`) or `mysqldump` (MySQL, `.sql.gz`). See [Backups](#backups-db-dump-and-s3-upload). |
|
|
141
142
|
| `dbProcedure` | `db-procedure` | Call a stored procedure; returns `{ rows, rowCount, procedure, sql }`. See [db-procedure](#db-procedure). |
|
|
142
143
|
| `dbListTables` | `db-list-tables` | `{ tables }` — base tables in a schema/database. |
|
|
143
144
|
| `dbListViews` | `db-list-views` | `{ views }`. |
|
|
@@ -152,6 +153,7 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
|
|
|
152
153
|
| `emailDownloadEmail` | `email-download-email` | Stream the raw message to an `.eml` file, named by `fileName`. `{ count, saved[], path, existing[] }`. |
|
|
153
154
|
| `emailDownloadAttachments` | `email-download-attachments` | Save attachments, optionally filtered by `extension` / `filenameContains`, named by `fileName` (see [Saving files](#saving-files-names-and-a-name-already-taken)). `{ count, saved[], path, skipped, existing[] }`. |
|
|
154
155
|
| `fileList` | `file-list` | The files in a folder, optionally matching `pattern` (`*.csv`, `Bank-2026-??.pdf`), sorted by name or date. `{ files[{ name, path, size, modifiedAt }], count }`. "Is it there?" is the exact name and `count`. See [Where files are](#where-files-are-one-data-root-a-folder-per-tenant). |
|
|
156
|
+
| `s3Upload` | `s3-upload` | One file to an S3 bucket (or an S3-compatible `endpoint`), in parts, with a SHA-256 checksum. Only writes. See [Backups](#backups-db-dump-and-s3-upload). |
|
|
155
157
|
| `fileRead` | `file-read` | A CSV, text, Excel, JSON or NDJSON file as rows (format from the extension). `{ rows, count, columns, truncated }`, at most `maxRows` (50,000). |
|
|
156
158
|
| `fileMove` | `file-move` | Move or copy a file into a folder (made if missing) or to a new name; `ifExists` as every saving step. `{ path, name, moved, skipped }`. |
|
|
157
159
|
| `fileDelete` | `file-delete` | Delete one file, **permanently** (no trash; `file-move` to a "processed" folder is the reversible one). Never a folder. `ifMissing`: `fail` (default) or `skip`. `{ path, deleted, skipped }`. |
|
|
@@ -206,6 +208,60 @@ For `mode: 'table'` the source's declared column types (`getTableSchema`) are pa
|
|
|
206
208
|
|
|
207
209
|
Message ids are IMAP UIDs **scoped to `folder`**, and a move changes the id — download and delete before moving.
|
|
208
210
|
|
|
211
|
+
## Backups: `db-dump` and `s3-upload`
|
|
212
|
+
|
|
213
|
+
A database's backup is two steps: dump it to a file, then send the file away.
|
|
214
|
+
|
|
215
|
+
```js
|
|
216
|
+
var dumped = await runAction({ name: 'db-dump', input: {
|
|
217
|
+
dbType: 'postgres', connectionInfoId: 'main-server', // the server's login, resolved by the host
|
|
218
|
+
database: 'shop'
|
|
219
|
+
} })
|
|
220
|
+
// → { filePath: '<tenant>/backups/shop-2026-11-13T02-00-05.dump', fileName, bytes, sha256, dbType, database, durationMs }
|
|
221
|
+
|
|
222
|
+
await runAction({ name: 's3-upload', input: {
|
|
223
|
+
connectionInfoId: 'backup-bucket', // { bucket, region, accessKeyId, secretAccessKey, prefix? }
|
|
224
|
+
filePath: dumped.output.filePath, deleteAfter: true
|
|
225
|
+
} })
|
|
226
|
+
// → { bucket, key: 'backups/shop-2026-11-13T02-00-05.dump', bytes, etag, versionId, deleted: true }
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
**db-dump**
|
|
230
|
+
|
|
231
|
+
| input | |
|
|
232
|
+
|---|---|
|
|
233
|
+
| `dbType` | `postgres` or `mysql`. `mssql` is refused: not supported yet. |
|
|
234
|
+
| `connection` | The server's login `{ host, port, user, password }`. |
|
|
235
|
+
| `database` | Required: which database on that server. |
|
|
236
|
+
| `folder` | In the tenant's folder; default `backups`. |
|
|
237
|
+
| `fileName` | Default `<database>-<UTC time>.dump` (Postgres) or `.sql.gz` (MySQL). A name, not a path. |
|
|
238
|
+
|
|
239
|
+
- **Postgres:** `pg_dump -Fc --no-owner --no-acl` — the custom format, already compressed, restored with `pg_restore`. `pg_dump` must be **at least the server's version** (it refuses an older one, and the step says so).
|
|
240
|
+
- **MySQL / MariaDB:** `mysqldump --single-transaction --quick --routines --triggers --events --databases <db>`, gzipped as it is written.
|
|
241
|
+
- **No shell, ever.** The program is run with an argument list built here; the password goes in its environment (`PGPASSWORD`, `MYSQL_PWD`), never on the command line where `ps` shows it. A database, host or user starting with `-` is refused (it would be read as an option).
|
|
242
|
+
- **Written under `<name>.partial`, renamed only when the program ends cleanly.** A failure removes it and fails the step with the end of what the program said, the password taken out. A program that is not installed is named ("pg_dump is not installed on this server").
|
|
243
|
+
- `system.signal` stops it (the program is killed); `system.onProgress({ bytes })` every second.
|
|
244
|
+
|
|
245
|
+
**s3-upload**
|
|
246
|
+
|
|
247
|
+
| input | |
|
|
248
|
+
|---|---|
|
|
249
|
+
| `connection` | `{ bucket, region, accessKeyId, secretAccessKey, prefix?, sessionToken?, endpoint?, forcePathStyle? }`. Without `accessKeyId` the SDK's own chain is used (an EC2 role). `endpoint` for an S3-compatible store. |
|
|
250
|
+
| `filePath` | The file, in the tenant's folder. |
|
|
251
|
+
| `key` | Default `<prefix>/<file name>`. |
|
|
252
|
+
| `deleteAfter` | `true`: the local file is deleted once it is uploaded. |
|
|
253
|
+
| `contentType` | Default `application/octet-stream`. |
|
|
254
|
+
|
|
255
|
+
- **Streamed in 16 MB parts** (`@aws-sdk/lib-storage`), so any size needs little memory. Every part carries a **SHA-256 checksum** — required by a bucket with Object Lock, and checked by S3.
|
|
256
|
+
- **It only writes.** No list, get or head of the bucket, so the key it uses needs `s3:PutObject` and `s3:AbortMultipartUpload` (a failed upload is aborted) on the prefix, and nothing else:
|
|
257
|
+
|
|
258
|
+
```json
|
|
259
|
+
{ "Effect": "Allow", "Action": ["s3:PutObject", "s3:AbortMultipartUpload"], "Resource": "arn:aws:s3:::<bucket>/backups/*" }
|
|
260
|
+
```
|
|
261
|
+
- Peers, optional: `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` (`^3`). Missing → the step says to install them.
|
|
262
|
+
|
|
263
|
+
`test/backup/` checks both: the dump programs with a stand-in on `PATH` (arguments, password only in the environment, a failure leaving no file), a real `pg_dump` restored with `pg_restore --list` when Postgres is here, and the upload against an S3 stand-in (one PUT, then parts, a SHA-256 checksum, never a read).
|
|
264
|
+
|
|
209
265
|
## Data operation
|
|
210
266
|
|
|
211
267
|
The step between steps. A fetch returns everything a table holds; an email
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// db-dump action — a whole database to one backup file, by the database's own
|
|
2
|
+
// dump program: pg_dump (Postgres) or mysqldump (MySQL / MariaDB).
|
|
3
|
+
//
|
|
4
|
+
// postgres pg_dump -Fc --no-owner --no-acl → <db>-<when>.dump (custom
|
|
5
|
+
// format: already compressed, restored with pg_restore)
|
|
6
|
+
// mysql mysqldump --single-transaction --quick --routines --triggers
|
|
7
|
+
// --events, gzipped here → <db>-<when>.sql.gz
|
|
8
|
+
// mssql refused: not supported yet
|
|
9
|
+
//
|
|
10
|
+
// THE PROGRAM IS RUN WITH AN ARGUMENT LIST, never a shell, and only these two
|
|
11
|
+
// programs, with arguments built here — nothing from the input reaches the
|
|
12
|
+
// command line except the host, port, user and database, each passed as its
|
|
13
|
+
// own argument. The password goes in the program's environment (PGPASSWORD,
|
|
14
|
+
// MYSQL_PWD), never on the command line, where `ps` would show it.
|
|
15
|
+
//
|
|
16
|
+
// THE FILE is written under a temporary name and renamed only once the
|
|
17
|
+
// program has finished cleanly, so a half-written dump never looks like a
|
|
18
|
+
// backup. A failure removes it and fails the step with the end of what the
|
|
19
|
+
// program said (stderr), the password taken out. Inside the tenant's own
|
|
20
|
+
// folder (lib/dataDir.js), `backups/` unless `folder` says otherwise.
|
|
21
|
+
//
|
|
22
|
+
// A stop (system.signal) kills the program. system.onProgress({ bytes })
|
|
23
|
+
// reports how much is written, every second or so.
|
|
24
|
+
|
|
25
|
+
var fs = require('fs');
|
|
26
|
+
var path = require('path');
|
|
27
|
+
var zlib = require('zlib');
|
|
28
|
+
var crypto = require('crypto');
|
|
29
|
+
var { spawn } = require('child_process');
|
|
30
|
+
var dataDir = require('../../dataDir');
|
|
31
|
+
|
|
32
|
+
var PROGRAMS = {
|
|
33
|
+
postgres: { bin: 'pg_dump', ext: '.dump', gzip: false, port: 5432 },
|
|
34
|
+
mysql: { bin: 'mysqldump', ext: '.sql.gz', gzip: true, port: 3306 }
|
|
35
|
+
};
|
|
36
|
+
var DEFAULT_FOLDER = 'backups';
|
|
37
|
+
var STDERR_KEEP = 2048;
|
|
38
|
+
var PROGRESS_MS = 1000;
|
|
39
|
+
|
|
40
|
+
/** "2026-11-13T02-00-05" — a file name's time, UTC. */
|
|
41
|
+
function stamp(date) {
|
|
42
|
+
return date.toISOString().slice(0, 19).replace(/:/g, '-');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A database name made safe for a file name. */
|
|
46
|
+
function fileSafe(name) {
|
|
47
|
+
return String(name).replace(/[^A-Za-z0-9._-]+/g, '_').replace(/^\.+/, '') || 'database';
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** The program's arguments and environment — the password only in the environment. */
|
|
51
|
+
function commandFor(dbType, conn, database) {
|
|
52
|
+
var p = PROGRAMS[dbType];
|
|
53
|
+
var host = String(conn.host || 'localhost');
|
|
54
|
+
var port = String(conn.port || p.port);
|
|
55
|
+
var user = conn.user ? String(conn.user) : null;
|
|
56
|
+
var env = Object.assign({}, process.env);
|
|
57
|
+
if (dbType === 'postgres') {
|
|
58
|
+
var args = ['-Fc', '--no-owner', '--no-acl', '-h', host, '-p', port];
|
|
59
|
+
if (user) args.push('-U', user);
|
|
60
|
+
args.push('-d', database);
|
|
61
|
+
delete env.PGPASSWORD;
|
|
62
|
+
if (conn.password) env.PGPASSWORD = String(conn.password);
|
|
63
|
+
return { bin: p.bin, args: args, env: env };
|
|
64
|
+
}
|
|
65
|
+
var margs = ['--single-transaction', '--quick', '--routines', '--triggers', '--events', '-h', host, '-P', port];
|
|
66
|
+
if (user) margs.push('-u', user);
|
|
67
|
+
margs.push('--databases', database);
|
|
68
|
+
delete env.MYSQL_PWD;
|
|
69
|
+
if (conn.password) env.MYSQL_PWD = String(conn.password);
|
|
70
|
+
return { bin: p.bin, args: margs, env: env };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function refuse(message, code) {
|
|
74
|
+
var err = new Error(message);
|
|
75
|
+
err.code = code;
|
|
76
|
+
return err;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
module.exports = {
|
|
80
|
+
name: 'db-dump',
|
|
81
|
+
// It writes a file, so it is never previewed.
|
|
82
|
+
example: { filePath: '/data/app/backups/shop-2026-11-13T02-00-05.dump', fileName: 'shop-2026-11-13T02-00-05.dump', bytes: 104857600,
|
|
83
|
+
sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', dbType: 'postgres', database: 'shop', durationMs: 42000 },
|
|
84
|
+
description: 'Back up a whole database to one file with its own dump program — pg_dump (Postgres, custom format) or mysqldump (MySQL, gzipped). Returns { filePath, fileName, bytes, sha256, dbType, database, durationMs }.',
|
|
85
|
+
requires: [], // the dump programs are checked when it runs
|
|
86
|
+
|
|
87
|
+
inputSchema: [
|
|
88
|
+
{ name: 'dbType', type: 'string', required: true, order: 1,
|
|
89
|
+
description: 'postgres or mysql (SQL Server: not supported yet).' },
|
|
90
|
+
{ name: 'connection', type: 'object', required: true, order: 2,
|
|
91
|
+
description: '{ host, port, user, password } — the server\'s login, a saved connection (connectionInfoId) resolved by the host.' },
|
|
92
|
+
{ name: 'database', type: 'string', required: true, order: 3,
|
|
93
|
+
description: 'The database on that server to back up.' },
|
|
94
|
+
{ name: 'folder', type: 'string', order: 4, group: 'File',
|
|
95
|
+
description: 'Where in this company\'s data folder the file goes. Default: backups.' },
|
|
96
|
+
{ name: 'fileName', type: 'string', order: 5, group: 'File',
|
|
97
|
+
description: 'The file\'s name. Default: <database>-<UTC time>.dump (Postgres) or .sql.gz (MySQL).' }
|
|
98
|
+
],
|
|
99
|
+
outputSchema: [
|
|
100
|
+
{ name: 'filePath', type: 'string' }, { name: 'fileName', type: 'string' }, { name: 'bytes', type: 'number' },
|
|
101
|
+
{ name: 'sha256', type: 'string' }, { name: 'dbType', type: 'string' }, { name: 'database', type: 'string' },
|
|
102
|
+
{ name: 'durationMs', type: 'number' }
|
|
103
|
+
],
|
|
104
|
+
|
|
105
|
+
execute: async function(ctx) {
|
|
106
|
+
var input = ctx.input || {};
|
|
107
|
+
var system = ctx.system || {};
|
|
108
|
+
var dbType = String(input.dbType || '').toLowerCase();
|
|
109
|
+
if (dbType === 'mssql') throw refuse('db-dump: SQL Server is not supported yet — postgres or mysql.', 'DB_TYPE_NOT_SUPPORTED');
|
|
110
|
+
if (!PROGRAMS[dbType]) throw refuse('db-dump: dbType must be postgres or mysql, not "' + input.dbType + '".', 'DB_TYPE_NOT_SUPPORTED');
|
|
111
|
+
var conn = input.connection || {};
|
|
112
|
+
var database = input.database;
|
|
113
|
+
if (!database) throw refuse('db-dump: say which database to back up.', 'NO_DATABASE');
|
|
114
|
+
// An argument starting with "-" would be read as an option, not a name.
|
|
115
|
+
[['database', database], ['host', conn.host], ['user', conn.user]].forEach(function(f) {
|
|
116
|
+
if (f[1] && /^-/.test(String(f[1]))) throw refuse('db-dump: the ' + f[0] + ' "' + f[1] + '" cannot start with "-".', 'BAD_ARGUMENT');
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
var started = Date.now();
|
|
120
|
+
var program = PROGRAMS[dbType];
|
|
121
|
+
var folder = dataDir.resolve(input.folder || DEFAULT_FOLDER, 'The backup folder');
|
|
122
|
+
fs.mkdirSync(folder, { recursive: true });
|
|
123
|
+
var name = input.fileName ? String(input.fileName) : fileSafe(database) + '-' + stamp(new Date()) + program.ext;
|
|
124
|
+
var filePath = dataDir.resolve(path.join(folder, name), 'The backup file');
|
|
125
|
+
if (path.dirname(filePath) !== fs.realpathSync(folder)) throw refuse('db-dump: fileName "' + name + '" must be a name, not a path.', 'PATH_NOT_ALLOWED');
|
|
126
|
+
var partial = filePath + '.partial';
|
|
127
|
+
|
|
128
|
+
var cmd = commandFor(dbType, conn, String(database));
|
|
129
|
+
var secret = conn.password ? String(conn.password) : null;
|
|
130
|
+
var out = fs.createWriteStream(partial);
|
|
131
|
+
var hash = crypto.createHash('sha256');
|
|
132
|
+
var bytes = 0;
|
|
133
|
+
var stderr = '';
|
|
134
|
+
var child;
|
|
135
|
+
|
|
136
|
+
var done = new Promise(function(resolve, reject) {
|
|
137
|
+
var settled = false;
|
|
138
|
+
function fail(err) { if (settled) return; settled = true; reject(err); }
|
|
139
|
+
function ok() { if (settled) return; settled = true; resolve(); }
|
|
140
|
+
|
|
141
|
+
try {
|
|
142
|
+
child = spawn(cmd.bin, cmd.args, { env: cmd.env, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
143
|
+
} catch (e) {
|
|
144
|
+
fail(e);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
child.on('error', function(e) {
|
|
148
|
+
fail(e.code === 'ENOENT' ? refuse(cmd.bin + ' is not installed on this server (needed to back up ' + dbType + ').', 'PROGRAM_NOT_FOUND') : e);
|
|
149
|
+
});
|
|
150
|
+
child.stderr.on('data', function(d) { stderr = (stderr + d.toString()).slice(-STDERR_KEEP); });
|
|
151
|
+
|
|
152
|
+
var source = program.gzip ? child.stdout.pipe(zlib.createGzip()) : child.stdout;
|
|
153
|
+
source.on('data', function(chunk) { hash.update(chunk); bytes += chunk.length; });
|
|
154
|
+
source.on('error', fail);
|
|
155
|
+
out.on('error', fail);
|
|
156
|
+
source.pipe(out);
|
|
157
|
+
|
|
158
|
+
var exit = null;
|
|
159
|
+
var written = false;
|
|
160
|
+
function finish() {
|
|
161
|
+
if (exit === null || !written) return;
|
|
162
|
+
if (exit.code === 0) return ok();
|
|
163
|
+
var said = stderr.trim();
|
|
164
|
+
if (secret) said = said.split(secret).join('****');
|
|
165
|
+
var why = exit.signal ? 'was stopped (' + exit.signal + ')' : 'ended with code ' + exit.code;
|
|
166
|
+
fail(refuse(cmd.bin + ' ' + why + (said ? ': ' + said : ''), 'DUMP_FAILED'));
|
|
167
|
+
}
|
|
168
|
+
child.on('close', function(code, signal) { exit = { code: code, signal: signal }; finish(); });
|
|
169
|
+
out.on('finish', function() { written = true; finish(); });
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
var timer = system.onProgress ? setInterval(function() { system.onProgress({ bytes: bytes }); }, PROGRESS_MS) : null;
|
|
173
|
+
var onAbort = function() { if (child && child.exitCode === null) child.kill('SIGTERM'); };
|
|
174
|
+
if (system.signal) {
|
|
175
|
+
if (system.signal.aborted) onAbort();
|
|
176
|
+
else system.signal.addEventListener('abort', onAbort, { once: true });
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
try {
|
|
180
|
+
await done;
|
|
181
|
+
} catch (err) {
|
|
182
|
+
out.destroy();
|
|
183
|
+
try { fs.unlinkSync(partial); } catch (_) {}
|
|
184
|
+
throw err;
|
|
185
|
+
} finally {
|
|
186
|
+
if (timer) clearInterval(timer);
|
|
187
|
+
if (system.signal) system.signal.removeEventListener('abort', onAbort);
|
|
188
|
+
}
|
|
189
|
+
fs.renameSync(partial, filePath);
|
|
190
|
+
if (system.onProgress) system.onProgress({ bytes: bytes });
|
|
191
|
+
return {
|
|
192
|
+
filePath: filePath, fileName: path.basename(filePath), bytes: bytes, sha256: hash.digest('hex'),
|
|
193
|
+
dbType: dbType, database: String(database), durationMs: Date.now() - started
|
|
194
|
+
};
|
|
195
|
+
},
|
|
196
|
+
|
|
197
|
+
// For tests: the command a dump runs, without running it.
|
|
198
|
+
_commandFor: commandFor
|
|
199
|
+
};
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// s3-upload action — one file from this company's data folder to an S3 bucket
|
|
2
|
+
// (or an S3-compatible store: `endpoint`).
|
|
3
|
+
//
|
|
4
|
+
// Streamed in parts (@aws-sdk/lib-storage Upload), so a file of any size
|
|
5
|
+
// needs no more memory than a few parts. Every part carries a SHA-256
|
|
6
|
+
// checksum: a bucket with Object Lock refuses a PUT without one, and S3
|
|
7
|
+
// checks the bytes it got against it.
|
|
8
|
+
//
|
|
9
|
+
// IT ONLY WRITES. No list, read or head of the bucket — a backup key should be
|
|
10
|
+
// allowed s3:PutObject (and s3:AbortMultipartUpload, so a failed upload is
|
|
11
|
+
// cleaned up) and nothing else, so a leaked key can add files and do no more.
|
|
12
|
+
//
|
|
13
|
+
// Peer deps, optional and lazy-required: @aws-sdk/client-s3 and
|
|
14
|
+
// @aws-sdk/lib-storage.
|
|
15
|
+
//
|
|
16
|
+
// The connection: { bucket, region, accessKeyId, secretAccessKey, prefix?,
|
|
17
|
+
// sessionToken?, endpoint?, forcePathStyle? } — a saved connection
|
|
18
|
+
// (connectionInfoId) the host resolves; never typed into a step.
|
|
19
|
+
|
|
20
|
+
var fs = require('fs');
|
|
21
|
+
var path = require('path');
|
|
22
|
+
var dataDir = require('../../dataDir');
|
|
23
|
+
|
|
24
|
+
var PART_SIZE = 16 * 1024 * 1024;
|
|
25
|
+
var QUEUE = 4;
|
|
26
|
+
|
|
27
|
+
function refuse(message, code) {
|
|
28
|
+
var err = new Error(message);
|
|
29
|
+
err.code = code;
|
|
30
|
+
return err;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function aws() {
|
|
34
|
+
try {
|
|
35
|
+
return { s3: require('@aws-sdk/client-s3'), storage: require('@aws-sdk/lib-storage') };
|
|
36
|
+
} catch (_) {
|
|
37
|
+
throw refuse("s3-upload: '@aws-sdk/client-s3' and '@aws-sdk/lib-storage' are not installed. Run: npm install @aws-sdk/client-s3 @aws-sdk/lib-storage", 'MISSING_DEPENDENCY');
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** The object's key: `key` as given, else <prefix>/<file name>; never starting with "/". */
|
|
42
|
+
function keyFor(input, conn, filePath) {
|
|
43
|
+
if (input.key) return String(input.key).replace(/^\/+/, '');
|
|
44
|
+
var prefix = String(conn.prefix || '').replace(/^\/+|\/+$/g, '');
|
|
45
|
+
return (prefix ? prefix + '/' : '') + path.basename(filePath);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
module.exports = {
|
|
49
|
+
name: 's3-upload',
|
|
50
|
+
// It writes, so it is never previewed.
|
|
51
|
+
example: { bucket: 'acme-backups', key: 'backups/shop/shop-2026-11-13T02-00-05.dump', bytes: 104857600, etag: '"5d41402abc4b2a76b9719d911017c592-7"', versionId: '3HL4kqtJlcpXroDTDmJ.rmSpXd3dIbrHY', deleted: true },
|
|
52
|
+
description: 'Upload one file from this company\'s data folder to an S3 bucket, in parts, with a SHA-256 checksum. Only writes: the key needs s3:PutObject (and s3:AbortMultipartUpload), nothing else. Returns { bucket, key, bytes, etag, versionId, deleted }.',
|
|
53
|
+
requires: [], // @aws-sdk/client-s3 and @aws-sdk/lib-storage checked at runtime
|
|
54
|
+
|
|
55
|
+
inputSchema: [
|
|
56
|
+
{ name: 'connection', type: 'object', required: true, order: 1,
|
|
57
|
+
description: '{ bucket, region, accessKeyId, secretAccessKey, prefix? } — a saved connection (connectionInfoId), resolved by the host.' },
|
|
58
|
+
{ name: 'filePath', type: 'string', required: true, order: 2,
|
|
59
|
+
description: 'The file, in this company\'s data folder, e.g. {steps.dump.filePath}.' },
|
|
60
|
+
{ name: 'key', type: 'string', order: 3,
|
|
61
|
+
description: 'The object\'s key in the bucket. Default: <prefix>/<file name>.' },
|
|
62
|
+
{ name: 'deleteAfter', type: 'boolean', default: false, order: 4,
|
|
63
|
+
description: 'Delete the local file once it is uploaded.' },
|
|
64
|
+
{ name: 'contentType', type: 'string', order: 5, group: 'Upload options',
|
|
65
|
+
description: 'Default: application/octet-stream.' }
|
|
66
|
+
],
|
|
67
|
+
outputSchema: [
|
|
68
|
+
{ name: 'bucket', type: 'string' }, { name: 'key', type: 'string' }, { name: 'bytes', type: 'number' },
|
|
69
|
+
{ name: 'etag', type: 'string' }, { name: 'versionId', type: 'string' }, { name: 'deleted', type: 'boolean' }
|
|
70
|
+
],
|
|
71
|
+
|
|
72
|
+
execute: async function(ctx) {
|
|
73
|
+
var input = ctx.input || {};
|
|
74
|
+
var system = ctx.system || {};
|
|
75
|
+
var conn = input.connection || {};
|
|
76
|
+
if (!conn.bucket) throw refuse('s3-upload: the connection names no bucket.', 'NO_BUCKET');
|
|
77
|
+
if (!conn.region && !conn.endpoint) throw refuse('s3-upload: the connection names no region.', 'NO_REGION');
|
|
78
|
+
var file = dataDir.resolve(input.filePath, 'The file to upload');
|
|
79
|
+
if (!fs.existsSync(file) || !fs.statSync(file).isFile()) throw refuse('s3-upload: "' + input.filePath + '" is not a file that exists.', 'FILE_NOT_FOUND');
|
|
80
|
+
var bytes = fs.statSync(file).size;
|
|
81
|
+
var key = keyFor(input, conn, file);
|
|
82
|
+
|
|
83
|
+
var lib = aws();
|
|
84
|
+
var clientConfig = { region: conn.region || 'us-east-1' };
|
|
85
|
+
if (conn.accessKeyId) {
|
|
86
|
+
clientConfig.credentials = { accessKeyId: String(conn.accessKeyId), secretAccessKey: String(conn.secretAccessKey || '') };
|
|
87
|
+
if (conn.sessionToken) clientConfig.credentials.sessionToken = String(conn.sessionToken);
|
|
88
|
+
}
|
|
89
|
+
if (conn.endpoint) clientConfig.endpoint = String(conn.endpoint);
|
|
90
|
+
if (conn.forcePathStyle) clientConfig.forcePathStyle = true;
|
|
91
|
+
var client = new lib.s3.S3Client(clientConfig);
|
|
92
|
+
|
|
93
|
+
var upload = new lib.storage.Upload({
|
|
94
|
+
client: client,
|
|
95
|
+
partSize: PART_SIZE,
|
|
96
|
+
queueSize: QUEUE,
|
|
97
|
+
leavePartsOnError: false, // a failed upload is aborted (s3:AbortMultipartUpload)
|
|
98
|
+
params: {
|
|
99
|
+
Bucket: String(conn.bucket),
|
|
100
|
+
Key: key,
|
|
101
|
+
Body: fs.createReadStream(file),
|
|
102
|
+
ContentLength: bytes,
|
|
103
|
+
ContentType: input.contentType || 'application/octet-stream',
|
|
104
|
+
ChecksumAlgorithm: 'SHA256'
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
if (system.onProgress) upload.on('httpUploadProgress', function(p) { system.onProgress({ bytes: p.loaded || 0 }); });
|
|
108
|
+
var onAbort = function() { upload.abort().catch(function() {}); };
|
|
109
|
+
if (system.signal) {
|
|
110
|
+
if (system.signal.aborted) onAbort();
|
|
111
|
+
else system.signal.addEventListener('abort', onAbort, { once: true });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
var res;
|
|
115
|
+
try {
|
|
116
|
+
res = await upload.done();
|
|
117
|
+
} finally {
|
|
118
|
+
if (system.signal) system.signal.removeEventListener('abort', onAbort);
|
|
119
|
+
if (client.destroy) client.destroy();
|
|
120
|
+
}
|
|
121
|
+
var deleted = false;
|
|
122
|
+
if (input.deleteAfter === true) {
|
|
123
|
+
fs.unlinkSync(file);
|
|
124
|
+
deleted = true;
|
|
125
|
+
}
|
|
126
|
+
return { bucket: String(conn.bucket), key: key, bytes: bytes, etag: (res && res.ETag) || null, versionId: (res && res.VersionId) || null, deleted: deleted };
|
|
127
|
+
},
|
|
128
|
+
|
|
129
|
+
// For tests.
|
|
130
|
+
_keyFor: keyFor
|
|
131
|
+
};
|
package/lib/builtins/index.js
CHANGED
|
@@ -40,6 +40,9 @@ module.exports = {
|
|
|
40
40
|
// fetchStream piped straight into upload() rather than staged through an
|
|
41
41
|
// NDJSON file the way dbFetch → dbPush does. See db/move.js.
|
|
42
42
|
dbMove: require('./db/move'),
|
|
43
|
+
// A whole database to one backup file, by its own dump program (pg_dump,
|
|
44
|
+
// mysqldump) — run with an argument list, never a shell. See db/dump.js.
|
|
45
|
+
dbDump: require('./db/dump'),
|
|
43
46
|
|
|
44
47
|
// DB introspection — list tables/views/procedures, or one table's columns.
|
|
45
48
|
dbListTables: require('./db/list-tables'),
|
|
@@ -53,6 +56,9 @@ module.exports = {
|
|
|
53
56
|
fileDelete: require('./file/delete'),
|
|
54
57
|
fileList: require('./file/list'),
|
|
55
58
|
fileRead: require('./file/read'),
|
|
59
|
+
// One file to an S3 bucket, in parts, write-only. Peers: @aws-sdk/client-s3
|
|
60
|
+
// and @aws-sdk/lib-storage — optional, lazy-required.
|
|
61
|
+
s3Upload: require('./file/s3-upload'),
|
|
56
62
|
|
|
57
63
|
// Email actions. Inbound routes to getInboundProvider(input.emailType)
|
|
58
64
|
// (imap today), outbound to getOutboundProvider (smtp) — see
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xeplr/actions",
|
|
3
|
-
"version": "2.2.
|
|
3
|
+
"version": "2.2.3",
|
|
4
4
|
"description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -39,6 +39,8 @@
|
|
|
39
39
|
"@xeplr/schema-handler": "^1.0.0"
|
|
40
40
|
},
|
|
41
41
|
"peerDependencies": {
|
|
42
|
+
"@aws-sdk/client-s3": "^3.0.0",
|
|
43
|
+
"@aws-sdk/lib-storage": "^3.0.0",
|
|
42
44
|
"@duckdb/node-api": "1.5.5-r.3",
|
|
43
45
|
"@xeplr/db": "^1.1.0",
|
|
44
46
|
"@xeplr/email": "^1.0.0",
|
|
@@ -62,6 +64,12 @@
|
|
|
62
64
|
"unzipper": "^0.12.5"
|
|
63
65
|
},
|
|
64
66
|
"peerDependenciesMeta": {
|
|
67
|
+
"@aws-sdk/client-s3": {
|
|
68
|
+
"optional": true
|
|
69
|
+
},
|
|
70
|
+
"@aws-sdk/lib-storage": {
|
|
71
|
+
"optional": true
|
|
72
|
+
},
|
|
65
73
|
"@duckdb/node-api": {
|
|
66
74
|
"optional": true
|
|
67
75
|
},
|
|
@@ -124,6 +132,8 @@
|
|
|
124
132
|
}
|
|
125
133
|
},
|
|
126
134
|
"devDependencies": {
|
|
135
|
+
"@aws-sdk/client-s3": "^3.1168.0",
|
|
136
|
+
"@aws-sdk/lib-storage": "^3.1168.0",
|
|
127
137
|
"@duckdb/node-api": "^1.5.5-r.3",
|
|
128
138
|
"@xeplr/db": "^1.1.0",
|
|
129
139
|
"@xeplr/utils": "^1.0.0",
|