@xeplr/actions 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -1
- package/index.js +4 -0
- package/lib/builtins/data-operation.js +7 -0
- package/lib/builtins/db/fetch.js +25 -0
- package/lib/builtins/db/list-columns.js +3 -0
- package/lib/builtins/db/list-procedures.js +3 -0
- package/lib/builtins/db/list-tables.js +3 -0
- package/lib/builtins/db/list-views.js +3 -0
- package/lib/builtins/db/procedure.js +2 -0
- package/lib/builtins/email/delete.js +2 -0
- package/lib/builtins/email/download-attachments.js +2 -0
- package/lib/builtins/email/download-email.js +2 -0
- package/lib/builtins/email/move.js +2 -0
- package/lib/builtins/email/read.js +8 -0
- package/lib/builtins/export/pages.js +2 -0
- package/lib/builtins/export/table.js +2 -0
- package/lib/builtins/file/delete.js +2 -0
- package/lib/builtins/file/list.js +6 -0
- package/lib/builtins/file/move.js +2 -0
- package/lib/builtins/file/read.js +6 -0
- package/lib/builtins/send-email.js +2 -0
- package/lib/drivers/db/duckdb.js +37 -3
- package/lib/drivers/db/mssql.js +32 -4
- package/lib/drivers/db/mysql.js +67 -5
- package/lib/drivers/db/postgres.js +92 -11
- package/lib/drivers/db/stop.js +89 -0
- package/lib/drivers/email/imap.js +5 -3
- package/lib/registry.js +16 -1
- package/lib/runner.js +35 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -64,10 +64,12 @@ var moved = await actions.runAction(actions.builtins.dbMove, {
|
|
|
64
64
|
|---|---|
|
|
65
65
|
| `register(def)` / `register(name, def)` | Add an action. Throws without `name` or `execute`, or with a `requires` module that is not installed (`ActionMissingDependencyError`). Re-registering a name replaces it. |
|
|
66
66
|
| `get(name)` / `has(name)` | Look one up. |
|
|
67
|
-
| `list()` | `[{ name, description, inputSchema, outputSchema, requires, event }]` — serializable, for an actions endpoint. `event` is `null` for an ordinary action. |
|
|
67
|
+
| `list()` | `[{ name, description, inputSchema, outputSchema, requires, event, preview, example }]` — serializable, for an actions endpoint. `event` is `null` for an ordinary action; `preview` is `true` when the action can be previewed (the function itself never leaves the process). |
|
|
68
68
|
| `clear()` | Empty the registry. |
|
|
69
69
|
| `runAction({ name, input, system?, timeoutMs? })` | Run a registered action. Never throws. |
|
|
70
70
|
| `runAction(nameOrDef, input, { system, timeoutMs, metaStore }?)` | Same, by name or with an action module directly. |
|
|
71
|
+
| `previewAction({ name, input, system? })` / `previewAction(nameOrDef, input, opts?)` | What the step **would** hand on, without running it — see [Previews](#previews-what-a-step-would-hand-on). Never calls `execute`. Never throws. |
|
|
72
|
+
| `PREVIEW_ROWS` | `5`: how many rows or items a preview reads. |
|
|
71
73
|
| `ActionNotRegisteredError` | Returned (in `error`) for an unknown name. |
|
|
72
74
|
| `ActionMissingDependencyError` | Thrown by `register()` for a missing `requires` module. |
|
|
73
75
|
| `TransientError` | For an action to signal "retry me". Nothing in this package (or `@xeplr/jobs`) acts on it yet. |
|
|
@@ -89,6 +91,8 @@ var moved = await actions.runAction(actions.builtins.dbMove, {
|
|
|
89
91
|
| `description` | Text for pickers. |
|
|
90
92
|
| `inputSchema` | `[{ name, type, required, default, description, order, ... }]`, validated by `@xeplr/schema-handler` `applySchema`. Built-ins also use `group`, `showWhen`, `system`, `options`, `optionsFrom`, `dependsOn`, `label`, `hidden` (left out of a form unless a value is already set), `suggestFrom` / `suggestWith` (a host option source that suggests values, asked with the named sibling inputs; `db-procedure`'s parameters ask `procedureParams` with the connection, database and procedure) and `control` (`'parameters'`: `[{ name, value }]` shown as rows, a value fixed or bound to the run's window: `{window.from}` / `{window.to}`, see `@xeplr/jobs`) — hints for a form; validation treats every field the same. |
|
|
91
93
|
| `outputSchema` | Optional documentation of the return value. |
|
|
94
|
+
| `preview({ input, system })` | Optional. Resolves to `{ output, rows? }`: what `execute` would return, read **without a side effect** — at most `PREVIEW_ROWS` rows or items, never a write, a send, a move or a delete. Only for an action that can read safely. |
|
|
95
|
+
| `example` | Optional. For an action that writes or sends: its output with example values, so a designer can show the shape without running it. |
|
|
92
96
|
| `requires` | Module names checked with `require.resolve` at `register()`. |
|
|
93
97
|
| `execute({ input, system, metaStore })` | Returns (or resolves to) the output. |
|
|
94
98
|
| `event` | Optional. Makes this a **screen function** rather than a step: the event it is written for — `beforeLoad`, `beforeSave`, `afterSave`, `beforeDelete`, `onError` (run by `@xeplr/factory` around a screen's records) or `decide` (run by `@xeplr/workflow-engine` to choose a flow's next step). A screen names one per event and pickers offer only the functions for that event. It is called with the event's context — `execute(ctx)`, not `{ input, system }` — and never as a workflow step. |
|
|
@@ -101,6 +105,26 @@ var moved = await actions.runAction(actions.builtins.dbMove, {
|
|
|
101
105
|
|
|
102
106
|
An unknown action, invalid input, a thrown or rejected `execute`, and a timeout (`timeoutMs`, message `Action timed out after Nms`) all come back as `status: 'failed'`. A timeout does not cancel the running work.
|
|
103
107
|
|
|
108
|
+
### Previews: what a step would hand on
|
|
109
|
+
|
|
110
|
+
A workflow designer shows a step's output before anything has run, so later steps can pick from it. Typing an example by hand was the only way, even where the output can simply be read. `previewAction` reads it instead, and **only ever calls an action's `preview`, never its `execute`**:
|
|
111
|
+
|
|
112
|
+
```js
|
|
113
|
+
{ status: 'success' | 'example' | 'failed', output, rows, error, durationMs }
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
- `success`: `output` is what the step would hand on (its shape, sample values); `rows` the first few rows or items when it reads some.
|
|
117
|
+
- `example`: the action has no preview (it writes or sends); `output` is its declared `example`, or `null`. **Nothing ran.**
|
|
118
|
+
- `failed`: the input is incomplete or the read failed; `error` says which.
|
|
119
|
+
|
|
120
|
+
The input is checked exactly as a run checks it.
|
|
121
|
+
|
|
122
|
+
| previewed (reads, up to 5) | example only (writes or sends; never run) |
|
|
123
|
+
|---|---|
|
|
124
|
+
| `db-fetch` (through the run's own cursor, then stops; nothing spooled), `db-list-tables` / `-views` / `-procedures` / `-columns`, `file-list`, `file-read`, `data-operation` (the run itself: it only computes), `email-read` (the mailbox opened **read-only**, IMAP `EXAMINE`, so nothing is marked read) | `file-move`, `file-delete`, `email-send`, `email-move`, `email-delete`, `email-download-email`, `email-download-attachments`, `export-table`, `export-pages`, `db-procedure` (a procedure may write) |
|
|
125
|
+
|
|
126
|
+
Neither: `db-push`, `db-move`, `http-request` (an outside system, which may change something), `spawn-program`, `file-upload`; a designer asks for an example by hand. `test/previews.test.js` checks it on a real folder, file and Postgres: five rows read and no file spooled, the real count kept, a deleting step's file still there.
|
|
127
|
+
|
|
104
128
|
## Built-in actions
|
|
105
129
|
|
|
106
130
|
Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a module straight to `runAction`. `@xeplr/jobs` registers all implemented ones except `spawnProgram` and `dbMove`.
|
|
@@ -558,6 +582,32 @@ await driver.query(pool, userSql, params) // reads only
|
|
|
558
582
|
|
|
559
583
|
**Values are bound, never written into SQL.** On MySQL, `stringifyObjects` is always on, so an object or array passed as a value is compared as text instead of being expanded into `key = value` pairs. **Procedure parameter names** are written into the call (`@region = @p0`, `region => $1`) and must be plain names — letters, digits and `_`.
|
|
560
584
|
|
|
585
|
+
## Stopping a query — the stop reaches the database
|
|
586
|
+
|
|
587
|
+
A query nobody is waiting for must not keep running. Every driver's `query` and `fetchStream` take a stop signal and a time limit:
|
|
588
|
+
|
|
589
|
+
```js
|
|
590
|
+
const stop = new AbortController()
|
|
591
|
+
await driver.query(pool, sql, params, { signal: stop.signal, timeoutMs: 120000 })
|
|
592
|
+
for await (const row of driver.fetchStream(pool, { sql, params, signal: stop.signal, timeoutMs: 120000 })) { … }
|
|
593
|
+
stop.abort(new Error('Nobody is waiting for this')) // cancels it IN THE DATABASE
|
|
594
|
+
```
|
|
595
|
+
|
|
596
|
+
- **A stop cancels the query in the database, not just our read.** A `GROUP BY` or `ORDER BY` is worked out in full before the first row comes back, so a stop that only ended our read loop left the database working, holding its CPU and a connection, until it finished.
|
|
597
|
+
- **The two errors:** `code: 'QUERY_STOPPED'`, whose message is the signal's reason; and `code: 'QUERY_TIMEOUT'`, "The database stopped this query after 2 minutes, its time limit…".
|
|
598
|
+
- **A signal that has already fired:** the query is never sent.
|
|
599
|
+
- **Neither option given:** the query runs exactly as before.
|
|
600
|
+
|
|
601
|
+
| Driver | How it is stopped | Time limit |
|
|
602
|
+
|---|---|---|
|
|
603
|
+
| postgres | `pg_cancel_backend(<the query's backend>)`, sent on a connection of its own, so a pool whose connections are all busy (often with the very queries being stopped) cannot block it. The stopped query's connection is destroyed, not reused | `statement_timeout`, enforced by Postgres even if this process dies. For `fetchStream` it applies to each FETCH. On a writable connection it is set for the query and reset after, without adding a transaction |
|
|
604
|
+
| duckdb | `interrupt()` on the query's own connection. A query still waiting for a free connection is let go before it runs | a timer that interrupts the same way |
|
|
605
|
+
| mssql | `request.cancel()`, an attention on the query's own connection | a timer that cancels the same way (SQL Server has no per-statement limit) |
|
|
606
|
+
| mysql | `KILL QUERY <thread id>`, sent on a connection of its own. The stopped query's connection is destroyed | a timer that kills the same way (`max_execution_time` covers SELECTs only, and MariaDB names it differently) |
|
|
607
|
+
| mongo | not yet | — |
|
|
608
|
+
|
|
609
|
+
The tests prove that the database itself let go: Postgres by `pg_stat_activity`, SQL Server by `sys.dm_exec_requests`, MySQL by the processlist, and DuckDB by its one connection answering at once (`test/drivers/stop.*.test.js`). The SQL Server and MySQL ones run when `MSSQL_*` and `MYSQL_*` are set.
|
|
610
|
+
|
|
561
611
|
## Rules the code enforces, and why
|
|
562
612
|
|
|
563
613
|
| rule | why |
|
package/index.js
CHANGED
|
@@ -21,6 +21,10 @@ module.exports = {
|
|
|
21
21
|
// runAction({ name, input, system?, timeoutMs? })
|
|
22
22
|
// runAction(nameOrAction, input, opts?)
|
|
23
23
|
runAction: runner.runAction,
|
|
24
|
+
// What a step would hand on, read without a side effect (an action's
|
|
25
|
+
// `preview`), or its declared example. Never runs `execute`.
|
|
26
|
+
previewAction: runner.previewAction,
|
|
27
|
+
PREVIEW_ROWS: runner.PREVIEW_ROWS,
|
|
24
28
|
|
|
25
29
|
// Saved connections: an id in the definition, the login only in the call.
|
|
26
30
|
// Shared by @xeplr/jobs and @xeplr/workflow-engine. See lib/connections.js.
|
|
@@ -114,6 +114,13 @@ module.exports = {
|
|
|
114
114
|
// The same call every action gets from runAction: { input, system }. It
|
|
115
115
|
// once took (input, context) — the runner's object landed in `input`,
|
|
116
116
|
// `from` was undefined, and every workflow step returned no rows.
|
|
117
|
+
// Works only on the rows it is given — no database, no file, no send — so
|
|
118
|
+
// its preview is the run, over the sample rows the step before hands on.
|
|
119
|
+
preview: async function(call) {
|
|
120
|
+
var out = await module.exports.execute(call);
|
|
121
|
+
return { output: out, rows: (out && out.rows || []).slice(0, require('../runner').PREVIEW_ROWS) };
|
|
122
|
+
},
|
|
123
|
+
|
|
117
124
|
execute: async function(call) {
|
|
118
125
|
var input = (call && call.input) || {};
|
|
119
126
|
var system = (call && call.system) || {};
|
package/lib/builtins/db/fetch.js
CHANGED
|
@@ -79,6 +79,31 @@ module.exports = {
|
|
|
79
79
|
description: 'Rows pulled per cursor round-trip / stream highWaterMark.' }
|
|
80
80
|
],
|
|
81
81
|
|
|
82
|
+
// THE FIRST ROWS, through the same statement and cursor a run uses, then
|
|
83
|
+
// stop: nothing is spooled. The output is what a run hands on, with the
|
|
84
|
+
// file it would write named but not written.
|
|
85
|
+
preview: async function(ctx) {
|
|
86
|
+
var input = ctx.input || {};
|
|
87
|
+
var driver = dbDrivers.getDriver(input.dbType);
|
|
88
|
+
dbDrivers.checkDriverRequires('db-fetch', input.dbType, driver);
|
|
89
|
+
var built = buildFetchSql(driver, input);
|
|
90
|
+
var pool = await driver.connect(input.connection);
|
|
91
|
+
var rows = [];
|
|
92
|
+
try {
|
|
93
|
+
var stream = driver.fetchStream(pool, { sql: built.sql, params: built.params, batchSize: require('../../runner').PREVIEW_ROWS });
|
|
94
|
+
for await (var row of stream) {
|
|
95
|
+
rows.push(row);
|
|
96
|
+
if (rows.length >= require('../../runner').PREVIEW_ROWS) break;
|
|
97
|
+
}
|
|
98
|
+
} finally {
|
|
99
|
+
await driver.close(pool);
|
|
100
|
+
}
|
|
101
|
+
var output = input.streaming_mode === false
|
|
102
|
+
? { rows: rows, rowCount: rows.length, streaming: false }
|
|
103
|
+
: { filePath: '(a file in this company\'s tmp/, written when it runs)', format: 'jsonl', bytes: 0, rows: rows.length, streaming: true };
|
|
104
|
+
return { output: output, rows: rows };
|
|
105
|
+
},
|
|
106
|
+
|
|
82
107
|
execute: async function(ctx) {
|
|
83
108
|
var input = ctx.input || {};
|
|
84
109
|
var system = ctx.system || {};
|
|
@@ -20,6 +20,9 @@ module.exports = {
|
|
|
20
20
|
description: 'Schema the table lives in (postgres/mssql — default public/all; mysql ignores this in favor of connection.database)' }
|
|
21
21
|
],
|
|
22
22
|
|
|
23
|
+
// Only reads the database's own catalogue, so the preview is the run.
|
|
24
|
+
preview: async function(ctx) { return { output: await module.exports.execute(ctx) }; },
|
|
25
|
+
|
|
23
26
|
execute: async function(ctx) {
|
|
24
27
|
var input = ctx.input || {};
|
|
25
28
|
var driver = dbDrivers.getDriver(input.dbType);
|
|
@@ -17,6 +17,9 @@ module.exports = {
|
|
|
17
17
|
description: 'Schema to list (postgres/mssql — default public/all; mysql ignores this in favor of connection.database)' }
|
|
18
18
|
],
|
|
19
19
|
|
|
20
|
+
// Only reads the database's own catalogue, so the preview is the run.
|
|
21
|
+
preview: async function(ctx) { return { output: await module.exports.execute(ctx) }; },
|
|
22
|
+
|
|
20
23
|
execute: async function(ctx) {
|
|
21
24
|
var input = ctx.input || {};
|
|
22
25
|
var driver = dbDrivers.getDriver(input.dbType);
|
|
@@ -19,6 +19,9 @@ module.exports = {
|
|
|
19
19
|
description: 'Schema to list (postgres/mssql — default public/all; mysql ignores this in favor of connection.database)' }
|
|
20
20
|
],
|
|
21
21
|
|
|
22
|
+
// Only reads the database's own catalogue, so the preview is the run.
|
|
23
|
+
preview: async function(ctx) { return { output: await module.exports.execute(ctx) }; },
|
|
24
|
+
|
|
22
25
|
execute: async function(ctx) {
|
|
23
26
|
var input = ctx.input || {};
|
|
24
27
|
var driver = dbDrivers.getDriver(input.dbType);
|
|
@@ -16,6 +16,9 @@ module.exports = {
|
|
|
16
16
|
description: 'Schema to list (postgres/mssql — default public/all; mysql ignores this in favor of connection.database)' }
|
|
17
17
|
],
|
|
18
18
|
|
|
19
|
+
// Only reads the database's own catalogue, so the preview is the run.
|
|
20
|
+
preview: async function(ctx) { return { output: await module.exports.execute(ctx) }; },
|
|
21
|
+
|
|
19
22
|
execute: async function(ctx) {
|
|
20
23
|
var input = ctx.input || {};
|
|
21
24
|
var driver = dbDrivers.getDriver(input.dbType);
|
|
@@ -32,6 +32,8 @@ var { buildProcedureCall } = require('../../drivers/db/procedure');
|
|
|
32
32
|
|
|
33
33
|
module.exports = {
|
|
34
34
|
name: 'db-procedure',
|
|
35
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
36
|
+
example: { rows: [{ id: 1, status: 'done' }], rowCount: 1, procedure: 'close_period', sql: 'CALL close_period(?)' },
|
|
35
37
|
description: 'Call a stored procedure with parameters. Returns its rows when it produces any — ' +
|
|
36
38
|
'no target table, no movement. For a procedure whose rows you want written to a ' +
|
|
37
39
|
'table, use db-move with mode="procedure".',
|
|
@@ -13,6 +13,8 @@ var emailConfig = require('../../drivers/email/config');
|
|
|
13
13
|
|
|
14
14
|
module.exports = {
|
|
15
15
|
name: 'email-delete',
|
|
16
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
17
|
+
example: { deleted: true, messageId: '4182', folder: 'INBOX' },
|
|
16
18
|
description: 'Permanently delete a message (flags \\Deleted and expunges — NOT a move to Trash). ' +
|
|
17
19
|
'The message id is folder-scoped, so input.folder must name the folder it is in.',
|
|
18
20
|
requires: [],
|
|
@@ -17,6 +17,8 @@ var fileNames = require('../../fileNames');
|
|
|
17
17
|
|
|
18
18
|
module.exports = {
|
|
19
19
|
name: 'email-download-attachments',
|
|
20
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
21
|
+
example: { count: 1, saved: [{ filename: 'invoice.pdf', path: 'mail/invoice.pdf', size: 91022, contentType: 'application/pdf' }], path: 'mail/invoice.pdf', existing: [], subject: 'Invoice 450103', messageId: '4182' },
|
|
20
22
|
description: 'Save a message\'s attachments to disk, optionally filtered by extension or ' +
|
|
21
23
|
'filename substring, named as you say. Returns { count, saved[], path, skipped, existing[] }.',
|
|
22
24
|
requires: [],
|
|
@@ -15,6 +15,8 @@ var fileNames = require('../../fileNames');
|
|
|
15
15
|
|
|
16
16
|
module.exports = {
|
|
17
17
|
name: 'email-download-email',
|
|
18
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
19
|
+
example: { count: 1, saved: [{ filename: 'Invoice 450103.eml', path: 'mail/Invoice 450103.eml', size: 48213, contentType: 'message/rfc822' }], path: 'mail/Invoice 450103.eml', existing: [], subject: 'Invoice 450103', messageId: '4182' },
|
|
18
20
|
description: 'Save an entire message to disk as a .eml (RFC-822) file, streamed. ' +
|
|
19
21
|
'Returns { count, saved[], path } like email-download-attachments.',
|
|
20
22
|
requires: [],
|
|
@@ -14,6 +14,8 @@ var emailConfig = require('../../drivers/email/config');
|
|
|
14
14
|
|
|
15
15
|
module.exports = {
|
|
16
16
|
name: 'email-move',
|
|
17
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
18
|
+
example: { moved: true, messageId: '4182', fromFolder: 'INBOX', toFolder: 'Processed' },
|
|
17
19
|
description: 'Move a message to another mailbox folder. The message id is folder-scoped, ' +
|
|
18
20
|
'so input.folder must name the folder the message is currently in.',
|
|
19
21
|
requires: [],
|
|
@@ -59,6 +59,14 @@ module.exports = {
|
|
|
59
59
|
description: 'true → spool messages to an NDJSON file (output.filePath). false → inline output.messages.' }
|
|
60
60
|
],
|
|
61
61
|
|
|
62
|
+
// The first messages, with the mailbox opened READ-ONLY (IMAP EXAMINE), so
|
|
63
|
+
// a preview never marks anything read — whatever a full download does.
|
|
64
|
+
preview: async function(ctx) {
|
|
65
|
+
var input = Object.assign({}, ctx.input, { limit: require('../../runner').PREVIEW_ROWS, readOnly: true, streaming_mode: false });
|
|
66
|
+
var out = await module.exports.execute({ input: input, system: ctx.system });
|
|
67
|
+
return { output: out, rows: out.messages };
|
|
68
|
+
},
|
|
69
|
+
|
|
62
70
|
execute: async function(ctx) {
|
|
63
71
|
var input = ctx.input || {};
|
|
64
72
|
var system = ctx.system || {};
|
|
@@ -66,6 +66,8 @@ function validate(spec) {
|
|
|
66
66
|
|
|
67
67
|
module.exports = {
|
|
68
68
|
name: 'export-pages',
|
|
69
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
70
|
+
example: { filePath: 'exports/board.pdf', fileName: 'board.pdf', contentType: 'application/pdf', bytes: 220514, pages: 3 },
|
|
69
71
|
description: 'Lay out pages of placed blocks (text, tables, images, shapes) as a PDF — a dashboard as it looked.',
|
|
70
72
|
requires: [],
|
|
71
73
|
LIMITS: LIMITS,
|
|
@@ -57,6 +57,8 @@ function columnsOf(rows) {
|
|
|
57
57
|
|
|
58
58
|
module.exports = {
|
|
59
59
|
name: 'export-table',
|
|
60
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
61
|
+
example: { filePath: 'exports/sales.xlsx', fileName: 'sales.xlsx', contentType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', bytes: 18432, rows: 1420, truncated: false, skipped: false },
|
|
60
62
|
description: 'Write rows to a CSV, Excel (.xlsx) or PDF file. Excel cells stay numbers with their number format; ' +
|
|
61
63
|
'a PDF shows each cell as it was displayed.',
|
|
62
64
|
requires: [],
|
|
@@ -11,6 +11,8 @@ var dataDir = require('../../dataDir');
|
|
|
11
11
|
|
|
12
12
|
module.exports = {
|
|
13
13
|
name: 'file-delete',
|
|
14
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
15
|
+
example: { path: 'bank/inbox/sept.csv', deleted: true, skipped: false },
|
|
14
16
|
description: 'Delete one file, permanently (file-move to a "processed" folder is the reversible one). Returns { path, deleted, skipped }.',
|
|
15
17
|
requires: [],
|
|
16
18
|
inputSchema: [
|
|
@@ -36,6 +36,12 @@ module.exports = {
|
|
|
36
36
|
{ name: 'count', type: 'number', description: 'How many, so the next step can branch on none.' }
|
|
37
37
|
],
|
|
38
38
|
|
|
39
|
+
// Listing only reads the folder; the preview is the run, its first files.
|
|
40
|
+
preview: async function(ctx) {
|
|
41
|
+
var out = await module.exports.execute(ctx);
|
|
42
|
+
return { output: { files: out.files.slice(0, require('../../runner').PREVIEW_ROWS), count: out.count }, rows: out.files.slice(0, require('../../runner').PREVIEW_ROWS) };
|
|
43
|
+
},
|
|
44
|
+
|
|
39
45
|
execute: async function(ctx) {
|
|
40
46
|
var input = ctx.input || {};
|
|
41
47
|
var dir = dataDir.resolve(input.folder, 'The folder');
|
|
@@ -12,6 +12,8 @@ var fileNames = require('../../fileNames');
|
|
|
12
12
|
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: 'file-move',
|
|
15
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
16
|
+
example: { path: 'bank/processed/sept.csv', name: 'sept.csv', moved: true, skipped: false },
|
|
15
17
|
description: 'Move or copy a file into a folder, or to a new name. Returns { path, name, moved, skipped }.',
|
|
16
18
|
requires: [],
|
|
17
19
|
inputSchema: [
|
|
@@ -38,6 +38,12 @@ module.exports = {
|
|
|
38
38
|
{ name: 'truncated', type: 'boolean' }
|
|
39
39
|
],
|
|
40
40
|
|
|
41
|
+
// The file's first rows: reading never changes it.
|
|
42
|
+
preview: async function(ctx) {
|
|
43
|
+
var out = await module.exports.execute({ input: Object.assign({}, ctx.input, { maxRows: require('../../runner').PREVIEW_ROWS }), system: ctx.system });
|
|
44
|
+
return { output: out, rows: out.rows };
|
|
45
|
+
},
|
|
46
|
+
|
|
41
47
|
execute: async function(ctx) {
|
|
42
48
|
var input = ctx.input || {};
|
|
43
49
|
var file = dataDir.resolve(input.path, 'The file');
|
|
@@ -48,6 +48,8 @@ var emailConfig = require('../drivers/email/config');
|
|
|
48
48
|
|
|
49
49
|
module.exports = {
|
|
50
50
|
name: 'email-send',
|
|
51
|
+
// What it hands on, with example values: it writes, so it is never previewed.
|
|
52
|
+
example: { sent: true, messageId: '<4182@mail.acme.com>', accepted: ['asha@acme.com'], rejected: [], response: '250 OK' },
|
|
51
53
|
description: 'Send an email over SMTP. Attachments accept { filename, path } descriptors, ' +
|
|
52
54
|
'so email-download-attachments output can be forwarded directly.',
|
|
53
55
|
requires: [], // nodemailer checked at runtime
|
package/lib/drivers/db/duckdb.js
CHANGED
|
@@ -349,6 +349,33 @@ async function acquire(entry) {
|
|
|
349
349
|
return new Promise(function(resolve) { entry.waiters.push(resolve); });
|
|
350
350
|
}
|
|
351
351
|
|
|
352
|
+
// ── STOPPING A QUERY (stop.js) ─────────────────────────────────────────────
|
|
353
|
+
// DuckDB runs in this process, so a stop is conn.interrupt() on the query's
|
|
354
|
+
// own connection, and the time limit is a timer that does the same. A query
|
|
355
|
+
// still waiting for a free connection is let go of before it ever runs.
|
|
356
|
+
var stop = require('./stop');
|
|
357
|
+
|
|
358
|
+
/** acquire(), unless the stop comes first; a connection that arrives too late goes straight back. */
|
|
359
|
+
function acquireUnlessStopped(entry, signal) {
|
|
360
|
+
if (!signal) return acquire(entry);
|
|
361
|
+
stop.throwIfStopped(signal);
|
|
362
|
+
return new Promise(function(resolve, reject) {
|
|
363
|
+
var gaveUp = false;
|
|
364
|
+
function onAbort() { gaveUp = true; reject(stop.stoppedError(signal)); }
|
|
365
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
366
|
+
acquire(entry).then(function(conn) {
|
|
367
|
+
signal.removeEventListener('abort', onAbort);
|
|
368
|
+
if (gaveUp) release(entry, conn);
|
|
369
|
+
else resolve(conn);
|
|
370
|
+
}, function(err) { signal.removeEventListener('abort', onAbort); reject(err); });
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** Watch one connection's query: interrupt it on a stop or past the limit. */
|
|
375
|
+
function watchConn(conn, opts) {
|
|
376
|
+
return stop.watch(opts && opts.signal, opts && opts.timeoutMs, function() { conn.interrupt(); });
|
|
377
|
+
}
|
|
378
|
+
|
|
352
379
|
function release(entry, conn) {
|
|
353
380
|
var waiter = entry.waiters.shift();
|
|
354
381
|
if (waiter) waiter(conn);
|
|
@@ -421,10 +448,11 @@ function plainDeep(v) {
|
|
|
421
448
|
* The result's OWN return type is what distinguishes the two cases, so that is
|
|
422
449
|
* what this reads. CHANGED_ROWS is DML; QUERY_RESULT is a select.
|
|
423
450
|
*/
|
|
424
|
-
async function query(pool, sql, params) {
|
|
451
|
+
async function query(pool, sql, params, opts) {
|
|
425
452
|
var entry = entryOf(pool);
|
|
426
453
|
refuseIndex(entry, sql);
|
|
427
|
-
var conn = await
|
|
454
|
+
var conn = await acquireUnlessStopped(entry, opts && opts.signal);
|
|
455
|
+
var w = watchConn(conn, opts);
|
|
428
456
|
try {
|
|
429
457
|
var api = require('@duckdb/node-api');
|
|
430
458
|
var reader = await conn.runAndReadAll(sql, params && params.length ? params : undefined);
|
|
@@ -441,8 +469,10 @@ async function query(pool, sql, params) {
|
|
|
441
469
|
});
|
|
442
470
|
return { rows: rows, rowCount: rows.length, columns: columns };
|
|
443
471
|
} catch (err) {
|
|
472
|
+
if (w.why()) throw w.why();
|
|
444
473
|
throw explainLockError(err, entry.key, entry.access, 0);
|
|
445
474
|
} finally {
|
|
475
|
+
w.done();
|
|
446
476
|
release(entry, conn);
|
|
447
477
|
}
|
|
448
478
|
}
|
|
@@ -756,7 +786,8 @@ async function plainValuesPlan(conn, sql) {
|
|
|
756
786
|
*/
|
|
757
787
|
async function* fetchStream(pool, opts) {
|
|
758
788
|
var entry = entryOf(pool);
|
|
759
|
-
var conn = await
|
|
789
|
+
var conn = await acquireUnlessStopped(entry, opts.signal);
|
|
790
|
+
var w = watchConn(conn, opts);
|
|
760
791
|
try {
|
|
761
792
|
var plan = await plainValuesPlan(conn, opts.sql);
|
|
762
793
|
var result = await conn.stream(plan.sql, (opts.params && opts.params.length) ? opts.params : undefined);
|
|
@@ -764,6 +795,7 @@ async function* fetchStream(pool, opts) {
|
|
|
764
795
|
var dates = plan.dateColumns;
|
|
765
796
|
var jsons = plan.jsonColumns;
|
|
766
797
|
while (true) {
|
|
798
|
+
if (w.why()) throw w.why();
|
|
767
799
|
var chunk = await result.fetchChunk();
|
|
768
800
|
if (!chunk || chunk.rowCount === 0) break;
|
|
769
801
|
var rows = chunk.getRowObjects(names);
|
|
@@ -783,8 +815,10 @@ async function* fetchStream(pool, opts) {
|
|
|
783
815
|
}
|
|
784
816
|
}
|
|
785
817
|
} catch (err) {
|
|
818
|
+
if (w.why()) throw w.why();
|
|
786
819
|
throw explainLockError(err, entry.key, entry.access, 0);
|
|
787
820
|
} finally {
|
|
821
|
+
w.done();
|
|
788
822
|
release(entry, conn);
|
|
789
823
|
}
|
|
790
824
|
}
|
package/lib/drivers/db/mssql.js
CHANGED
|
@@ -96,17 +96,34 @@ async function close(pool) {
|
|
|
96
96
|
if (pool) await pool.close();
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
// ── STOPPING A QUERY (stop.js) ─────────────────────────────────────────────
|
|
100
|
+
// A stop is request.cancel(): an ATTENTION sent on the query's own
|
|
101
|
+
// connection, which SQL Server answers by abandoning the batch. SQL Server
|
|
102
|
+
// has no per-statement time limit of its own, so the limit is a timer here
|
|
103
|
+
// that cancels the same way.
|
|
104
|
+
var stop = require('./stop');
|
|
105
|
+
|
|
99
106
|
// Bind params as @p0..@pN and normalize `{ recordset, rowsAffected }` to
|
|
100
107
|
// Postgres shape. NOTE: mssql infers the SQL type from each JS value; a null
|
|
101
108
|
// with no sibling non-null value in the batch falls back to NVARCHAR, which is
|
|
102
109
|
// fine for our INSERTs since the column type is already declared.
|
|
103
|
-
async function query(pool, sql, params) {
|
|
110
|
+
async function query(pool, sql, params, opts) {
|
|
111
|
+
opts = opts || {};
|
|
112
|
+
stop.throwIfStopped(opts.signal);
|
|
104
113
|
var request = pool.request();
|
|
105
114
|
params = params || [];
|
|
106
115
|
for (var i = 0; i < params.length; i++) {
|
|
107
116
|
request.input('p' + i, params[i]);
|
|
108
117
|
}
|
|
109
|
-
var
|
|
118
|
+
var w = stop.watch(opts.signal, opts.timeoutMs, function() { request.cancel(); });
|
|
119
|
+
var r;
|
|
120
|
+
try {
|
|
121
|
+
r = await request.query(sql);
|
|
122
|
+
} catch (err) {
|
|
123
|
+
throw w.why() || err;
|
|
124
|
+
} finally {
|
|
125
|
+
w.done();
|
|
126
|
+
}
|
|
110
127
|
var rows = r.recordset || [];
|
|
111
128
|
var rowCount = Array.isArray(r.rowsAffected)
|
|
112
129
|
? r.rowsAffected.reduce(function(a, b) { return a + b; }, 0)
|
|
@@ -382,8 +399,9 @@ async function ensureUpsertIndex(pool, tableName, primaryKeys) {
|
|
|
382
399
|
// with real backpressure: pause the request once `batchSize` rows are buffered,
|
|
383
400
|
// resume after the consumer drains them.
|
|
384
401
|
//
|
|
385
|
-
// opts: { sql, params?, batchSize? } → async iterable<row>
|
|
402
|
+
// opts: { sql, params?, batchSize?, signal?, timeoutMs? } → async iterable<row>
|
|
386
403
|
async function* fetchStream(pool, opts) {
|
|
404
|
+
stop.throwIfStopped(opts.signal);
|
|
387
405
|
var high = Math.max(1, parseInt(opts.batchSize || 1000, 10));
|
|
388
406
|
var request = pool.request();
|
|
389
407
|
var params = opts.params || [];
|
|
@@ -406,12 +424,21 @@ async function* fetchStream(pool, opts) {
|
|
|
406
424
|
request.on('error', function(err) { error = err; signal(); });
|
|
407
425
|
request.on('done', function() { done = true; signal(); });
|
|
408
426
|
|
|
427
|
+
// A stop cancels the request and wakes the consumer, which may be waiting
|
|
428
|
+
// for a first row the server is still working out.
|
|
429
|
+
var w = stop.watch(opts.signal, opts.timeoutMs, function() {
|
|
430
|
+
try { request.cancel(); } catch (_) {}
|
|
431
|
+
error = error || w.why();
|
|
432
|
+
signal();
|
|
433
|
+
});
|
|
434
|
+
|
|
409
435
|
// Fire the query — with stream=true this drives the events above. We don't
|
|
410
436
|
// await the returned promise; the events are the source of truth.
|
|
411
437
|
request.query(opts.sql).catch(function(err) { error = error || err; signal(); });
|
|
412
438
|
|
|
413
439
|
try {
|
|
414
440
|
while (true) {
|
|
441
|
+
if (w.why()) throw w.why();
|
|
415
442
|
if (buffer.length) {
|
|
416
443
|
var batch = buffer;
|
|
417
444
|
buffer = [];
|
|
@@ -419,11 +446,12 @@ async function* fetchStream(pool, opts) {
|
|
|
419
446
|
for (var j = 0; j < batch.length; j++) yield batch[j];
|
|
420
447
|
continue;
|
|
421
448
|
}
|
|
422
|
-
if (error) throw error;
|
|
449
|
+
if (error) throw w.why() || error;
|
|
423
450
|
if (done) return;
|
|
424
451
|
await new Promise(function(res) { wake = res; });
|
|
425
452
|
}
|
|
426
453
|
} finally {
|
|
454
|
+
w.done();
|
|
427
455
|
// If the consumer breaks out early, stop the underlying request.
|
|
428
456
|
if (!done && !error) { try { request.cancel(); } catch (_) {} }
|
|
429
457
|
}
|
package/lib/drivers/db/mysql.js
CHANGED
|
@@ -106,13 +106,67 @@ async function close(pool) {
|
|
|
106
106
|
if (pool) await pool.end();
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
+
// ── STOPPING A QUERY (stop.js) ─────────────────────────────────────────────
|
|
110
|
+
// A stop is KILL QUERY <the connection's thread id>, sent on a connection of
|
|
111
|
+
// its own: the pool's may all be busy with the very queries being stopped.
|
|
112
|
+
// The time limit is a timer here that kills the same way — MySQL's own
|
|
113
|
+
// max_execution_time covers SELECTs only, and MariaDB spells it differently.
|
|
114
|
+
var stop = require('./stop');
|
|
115
|
+
|
|
116
|
+
/** Kill whatever thread `threadId` is running, from a connection of its own. */
|
|
117
|
+
async function killQuery(pool, threadId) {
|
|
118
|
+
var mysql = require('mysql2/promise');
|
|
119
|
+
var c = pool.pool.config.connectionConfig;
|
|
120
|
+
var conn = await mysql.createConnection({ host: c.host, port: c.port, user: c.user, password: c.password, connectTimeout: 5000 });
|
|
121
|
+
try {
|
|
122
|
+
await conn.query('KILL QUERY ' + Number(threadId));
|
|
123
|
+
} finally {
|
|
124
|
+
await conn.end().catch(function() {});
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Watch one checked-out connection's query; `killed` is the pending kill. */
|
|
129
|
+
function watchConn(pool, conn, opts) {
|
|
130
|
+
var state = { killed: null };
|
|
131
|
+
state.w = stop.watch(opts.signal, opts.timeoutMs, function() {
|
|
132
|
+
state.killed = killQuery(pool, conn.connection.threadId).catch(function() { /* it finished first, or the server is gone */ });
|
|
133
|
+
});
|
|
134
|
+
return state;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Give the connection back — destroyed when a query on it was stopped. */
|
|
138
|
+
async function giveBack(state, conn) {
|
|
139
|
+
state.w.done();
|
|
140
|
+
if (state.killed) await state.killed;
|
|
141
|
+
if (state.w.why()) conn.destroy();
|
|
142
|
+
else conn.release();
|
|
143
|
+
}
|
|
144
|
+
|
|
109
145
|
// Normalize mysql2's `[rows|ResultSetHeader, fields]` to Postgres shape.
|
|
110
146
|
//
|
|
111
147
|
// The second element is the field list, which mysql2 returns even for a
|
|
112
148
|
// SELECT that matched nothing — so `columns` describes the result's shape
|
|
113
149
|
// when there is not a single row to infer it from.
|
|
114
|
-
async function query(pool, sql, params) {
|
|
115
|
-
|
|
150
|
+
async function query(pool, sql, params, opts) {
|
|
151
|
+
opts = opts || {};
|
|
152
|
+
stop.throwIfStopped(opts.signal);
|
|
153
|
+
var res;
|
|
154
|
+
if (opts.signal || stop.limitOf(opts.timeoutMs)) {
|
|
155
|
+
var conn = await pool.getConnection();
|
|
156
|
+
var state = watchConn(pool, conn, opts);
|
|
157
|
+
try {
|
|
158
|
+
res = await conn.query(sql, params || []);
|
|
159
|
+
// A killed statement can still come back normally (SLEEP() answers 1
|
|
160
|
+
// when killed): stopped is stopped, whatever it returned.
|
|
161
|
+
if (state.w.why()) throw state.w.why();
|
|
162
|
+
} catch (err) {
|
|
163
|
+
throw state.w.why() || err;
|
|
164
|
+
} finally {
|
|
165
|
+
await giveBack(state, conn);
|
|
166
|
+
}
|
|
167
|
+
} else {
|
|
168
|
+
res = await pool.query(sql, params || []);
|
|
169
|
+
}
|
|
116
170
|
var out = res[0];
|
|
117
171
|
var fields = res[1];
|
|
118
172
|
|
|
@@ -412,10 +466,12 @@ function upsertIndexName(tableName) {
|
|
|
412
466
|
// `for await` over it applies backpressure (highWaterMark bounds the rows
|
|
413
467
|
// buffered in the driver before it pauses reading from the socket).
|
|
414
468
|
//
|
|
415
|
-
// opts: { sql, params?, batchSize? } → async iterable<row>
|
|
469
|
+
// opts: { sql, params?, batchSize?, signal?, timeoutMs? } → async iterable<row>
|
|
416
470
|
async function* fetchStream(pool, opts) {
|
|
471
|
+
stop.throwIfStopped(opts.signal);
|
|
417
472
|
var batchSize = Math.max(1, parseInt(opts.batchSize || 1000, 10));
|
|
418
473
|
var conn = await pool.getConnection();
|
|
474
|
+
var state = watchConn(pool, conn, opts);
|
|
419
475
|
try {
|
|
420
476
|
// .stream() lives on the core (non-promise) connection under .connection.
|
|
421
477
|
var core = conn.connection;
|
|
@@ -423,9 +479,15 @@ async function* fetchStream(pool, opts) {
|
|
|
423
479
|
? { sql: opts.sql, values: opts.params }
|
|
424
480
|
: opts.sql;
|
|
425
481
|
var stream = core.query(q).stream({ highWaterMark: batchSize });
|
|
426
|
-
for await (var row of stream)
|
|
482
|
+
for await (var row of stream) {
|
|
483
|
+
if (state.w.why()) throw state.w.why();
|
|
484
|
+
yield row;
|
|
485
|
+
}
|
|
486
|
+
if (state.w.why()) throw state.w.why();
|
|
487
|
+
} catch (err) {
|
|
488
|
+
throw state.w.why() || err;
|
|
427
489
|
} finally {
|
|
428
|
-
conn
|
|
490
|
+
await giveBack(state, conn);
|
|
429
491
|
}
|
|
430
492
|
}
|
|
431
493
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
//
|
|
9
9
|
// requires: ['pg']
|
|
10
10
|
// async connect(config) → pool
|
|
11
|
-
// async query(pool, sql, params?)
|
|
11
|
+
// async query(pool, sql, params?, { signal?, timeoutMs }?) → { rows, rowCount, ... }
|
|
12
12
|
// async close(pool) → void
|
|
13
13
|
//
|
|
14
14
|
// buildCreateTableSql(tableName, columns, primaryKeys?) → string
|
|
@@ -82,6 +82,56 @@ async function close(pool) {
|
|
|
82
82
|
if (pool) await pool.end();
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
// ── STOPPING A QUERY (stop.js) ─────────────────────────────────────────────
|
|
86
|
+
// A stop is pg_cancel_backend(<its backend's pid>), sent on a connection of
|
|
87
|
+
// its own: the pool's may all be busy — quite possibly with the very queries
|
|
88
|
+
// being stopped — and a cancel that waits for a free one never arrives. The
|
|
89
|
+
// time limit is statement_timeout, so Postgres enforces it even if this
|
|
90
|
+
// process dies; our own timer, a little later, is only the backstop.
|
|
91
|
+
var stop = require('./stop');
|
|
92
|
+
var QUERY_CANCELED = '57014';
|
|
93
|
+
var BACKSTOP_MS = 2000;
|
|
94
|
+
|
|
95
|
+
/** Cancel whatever `pid` is running, from a connection of its own. */
|
|
96
|
+
async function cancelBackend(pool, pid) {
|
|
97
|
+
var pg = require('pg');
|
|
98
|
+
var o = pool.options || {};
|
|
99
|
+
var c = new pg.Client({ host: o.host, port: o.port, user: o.user, password: o.password, database: o.database, connectionTimeoutMillis: 5000 });
|
|
100
|
+
try {
|
|
101
|
+
await c.connect();
|
|
102
|
+
await c.query('SELECT pg_cancel_backend($1)', [pid]);
|
|
103
|
+
} finally {
|
|
104
|
+
await c.end().catch(function() {});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Watch one checked-out client's query for `opts` (signal, timeoutMs).
|
|
110
|
+
* `cancelled` is the pending cancel, awaited before the client goes back.
|
|
111
|
+
*/
|
|
112
|
+
function watchClient(pool, client, opts) {
|
|
113
|
+
var limit = stop.limitOf(opts.timeoutMs);
|
|
114
|
+
var state = { cancelled: null, limit: limit };
|
|
115
|
+
state.w = stop.watch(opts.signal, limit ? limit + BACKSTOP_MS : 0, function() {
|
|
116
|
+
state.cancelled = cancelBackend(pool, client.processID).catch(function() { /* it finished first, or the server is gone */ });
|
|
117
|
+
});
|
|
118
|
+
return state;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The error to reject with: ours when we stopped it, the time limit when Postgres did. */
|
|
122
|
+
function stopReason(state, err) {
|
|
123
|
+
if (state.w.why()) return state.w.why();
|
|
124
|
+
if (err && err.code === QUERY_CANCELED && state.limit) return stop.timeoutError(state.limit);
|
|
125
|
+
return err;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Give the client back — destroyed when a query on it was stopped, so no half-cancelled state is reused. */
|
|
129
|
+
async function giveBack(state, client) {
|
|
130
|
+
state.w.done();
|
|
131
|
+
if (state.cancelled) await state.cancelled;
|
|
132
|
+
client.release(state.w.why() ? true : undefined);
|
|
133
|
+
}
|
|
134
|
+
|
|
85
135
|
// `columns` is the query's OUTPUT COLUMNS, which the server describes whether
|
|
86
136
|
// or not any row matched. Callers that need the shape of a result — a report
|
|
87
137
|
// listing its columns, say — must not have to infer it from the first row,
|
|
@@ -90,16 +140,33 @@ async function close(pool) {
|
|
|
90
140
|
// Attached to pg's own Result rather than wrapping it: callers already treat
|
|
91
141
|
// this as the canonical shape the other drivers normalize TO, so replacing
|
|
92
142
|
// the object risks dropping a property something else reads.
|
|
93
|
-
async function query(pool, sql, params) {
|
|
143
|
+
async function query(pool, sql, params, opts) {
|
|
144
|
+
opts = opts || {};
|
|
145
|
+
stop.throwIfStopped(opts.signal);
|
|
94
146
|
var result;
|
|
95
|
-
|
|
147
|
+
var limit = stop.limitOf(opts.timeoutMs);
|
|
148
|
+
if (pool.__xeplrReadOnly || opts.signal || limit) {
|
|
149
|
+
var readOnly = Boolean(pool.__xeplrReadOnly);
|
|
96
150
|
var client = await pool.connect();
|
|
151
|
+
var state = watchClient(pool, client, opts);
|
|
97
152
|
try {
|
|
98
|
-
|
|
99
|
-
|
|
153
|
+
// Read-only: inside its own transaction, the limit with it (LOCAL).
|
|
154
|
+
// Otherwise the statement runs as it always did — no transaction added,
|
|
155
|
+
// which some statements refuse — and the limit is on the session until
|
|
156
|
+
// it is reset below.
|
|
157
|
+
if (readOnly) await client.query('BEGIN TRANSACTION READ ONLY');
|
|
158
|
+
if (limit) await client.query((readOnly ? 'SET LOCAL' : 'SET') + ' statement_timeout = ' + limit);
|
|
159
|
+
result = await client.query(readOnly ? { text: sql, values: params || [], queryMode: 'extended' } : sql, readOnly ? undefined : (params || []));
|
|
160
|
+
} catch (err) {
|
|
161
|
+
throw stopReason(state, err);
|
|
100
162
|
} finally {
|
|
101
|
-
|
|
102
|
-
|
|
163
|
+
if (!state.w.why()) {
|
|
164
|
+
try {
|
|
165
|
+
if (readOnly) await client.query('ROLLBACK');
|
|
166
|
+
else if (limit) await client.query('RESET statement_timeout');
|
|
167
|
+
} catch (_) { /* the connection is released either way */ }
|
|
168
|
+
}
|
|
169
|
+
await giveBack(state, client);
|
|
103
170
|
}
|
|
104
171
|
} else {
|
|
105
172
|
result = await pool.query(sql, params || []);
|
|
@@ -342,14 +409,23 @@ async function listProcedures(pool, schema) {
|
|
|
342
409
|
// rows at a time. The async generator naturally applies backpressure: the next
|
|
343
410
|
// FETCH only fires when the consumer asks for more.
|
|
344
411
|
//
|
|
345
|
-
// opts: { sql, params?, batchSize? } → async iterable<row>
|
|
412
|
+
// opts: { sql, params?, batchSize?, signal?, timeoutMs? } → async iterable<row>
|
|
413
|
+
//
|
|
414
|
+
// A GROUP BY or ORDER BY is worked out in full by the FIRST FETCH, which can
|
|
415
|
+
// take minutes — so a stop must cancel that FETCH in Postgres, not wait for
|
|
416
|
+
// it to come back. timeoutMs is per statement (each FETCH), as Postgres
|
|
417
|
+
// counts it.
|
|
346
418
|
async function* fetchStream(pool, opts) {
|
|
419
|
+
stop.throwIfStopped(opts.signal);
|
|
347
420
|
var batchSize = Math.max(1, parseInt(opts.batchSize || 1000, 10));
|
|
421
|
+
var limit = stop.limitOf(opts.timeoutMs);
|
|
348
422
|
var client = await pool.connect();
|
|
423
|
+
var state = watchClient(pool, client, opts);
|
|
349
424
|
var declared = false;
|
|
350
425
|
try {
|
|
351
426
|
var readOnly = Boolean(pool.__xeplrReadOnly);
|
|
352
427
|
await client.query(readOnly ? 'BEGIN TRANSACTION READ ONLY' : 'BEGIN');
|
|
428
|
+
if (limit) await client.query('SET LOCAL statement_timeout = ' + limit);
|
|
353
429
|
// Extended protocol when read-only: one statement, so the caller's SQL
|
|
354
430
|
// cannot close the cursor declaration and carry on with its own.
|
|
355
431
|
await client.query(readOnly
|
|
@@ -357,14 +433,19 @@ async function* fetchStream(pool, opts) {
|
|
|
357
433
|
: 'DECLARE __xeplr_fetch_cur NO SCROLL CURSOR FOR ' + opts.sql, readOnly ? undefined : (opts.params || []));
|
|
358
434
|
declared = true;
|
|
359
435
|
while (true) {
|
|
436
|
+
if (state.w.why()) throw state.w.why();
|
|
360
437
|
var res = await client.query('FETCH FORWARD ' + batchSize + ' FROM __xeplr_fetch_cur');
|
|
361
438
|
if (res.rows.length === 0) break;
|
|
362
439
|
for (var i = 0; i < res.rows.length; i++) yield res.rows[i];
|
|
363
440
|
}
|
|
441
|
+
} catch (err) {
|
|
442
|
+
throw stopReason(state, err);
|
|
364
443
|
} finally {
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
444
|
+
if (!state.w.why()) {
|
|
445
|
+
try { if (declared) await client.query('CLOSE __xeplr_fetch_cur'); } catch (_) {}
|
|
446
|
+
try { await client.query(pool.__xeplrReadOnly ? 'ROLLBACK' : 'COMMIT'); } catch (_) {}
|
|
447
|
+
}
|
|
448
|
+
await giveBack(state, client);
|
|
368
449
|
}
|
|
369
450
|
}
|
|
370
451
|
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// STOPPING A QUERY — the part every driver shares.
|
|
2
|
+
//
|
|
3
|
+
// A query nobody is waiting for must not run (see CLAUDE.md, "No query runs
|
|
4
|
+
// that nobody needs"). Every driver's query() and fetchStream() take
|
|
5
|
+
//
|
|
6
|
+
// signal an AbortSignal: when it fires, the query is CANCELLED IN THE
|
|
7
|
+
// DATABASE (each driver's own way), not just abandoned here —
|
|
8
|
+
// a GROUP BY the database is still working on would otherwise
|
|
9
|
+
// keep its CPU and its connection until it finished
|
|
10
|
+
// timeoutMs a time limit; past it the query is cancelled the same way
|
|
11
|
+
//
|
|
12
|
+
// and reject with one of two errors a caller can tell apart:
|
|
13
|
+
//
|
|
14
|
+
// code QUERY_STOPPED the signal fired; the message is its reason's
|
|
15
|
+
// code QUERY_TIMEOUT the time limit passed
|
|
16
|
+
//
|
|
17
|
+
// Neither is given: the query runs as it always did.
|
|
18
|
+
|
|
19
|
+
/** The error a stopped query rejects with. */
|
|
20
|
+
function stoppedError(signal) {
|
|
21
|
+
var reason = signal && signal.reason;
|
|
22
|
+
// abort() with no reason gives a generic AbortError: say it in our words.
|
|
23
|
+
var said = reason && reason.name !== 'AbortError' && reason.message;
|
|
24
|
+
var err = new Error(said || 'The query was stopped');
|
|
25
|
+
err.code = 'QUERY_STOPPED';
|
|
26
|
+
return err;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** "45 seconds", "2 minutes" — how long a limit is, as a person says it. */
|
|
30
|
+
function durationText(ms) {
|
|
31
|
+
var secs = Math.max(1, Math.round(ms / 1000));
|
|
32
|
+
if (secs < 60) return secs + ' second' + (secs === 1 ? '' : 's');
|
|
33
|
+
var mins = Math.round(secs / 60);
|
|
34
|
+
return mins + ' minute' + (mins === 1 ? '' : 's');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The error a query past its time limit rejects with. */
|
|
38
|
+
function timeoutError(timeoutMs) {
|
|
39
|
+
var err = new Error('The database stopped this query after ' + durationText(timeoutMs) +
|
|
40
|
+
', its time limit. Narrow it with a filter or a shorter date range, then run it again.');
|
|
41
|
+
err.code = 'QUERY_TIMEOUT';
|
|
42
|
+
return err;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A positive whole number of milliseconds, or 0 for no limit. */
|
|
46
|
+
function limitOf(timeoutMs) {
|
|
47
|
+
var n = Math.floor(Number(timeoutMs));
|
|
48
|
+
return Number.isFinite(n) && n > 0 ? n : 0;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Watch one query: call `cancel()` once — when `signal` fires or `timeoutMs`
|
|
53
|
+
* passes, whichever is first — and remember why. `why()` is the error to
|
|
54
|
+
* reject with after the database has given up the query (null: not stopped).
|
|
55
|
+
* `done()` stops watching; call it in a finally.
|
|
56
|
+
*/
|
|
57
|
+
function watch(signal, timeoutMs, cancel) {
|
|
58
|
+
var limit = limitOf(timeoutMs);
|
|
59
|
+
var reason = null;
|
|
60
|
+
var timer = null;
|
|
61
|
+
function fire(err) {
|
|
62
|
+
if (reason) return;
|
|
63
|
+
reason = err;
|
|
64
|
+
try { cancel(); } catch (_) { /* the driver's cancel reports its own failures */ }
|
|
65
|
+
}
|
|
66
|
+
function onAbort() { fire(stoppedError(signal)); }
|
|
67
|
+
if (signal) {
|
|
68
|
+
if (signal.aborted) onAbort();
|
|
69
|
+
else signal.addEventListener('abort', onAbort, { once: true });
|
|
70
|
+
}
|
|
71
|
+
if (limit && !reason) {
|
|
72
|
+
timer = setTimeout(function() { fire(timeoutError(limit)); }, limit);
|
|
73
|
+
if (timer.unref) timer.unref();
|
|
74
|
+
}
|
|
75
|
+
return {
|
|
76
|
+
why: function() { return reason; },
|
|
77
|
+
done: function() {
|
|
78
|
+
if (timer) clearTimeout(timer);
|
|
79
|
+
if (signal) signal.removeEventListener('abort', onAbort);
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Throw now when the signal has already fired: never start a query nobody wants. */
|
|
85
|
+
function throwIfStopped(signal) {
|
|
86
|
+
if (signal && signal.aborted) throw stoppedError(signal);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
module.exports = { watch: watch, throwIfStopped: throwIfStopped, stoppedError: stoppedError, timeoutError: timeoutError, limitOf: limitOf, durationText: durationText };
|
|
@@ -95,7 +95,7 @@ function buildClientConfig(action, connection) {
|
|
|
95
95
|
// through this so the release/logout pair can never be forgotten in one of
|
|
96
96
|
// them. `logout()` is best-effort in its own catch: a failure there must not
|
|
97
97
|
// mask the real error from the work, and must not fail an op that succeeded.
|
|
98
|
-
async function withMailbox(action, connection, folder, fn) {
|
|
98
|
+
async function withMailbox(action, connection, folder, fn, readOnly) {
|
|
99
99
|
// Config FIRST, deps second. Validating the connection needs no peer dep, so
|
|
100
100
|
// a misconfigured connection reports the missing field even on a host that
|
|
101
101
|
// hasn't installed imapflow — otherwise the dep error masks the real one.
|
|
@@ -105,7 +105,9 @@ async function withMailbox(action, connection, folder, fn) {
|
|
|
105
105
|
|
|
106
106
|
await client.connect();
|
|
107
107
|
try {
|
|
108
|
-
|
|
108
|
+
// Read-only (IMAP EXAMINE) for a preview: nothing it fetches can mark a
|
|
109
|
+
// message read, whatever the server does on a full download.
|
|
110
|
+
var lock = await client.getMailboxLock(folder || 'INBOX', readOnly ? { readOnly: true } : undefined);
|
|
109
111
|
try {
|
|
110
112
|
return await fn(client);
|
|
111
113
|
} finally {
|
|
@@ -257,7 +259,7 @@ async function read(connection, input) {
|
|
|
257
259
|
});
|
|
258
260
|
}
|
|
259
261
|
return out;
|
|
260
|
-
});
|
|
262
|
+
}, input.readOnly === true);
|
|
261
263
|
}
|
|
262
264
|
|
|
263
265
|
/** Move one message to another folder. Returns the shape the action wraps. */
|
package/lib/registry.js
CHANGED
|
@@ -9,6 +9,15 @@
|
|
|
9
9
|
// outputSchema : array? — optional; documents what the action returns
|
|
10
10
|
// requires : string[]? — peer-dep module names checked at register time
|
|
11
11
|
// execute : function — async ({ input, system }) => output
|
|
12
|
+
// preview : function? — async ({ input, system }) => { output, rows? }:
|
|
13
|
+
// what execute WOULD hand on, read without a
|
|
14
|
+
// side effect — a few rows, never a write, a
|
|
15
|
+
// send, a move or a delete. A designer shows it
|
|
16
|
+
// before anything has run (runner.previewAction).
|
|
17
|
+
// example : object? — for an action that cannot be previewed
|
|
18
|
+
// safely (it writes or sends): its output with
|
|
19
|
+
// example values, so a designer still shows
|
|
20
|
+
// the shape.
|
|
12
21
|
// event : string? — a SCREEN FUNCTION, not a step: the event it
|
|
13
22
|
// is written for ('beforeSave', 'decide', …).
|
|
14
23
|
// Called with that event's own context instead
|
|
@@ -61,6 +70,8 @@ function register(nameOrDef, def) {
|
|
|
61
70
|
outputSchema: action.outputSchema || null,
|
|
62
71
|
requires: Array.isArray(action.requires) ? action.requires.slice() : [],
|
|
63
72
|
event: action.event || null,
|
|
73
|
+
preview: typeof action.preview === 'function' ? action.preview : null,
|
|
74
|
+
example: action.example && typeof action.example === 'object' ? action.example : null,
|
|
64
75
|
execute: action.execute
|
|
65
76
|
});
|
|
66
77
|
}
|
|
@@ -78,7 +89,11 @@ function list() {
|
|
|
78
89
|
inputSchema: def.inputSchema,
|
|
79
90
|
outputSchema: def.outputSchema,
|
|
80
91
|
requires: def.requires,
|
|
81
|
-
event: def.event
|
|
92
|
+
event: def.event,
|
|
93
|
+
// Whether a designer can ask for a preview — the function itself never
|
|
94
|
+
// leaves the process.
|
|
95
|
+
preview: Boolean(def.preview),
|
|
96
|
+
example: def.example
|
|
82
97
|
});
|
|
83
98
|
});
|
|
84
99
|
return out;
|
package/lib/runner.js
CHANGED
|
@@ -43,6 +43,40 @@ async function runAction(first, second, third) {
|
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* WHAT A STEP WOULD HAND ON, without running it. Calls the action's `preview`
|
|
48
|
+
* — never `execute` — with the input checked exactly as a run checks it.
|
|
49
|
+
*
|
|
50
|
+
* Returns { status, output, rows, error, durationMs }:
|
|
51
|
+
* status 'success' output is what execute would return (its shape and
|
|
52
|
+
* sample values); rows, when the action reads rows, the
|
|
53
|
+
* first few of them.
|
|
54
|
+
* status 'example' the action cannot be previewed safely; output is its
|
|
55
|
+
* declared example (or null).
|
|
56
|
+
* status 'failed' the input is incomplete or the read failed — error says.
|
|
57
|
+
* Never throws.
|
|
58
|
+
*/
|
|
59
|
+
async function previewAction(first, second, third) {
|
|
60
|
+
var start = Date.now();
|
|
61
|
+
var opts = normalizeArgs(first, second, third);
|
|
62
|
+
var def = opts.action || get(opts.name);
|
|
63
|
+
if (!def) return Object.assign(finish(start, 'failed', null, new ActionNotRegisteredError(opts.name)), { rows: null });
|
|
64
|
+
if (typeof def.preview !== 'function') {
|
|
65
|
+
return Object.assign(finish(start, 'example', def.example || null, null), { rows: null });
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
var input = applySchema(def.inputSchema, coerceScalars(def.inputSchema, opts.input || {}), 'input');
|
|
69
|
+
var seen = await def.preview({ input: input, system: opts.system || {}, metaStore: opts.metaStore });
|
|
70
|
+
seen = seen || {};
|
|
71
|
+
return Object.assign(finish(start, 'success', seen.output === undefined ? null : seen.output, null), { rows: seen.rows || null });
|
|
72
|
+
} catch (err) {
|
|
73
|
+
return Object.assign(finish(start, 'failed', null, err), { rows: null });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** How many rows a preview reads: enough to see, never the data. */
|
|
78
|
+
var PREVIEW_ROWS = 5;
|
|
79
|
+
|
|
46
80
|
/**
|
|
47
81
|
* ONE VALUE WHERE A LIST WAS DECLARED MEANS A LIST OF ONE.
|
|
48
82
|
*
|
|
@@ -125,4 +159,4 @@ function withTimeout(promise, ms) {
|
|
|
125
159
|
});
|
|
126
160
|
}
|
|
127
161
|
|
|
128
|
-
module.exports = { runAction: runAction };
|
|
162
|
+
module.exports = { runAction: runAction, previewAction: previewAction, PREVIEW_ROWS: PREVIEW_ROWS };
|
package/package.json
CHANGED