@xeplr/imports 0.0.0-stage → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +126 -2
- package/index.js +28 -0
- package/lib/imports.js +240 -0
- package/lib/router.js +321 -0
- package/lib/runs.js +93 -0
- package/migrations-auth/0001_imports_access.sql +49 -0
- package/package.json +52 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Xeplr
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,127 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @xeplr/imports
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Data imports as a drop-in module. Upload a CSV, Excel or JSON file and check it against the table it is going into. Then load it as a **New table**, an **Append** or a **Replace**. Afterwards you can see and download the rows the database refused, and roll the import back. One package holds the routes, the access catalog and the settings. The screen is [`ImportsPage` in `@xeplr/ui-utils`](https://www.npmjs.com/package/@xeplr/ui-utils).
|
|
4
|
+
|
|
5
|
+
## What you get
|
|
6
|
+
|
|
7
|
+
- **Big files, no waiting.** CSV, Excel and JSON files of hundreds of megabytes upload and load in the background, with live progress and a stop button.
|
|
8
|
+
- **New table, append or replace.** You choose what happens to the table. A typo in a table's name is caught, so it never turns into a new table.
|
|
9
|
+
- **Replace without risk.** The table is only swapped once every row has loaded. An import that fails or is stopped leaves your data exactly as it was.
|
|
10
|
+
- **Checked before it loads.** The file's columns are compared with the table first. You see what will be refused, what will be left empty and what matches, before anything is touched.
|
|
11
|
+
- **Copy-paste damage repaired.** Rows that a paste from another system folded into one cell are found and put back.
|
|
12
|
+
- **Nothing silently lost.** Every row the database refuses is kept with the reason. View it, copy it, or download it as Excel to fix and load again.
|
|
13
|
+
- **Full history and undo.** Every import is recorded with who ran it, where it went and how it went, and can be rolled back.
|
|
14
|
+
- **Drop-in.** One router and one screen, and the app supplies only its database connections.
|
|
15
|
+
|
|
16
|
+
Every step underneath is [`@xeplr/actions`](https://www.npmjs.com/package/@xeplr/actions): its file readers, the uploader, the error table and staged loads. This package puts those steps behind HTTP and leaves the app only what is its own.
|
|
17
|
+
|
|
18
|
+
## Using it
|
|
19
|
+
|
|
20
|
+
```js
|
|
21
|
+
var imports = require('@xeplr/imports');
|
|
22
|
+
|
|
23
|
+
// routes — mounted at /imports (the access catalog assumes that path)
|
|
24
|
+
router.use('/imports', imports.router({
|
|
25
|
+
auth: auth.mtMembershipGate, // the app's gate, run before every route
|
|
26
|
+
service: 'my-app', // names this app's import records
|
|
27
|
+
resolveTarget: async function(connectionId, dbInfoId, req) {
|
|
28
|
+
// the database a file loads into, as a login the @xeplr/actions drivers take
|
|
29
|
+
return { dbType: 'postgres', dbConnection: { host, port, user, password, database } }; // or null
|
|
30
|
+
},
|
|
31
|
+
listTargets: async function(req) {
|
|
32
|
+
// the databases this caller may import into, for the picker and for history's labels
|
|
33
|
+
return [{ connectionId: 'c1', label: 'Warehouse', databases: [{ id: 'd1', label: 'sales' }] }];
|
|
34
|
+
}
|
|
35
|
+
}));
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
```js
|
|
39
|
+
// env.required.js
|
|
40
|
+
module.exports = [].concat(imports.requiredEnv /* , … */);
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
# .env
|
|
45
|
+
XEPLR_DATA_DIR=/srv/xeplr-data # required, absolute: each tenant's files under it
|
|
46
|
+
XCFG_DB_NAME=xeplr_configs # required: where import records live
|
|
47
|
+
XEPLR_AUTH_MIGRATIONS=…,/…/node_modules/@xeplr/imports/migrations-auth
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
On the screen side, add one route for `<ImportsPage/>` and a drawer entry. That is the whole of it.
|
|
51
|
+
|
|
52
|
+
### What the app supplies, and why only that
|
|
53
|
+
|
|
54
|
+
| | why it is the app's |
|
|
55
|
+
|---|---|
|
|
56
|
+
| `auth` | who may call these routes is decided by the app's sign-in, as for every other route |
|
|
57
|
+
| `resolveTarget(connectionId, dbInfoId, req)` | where the app keeps its connections, and which of them a caller may use, is its own business. Return `null` for one the caller may not use, and the route answers 404 |
|
|
58
|
+
| `listTargets(req)` | the same connections, as a list for the picker |
|
|
59
|
+
| `service` | names this app's rows in the shared `import_meta` table. Or pass `config`, an `attachConfig()` instance the app already has |
|
|
60
|
+
| `userNames(ids)` (optional) | names for records made before the importer's name was stored on them |
|
|
61
|
+
|
|
62
|
+
Tenancy comes from the app's `mtMiddleware`, as it does for its other routes. Each import record carries `mtId1`/`mtId2`, and history reads them back through the tenant filter, so another company's or workspace's imports are simply not found. Import records live in `xeplr_configs` and are application-scoped (`registerApplication()` at startup).
|
|
63
|
+
|
|
64
|
+
Database drivers are the app's own: install `pg`, `mysql2`, `mssql` or `@duckdb/node-api` for each kind of database it imports into. The file readers (CSV, Excel) come with this package.
|
|
65
|
+
|
|
66
|
+
## Settings
|
|
67
|
+
|
|
68
|
+
**From 1.x (2.0.0):** `UPLOAD_DIR` is gone — set `XEPLR_DATA_DIR` (in `requiredEnv`), and call `@xeplr/db`'s `registerApplication()` at boot. Uploads now live per tenant.
|
|
69
|
+
|
|
70
|
+
| Variable | Required | Meaning |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `XEPLR_DATA_DIR` | **yes**, absolute (`@xeplr/actions`) | The data root. An upload is staged in the requesting tenant's own `uploads/` folder under it (`<root>/<app>/<company>/<workspace>/uploads`) until imported; a dropped-rows download is written in its `tmp/` for the moment it is sent. Asked per request, so one company's files never sit beside another's. Never defaulted: a missing, relative or absent one is refused by name. |
|
|
73
|
+
| `XCFG_DB_NAME` | **yes** | The `xeplr_configs` database holding `import_meta`. The login is `XCFG_DB_CONNECTION_INFO_ENCRYPTED`, or the shared `XEPLR_DB_CONNECTION` (`@xeplr/db`) |
|
|
74
|
+
| `UPLOAD_MAX_MB` | no | The largest file accepted, default 500. Files go to disk and are streamed, so this costs disk, not memory. A proxy in front (nginx `client_max_body_size`) must allow at least as much |
|
|
75
|
+
| `IMPORT_TIME_LIMIT_MINUTES` | no | How long an import may run before it is stopped, default 60 |
|
|
76
|
+
|
|
77
|
+
## Routes
|
|
78
|
+
|
|
79
|
+
Mounted at `/imports`:
|
|
80
|
+
|
|
81
|
+
| Method | Path | |
|
|
82
|
+
|---|---|---|
|
|
83
|
+
| POST | `/files` | The file (multipart, field `file`) → `{ filePath, fileName, originalName, size }` |
|
|
84
|
+
| POST | `/inspect` | `{ filePath }` → `{ format, sheets?, checks }`. `checks` lists what is wrong before anything loads: rows pasted into one cell, with "Recover them" as a choice |
|
|
85
|
+
| POST | `/check-target` | `{ filePath, sheet?, connectionId, dbInfoId, targetTable }` → `{ exists, matched, extra, missing: [{ name, required }] }` |
|
|
86
|
+
| POST | `/commit` | `{ filePath, connectionId, dbInfoId, originalFileName?, mappings: [{ sheet?, targetTable, mode, repairPastedRows?, primaryKeys? }] }` → **202** `{ runId }`. The import runs in the background. The same file is never imported twice at once (409) |
|
|
87
|
+
| GET | `/runs/:id` | `{ status: running \| complete \| error \| stopped, progress, result: { results }, error }`. A run is shown only to the person who started it. Kept an hour after it ends |
|
|
88
|
+
| POST | `/runs/:id/stop` | Stops it within a batch |
|
|
89
|
+
| GET | `/targets` | `listTargets` |
|
|
90
|
+
| GET | `/history` | Past imports, newest first: `{ id, originalFileName, part, targetTable, destination, status, mode, totalRows, rowsLoaded, rowsDropped, errorMessage, createdByName, startedAt }` |
|
|
91
|
+
| GET | `/history/:id/dropped` | The rows the database refused, from `<table>_import_errors`: `{ columns, rows: [{ rowNum, reason, values }], total, dropped }`, the first 1,000 |
|
|
92
|
+
| GET | `/history/:id/dropped/file` | The same rows as .xlsx: the file's columns only, header in row 1, so it imports again once fixed. The reason is not a column, because it would be refused on the way back in |
|
|
93
|
+
| POST | `/history/:id/rollback` | Deletes what the import loaded |
|
|
94
|
+
| POST | `/history/:id/delete` | Removes the record only. The data stays |
|
|
95
|
+
|
|
96
|
+
A commit result per part is `{ targetTable, sheet, success, rows, dropped, jobId, recovered, error }`.
|
|
97
|
+
|
|
98
|
+
## The rules
|
|
99
|
+
|
|
100
|
+
- **Modes.** A **New table** must not exist yet. **Append** and **Replace** need one that does, so a typo is never a fresh table. A **Replace** is staged: the rows load into a staging table, then one transaction empties the table and moves them in. A Replace that fails or is stopped leaves the table as it was. See [`@xeplr/actions` Import modes](https://www.npmjs.com/package/@xeplr/actions#import-modes).
|
|
101
|
+
- **Rollback is refused for a Replace**, because the rows it replaced are gone. It is also refused for an upsert (`primaryKeys`), which may only have updated rows that already existed.
|
|
102
|
+
- **Refused rows are kept.** Each row the database refuses goes to `<table>_import_errors` with its place in the file and the reason, so it can be seen, fixed and loaded again. An import gives up after five refusals in a row, and rows after that are counted but not kept.
|
|
103
|
+
- **A filePath is checked.** It comes from the client, so only a file in the requesting tenant's own `uploads/` is accepted — never a file elsewhere on the server, and never another company's or workspace's upload (`../` and links followed first). A file the import cannot read is removed at inspect, and an imported file is removed when its import ends.
|
|
104
|
+
|
|
105
|
+
## Access
|
|
106
|
+
|
|
107
|
+
`migrations-auth/0001_imports_access.sql` registers every route by group: `imports:view` (history, dropped rows, runs, targets), `imports:create` (upload, inspect, check, commit, stop) and `imports:edit` (rollback, delete). It maps all of them to Super Admin and CompanyAdmin. Loading data into a company's databases is an administrator's job, so Creator and Viewer get none. An app that wants them to can map those roles in its own migration. Inserts are idempotent.
|
|
108
|
+
|
|
109
|
+
## Tests
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
PG_HOST=… PG_PORT=… PG_USER=… PG_PASSWORD=… PG_DATABASE=… npm test
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The real router runs in an Express app, against real Postgres and a throwaway `xeplr_config_imports_test`. The tests cover:
|
|
116
|
+
|
|
117
|
+
- refused paths and types
|
|
118
|
+
- pasted rows found and recovered
|
|
119
|
+
- each mode and its refusals
|
|
120
|
+
- refused rows shown and downloaded
|
|
121
|
+
- history scoped to a workspace
|
|
122
|
+
- rollback, with a Replace refused
|
|
123
|
+
- delete
|
|
124
|
+
- a stopped Replace leaving the table untouched
|
|
125
|
+
- runs that only their importer can see
|
|
126
|
+
|
|
127
|
+
Without `PG_HOST`, only the startup checks run.
|
package/index.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// @xeplr/imports — data imports as a drop-in module. See README.md.
|
|
2
|
+
//
|
|
3
|
+
// var imports = require('@xeplr/imports');
|
|
4
|
+
// router.use('/imports', imports.router({ auth, resolveTarget, listTargets, service: 'my-app' }));
|
|
5
|
+
//
|
|
6
|
+
// plus `imports.requiredEnv` in the app's env.required.js and
|
|
7
|
+
// `imports.authMigrationsDir` in XEPLR_AUTH_MIGRATIONS.
|
|
8
|
+
|
|
9
|
+
var path = require('path');
|
|
10
|
+
var actions = require('@xeplr/actions');
|
|
11
|
+
var { requiredEnv: configRequiredEnv } = require('@xeplr/actions/lib/uploader/attach');
|
|
12
|
+
|
|
13
|
+
// What this package needs from the environment, for the app's checkEnv.
|
|
14
|
+
// XEPLR_DATA_DIR (@xeplr/actions): uploads are staged in each tenant's own
|
|
15
|
+
// uploads/ under it — absolute, never defaulted (a wrong-but-present location
|
|
16
|
+
// reads as "no file", not as an error). The import records live in
|
|
17
|
+
// xeplr_configs (XCFG_DB_NAME, and the login in XCFG_DB_CONNECTION_INFO_ENCRYPTED
|
|
18
|
+
// or the shared XEPLR_DB_CONNECTION).
|
|
19
|
+
var requiredEnv = actions.requiredEnv.concat(configRequiredEnv);
|
|
20
|
+
|
|
21
|
+
module.exports = {
|
|
22
|
+
router: require('./lib/router'),
|
|
23
|
+
requiredEnv: requiredEnv,
|
|
24
|
+
// This package's access catalog (apis, role mappings) — add it to the app's
|
|
25
|
+
// XEPLR_AUTH_MIGRATIONS so its auth database knows these routes.
|
|
26
|
+
authMigrationsDir: path.join(__dirname, 'migrations-auth'),
|
|
27
|
+
MODES: require('./lib/imports').MODES
|
|
28
|
+
};
|
package/lib/imports.js
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
// WHAT AN IMPORT DOES, apart from HTTP: what a file holds, how it compares
|
|
2
|
+
// with the table it names, loading it, and reading back what it refused.
|
|
3
|
+
// Every step is @xeplr/actions — this file only puts them in order. The
|
|
4
|
+
// routes (router.js) call these; nothing here knows about a request.
|
|
5
|
+
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
var { runAction, builtins, formats, uploader, drivers, dataDir } = require('@xeplr/actions');
|
|
9
|
+
|
|
10
|
+
var EXT_TO_FORMAT = { '.csv': 'csv', '.json': 'json', '.xlsx': 'excel', '.xls': 'excel' };
|
|
11
|
+
|
|
12
|
+
// Rows the dropped-rows view shows at once, and the most a download carries
|
|
13
|
+
// (Excel's own limit — past it the sheet would be cut anyway).
|
|
14
|
+
var DROPPED_VIEW_ROWS = 1000;
|
|
15
|
+
var DROPPED_FILE_ROWS = 1048575;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* WHERE AN UPLOAD IS STAGED: `uploads/` in the folder of the tenant the
|
|
19
|
+
* request is for (@xeplr/actions' dataDir, under XEPLR_DATA_DIR). Asked per
|
|
20
|
+
* request — never a folder fixed at startup, which would put every company's
|
|
21
|
+
* files side by side. Made if missing.
|
|
22
|
+
*/
|
|
23
|
+
function uploadDir() {
|
|
24
|
+
var dir = dataDir.resolve(UPLOADS, 'The upload folder');
|
|
25
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
26
|
+
return dir;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
var UPLOADS = 'uploads';
|
|
30
|
+
|
|
31
|
+
/** How big an upload may be and how long an import may run, from the environment. */
|
|
32
|
+
function settings() {
|
|
33
|
+
return {
|
|
34
|
+
// Client exports run to a few hundred MB (a 773k-row ERP CSV is 270 MB);
|
|
35
|
+
// the file goes to disk and is streamed, so the size costs disk, not memory.
|
|
36
|
+
maxUploadMb: Number(process.env.UPLOAD_MAX_MB) || 500,
|
|
37
|
+
timeLimitMs: (Number(process.env.IMPORT_TIME_LIMIT_MINUTES) || 60) * 60 * 1000
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function formatOf(filename) {
|
|
42
|
+
return EXT_TO_FORMAT[path.extname(filename).toLowerCase()] || null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* A filePath comes from the client, relaying what the upload answered — so
|
|
47
|
+
* only one inside THIS tenant's upload folder is accepted: never a file
|
|
48
|
+
* elsewhere on the server, and never another company's upload. The real
|
|
49
|
+
* path (../ and links followed), or null.
|
|
50
|
+
*/
|
|
51
|
+
function uploadedFile(filePath) {
|
|
52
|
+
var dir = uploadDir();
|
|
53
|
+
var real;
|
|
54
|
+
try { real = dataDir.resolve(String(filePath || ''), 'The uploaded file'); } catch (e) { return null; }
|
|
55
|
+
return real.indexOf(dir + path.sep) === 0 ? real : null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function listSheets(filePath) {
|
|
59
|
+
return formats.getFormat('excel').listSheets(filePath);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* What is wrong with the file before anything is loaded — today, rows a paste
|
|
64
|
+
* folded into one cell (see @xeplr/actions formats/pastedRows.js), with the
|
|
65
|
+
* fix offered as a choice. Per sheet for Excel, once ('') for a CSV.
|
|
66
|
+
*
|
|
67
|
+
* @returns {{ [part]: [{ key, message, fixLabel, examples }] }}
|
|
68
|
+
*/
|
|
69
|
+
async function inspectChecks(filePath, format, sheets) {
|
|
70
|
+
var checks = {};
|
|
71
|
+
if (format !== 'excel' && format !== 'csv') return checks;
|
|
72
|
+
var parts = format === 'excel' ? (sheets || []) : [null];
|
|
73
|
+
for (var i = 0; i < parts.length; i++) {
|
|
74
|
+
var found = await formats.scanPastedRows(format, filePath, parts[i] ? { sheet: parts[i], examples: 3 } : { examples: 3 });
|
|
75
|
+
if (!found.cells) continue;
|
|
76
|
+
checks[parts[i] == null ? '' : parts[i]] = [{
|
|
77
|
+
key: 'pastedRows',
|
|
78
|
+
message: plural(found.rows, 'row') + (found.rows === 1 ? ' was' : ' were') + ' pasted into ' + plural(found.cells, 'cell') +
|
|
79
|
+
' — they exist nowhere else in the file. This happens when data copied out of another system is pasted into Excel and a value starts with a quote mark (an inch mark).',
|
|
80
|
+
fixLabel: 'Recover them as ' + plural(found.rows, 'separate row'),
|
|
81
|
+
examples: found.examples.map(function(e) {
|
|
82
|
+
return 'Row ' + e.row + ', ' + e.column + ': holds ' + plural(e.rows, 'more row') + ' — begins with ' + e.first;
|
|
83
|
+
})
|
|
84
|
+
}];
|
|
85
|
+
}
|
|
86
|
+
return checks;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* One part of a file against the table it names — said before loading: does
|
|
91
|
+
* the table exist, which file columns does it lack (the import would be
|
|
92
|
+
* refused), which of its columns does the file lack (left empty, or every row
|
|
93
|
+
* refused when required).
|
|
94
|
+
*
|
|
95
|
+
* @returns {Promise<{ exists, matched, extra, missing: [{ name, required }] }>}
|
|
96
|
+
*/
|
|
97
|
+
async function checkTarget(opts) {
|
|
98
|
+
var columns = await formats.columnsOf(opts.format, opts.filePath, opts.sheet ? { sheet: opts.sheet } : {});
|
|
99
|
+
return uploader.compareToTable({
|
|
100
|
+
driver: drivers.db.getDriver(opts.dbType),
|
|
101
|
+
connection: opts.dbConnection,
|
|
102
|
+
targetTable: opts.targetTable,
|
|
103
|
+
columns: columns
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Load one part of a file into one table: the file-upload action, which
|
|
109
|
+
* records the import (import_meta) before writing a row. Never throws — the
|
|
110
|
+
* caller reports each part's outcome without stopping the others.
|
|
111
|
+
*/
|
|
112
|
+
async function loadPart(opts, metaStore) {
|
|
113
|
+
var outcome = await runAction(builtins.fileUpload, {
|
|
114
|
+
sourceType: 'local',
|
|
115
|
+
sourcePath: opts.filePath,
|
|
116
|
+
format: opts.format,
|
|
117
|
+
formatConfig: Object.assign(opts.sheet ? { sheet: opts.sheet } : {}, opts.repairPastedRows ? { repairPastedRows: true } : {}),
|
|
118
|
+
dbType: opts.dbType,
|
|
119
|
+
dbConnection: opts.dbConnection,
|
|
120
|
+
targetTable: opts.targetTable,
|
|
121
|
+
primaryKeys: opts.primaryKeys,
|
|
122
|
+
mode: opts.mode || undefined,
|
|
123
|
+
movementId: opts.movementId
|
|
124
|
+
}, {
|
|
125
|
+
system: { mtId1: opts.mtId1, mtId2: opts.mtId2, details: opts.details, onProgress: opts.onProgress || null, signal: opts.signal || null },
|
|
126
|
+
metaStore: metaStore
|
|
127
|
+
});
|
|
128
|
+
// An action that failed before its own ending was recorded would leave the
|
|
129
|
+
// import 'running' forever. recordEnd is an upsert, so this is harmless when
|
|
130
|
+
// the uploader already closed it.
|
|
131
|
+
if (outcome.status !== 'success') {
|
|
132
|
+
await metaStore.recordEnd(opts.movementId, {
|
|
133
|
+
status: 'failed',
|
|
134
|
+
error: outcome.error && outcome.error.message,
|
|
135
|
+
targetTable: opts.targetTable, connectionKey: opts.targetTable, dbType: opts.dbType,
|
|
136
|
+
mtId1: opts.mtId1, mtId2: opts.mtId2, details: opts.details
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
return outcome;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Undo an import: every row it loaded, and its refused rows. See uploader.rollback for what it refuses. */
|
|
143
|
+
function rollback(opts, metaStore) {
|
|
144
|
+
return uploader.rollback({
|
|
145
|
+
movementId: opts.movementId,
|
|
146
|
+
driver: drivers.db.getDriver(opts.dbType),
|
|
147
|
+
connection: opts.dbConnection,
|
|
148
|
+
targetTable: opts.targetTable,
|
|
149
|
+
metaStore: metaStore
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* The rows an import dropped — refused by the database — from
|
|
155
|
+
* `<table>_import_errors`, in the file's column order (`columns`, the
|
|
156
|
+
* import's own record). An import made before refused rows were kept has
|
|
157
|
+
* only its count: total 0.
|
|
158
|
+
*
|
|
159
|
+
* @returns {Promise<{ columns: string[], rows: [{ rowNum, reason, values }], total }>}
|
|
160
|
+
*/
|
|
161
|
+
async function droppedRows(opts) {
|
|
162
|
+
var driver = drivers.db.getDriver(opts.dbType);
|
|
163
|
+
var pool = await driver.connect(opts.dbConnection);
|
|
164
|
+
try {
|
|
165
|
+
var errTable = opts.targetTable + '_import_errors';
|
|
166
|
+
var columns = (opts.columns || []).map(function(c) { return typeof c === 'string' ? c : c.name; })
|
|
167
|
+
.filter(function(n) { return n && n.indexOf('__xeplr_') !== 0; });
|
|
168
|
+
var schema = await driver.getTableSchema(pool, errTable);
|
|
169
|
+
if (!schema || !schema.length) return { columns: columns, rows: [], total: 0 };
|
|
170
|
+
|
|
171
|
+
var mark = require('@xeplr/actions/lib/drivers/db/procedure').placeholders(opts.dbType)(0);
|
|
172
|
+
var t = driver.quoteIdent(errTable);
|
|
173
|
+
var counted = await driver.query(pool, 'SELECT COUNT(*) AS n FROM ' + t + ' WHERE movement_id = ' + mark, [opts.movementId]);
|
|
174
|
+
var total = Number(counted.rows[0].n || counted.rows[0].N || 0);
|
|
175
|
+
|
|
176
|
+
var limit = Math.max(1, Math.min(Number(opts.limit) || DROPPED_VIEW_ROWS, DROPPED_FILE_ROWS));
|
|
177
|
+
var top = opts.dbType === 'mssql' ? 'TOP (' + limit + ') ' : '';
|
|
178
|
+
var tail = opts.dbType === 'mssql' ? '' : ' LIMIT ' + limit;
|
|
179
|
+
var found = await driver.query(pool, 'SELECT ' + top + 'row_num, error_description, raw_row FROM ' + t +
|
|
180
|
+
' WHERE movement_id = ' + mark + ' ORDER BY row_num' + tail, [opts.movementId]);
|
|
181
|
+
|
|
182
|
+
var rows = found.rows.map(function(r) {
|
|
183
|
+
var values = r.raw_row;
|
|
184
|
+
if (typeof values === 'string') { try { values = JSON.parse(values); } catch (_) { values = {}; } }
|
|
185
|
+
values = values || {};
|
|
186
|
+
// A value the import's record does not name is shown, not lost.
|
|
187
|
+
Object.keys(values).forEach(function(k) { if (columns.indexOf(k) === -1) columns.push(k); });
|
|
188
|
+
return { rowNum: r.row_num == null ? null : Number(r.row_num), reason: r.error_description || '', values: values };
|
|
189
|
+
});
|
|
190
|
+
return { columns: columns, rows: rows, total: total };
|
|
191
|
+
} finally {
|
|
192
|
+
await driver.close(pool);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* droppedRows' answer as an .xlsx: the file's columns only, header in row 1,
|
|
198
|
+
* no title — the shape an import reads, so the rows go back in once fixed.
|
|
199
|
+
* The reason is not a column: it would be refused on the way back in.
|
|
200
|
+
*/
|
|
201
|
+
async function droppedRowsFile(found, outputPath) {
|
|
202
|
+
var result = await runAction(builtins.exportTable, {
|
|
203
|
+
format: 'xlsx',
|
|
204
|
+
document: { columns: found.columns.map(function(c) { return { header: c }; }) },
|
|
205
|
+
rows: found.rows.map(function(r) {
|
|
206
|
+
return { values: found.columns.map(function(c) { return r.values[c] == null ? null : r.values[c]; }) };
|
|
207
|
+
}),
|
|
208
|
+
outputPath: outputPath
|
|
209
|
+
});
|
|
210
|
+
if (result.status !== 'success') throw new Error((result.error && result.error.message) || 'The file could not be written');
|
|
211
|
+
return result.output.filePath;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** How an import went in — 'new' | 'append' | 'replace', or null for one from before modes. */
|
|
215
|
+
function modeOf(job) {
|
|
216
|
+
var meta = job && job.meta;
|
|
217
|
+
if (typeof meta === 'string') { try { meta = JSON.parse(meta); } catch (e) { meta = null; } }
|
|
218
|
+
return (meta && meta.mode) || null;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function plural(n, word) {
|
|
222
|
+
return n + ' ' + word + (n === 1 ? '' : 's');
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
module.exports = {
|
|
226
|
+
MODES: uploader.MODES,
|
|
227
|
+
DROPPED_FILE_ROWS: DROPPED_FILE_ROWS,
|
|
228
|
+
settings: settings,
|
|
229
|
+
uploadDir: uploadDir,
|
|
230
|
+
formatOf: formatOf,
|
|
231
|
+
uploadedFile: uploadedFile,
|
|
232
|
+
listSheets: listSheets,
|
|
233
|
+
inspectChecks: inspectChecks,
|
|
234
|
+
checkTarget: checkTarget,
|
|
235
|
+
loadPart: loadPart,
|
|
236
|
+
rollback: rollback,
|
|
237
|
+
droppedRows: droppedRows,
|
|
238
|
+
droppedRowsFile: droppedRowsFile,
|
|
239
|
+
modeOf: modeOf
|
|
240
|
+
};
|
package/lib/router.js
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
// THE IMPORT ROUTES, mounted whole by an app:
|
|
2
|
+
//
|
|
3
|
+
// router.use('/imports', imports.router({ auth, resolveTarget, listTargets, service }))
|
|
4
|
+
//
|
|
5
|
+
// POST /files the file (multipart "file") → { filePath, … }
|
|
6
|
+
// POST /inspect { filePath } → { format, sheets?, checks }
|
|
7
|
+
// POST /check-target { filePath, sheet?, connectionId, dbInfoId, targetTable } → { exists, matched, extra, missing }
|
|
8
|
+
// POST /commit { filePath, connectionId, dbInfoId, originalFileName?, mappings } → 202 { runId }
|
|
9
|
+
// GET /runs/:id { status, progress, result, error } — the importer's own runs only
|
|
10
|
+
// POST /runs/:id/stop
|
|
11
|
+
// GET /targets the databases a person may import into (the app's listTargets)
|
|
12
|
+
// GET /history past imports, newest first
|
|
13
|
+
// GET /history/:id/dropped the rows the database refused, with reasons
|
|
14
|
+
// GET /history/:id/dropped/file the same rows as .xlsx, to fix and import again
|
|
15
|
+
// POST /history/:id/rollback delete what the import loaded (not a Replace, not an upsert)
|
|
16
|
+
// POST /history/:id/delete remove the record only
|
|
17
|
+
//
|
|
18
|
+
// The app supplies only what is its own: who may call these (auth), and its
|
|
19
|
+
// connections (resolveTarget, listTargets). Tenancy comes from the app's
|
|
20
|
+
// mtMiddleware, as for every other route: import records carry mtId1/mtId2 and
|
|
21
|
+
// are read back through the tenant filter.
|
|
22
|
+
|
|
23
|
+
var fs = require('fs');
|
|
24
|
+
var path = require('path');
|
|
25
|
+
var crypto = require('crypto');
|
|
26
|
+
var express = require('express');
|
|
27
|
+
var { uploadRoute } = require('@xeplr/base-apis');
|
|
28
|
+
var { getMtContext } = require('@xeplr/db');
|
|
29
|
+
var { attachConfig, dataDir } = require('@xeplr/actions');
|
|
30
|
+
var imports = require('./imports');
|
|
31
|
+
var runs = require('./runs');
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {object} options
|
|
35
|
+
* @param {Function|Function[]} options.auth - middleware run before every route (the app's gate)
|
|
36
|
+
* @param {(connectionId, dbInfoId, req) => Promise<{ dbType, dbConnection, label? } | null>} options.resolveTarget
|
|
37
|
+
* the database a part loads into, as a login the drivers take; null when it is not one this caller may use
|
|
38
|
+
* @param {(req) => Promise<[{ connectionId, label, databases: [{ id, label }] }]>} options.listTargets
|
|
39
|
+
* the databases this caller may import into, for the picker and for history's labels
|
|
40
|
+
* @param {string} [options.service] - names this app's import records (import_meta.service). Required without `config`.
|
|
41
|
+
* @param {object} [options.config] - an @xeplr/actions attachConfig() instance the app already has; else one is made
|
|
42
|
+
* @param {(userIds: string[]) => Promise<{ [id]: string }>} [options.userNames] - names for records made before
|
|
43
|
+
* the importer's name was kept on them
|
|
44
|
+
*/
|
|
45
|
+
function createRouter(options) {
|
|
46
|
+
options = options || {};
|
|
47
|
+
if (!options.resolveTarget) throw new Error('@xeplr/imports: router() needs resolveTarget(connectionId, dbInfoId, req)');
|
|
48
|
+
if (!options.listTargets) throw new Error('@xeplr/imports: router() needs listTargets(req)');
|
|
49
|
+
var config = options.config || (options.service ? attachConfig({ service: options.service }) : null);
|
|
50
|
+
if (!config) throw new Error('@xeplr/imports: router() needs `service` (names this app\'s import records) or `config`');
|
|
51
|
+
var s = imports.settings();
|
|
52
|
+
var router = express.Router();
|
|
53
|
+
var auth = options.auth ? (Array.isArray(options.auth) ? options.auth : [options.auth]) : [];
|
|
54
|
+
var inFlight = new Set();
|
|
55
|
+
|
|
56
|
+
if (auth.length) router.use(auth);
|
|
57
|
+
router.use(function(req, res, next) { config.ready().then(function() { next(); }, next); });
|
|
58
|
+
router.use(express.json({ limit: '1mb' }));
|
|
59
|
+
|
|
60
|
+
// The file itself — @xeplr/base-apis' generic receive-and-store.
|
|
61
|
+
// Into the requesting tenant's own uploads/ (imports.uploadDir), asked per
|
|
62
|
+
// upload: the folder depends on who is asking.
|
|
63
|
+
router.use('/files', uploadRoute({ destination: function() { return imports.uploadDir(); }, maxSize: s.maxUploadMb * 1024 * 1024 }));
|
|
64
|
+
|
|
65
|
+
/** The uploaded file a body names, checked: in this tenant's uploads, there, and a format we read. */
|
|
66
|
+
function fileOf(body) {
|
|
67
|
+
var given = body && body.filePath;
|
|
68
|
+
if (!given) return { status: 400, error: 'filePath is required' };
|
|
69
|
+
var filePath = imports.uploadedFile(given);
|
|
70
|
+
if (!filePath) return { status: 400, error: 'Invalid filePath' };
|
|
71
|
+
if (!fs.existsSync(filePath)) return { status: 404, error: 'Upload not found — it may have expired. Please upload the file again.' };
|
|
72
|
+
var format = imports.formatOf(filePath);
|
|
73
|
+
if (!format) return { status: 400, error: 'Unsupported file type — use CSV, JSON, or Excel (.xlsx/.xls)' };
|
|
74
|
+
return { filePath: filePath, format: format };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function targetOf(connectionId, dbInfoId, req) {
|
|
78
|
+
if (!connectionId || !dbInfoId) return null;
|
|
79
|
+
return options.resolveTarget(connectionId, dbInfoId, req);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
router.post('/inspect', handle(async function(req, res) {
|
|
83
|
+
var file = fileOf(req.body);
|
|
84
|
+
if (file.error) {
|
|
85
|
+
if (file.status === 400 && req.body && req.body.filePath && /Unsupported/.test(file.error)) {
|
|
86
|
+
await fs.promises.unlink(req.body.filePath).catch(function() {});
|
|
87
|
+
}
|
|
88
|
+
return res.status(file.status).json({ error: file.error });
|
|
89
|
+
}
|
|
90
|
+
var sheets = file.format === 'excel' ? await imports.listSheets(file.filePath) : undefined;
|
|
91
|
+
res.json({ format: file.format, sheets: sheets, checks: await imports.inspectChecks(file.filePath, file.format, sheets) });
|
|
92
|
+
}));
|
|
93
|
+
|
|
94
|
+
router.post('/check-target', handle(async function(req, res) {
|
|
95
|
+
var b = req.body || {};
|
|
96
|
+
var file = fileOf(b);
|
|
97
|
+
if (file.error) return res.status(file.status).json({ error: file.error });
|
|
98
|
+
if (!b.targetTable) return res.status(400).json({ error: 'targetTable is required' });
|
|
99
|
+
var target = await targetOf(b.connectionId, b.dbInfoId, req);
|
|
100
|
+
if (!target) return res.status(404).json({ error: 'Database not found' });
|
|
101
|
+
res.json(await imports.checkTarget({ filePath: file.filePath, format: file.format, sheet: b.sheet || null,
|
|
102
|
+
dbType: target.dbType, dbConnection: target.dbConnection, targetTable: String(b.targetTable).trim() }));
|
|
103
|
+
}));
|
|
104
|
+
|
|
105
|
+
router.post('/commit', handle(async function(req, res) {
|
|
106
|
+
var b = req.body || {};
|
|
107
|
+
var mappings = b.mappings;
|
|
108
|
+
if (!Array.isArray(mappings) || !mappings.length) return res.status(400).json({ error: 'At least one mapping is required' });
|
|
109
|
+
var file = fileOf(b);
|
|
110
|
+
if (file.error) return res.status(file.status).json({ error: file.error });
|
|
111
|
+
var badMode = mappings.filter(function(m) { return m.mode != null && imports.MODES.indexOf(m.mode) === -1; })[0];
|
|
112
|
+
if (badMode) return res.status(400).json({ error: 'mode must be one of ' + imports.MODES.join(', ') });
|
|
113
|
+
if (mappings.some(function(m) { return !m.targetTable || !String(m.targetTable).trim(); })) {
|
|
114
|
+
return res.status(400).json({ error: 'Every mapping needs a targetTable' });
|
|
115
|
+
}
|
|
116
|
+
var target = await targetOf(b.connectionId, b.dbInfoId, req);
|
|
117
|
+
if (!target) return res.status(404).json({ error: 'Database not found' });
|
|
118
|
+
// ONE IMPORT PER FILE AT A TIME: a second press, or a retry after the
|
|
119
|
+
// browser gave up waiting, would load the same rows twice.
|
|
120
|
+
if (inFlight.has(file.filePath)) {
|
|
121
|
+
return res.status(409).json({ error: 'This file is already being imported. Wait for it to finish.' });
|
|
122
|
+
}
|
|
123
|
+
inFlight.add(file.filePath);
|
|
124
|
+
|
|
125
|
+
var mt = getMtContext() || {};
|
|
126
|
+
var user = req.user || {};
|
|
127
|
+
var count = mappings.length;
|
|
128
|
+
var runId = runs.startRun(async function(onProgress, signal) {
|
|
129
|
+
try {
|
|
130
|
+
var results = [];
|
|
131
|
+
for (var i = 0; i < mappings.length; i++) {
|
|
132
|
+
if (signal.aborted) break;
|
|
133
|
+
var m = mappings[i];
|
|
134
|
+
var movementId = crypto.randomBytes(12).toString('hex');
|
|
135
|
+
var outcome = await imports.loadPart({
|
|
136
|
+
filePath: file.filePath, format: file.format, sheet: m.sheet || null,
|
|
137
|
+
dbType: target.dbType, dbConnection: target.dbConnection,
|
|
138
|
+
targetTable: String(m.targetTable).trim(), primaryKeys: m.primaryKeys, mode: m.mode,
|
|
139
|
+
repairPastedRows: Boolean(m.repairPastedRows),
|
|
140
|
+
movementId: movementId, signal: signal,
|
|
141
|
+
onProgress: progressFor(onProgress, i, count, m.sheet),
|
|
142
|
+
mtId1: mt.mtId1, mtId2: mt.mtId2,
|
|
143
|
+
details: {
|
|
144
|
+
connectionId: b.connectionId, dbInfoId: b.dbInfoId,
|
|
145
|
+
// The person's own name for the file — never the stored one, a
|
|
146
|
+
// random id that means nothing to anybody.
|
|
147
|
+
originalFileName: b.originalFileName || null,
|
|
148
|
+
part: m.sheet || null,
|
|
149
|
+
recordCreatedBy: user.id || null,
|
|
150
|
+
// Kept on the record, so history can say who without asking
|
|
151
|
+
// the app's user store.
|
|
152
|
+
recordCreatedByName: user.name || user.email || null
|
|
153
|
+
}
|
|
154
|
+
}, config.metaStore);
|
|
155
|
+
results.push({
|
|
156
|
+
targetTable: String(m.targetTable).trim(),
|
|
157
|
+
sheet: m.sheet || null,
|
|
158
|
+
success: outcome.status === 'success',
|
|
159
|
+
rows: outcome.output ? outcome.output.totalRows : null,
|
|
160
|
+
dropped: outcome.output ? (outcome.output.dropped || 0) : null,
|
|
161
|
+
jobId: movementId,
|
|
162
|
+
recovered: outcome.output && outcome.output.repairedPastedRows ? outcome.output.repairedPastedRows.rows : null,
|
|
163
|
+
error: outcome.error ? outcome.error.message : null
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
return { results: results };
|
|
167
|
+
} finally {
|
|
168
|
+
inFlight.delete(file.filePath);
|
|
169
|
+
await fs.promises.unlink(file.filePath).catch(function() {});
|
|
170
|
+
}
|
|
171
|
+
}, { userId: user.id, timeLimitMs: s.timeLimitMs });
|
|
172
|
+
res.status(202).json({ runId: runId });
|
|
173
|
+
}));
|
|
174
|
+
|
|
175
|
+
router.get('/runs/:id', function(req, res) {
|
|
176
|
+
var run = runs.view(req.params.id, req.user && req.user.id);
|
|
177
|
+
if (!run) return res.status(404).json({ error: 'No such import run' });
|
|
178
|
+
res.json(run);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
router.post('/runs/:id/stop', function(req, res) {
|
|
182
|
+
var run = runs.view(req.params.id, req.user && req.user.id);
|
|
183
|
+
if (!run) return res.status(404).json({ error: 'No such import run' });
|
|
184
|
+
res.json({ stopped: runs.stop(req.params.id, 'Stopped') });
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
router.get('/targets', handle(async function(req, res) {
|
|
188
|
+
res.json({ dataArray: (await options.listTargets(req)) || [] });
|
|
189
|
+
}));
|
|
190
|
+
|
|
191
|
+
/** One import record of this app, through the tenant filter — another company's is simply not found. */
|
|
192
|
+
async function jobOf(id) {
|
|
193
|
+
return config.ImportMeta.query().where('service', config.SERVICE_NAME).findById(id);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
router.get('/history', handle(async function(req, res) {
|
|
197
|
+
var jobs = await config.ImportMeta.query().where('service', config.SERVICE_NAME).orderBy('started_at', 'desc').limit(200);
|
|
198
|
+
var targets = (await options.listTargets(req)) || [];
|
|
199
|
+
var labels = {};
|
|
200
|
+
targets.forEach(function(t) {
|
|
201
|
+
(t.databases || []).forEach(function(d) { labels[t.connectionId + '\u0000' + d.id] = [t.label, d.label].filter(Boolean).join(' / '); });
|
|
202
|
+
});
|
|
203
|
+
var unnamed = jobs.map(function(j) { var d = j.details || {}; return !d.recordCreatedByName && d.recordCreatedBy; }).filter(Boolean);
|
|
204
|
+
var names = unnamed.length && options.userNames ? (await options.userNames(Array.from(new Set(unnamed)))) || {} : {};
|
|
205
|
+
res.json({ dataArray: jobs.map(function(job) {
|
|
206
|
+
var d = job.details || {};
|
|
207
|
+
return {
|
|
208
|
+
id: job.id,
|
|
209
|
+
originalFileName: d.originalFileName,
|
|
210
|
+
part: d.part,
|
|
211
|
+
targetTable: job.target_table,
|
|
212
|
+
destination: labels[d.connectionId + '\u0000' + d.dbInfoId] || null,
|
|
213
|
+
status: job.status,
|
|
214
|
+
mode: imports.modeOf(job),
|
|
215
|
+
totalRows: job.total_rows == null ? null : Number(job.total_rows),
|
|
216
|
+
rowsLoaded: job.completed == null ? null : Number(job.completed),
|
|
217
|
+
rowsDropped: job.dropped == null ? null : Number(job.dropped),
|
|
218
|
+
errorMessage: job.error,
|
|
219
|
+
createdByName: d.recordCreatedByName || names[d.recordCreatedBy] || null,
|
|
220
|
+
startedAt: job.started_at
|
|
221
|
+
};
|
|
222
|
+
}) });
|
|
223
|
+
}));
|
|
224
|
+
|
|
225
|
+
/** A record and the database it loaded into, or { status, error }. */
|
|
226
|
+
async function jobAndTarget(req) {
|
|
227
|
+
var job = await jobOf(req.params.id);
|
|
228
|
+
if (!job) return { status: 404, error: 'Import not found' };
|
|
229
|
+
var d = job.details || {};
|
|
230
|
+
var target = await targetOf(d.connectionId, d.dbInfoId, req);
|
|
231
|
+
if (!target) return { status: 404, error: 'The database this import loaded into is no longer available' };
|
|
232
|
+
return { job: job, target: target };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async function droppedOf(req, limit) {
|
|
236
|
+
var got = await jobAndTarget(req);
|
|
237
|
+
if (got.error) return got;
|
|
238
|
+
got.found = await imports.droppedRows({
|
|
239
|
+
dbType: got.target.dbType, dbConnection: got.target.dbConnection,
|
|
240
|
+
targetTable: got.job.target_table, movementId: got.job.id, columns: got.job.columns, limit: limit
|
|
241
|
+
});
|
|
242
|
+
return got;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
router.get('/history/:id/dropped', handle(async function(req, res) {
|
|
246
|
+
var got = await droppedOf(req);
|
|
247
|
+
if (got.error) return res.status(got.status).json({ error: got.error });
|
|
248
|
+
res.json({ columns: got.found.columns, rows: got.found.rows, total: got.found.total, dropped: Number(got.job.dropped || 0) });
|
|
249
|
+
}));
|
|
250
|
+
|
|
251
|
+
router.get('/history/:id/dropped/file', handle(async function(req, res) {
|
|
252
|
+
var got = await droppedOf(req, imports.DROPPED_FILE_ROWS);
|
|
253
|
+
if (got.error) return res.status(got.status).json({ error: got.error });
|
|
254
|
+
if (!got.found.rows.length) return res.status(404).json({ error: 'No dropped rows were kept for this import' });
|
|
255
|
+
var d = got.job.details || {};
|
|
256
|
+
var base = String(d.originalFileName || got.job.target_table).replace(/\.[^.]+$/, '');
|
|
257
|
+
var name = (base + ' - dropped rows').replace(/[\\/:*?"<>|\u0000-\u001F]+/g, ' ').trim().slice(0, 120) + '.xlsx';
|
|
258
|
+
// In the tenant's own temporary folder — these are customer rows too —
|
|
259
|
+
// and gone once sent. No leading dot in the name: Express will not send a
|
|
260
|
+
// file under one.
|
|
261
|
+
var dir = fs.mkdtempSync(path.join(dataDir.scratch(), 'dropped-'));
|
|
262
|
+
try {
|
|
263
|
+
var filePath = await imports.droppedRowsFile(got.found, path.join(dir, name));
|
|
264
|
+
// With a callback Express leaves a failed send to us — answer it, or
|
|
265
|
+
// the request hangs.
|
|
266
|
+
res.download(filePath, name, function(err) {
|
|
267
|
+
fs.rm(dir, { recursive: true, force: true }, function() {});
|
|
268
|
+
if (err && !res.headersSent) res.status(500).json({ error: 'The file could not be sent: ' + err.message });
|
|
269
|
+
});
|
|
270
|
+
} catch (err) {
|
|
271
|
+
fs.rm(dir, { recursive: true, force: true }, function() {});
|
|
272
|
+
throw err;
|
|
273
|
+
}
|
|
274
|
+
}));
|
|
275
|
+
|
|
276
|
+
router.post('/history/:id/rollback', handle(async function(req, res) {
|
|
277
|
+
var job = await jobOf(req.params.id);
|
|
278
|
+
if (!job) return res.status(404).json({ error: 'Import not found' });
|
|
279
|
+
if (job.status !== 'completed') return res.status(400).json({ error: 'Only a completed import can be rolled back' });
|
|
280
|
+
if (imports.modeOf(job) === 'replace') {
|
|
281
|
+
return res.status(400).json({ error: 'This import replaced the table\'s rows, and the rows it replaced are gone — ' +
|
|
282
|
+
'deleting the new ones would leave the table empty, not as it was. Import the earlier data again to restore it.' });
|
|
283
|
+
}
|
|
284
|
+
if (job.primary_keys && job.primary_keys.length) {
|
|
285
|
+
return res.status(400).json({ error: 'This import used upsert (matched on ' + job.primary_keys.join(', ') + ') — rollback is ' +
|
|
286
|
+
'disabled because it may have only updated rows that already existed, not created them.' });
|
|
287
|
+
}
|
|
288
|
+
var got = await jobAndTarget(req);
|
|
289
|
+
if (got.error) return res.status(got.status).json({ error: got.error });
|
|
290
|
+
var result = await imports.rollback({ movementId: job.id, dbType: got.target.dbType, dbConnection: got.target.dbConnection,
|
|
291
|
+
targetTable: job.target_table }, config.metaStore);
|
|
292
|
+
res.json({ success: true, mainDeleted: result.mainDeleted, errorDeleted: result.errorDeleted });
|
|
293
|
+
}));
|
|
294
|
+
|
|
295
|
+
router.post('/history/:id/delete', handle(async function(req, res) {
|
|
296
|
+
var job = await jobOf(req.params.id);
|
|
297
|
+
if (!job) return res.status(404).json({ error: 'Import not found' });
|
|
298
|
+
await config.ImportMeta.query().findById(job.id).patch({ isActive: false });
|
|
299
|
+
res.json({ success: true });
|
|
300
|
+
}));
|
|
301
|
+
|
|
302
|
+
return router;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Which part of how many, with the uploader's own row counts. */
|
|
306
|
+
function progressFor(onProgress, index, count, sheet) {
|
|
307
|
+
return function(p) {
|
|
308
|
+
onProgress(Object.assign({}, p, { mapping: index + 1, mappings: count, sheet: sheet || null }));
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** An async route whose failure answers 500 (or the error's own status) with its message. */
|
|
313
|
+
function handle(fn) {
|
|
314
|
+
return function(req, res) {
|
|
315
|
+
Promise.resolve(fn(req, res)).catch(function(err) {
|
|
316
|
+
if (!res.headersSent) res.status(err.status || 500).json({ error: err.message });
|
|
317
|
+
});
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
module.exports = createRouter;
|
package/lib/runs.js
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// IMPORTS RUN IN THE BACKGROUND. A 300 MB file takes minutes to load, far
|
|
2
|
+
// past any gateway's timeout — so the commit answers at once with a run id,
|
|
3
|
+
// and the page asks how far it has got (GET /runs/:id) until it ends.
|
|
4
|
+
//
|
|
5
|
+
// Held in this process's memory: a run belongs to the process doing the work,
|
|
6
|
+
// and dies with it. The import's durable record is import_meta (written by
|
|
7
|
+
// @xeplr/actions' uploader before the first row), so a run lost to a restart
|
|
8
|
+
// is still in the history, with its rows to roll back.
|
|
9
|
+
|
|
10
|
+
var crypto = require('crypto');
|
|
11
|
+
|
|
12
|
+
// A finished run is kept this long for its page to read the ending, then gone.
|
|
13
|
+
var KEEP_FINISHED_MS = 60 * 60 * 1000;
|
|
14
|
+
|
|
15
|
+
var runs = new Map();
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Start `work(onProgress, signal)` in the background.
|
|
19
|
+
*
|
|
20
|
+
* @param {object} opts - { userId, timeLimitMs? }
|
|
21
|
+
* @returns {string} the run id
|
|
22
|
+
*/
|
|
23
|
+
function startRun(work, opts) {
|
|
24
|
+
opts = opts || {};
|
|
25
|
+
var run = {
|
|
26
|
+
id: 'imp_' + crypto.randomBytes(10).toString('hex'),
|
|
27
|
+
userId: opts.userId == null ? null : String(opts.userId),
|
|
28
|
+
status: 'running',
|
|
29
|
+
progress: null,
|
|
30
|
+
result: null,
|
|
31
|
+
error: null,
|
|
32
|
+
startedAt: Date.now(),
|
|
33
|
+
finishedAt: null,
|
|
34
|
+
controller: new AbortController(),
|
|
35
|
+
timer: null
|
|
36
|
+
};
|
|
37
|
+
runs.set(run.id, run);
|
|
38
|
+
|
|
39
|
+
if (opts.timeLimitMs) {
|
|
40
|
+
run.timer = setTimeout(function() {
|
|
41
|
+
stop(run.id, 'The import took longer than ' + Math.round(opts.timeLimitMs / 60000) + ' minutes and was stopped.');
|
|
42
|
+
}, opts.timeLimitMs);
|
|
43
|
+
if (run.timer.unref) run.timer.unref();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
Promise.resolve().then(function() {
|
|
47
|
+
return work(function(progress) { run.progress = progress; }, run.controller.signal);
|
|
48
|
+
}).then(function(result) {
|
|
49
|
+
settle(run, run.controller.signal.aborted ? 'stopped' : 'complete', result, null);
|
|
50
|
+
}, function(err) {
|
|
51
|
+
settle(run, run.controller.signal.aborted ? 'stopped' : 'error', null, (err && err.message) || String(err));
|
|
52
|
+
});
|
|
53
|
+
return run.id;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function settle(run, status, result, error) {
|
|
57
|
+
if (run.timer) clearTimeout(run.timer);
|
|
58
|
+
run.status = status;
|
|
59
|
+
run.result = result;
|
|
60
|
+
run.error = status === 'stopped' ? (run.stopReason || 'Stopped') : error;
|
|
61
|
+
run.finishedAt = Date.now();
|
|
62
|
+
var forget = setTimeout(function() { runs.delete(run.id); }, KEEP_FINISHED_MS);
|
|
63
|
+
if (forget.unref) forget.unref();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Ask a running import to stop: it stops reading within a batch. */
|
|
67
|
+
function stop(id, reason) {
|
|
68
|
+
var run = runs.get(id);
|
|
69
|
+
if (!run || run.status !== 'running') return false;
|
|
70
|
+
run.stopReason = reason || 'Stopped';
|
|
71
|
+
run.controller.abort();
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A run as its page sees it — or null when there is none for this user. A
|
|
77
|
+
* run is only ever shown to the person who started it.
|
|
78
|
+
*/
|
|
79
|
+
function view(id, userId) {
|
|
80
|
+
var run = runs.get(id);
|
|
81
|
+
if (!run || run.userId !== (userId == null ? null : String(userId))) return null;
|
|
82
|
+
return {
|
|
83
|
+
id: run.id,
|
|
84
|
+
status: run.status,
|
|
85
|
+
progress: run.progress,
|
|
86
|
+
result: run.result,
|
|
87
|
+
error: run.error,
|
|
88
|
+
startedAt: run.startedAt,
|
|
89
|
+
finishedAt: run.finishedAt
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
module.exports = { startRun: startRun, stop: stop, view: view };
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
-- 0001_imports_access.sql
|
|
2
|
+
-- @xeplr/imports' own access catalog: every route lib/router.js serves, by
|
|
3
|
+
-- group, and the roles that reach them. The package never runs this itself —
|
|
4
|
+
-- a consuming app lists this directory in XEPLR_AUTH_MIGRATIONS
|
|
5
|
+
-- (imports.authMigrationsDir), so its auth database knows these routes exist.
|
|
6
|
+
--
|
|
7
|
+
-- The names assume the app mounts the router at /imports, as the README says.
|
|
8
|
+
--
|
|
9
|
+
-- Groups:
|
|
10
|
+
-- imports:view see past imports, their dropped rows, a run's progress, the databases
|
|
11
|
+
-- imports:create upload, inspect, check against a table, import, stop one's own import
|
|
12
|
+
-- imports:edit roll an import back, delete its record
|
|
13
|
+
--
|
|
14
|
+
-- Roles are the shared vocabulary every app's auth database uses (as in
|
|
15
|
+
-- @xeplr/jobs' 0001): Super Admin gets everything; CompanyAdmin gets
|
|
16
|
+
-- imports:* (its rows stay within its own company through mtId1). Loading
|
|
17
|
+
-- data into a company's databases is an administrator's job, so Creator and
|
|
18
|
+
-- Viewer get none — an app that wants them to can map them in its own
|
|
19
|
+
-- migration. Idempotent: insert-if-absent on name + group, and on each mapping.
|
|
20
|
+
|
|
21
|
+
INSERT INTO "roles" (id, name, "isActive", "mtId1", "recordCreatedDate", "recordModifiedDate")
|
|
22
|
+
SELECT encode(gen_random_bytes(12), 'hex'), name, true, '*', now(), now()
|
|
23
|
+
FROM (VALUES ('Super Admin'), ('CompanyAdmin')) AS v(name)
|
|
24
|
+
WHERE NOT EXISTS (SELECT 1 FROM "roles" r WHERE r.name = v.name);
|
|
25
|
+
|
|
26
|
+
INSERT INTO "apis" (id, name, "apiGroup", "isPublic", "isActive", "mtId1", "recordCreatedDate", "recordModifiedDate")
|
|
27
|
+
SELECT encode(gen_random_bytes(12), 'hex'), v.name, v.grp, false, true, '*', now(), now()
|
|
28
|
+
FROM (VALUES
|
|
29
|
+
('POST /imports/files', 'imports:create'),
|
|
30
|
+
('POST /imports/inspect', 'imports:create'),
|
|
31
|
+
('POST /imports/check-target', 'imports:create'),
|
|
32
|
+
('POST /imports/commit', 'imports:create'),
|
|
33
|
+
('POST /imports/runs/:id/stop', 'imports:create'),
|
|
34
|
+
('GET /imports/runs/:id', 'imports:view'),
|
|
35
|
+
('GET /imports/targets', 'imports:view'),
|
|
36
|
+
('GET /imports/history', 'imports:view'),
|
|
37
|
+
('GET /imports/history/:id/dropped', 'imports:view'),
|
|
38
|
+
('GET /imports/history/:id/dropped/file', 'imports:view'),
|
|
39
|
+
('POST /imports/history/:id/rollback', 'imports:edit'),
|
|
40
|
+
('POST /imports/history/:id/delete', 'imports:edit')
|
|
41
|
+
) AS v(name, grp)
|
|
42
|
+
WHERE NOT EXISTS (SELECT 1 FROM "apis" a WHERE a.name = v.name AND a."apiGroup" = v.grp);
|
|
43
|
+
|
|
44
|
+
INSERT INTO "apisRolesMapping" (id, "roleId", "apiId", "isActive", "mtId1", "recordCreatedDate", "recordModifiedDate")
|
|
45
|
+
SELECT encode(gen_random_bytes(12), 'hex'), r.id, a.id, true, '*', now(), now()
|
|
46
|
+
FROM "roles" r CROSS JOIN "apis" a
|
|
47
|
+
WHERE r.name IN ('Super Admin', 'CompanyAdmin')
|
|
48
|
+
AND a."apiGroup" LIKE 'imports:%'
|
|
49
|
+
AND NOT EXISTS (SELECT 1 FROM "apisRolesMapping" m WHERE m."roleId" = r.id AND m."apiId" = a.id);
|
package/package.json
CHANGED
|
@@ -1,6 +1,54 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xeplr/imports",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Data imports as a drop-in module: upload a CSV/Excel/JSON file, check it against the table, load it as a New table, Append or Replace, see and download the rows the database refused, and roll back — routes, access catalog and env in one package",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"files": [
|
|
7
|
+
"index.js",
|
|
8
|
+
"lib/",
|
|
9
|
+
"migrations-auth/"
|
|
10
|
+
],
|
|
11
|
+
"exports": {
|
|
12
|
+
".": "./index.js",
|
|
13
|
+
"./package.json": "./package.json"
|
|
14
|
+
},
|
|
15
|
+
"keywords": [
|
|
16
|
+
"import",
|
|
17
|
+
"upload",
|
|
18
|
+
"csv",
|
|
19
|
+
"excel",
|
|
20
|
+
"express"
|
|
21
|
+
],
|
|
22
|
+
"author": "xeplr",
|
|
23
|
+
"license": "MIT",
|
|
24
|
+
"repository": {
|
|
25
|
+
"type": "git",
|
|
26
|
+
"url": "https://github.com/Xeplr/imports"
|
|
27
|
+
},
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public"
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"@xeplr/actions": "^2.0.0",
|
|
33
|
+
"@xeplr/base-apis": "^2.0.3",
|
|
34
|
+
"@xeplr/db": "^1.0.8",
|
|
35
|
+
"@xeplr/utils": "^1.1.0",
|
|
36
|
+
"csv-parse": "^7.0.3",
|
|
37
|
+
"fflate": "^0.8.3",
|
|
38
|
+
"multer": "^2.4.0",
|
|
39
|
+
"saxes": "^6.0.0",
|
|
40
|
+
"unzipper": "^0.12.5"
|
|
41
|
+
},
|
|
42
|
+
"peerDependencies": {
|
|
43
|
+
"express": "^4.0.0 || ^5.0.0"
|
|
44
|
+
},
|
|
45
|
+
"scripts": {
|
|
46
|
+
"test": "node --test test/*.test.js"
|
|
47
|
+
},
|
|
48
|
+
"devDependencies": {
|
|
49
|
+
"express": "^5.3.0",
|
|
50
|
+
"knex": "^3.3.0",
|
|
51
|
+
"mysql2": "^3.24.5",
|
|
52
|
+
"pg": "^8.23.1"
|
|
53
|
+
}
|
|
54
|
+
}
|