@xeplr/imports 1.0.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/index.js +7 -5
- package/lib/imports.js +24 -16
- package/lib/router.js +14 -10
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -42,7 +42,7 @@ module.exports = [].concat(imports.requiredEnv /* , … */);
|
|
|
42
42
|
|
|
43
43
|
```
|
|
44
44
|
# .env
|
|
45
|
-
|
|
45
|
+
XEPLR_DATA_DIR=/srv/xeplr-data # required, absolute: each tenant's files under it
|
|
46
46
|
XCFG_DB_NAME=xeplr_configs # required: where import records live
|
|
47
47
|
XEPLR_AUTH_MIGRATIONS=…,/…/node_modules/@xeplr/imports/migrations-auth
|
|
48
48
|
```
|
|
@@ -65,9 +65,11 @@ Database drivers are the app's own: install `pg`, `mysql2`, `mssql` or `@duckdb/
|
|
|
65
65
|
|
|
66
66
|
## Settings
|
|
67
67
|
|
|
68
|
+
**From 1.x (2.0.0):** `UPLOAD_DIR` is gone — set `XEPLR_DATA_DIR` (in `requiredEnv`), and call `@xeplr/db`'s `registerApplication()` at boot. Uploads now live per tenant.
|
|
69
|
+
|
|
68
70
|
| Variable | Required | Meaning |
|
|
69
71
|
|---|---|---|
|
|
70
|
-
| `
|
|
72
|
+
| `XEPLR_DATA_DIR` | **yes**, absolute (`@xeplr/actions`) | The data root. An upload is staged in the requesting tenant's own `uploads/` folder under it (`<root>/<app>/<company>/<workspace>/uploads`) until imported; a dropped-rows download is written in its `tmp/` for the moment it is sent. Asked per request, so one company's files never sit beside another's. Never defaulted: a missing, relative or absent one is refused by name. |
|
|
71
73
|
| `XCFG_DB_NAME` | **yes** | The `xeplr_configs` database holding `import_meta`. The login is `XCFG_DB_CONNECTION_INFO_ENCRYPTED`, or the shared `XEPLR_DB_CONNECTION` (`@xeplr/db`) |
|
|
72
74
|
| `UPLOAD_MAX_MB` | no | The largest file accepted, default 500. Files go to disk and are streamed, so this costs disk, not memory. A proxy in front (nginx `client_max_body_size`) must allow at least as much |
|
|
73
75
|
| `IMPORT_TIME_LIMIT_MINUTES` | no | How long an import may run before it is stopped, default 60 |
|
|
@@ -98,7 +100,7 @@ A commit result per part is `{ targetTable, sheet, success, rows, dropped, jobId
|
|
|
98
100
|
- **Modes.** A **New table** must not exist yet. **Append** and **Replace** need one that does, so a typo is never a fresh table. A **Replace** is staged: the rows load into a staging table, then one transaction empties the table and moves them in. A Replace that fails or is stopped leaves the table as it was. See [`@xeplr/actions` Import modes](https://www.npmjs.com/package/@xeplr/actions#import-modes).
|
|
99
101
|
- **Rollback is refused for a Replace**, because the rows it replaced are gone. It is also refused for an upsert (`primaryKeys`), which may only have updated rows that already existed.
|
|
100
102
|
- **Refused rows are kept.** Each row the database refuses goes to `<table>_import_errors` with its place in the file and the reason, so it can be seen, fixed and loaded again. An import gives up after five refusals in a row, and rows after that are counted but not kept.
|
|
101
|
-
- **A filePath is checked.** It comes from the client, so only a
|
|
103
|
+
- **A filePath is checked.** It comes from the client, so only a file in the requesting tenant's own `uploads/` is accepted — never a file elsewhere on the server, and never another company's or workspace's upload (`../` and links followed first). A file the import cannot read is removed at inspect, and an imported file is removed when its import ends.
|
|
102
104
|
|
|
103
105
|
## Access
|
|
104
106
|
|
package/index.js
CHANGED
|
@@ -7,14 +7,16 @@
|
|
|
7
7
|
// `imports.authMigrationsDir` in XEPLR_AUTH_MIGRATIONS.
|
|
8
8
|
|
|
9
9
|
var path = require('path');
|
|
10
|
+
var actions = require('@xeplr/actions');
|
|
10
11
|
var { requiredEnv: configRequiredEnv } = require('@xeplr/actions/lib/uploader/attach');
|
|
11
12
|
|
|
12
13
|
// What this package needs from the environment, for the app's checkEnv.
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
|
|
14
|
+
// XEPLR_DATA_DIR (@xeplr/actions): uploads are staged in each tenant's own
|
|
15
|
+
// uploads/ under it — absolute, never defaulted (a wrong-but-present location
|
|
16
|
+
// reads as "no file", not as an error). The import records live in
|
|
17
|
+
// xeplr_configs (XCFG_DB_NAME, and the login in XCFG_DB_CONNECTION_INFO_ENCRYPTED
|
|
18
|
+
// or the shared XEPLR_DB_CONNECTION).
|
|
19
|
+
var requiredEnv = actions.requiredEnv.concat(configRequiredEnv);
|
|
18
20
|
|
|
19
21
|
module.exports = {
|
|
20
22
|
router: require('./lib/router'),
|
package/lib/imports.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
var fs = require('fs');
|
|
7
7
|
var path = require('path');
|
|
8
|
-
var { runAction, builtins, formats, uploader, drivers } = require('@xeplr/actions');
|
|
8
|
+
var { runAction, builtins, formats, uploader, drivers, dataDir } = require('@xeplr/actions');
|
|
9
9
|
|
|
10
10
|
var EXT_TO_FORMAT = { '.csv': 'csv', '.json': 'json', '.xlsx': 'excel', '.xls': 'excel' };
|
|
11
11
|
|
|
@@ -15,18 +15,22 @@ var DROPPED_VIEW_ROWS = 1000;
|
|
|
15
15
|
var DROPPED_FILE_ROWS = 1048575;
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* WHERE AN UPLOAD IS STAGED: `uploads/` in the folder of the tenant the
|
|
19
|
+
* request is for (@xeplr/actions' dataDir, under XEPLR_DATA_DIR). Asked per
|
|
20
|
+
* request — never a folder fixed at startup, which would put every company's
|
|
21
|
+
* files side by side. Made if missing.
|
|
22
22
|
*/
|
|
23
|
-
function
|
|
24
|
-
var dir =
|
|
25
|
-
if (!dir) throw new Error('@xeplr/imports: UPLOAD_DIR is not set — the absolute directory uploaded files are staged in');
|
|
26
|
-
if (!path.isAbsolute(dir)) throw new Error('@xeplr/imports: UPLOAD_DIR must be an absolute path, got "' + dir + '"');
|
|
23
|
+
function uploadDir() {
|
|
24
|
+
var dir = dataDir.resolve(UPLOADS, 'The upload folder');
|
|
27
25
|
fs.mkdirSync(dir, { recursive: true });
|
|
26
|
+
return dir;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
var UPLOADS = 'uploads';
|
|
30
|
+
|
|
31
|
+
/** How big an upload may be and how long an import may run, from the environment. */
|
|
32
|
+
function settings() {
|
|
28
33
|
return {
|
|
29
|
-
uploadDir: path.resolve(dir),
|
|
30
34
|
// Client exports run to a few hundred MB (a 773k-row ERP CSV is 270 MB);
|
|
31
35
|
// the file goes to disk and is streamed, so the size costs disk, not memory.
|
|
32
36
|
maxUploadMb: Number(process.env.UPLOAD_MAX_MB) || 500,
|
|
@@ -40,12 +44,15 @@ function formatOf(filename) {
|
|
|
40
44
|
|
|
41
45
|
/**
|
|
42
46
|
* A filePath comes from the client, relaying what the upload answered — so
|
|
43
|
-
* only one inside
|
|
44
|
-
* on the server.
|
|
47
|
+
* only one inside THIS tenant's upload folder is accepted: never a file
|
|
48
|
+
* elsewhere on the server, and never another company's upload. The real
|
|
49
|
+
* path (../ and links followed), or null.
|
|
45
50
|
*/
|
|
46
|
-
function
|
|
47
|
-
var
|
|
48
|
-
|
|
51
|
+
function uploadedFile(filePath) {
|
|
52
|
+
var dir = uploadDir();
|
|
53
|
+
var real;
|
|
54
|
+
try { real = dataDir.resolve(String(filePath || ''), 'The uploaded file'); } catch (e) { return null; }
|
|
55
|
+
return real.indexOf(dir + path.sep) === 0 ? real : null;
|
|
49
56
|
}
|
|
50
57
|
|
|
51
58
|
function listSheets(filePath) {
|
|
@@ -219,8 +226,9 @@ module.exports = {
|
|
|
219
226
|
MODES: uploader.MODES,
|
|
220
227
|
DROPPED_FILE_ROWS: DROPPED_FILE_ROWS,
|
|
221
228
|
settings: settings,
|
|
229
|
+
uploadDir: uploadDir,
|
|
222
230
|
formatOf: formatOf,
|
|
223
|
-
|
|
231
|
+
uploadedFile: uploadedFile,
|
|
224
232
|
listSheets: listSheets,
|
|
225
233
|
inspectChecks: inspectChecks,
|
|
226
234
|
checkTarget: checkTarget,
|
package/lib/router.js
CHANGED
|
@@ -26,7 +26,7 @@ var crypto = require('crypto');
|
|
|
26
26
|
var express = require('express');
|
|
27
27
|
var { uploadRoute } = require('@xeplr/base-apis');
|
|
28
28
|
var { getMtContext } = require('@xeplr/db');
|
|
29
|
-
var { attachConfig } = require('@xeplr/actions');
|
|
29
|
+
var { attachConfig, dataDir } = require('@xeplr/actions');
|
|
30
30
|
var imports = require('./imports');
|
|
31
31
|
var runs = require('./runs');
|
|
32
32
|
|
|
@@ -58,17 +58,20 @@ function createRouter(options) {
|
|
|
58
58
|
router.use(express.json({ limit: '1mb' }));
|
|
59
59
|
|
|
60
60
|
// The file itself — @xeplr/base-apis' generic receive-and-store.
|
|
61
|
-
|
|
61
|
+
// Into the requesting tenant's own uploads/ (imports.uploadDir), asked per
|
|
62
|
+
// upload: the folder depends on who is asking.
|
|
63
|
+
router.use('/files', uploadRoute({ destination: function() { return imports.uploadDir(); }, maxSize: s.maxUploadMb * 1024 * 1024 }));
|
|
62
64
|
|
|
63
|
-
/** The uploaded file a body names, checked:
|
|
65
|
+
/** The uploaded file a body names, checked: in this tenant's uploads, there, and a format we read. */
|
|
64
66
|
function fileOf(body) {
|
|
65
|
-
var
|
|
66
|
-
if (!
|
|
67
|
-
|
|
67
|
+
var given = body && body.filePath;
|
|
68
|
+
if (!given) return { status: 400, error: 'filePath is required' };
|
|
69
|
+
var filePath = imports.uploadedFile(given);
|
|
70
|
+
if (!filePath) return { status: 400, error: 'Invalid filePath' };
|
|
68
71
|
if (!fs.existsSync(filePath)) return { status: 404, error: 'Upload not found — it may have expired. Please upload the file again.' };
|
|
69
72
|
var format = imports.formatOf(filePath);
|
|
70
73
|
if (!format) return { status: 400, error: 'Unsupported file type — use CSV, JSON, or Excel (.xlsx/.xls)' };
|
|
71
|
-
return { filePath:
|
|
74
|
+
return { filePath: filePath, format: format };
|
|
72
75
|
}
|
|
73
76
|
|
|
74
77
|
async function targetOf(connectionId, dbInfoId, req) {
|
|
@@ -252,9 +255,10 @@ function createRouter(options) {
|
|
|
252
255
|
var d = got.job.details || {};
|
|
253
256
|
var base = String(d.originalFileName || got.job.target_table).replace(/\.[^.]+$/, '');
|
|
254
257
|
var name = (base + ' - dropped rows').replace(/[\\/:*?"<>|\u0000-\u001F]+/g, ' ').trim().slice(0, 120) + '.xlsx';
|
|
255
|
-
//
|
|
256
|
-
// No leading dot in the name: Express will not send a
|
|
257
|
-
|
|
258
|
+
// In the tenant's own temporary folder — these are customer rows too —
|
|
259
|
+
// and gone once sent. No leading dot in the name: Express will not send a
|
|
260
|
+
// file under one.
|
|
261
|
+
var dir = fs.mkdtempSync(path.join(dataDir.scratch(), 'dropped-'));
|
|
258
262
|
try {
|
|
259
263
|
var filePath = await imports.droppedRowsFile(got.found, path.join(dir, name));
|
|
260
264
|
// With a callback Express leaves a failed send to us — answer it, or
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xeplr/imports",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "Data imports as a drop-in module: upload a CSV/Excel/JSON file, check it against the table, load it as a New table, Append or Replace, see and download the rows the database refused, and roll back — routes, access catalog and env in one package",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"files": [
|
|
@@ -29,10 +29,10 @@
|
|
|
29
29
|
"access": "public"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@xeplr/actions": "^
|
|
32
|
+
"@xeplr/actions": "^2.0.0",
|
|
33
33
|
"@xeplr/base-apis": "^2.0.3",
|
|
34
34
|
"@xeplr/db": "^1.0.8",
|
|
35
|
-
"@xeplr/utils": "^1.0
|
|
35
|
+
"@xeplr/utils": "^1.1.0",
|
|
36
36
|
"csv-parse": "^7.0.3",
|
|
37
37
|
"fflate": "^0.8.3",
|
|
38
38
|
"multer": "^2.4.0",
|