@xeplr/imports 1.0.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -42,7 +42,7 @@ module.exports = [].concat(imports.requiredEnv /* , … */);
42
42
 
43
43
  ```
44
44
  # .env
45
- UPLOAD_DIR=/data/my-app/uploads # required, absolute
45
+ XEPLR_DATA_DIR=/srv/xeplr-data # required, absolute: each tenant's files under it
46
46
  XCFG_DB_NAME=xeplr_configs # required: where import records live
47
47
  XEPLR_AUTH_MIGRATIONS=…,/…/node_modules/@xeplr/imports/migrations-auth
48
48
  ```
@@ -65,9 +65,11 @@ Database drivers are the app's own: install `pg`, `mysql2`, `mssql` or `@duckdb/
65
65
 
66
66
  ## Settings
67
67
 
68
+ **From 1.x (2.0.0):** `UPLOAD_DIR` is gone — set `XEPLR_DATA_DIR` (in `requiredEnv`), and call `@xeplr/db`'s `registerApplication()` at boot. Uploads now live per tenant.
69
+
68
70
  | Variable | Required | Meaning |
69
71
  |---|---|---|
70
- | `UPLOAD_DIR` | **yes**, absolute | Where uploaded files are staged until imported, and where a dropped-rows download is written for the moment it is sent. Never defaulted: a missing or relative one would make the same install read different files depending on where it was started. `router()` refuses to start without it. |
72
+ | `XEPLR_DATA_DIR` | **yes**, absolute (`@xeplr/actions`) | The data root. An upload is staged in the requesting tenant's own `uploads/` folder under it (`<root>/<app>/<company>/<workspace>/uploads`) until imported; a dropped-rows download is written in its `tmp/` for the moment it is sent. Asked per request, so one company's files never sit beside another's. Never defaulted: a missing, relative or absent one is refused by name. |
71
73
  | `XCFG_DB_NAME` | **yes** | The `xeplr_configs` database holding `import_meta`. The login is `XCFG_DB_CONNECTION_INFO_ENCRYPTED`, or the shared `XEPLR_DB_CONNECTION` (`@xeplr/db`) |
72
74
  | `UPLOAD_MAX_MB` | no | The largest file accepted, default 500. Files go to disk and are streamed, so this costs disk, not memory. A proxy in front (nginx `client_max_body_size`) must allow at least as much |
73
75
  | `IMPORT_TIME_LIMIT_MINUTES` | no | How long an import may run before it is stopped, default 60 |
@@ -98,7 +100,7 @@ A commit result per part is `{ targetTable, sheet, success, rows, dropped, jobId
98
100
  - **Modes.** A **New table** must not exist yet. **Append** and **Replace** need one that does, so a typo is never a fresh table. A **Replace** is staged: the rows load into a staging table, then one transaction empties the table and moves them in. A Replace that fails or is stopped leaves the table as it was. See [`@xeplr/actions` Import modes](https://www.npmjs.com/package/@xeplr/actions#import-modes).
99
101
  - **Rollback is refused for a Replace**, because the rows it replaced are gone. It is also refused for an upsert (`primaryKeys`), which may only have updated rows that already existed.
100
102
  - **Refused rows are kept.** Each row the database refuses goes to `<table>_import_errors` with its place in the file and the reason, so it can be seen, fixed and loaded again. An import gives up after five refusals in a row, and rows after that are counted but not kept.
101
- - **A filePath is checked.** It comes from the client, so only a path inside `UPLOAD_DIR` is accepted. A file the import cannot read is removed at inspect, and an imported file is removed when its import ends.
103
+ - **A filePath is checked.** It comes from the client, so only a file in the requesting tenant's own `uploads/` is accepted — never a file elsewhere on the server, and never another company's or workspace's upload (`../` and links followed first). A file the import cannot read is removed at inspect, and an imported file is removed when its import ends.
102
104
 
103
105
  ## Access
104
106
 
package/index.js CHANGED
@@ -7,14 +7,16 @@
7
7
  // `imports.authMigrationsDir` in XEPLR_AUTH_MIGRATIONS.
8
8
 
9
9
  var path = require('path');
10
+ var actions = require('@xeplr/actions');
10
11
  var { requiredEnv: configRequiredEnv } = require('@xeplr/actions/lib/uploader/attach');
11
12
 
12
13
  // What this package needs from the environment, for the app's checkEnv.
13
- // UPLOAD_DIR: where uploaded files are staged — absolute, never defaulted (a
14
- // wrong-but-present location reads as "no file", not as an error). The
15
- // import records live in xeplr_configs (XCFG_DB_NAME, and the login in
16
- // XCFG_DB_CONNECTION_INFO_ENCRYPTED or the shared XEPLR_DB_CONNECTION).
17
- var requiredEnv = ['UPLOAD_DIR'].concat(configRequiredEnv);
14
+ // XEPLR_DATA_DIR (@xeplr/actions): uploads are staged in each tenant's own
15
+ // uploads/ under it — absolute, never defaulted (a wrong-but-present location
16
+ // reads as "no file", not as an error). The import records live in
17
+ // xeplr_configs (XCFG_DB_NAME, and the login in XCFG_DB_CONNECTION_INFO_ENCRYPTED
18
+ // or the shared XEPLR_DB_CONNECTION).
19
+ var requiredEnv = actions.requiredEnv.concat(configRequiredEnv);
18
20
 
19
21
  module.exports = {
20
22
  router: require('./lib/router'),
package/lib/imports.js CHANGED
@@ -5,7 +5,7 @@
5
5
 
6
6
  var fs = require('fs');
7
7
  var path = require('path');
8
- var { runAction, builtins, formats, uploader, drivers } = require('@xeplr/actions');
8
+ var { runAction, builtins, formats, uploader, drivers, dataDir } = require('@xeplr/actions');
9
9
 
10
10
  var EXT_TO_FORMAT = { '.csv': 'csv', '.json': 'json', '.xlsx': 'excel', '.xls': 'excel' };
11
11
 
@@ -15,18 +15,22 @@ var DROPPED_VIEW_ROWS = 1000;
15
15
  var DROPPED_FILE_ROWS = 1048575;
16
16
 
17
17
  /**
18
- * Where uploads are staged and how big one may be, from the environment.
19
- * UPLOAD_DIR is required and absolute: a missing one would default to
20
- * somewhere, and a relative one resolves against wherever the process was
21
- * started — the same install reading different files depending on that.
18
+ * WHERE AN UPLOAD IS STAGED: `uploads/` in the folder of the tenant the
19
+ * request is for (@xeplr/actions' dataDir, under XEPLR_DATA_DIR). Asked per
20
+ * request — never a folder fixed at startup, which would put every company's
21
+ * files side by side. Made if missing.
22
22
  */
23
- function settings() {
24
- var dir = process.env.UPLOAD_DIR;
25
- if (!dir) throw new Error('@xeplr/imports: UPLOAD_DIR is not set — the absolute directory uploaded files are staged in');
26
- if (!path.isAbsolute(dir)) throw new Error('@xeplr/imports: UPLOAD_DIR must be an absolute path, got "' + dir + '"');
23
+ function uploadDir() {
24
+ var dir = dataDir.resolve(UPLOADS, 'The upload folder');
27
25
  fs.mkdirSync(dir, { recursive: true });
26
+ return dir;
27
+ }
28
+
29
+ var UPLOADS = 'uploads';
30
+
31
+ /** How big an upload may be and how long an import may run, from the environment. */
32
+ function settings() {
28
33
  return {
29
- uploadDir: path.resolve(dir),
30
34
  // Client exports run to a few hundred MB (a 773k-row ERP CSV is 270 MB);
31
35
  // the file goes to disk and is streamed, so the size costs disk, not memory.
32
36
  maxUploadMb: Number(process.env.UPLOAD_MAX_MB) || 500,
@@ -40,12 +44,15 @@ function formatOf(filename) {
40
44
 
41
45
  /**
42
46
  * A filePath comes from the client, relaying what the upload answered — so
43
- * only one inside the upload directory is accepted, never an arbitrary file
44
- * on the server.
47
+ * only one inside THIS tenant's upload folder is accepted: never a file
48
+ * elsewhere on the server, and never another company's upload. The real
49
+ * path (../ and links followed), or null.
45
50
  */
46
- function isWithin(dir, filePath) {
47
- var resolved = path.resolve(String(filePath || ''));
48
- return resolved === dir || resolved.indexOf(dir + path.sep) === 0;
51
+ function uploadedFile(filePath) {
52
+ var dir = uploadDir();
53
+ var real;
54
+ try { real = dataDir.resolve(String(filePath || ''), 'The uploaded file'); } catch (e) { return null; }
55
+ return real.indexOf(dir + path.sep) === 0 ? real : null;
49
56
  }
50
57
 
51
58
  function listSheets(filePath) {
@@ -219,8 +226,9 @@ module.exports = {
219
226
  MODES: uploader.MODES,
220
227
  DROPPED_FILE_ROWS: DROPPED_FILE_ROWS,
221
228
  settings: settings,
229
+ uploadDir: uploadDir,
222
230
  formatOf: formatOf,
223
- isWithin: isWithin,
231
+ uploadedFile: uploadedFile,
224
232
  listSheets: listSheets,
225
233
  inspectChecks: inspectChecks,
226
234
  checkTarget: checkTarget,
package/lib/router.js CHANGED
@@ -26,7 +26,7 @@ var crypto = require('crypto');
26
26
  var express = require('express');
27
27
  var { uploadRoute } = require('@xeplr/base-apis');
28
28
  var { getMtContext } = require('@xeplr/db');
29
- var { attachConfig } = require('@xeplr/actions');
29
+ var { attachConfig, dataDir } = require('@xeplr/actions');
30
30
  var imports = require('./imports');
31
31
  var runs = require('./runs');
32
32
 
@@ -58,17 +58,20 @@ function createRouter(options) {
58
58
  router.use(express.json({ limit: '1mb' }));
59
59
 
60
60
  // The file itself — @xeplr/base-apis' generic receive-and-store.
61
- router.use('/files', uploadRoute({ destination: s.uploadDir, maxSize: s.maxUploadMb * 1024 * 1024 }));
61
+ // Into the requesting tenant's own uploads/ (imports.uploadDir), asked per
62
+ // upload: the folder depends on who is asking.
63
+ router.use('/files', uploadRoute({ destination: function() { return imports.uploadDir(); }, maxSize: s.maxUploadMb * 1024 * 1024 }));
62
64
 
63
- /** The uploaded file a body names, checked: inside the upload directory, there, and a format we read. */
65
+ /** The uploaded file a body names, checked: in this tenant's uploads, there, and a format we read. */
64
66
  function fileOf(body) {
65
- var filePath = body && body.filePath;
66
- if (!filePath) return { status: 400, error: 'filePath is required' };
67
- if (!imports.isWithin(s.uploadDir, filePath)) return { status: 400, error: 'Invalid filePath' };
67
+ var given = body && body.filePath;
68
+ if (!given) return { status: 400, error: 'filePath is required' };
69
+ var filePath = imports.uploadedFile(given);
70
+ if (!filePath) return { status: 400, error: 'Invalid filePath' };
68
71
  if (!fs.existsSync(filePath)) return { status: 404, error: 'Upload not found — it may have expired. Please upload the file again.' };
69
72
  var format = imports.formatOf(filePath);
70
73
  if (!format) return { status: 400, error: 'Unsupported file type — use CSV, JSON, or Excel (.xlsx/.xls)' };
71
- return { filePath: path.resolve(filePath), format: format };
74
+ return { filePath: filePath, format: format };
72
75
  }
73
76
 
74
77
  async function targetOf(connectionId, dbInfoId, req) {
@@ -252,9 +255,10 @@ function createRouter(options) {
252
255
  var d = got.job.details || {};
253
256
  var base = String(d.originalFileName || got.job.target_table).replace(/\.[^.]+$/, '');
254
257
  var name = (base + ' - dropped rows').replace(/[\\/:*?"<>|\u0000-\u001F]+/g, ' ').trim().slice(0, 120) + '.xlsx';
255
- // Beside the uploads — these are customer rows too — and gone once sent.
256
- // No leading dot in the name: Express will not send a file under one.
257
- var dir = fs.mkdtempSync(path.join(s.uploadDir, 'dropped-'));
258
+ // In the tenant's own temporary folder — these are customer rows too —
259
+ // and gone once sent. No leading dot in the name: Express will not send a
260
+ // file under one.
261
+ var dir = fs.mkdtempSync(path.join(dataDir.scratch(), 'dropped-'));
258
262
  try {
259
263
  var filePath = await imports.droppedRowsFile(got.found, path.join(dir, name));
260
264
  // With a callback Express leaves a failed send to us — answer it, or
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@xeplr/imports",
3
- "version": "1.0.1",
3
+ "version": "2.0.0",
4
4
  "description": "Data imports as a drop-in module: upload a CSV/Excel/JSON file, check it against the table, load it as a New table, Append or Replace, see and download the rows the database refused, and roll back — routes, access catalog and env in one package",
5
5
  "main": "index.js",
6
6
  "files": [
@@ -29,10 +29,10 @@
29
29
  "access": "public"
30
30
  },
31
31
  "dependencies": {
32
- "@xeplr/actions": "^1.1.23",
32
+ "@xeplr/actions": "^2.0.0",
33
33
  "@xeplr/base-apis": "^2.0.3",
34
34
  "@xeplr/db": "^1.0.8",
35
- "@xeplr/utils": "^1.0.13",
35
+ "@xeplr/utils": "^1.1.0",
36
36
  "csv-parse": "^7.0.3",
37
37
  "fflate": "^0.8.3",
38
38
  "multer": "^2.4.0",