@xeplr/actions 1.1.20 → 1.1.21

package/README.md

@@ -348,10 +348,16 @@ Upserts need no index: they are staged and match keys in the move. A read-only o
 | format | reads | notes |
 |---|---|---|
 | `csv` | stream | `formatConfig`: `columns` (default first row), `delimiter` (`,`), `cast` (true), `skipEmptyLines`, `trim`. |
-| `excel` | file path | `.xlsx` via `unzipper` + `saxes`; `sheet`, `startRow`. Date-styled cells arrive as `Date`. |
+| `excel` | file path | `.xlsx` via `unzipper` + `saxes`; `sheet`, `startRow`. Date-styled cells arrive as `Date`. Excel's own escapes in cell text are decoded (`_x000D_` → a real line break; `_x005F_` → `_`), and Japanese phonetic runs (`<rPh>`, furigana) are not joined into the value. |
 | `json` | stream | A JSON array or NDJSON; buffers the whole file. |
 | `txt` | — | Placeholder, not implemented. |
 
+**Rows pasted into one cell** (`lib/formats/pastedRows.js`). Data copied out of an ERP and pasted into Excel: a value starting with `"` (an inch mark) makes Excel put everything up to the next `"` — tabs, line breaks, whole rows — into that one cell. The rows it swallowed exist nowhere else. Seen for real: 465 rows folded into 11 cells.
+- `formats.scanPastedRows(format, filePath, { sheet })` → `{ cells, rows, examples: [{ row, column, rows, first }] }` — found, nothing changed. For an import's inspect step.
+- `formatConfig.repairPastedRows: true` on `file-upload` puts them back: the cell splits on its line breaks into the rest of its own row, whole rows, and the start of the last row, whose remaining values are the merged row's later cells. The eaten quotes go back where they opened and closed. The action's output says what it did: `repairedPastedRows: { cells, rows, at }`.
+- **Only an exact shape is repaired:** the pieces must have the sheet's column counts (and the cells the last row's values would leave empty must be empty). Ordinary multi-line text never matches, so a genuine note with line breaks is never touched. On the client's real file: 11 of 11 cells, 465 of 465 rows, no recovered entry number duplicating an existing one.
+- `pastedRows.decodeOoxml`, `splitPastedRows(rec, headers)`, `expandPastedRows(rows)`, `scanForPastedRows(rows)` for other readers.
+
 File sources: only `local` is implemented (`path`). `sftp`, `sharepoint` and `google` are placeholders.
 
 ## Movement metadata (`xeplr_configs`)
@@ -448,7 +454,7 @@ node --test test/uploader/reconcile.test.js
 
 | needs nothing | needs a database |
 |---|---|
-| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
+| `test/formats/pasted-rows.test.js` (its last case loads into Postgres when `PG_HOST`/`PG_PORT`/`PG_USER`/`PG_PASSWORD`/`PG_DATABASE` are set, else skipped), `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
 | `test/list-columns-length.test.js` (`XEPLR_TEST_PG_URL`, `MYSQL_TEST_URL`, `MSSQL_TEST_URL`; each skipped when unset), `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js`, `test/duckdb-index-guard.test.js`, `test/staged-load.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
 | | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
 | | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |

package/lib/builtins/file/upload.js

@@ -11,6 +11,7 @@
 var { upload } = require('../../uploader');
 var fileSources = require('../../drivers/file');
 var formats     = require('../../formats');
+var { expandPastedRows } = require('../../formats/pastedRows');
 var dbDrivers   = require('../../drivers/db');
 var { SqlQueue } = require('@xeplr/utils/lib/queue');
 var { generateId } = require('@xeplr/utils/lib/helpers');
@@ -82,6 +83,13 @@ module.exports = {
       var stream = await source.openStream(sourceCfg);
       rowsIterable = format.parseStream(stream, input.formatConfig || {});
     }
+    // Rows that a paste folded into one cell, put back — only when asked
+    // (formatConfig.repairPastedRows), after the person saw what was found.
+    // See lib/formats/pastedRows.js.
+    var repaired = [];
+    if (input.formatConfig && input.formatConfig.repairPastedRows) {
+      rowsIterable = expandPastedRows(rowsIterable, function(r) { repaired.push(r); });
+    }
 
     // Set up target
     var movementId = input.movementId || system.occurrenceId || ('mv_' + generateId());
@@ -97,7 +105,7 @@ module.exports = {
     });
 
     try {
-      return await upload({
+      var result = await upload({
         source:         rowsIterable,
         driver:         driver,
         connection:     pool,
@@ -120,6 +128,15 @@ module.exports = {
         onProgress:     system.onProgress || null,
         signal:         system.signal || null
       });
+      // What the repair put back, for the import's report.
+      if (repaired.length) {
+        result = Object.assign({}, result, { repairedPastedRows: {
+          cells: repaired.length,
+          rows: repaired.reduce(function(n, r) { return n + r.rows; }, 0),
+          at: repaired
+        } });
+      }
+      return result;
     } finally {
       queue.stop();
       await driver.close(pool);

package/lib/formats/_xlsx-stream.js

@@ -28,6 +28,7 @@
 var unzipper = require('unzipper');
 var { SaxesParser } = require('saxes');
 var { StringDecoder } = require('string_decoder');
+var { decodeOoxml } = require('./pastedRows');
 
 // Strip an XML namespace prefix: "x:row" -> "row", "row" -> "row".
 function localName(n) {
@@ -194,18 +195,25 @@ async function parseSharedStrings(directory) {
   var f = findFile(directory, 'xl/sharedStrings.xml');
   if (!f) return null;
   var strings = [];
-  var cur = null, inT = 0, buf = '';
+  // inRph: inside a phonetic run (<rPh>, the furigana Japanese Excel stores
+  // beside a cell's text). Its <t> is a reading aid, not the cell's value —
+  // joining it in gave "東京とうきょう" for 東京.
+  var cur = null, inT = 0, inRph = 0, buf = '';
   await saxParseStream(f.stream(), {
     opentag: function(tag) {
       var ln = localName(tag.name);
-      if      (ln === 'si') { cur = ''; }
-      else if (ln === 't')  { inT++; buf = ''; }
+      if      (ln === 'si')  { cur = ''; }
+      else if (ln === 'rPh') { inRph++; }
+      else if (ln === 't')   { inT++; buf = ''; }
     },
     text: function(t) { if (inT > 0) buf += t; },
     closetag: function(tag) {
       var ln = localName(tag.name);
-      if      (ln === 't')  { inT--; cur = (cur || '') + buf; buf = ''; }
-      else if (ln === 'si') { strings.push(cur || ''); cur = null; }
+      if      (ln === 't')   { inT--; if (!inRph) cur = (cur || '') + buf; buf = ''; }
+      else if (ln === 'rPh') { inRph--; }
+      // Excel's own escapes (_x000D_ for a line break inside the cell) decoded
+      // here, once per distinct string.
+      else if (ln === 'si')  { strings.push(decodeOoxml(cur || '')); cur = null; }
     }
   });
   return strings;
@@ -288,7 +296,7 @@ async function stream_xlsx_sheet_rows(filePath, sheet_name, handlers) {
     } else if (curType === 'b') {
       val = textBuf === '1';
     } else if (curType === 'str' || curType === 'inlineStr' || closingLocal === 't') {
-      val = textBuf;
+      val = decodeOoxml(textBuf);
     } else {
       var n = Number(textBuf);
       val = (textBuf !== '' && !Number.isNaN(n)) ? n : (textBuf === '' ? null : textBuf);

package/lib/formats/index.js

@@ -33,8 +33,27 @@ function checkFormatRequires(actionName, format, mod) {
   }
 }
 
+/**
+ * Which cells of a file hold rows a paste folded into them (pastedRows.js),
+ * read without changing anything — for an import's inspect step.
+ * opts: the format's own options (`sheet` for excel), plus `examples`.
+ */
+async function scanPastedRows(type, filePath, opts) {
+  var format = getFormat(type);
+  var config = opts || {};
+  var rows = format.inputMode === 'path'
+    ? format.parsePath(filePath, config)
+    : format.parseStream(require('fs').createReadStream(filePath), config);
+  return pastedRows.scanForPastedRows(rows, { examples: config.examples });
+}
+
+var pastedRows = require('./pastedRows');
+
 module.exports = {
   getFormat: getFormat,
   checkFormatRequires: checkFormatRequires,
-  SUPPORTED: Object.keys(FORMATS)
+  SUPPORTED: Object.keys(FORMATS),
+  scanPastedRows: scanPastedRows,
+  // decodeOoxml, splitPastedRows, expandPastedRows, scanForPastedRows
+  pastedRows: pastedRows
 };

package/lib/formats/pastedRows.js

@@ -0,0 +1,146 @@
+// ROWS PASTED INTO ONE CELL — found, and put back.
+//
+// Data copied out of an ERP is tab-separated text, a row per line. Pasted into
+// Excel, a value that STARTS with a double quote (an inch mark: `"4 INCH`)
+// switches Excel into quoted mode, and everything up to the next `"` — tabs,
+// line breaks, whole rows — lands in that one cell. The sheet then holds one
+// row whose cell carries the next N rows, and those rows exist nowhere else.
+// Seen for real: 465 rows folded into 11 Item Description cells.
+//
+// The damage has an exact shape, which is what makes it safe to undo:
+//
+//   cell = <its own row's value> \t <its own row's remaining columns>
+//          \n <a whole row> \n <a whole row> …
+//          \n <the last row's columns up to and including this one>
+//
+// and the merged row's OTHER columns after the cell belong to that last row.
+// So the cell splits on its line breaks into a first piece of (n - c) values,
+// whole rows of n values, and a last piece of L values — n columns, the cell
+// at column c. Usually L is c + 1 (the closing quote was in the same column);
+// when it closed in a later column k, L is k + 1, and the last row's
+// remaining (n - L) values sit in the merged row's cells right after c, the
+// cells beyond them empty. A cell that does not split exactly like that is
+// left alone: ordinary multi-line text never has those counts, so a genuine
+// note with line breaks is never "repaired".
+//
+// The quote that opened the cell and the one that closed it were eaten by the
+// paste; they are put back — at the start of the first row's value, where
+// quoted mode began, and at the end of the last row's value where it ended
+// (an inch mark: `Painting brush 4"`).
+
+// Excel's own escape inside cell text: `_xHHHH_` is the character with that
+// hex code. A line break inside a cell is saved as `_x000D_` + a real newline;
+// `_x005F_` is a literal underscore, which is how text that really says
+// "_x000D_" is stored ("_x005F_x000D_") — left to right, that decodes right.
+var ESCAPE = /_x([0-9A-Fa-f]{4})_/g;
+
+function decodeOoxml(s) {
+  if (typeof s !== 'string' || s.indexOf('_x') === -1) return s;
+  return s.replace(ESCAPE, function(_, hex) { return String.fromCharCode(parseInt(hex, 16)); });
+}
+
+var LINE = /\r\n|\r|\n/;
+var NUMBER = /^-?\d+(\.\d+)?$/;
+
+// A value from inside the cell: text, typed as the sheet's own cells would be.
+function cast(v) {
+  if (v === '') return null;
+  return NUMBER.test(v) ? Number(v) : v;
+}
+
+/**
+ * The rows folded into one of `rec`'s cells, put back — or null when no cell
+ * has that exact shape. `headers` in sheet order.
+ *
+ * @returns {{ column: string, rows: object[] } | null}  rows[0] is `rec` itself, rebuilt
+ */
+function splitPastedRows(rec, headers) {
+  var n = headers.length;
+  for (var c = 0; c < n; c++) {
+    var cell = decodeOoxml(rec[headers[c]]);
+    if (typeof cell !== 'string' || cell.indexOf('\t') === -1 || !LINE.test(cell)) continue;
+    var lines = cell.split(LINE);
+    var first = lines[0].split('\t');
+    var last = lines[lines.length - 1].split('\t');
+    var middle = lines.slice(1, -1).map(function(l) { return l.split('\t'); });
+    var L = last.length;
+    if (first.length !== n - c || L < c + 1 || L > n) continue;
+    if (middle.some(function(m) { return m.length !== n; })) continue;
+    // The last row's values after the closing quote: the merged row's cells
+    // right after c — and every cell beyond those must be empty, or this is
+    // not the shape a paste leaves.
+    var tail = headers.slice(c + 1, c + 1 + (n - L));
+    var beyond = headers.slice(c + 1 + (n - L));
+    if (beyond.some(function(h) { return rec[h] !== null && rec[h] !== undefined && rec[h] !== ''; })) continue;
+
+    var rows = [];
+    var top = {};
+    headers.forEach(function(h, i) { top[h] = i < c ? rec[h] : cast(first[i - c]); });
+    top[headers[c]] = '"' + first[0];
+    rows.push(top);
+    middle.forEach(function(m) {
+      var r = {};
+      headers.forEach(function(h, i) { r[h] = cast(m[i]); });
+      rows.push(r);
+    });
+    var end = {};
+    headers.forEach(function(h, i) { end[h] = i < L ? cast(last[i]) : rec[tail[i - L]]; });
+    // The closing quote, at the end of the value it closed.
+    end[headers[L - 1]] = String(last[L - 1]) + '"';
+    rows.push(end);
+    return { column: headers[c], rows: rows };
+  }
+  return null;
+}
+
+/**
+ * Rows as they were before the paste folded them: each record whose cell
+ * holds pasted rows becomes those rows. Everything else passes through.
+ * `onRepair({ row, column, rows })` hears each one (row: 1-based data row).
+ */
+async function* expandPastedRows(records, onRepair) {
+  var headers = null;
+  var index = 0;
+  for await (var rec of records) {
+    index++;
+    if (!headers) headers = Object.keys(rec);
+    var split = splitPastedRows(rec, headers);
+    if (!split) { yield rec; continue; }
+    if (onRepair) onRepair({ row: index, column: split.column, rows: split.rows.length - 1 });
+    for (var r of split.rows) yield r;
+  }
+}
+
+/**
+ * Read a whole file's rows and say which cells hold pasted rows, without
+ * changing anything — for the inspect step, before anything is loaded.
+ *
+ * @returns {{ cells: number, rows: number, examples: [{ row, column, rows, first }] }}
+ *          rows: how many rows the repair would recover
+ */
+async function scanForPastedRows(records, opts) {
+  var limit = (opts && opts.examples) || 5;
+  var headers = null;
+  var index = 0;
+  var found = { cells: 0, rows: 0, examples: [] };
+  for await (var rec of records) {
+    index++;
+    if (!headers) headers = Object.keys(rec);
+    var split = splitPastedRows(rec, headers);
+    if (!split) continue;
+    found.cells++;
+    found.rows += split.rows.length - 1;
+    if (found.examples.length < limit) {
+      found.examples.push({ row: index, column: split.column, rows: split.rows.length - 1,
+        first: String(split.rows[0][split.column]).slice(0, 80) });
+    }
+  }
+  return found;
+}
+
+module.exports = {
+  decodeOoxml: decodeOoxml,
+  splitPastedRows: splitPastedRows,
+  expandPastedRows: expandPastedRows,
+  scanForPastedRows: scanForPastedRows
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.1.20",
+  "version": "1.1.21",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "files": [