@ricsam/excel-to-formula-engine 0.1.0 → 0.2.0

package/README.md

@@ -1,6 +1,7 @@
 # excel-to-formula-engine
 
-Convert an Excel `.xlsx` file into [`@ricsam/formula-engine`](https://github.com/ricsam/formula-engine)
+Convert a spreadsheet file — `.xlsx`, `.csv` or `.tsv` — into
+[`@ricsam/formula-engine`](https://github.com/ricsam/formula-engine)
 `WorkbookData`, in the browser.
 
 ```ts
@@ -22,6 +23,54 @@ external.
 bun add @ricsam/excel-to-formula-engine @ricsam/formula-engine
 ```
 
+## Delimited text
+
+CSV and TSV go through `csvToFormulaEngine`, or through
+`spreadsheetToFormulaEngine` when you do not know which kind of file you have —
+a drop target, typically. That one settles the format from the bytes rather than
+the file name, since `.xlsx` is a ZIP and delimited text is not, so a workbook
+saved with a `.csv` extension still reads correctly.
+
+```ts
+import {
+  csvToFormulaEngine,
+  spreadsheetToFormulaEngine
+} from "@ricsam/excel-to-formula-engine";
+
+// Delimiter is detected: comma, tab, semicolon or pipe.
+const data = await csvToFormulaEngine(file);
+
+// Or let it work out which kind of file this is.
+const data = await spreadsheetToFormulaEngine(file);
+```
+
+Parsing follows RFC 4180, so a quoted field may contain the delimiter, doubled
+quotes, and line breaks. The UTF-8 BOM Excel writes on every CSV export is
+stripped — left in, it becomes an invisible part of the first header cell.
+
+### What a bare string becomes
+
+A CSV carries no types, so every cell is a judgement call. The rules err toward
+keeping data recoverable:
+
+| Field | Becomes | Why |
+| --- | --- | --- |
+| `42`, `-3.5`, `1.50`, `1e3` | number | Plain decimals. |
+| `TRUE` / `false` | boolean | Any case, as in a spreadsheet. |
+| `=B2*C2` | formula | What a spreadsheet does with the same file. |
+| `007`, `01234` | **text** | Part numbers and postcodes — the mangling Excel is notorious for. |
+| `12345678901234567890` | **text** | Beyond float precision; converting would silently change it. |
+| `1,000`, `$5`, `50%` | **text** | Converting would drop the formatting. |
+| `0x1f`, `Infinity` | **text** | `Number()` accepts these; a spreadsheet does not. |
+
+Empty fields are left out rather than stored as empty strings, which is what
+keeps a sparse sheet sparse.
+
+There is deliberately no option to keep an `=`-leading field as text: the engine
+stores a formula as a string beginning with `=` and has no escape for text that
+merely looks like one, so such an option could only be honoured by altering the
+value.
+
 ## Usage
 
 `excelToFormulaEngine` accepts a `Blob` or `File` straight from a file input, or

package/dist/index.cjs

@@ -41,7 +41,11 @@ var exports_excel_to_formula_engine = {};
 __export(exports_excel_to_formula_engine, {
   columnIndexToLabel: () => columnIndexToLabel,
   columnLabelToIndex: () => columnLabelToIndex,
+  convertDelimitedText: () => convertDelimitedText,
   convertWorkbook: () => convertWorkbook,
+  csvToFormulaEngine: () => csvToFormulaEngine,
+  csvToFormulaEngineDetailed: () => csvToFormulaEngineDetailed,
+  detectDelimiter: () => detectDelimiter,
   excelSerialToIso: () => excelSerialToIso,
   excelToFormulaEngine: () => excelToFormulaEngine,
   excelToFormulaEngineDetailed: () => excelToFormulaEngineDetailed,
@@ -49,6 +53,10 @@ __export(exports_excel_to_formula_engine, {
   offsetFormulaReferences: () => offsetFormulaReferences,
   parseA1: () => parseA1,
   parseA1Range: () => parseA1Range,
+  parseDelimitedText: () => parseDelimitedText,
+  spreadsheetToFormulaEngine: () => spreadsheetToFormulaEngine,
+  spreadsheetToFormulaEngineDetailed: () => spreadsheetToFormulaEngineDetailed,
+  stripBom: () => stripBom,
   toA1: () => toA1,
   translateFormula: () => translateFormula
 });
@@ -961,12 +969,14 @@ function getSupportedFunctions() {
   return supportedFunctionNames;
 }
 var XLFN_PREFIX = /_xlfn\.(_xlws\.)?/g;
-var IMPLICIT_INTERSECTION = /(?<![\w$"'\]])@(?=[A-Za-z_\\])/g;
+var IMPLICIT_INTERSECTION = /(?<![\w$"'\]\[])@(?=[A-Za-z_\\])/g;
+var THIS_ROW_REFERENCE = /(?:'((?:[^']|'')+)'|([A-Za-z_\\][\w.\\]*))?\[\[#This ?Row\],\s*\[([^\]]*)\](?:\s*:\s*\[([^\]]*)\])?\]/gi;
 function translateFormula(rawFormula, options = {}) {
   const diagnostics = [];
   let body = rawFormula.startsWith("=") ? rawFormula.slice(1) : rawFormula;
   body = body.replace(XLFN_PREFIX, "");
   body = body.replace(IMPLICIT_INTERSECTION, "");
+  body = contractThisRowReferences(body, options.containingTableName);
   for (const name of findUnsupportedFunctions(body)) {
     diagnostics.push({
       severity: "warning",
@@ -1079,6 +1089,22 @@ function columnIndexToLabel2(index) {
   }
   return label;
 }
+function contractThisRowReferences(formulaBody, containingTableName) {
+  return formulaBody.replace(THIS_ROW_REFERENCE, (whole, quotedTable, bareTable, startCol, endCol) => {
+    const tableName = quotedTable ? String(quotedTable).replace(/''/g, "'") : bareTable ? String(bareTable) : undefined;
+    const columns = endCol === undefined ? `[${startCol}]` : `[${startCol}]:[${endCol}]`;
+    const isPlainSingleColumn = endCol === undefined && /^[^[\]@#'"]+$/.test(String(startCol));
+    const reference = isPlainSingleColumn ? `@${startCol}` : `@${columns}`;
+    if (tableName && containingTableName && tableName.toLowerCase() === containingTableName.toLowerCase()) {
+      return `[${reference}]`;
+    }
+    if (!tableName) {
+      return `[${reference}]`;
+    }
+    const quoted = quotedTable ? `'${String(quotedTable)}'` : tableName;
+    return `${quoted}[${reference}]`;
+  });
+}
 
 // src/translate/conditional-format.ts
 var import_formula_engine2 = require("@ricsam/formula-engine");
@@ -1309,6 +1335,9 @@ async function convertWorkbook(source, options = {}) {
   for (const entry of selected) {
     const worksheet = parseWorksheet(pkg, entry.partPath, sharedStrings);
     importedSheetNames.add(entry.name);
+    const sheetTables = importTables ? parseTables(pkg, entry.partPath) : [];
+    const tableRanges = sheetTables.map((table) => ({ name: table.displayName, range: parseA1Range(table.ref) })).filter((entryTable) => entryTable.range !== undefined);
+    const tableAt = (colIndex, rowIndex) => tableRanges.find(({ range }) => colIndex >= range.start.colIndex && colIndex <= range.end.colIndex && rowIndex >= range.start.rowIndex && rowIndex <= range.end.rowIndex)?.name;
     const content = new Map;
     const cellMetadata = new Map;
     const styleGroups = new Map;
@@ -1318,7 +1347,8 @@ async function convertWorkbook(source, options = {}) {
         convertDates,
         date1904,
         sheetName: entry.name,
-        diagnostics
+        diagnostics,
+        containingTableName: tableAt(cell.colIndex, cell.rowIndex)
       });
       if (value !== undefined) {
         content.set(cell.reference, value);
@@ -1373,7 +1403,7 @@ async function convertWorkbook(source, options = {}) {
       }
     }
     if (importTables) {
-      for (const table of parseTables(pkg, entry.partPath)) {
+      for (const table of sheetTables) {
         const converted = convertTable(table, entry.name, diagnostics);
         if (converted) {
           tables.push(converted);
@@ -1423,7 +1453,8 @@ function cellValue(cell, format, context) {
     const body = cell.formulaOffset ? offsetFormulaReferences(cell.formula, cell.formulaOffset.cols, cell.formulaOffset.rows) : cell.formula;
     const translated = translateFormula(body, {
       sheetName: context.sheetName,
-      cellReference: cell.reference
+      cellReference: cell.reference,
+      containingTableName: context.containingTableName
     });
     context.diagnostics.push(...translated.diagnostics);
     return translated.formula;
@@ -1569,6 +1600,238 @@ function coalesceCellsIntoAreas(sheetName, cells) {
   return areas;
 }
 
+// src/csv/parse.ts
+var CANDIDATE_DELIMITERS = [",", "\t", ";", "|"];
+function parseDelimitedText(text, delimiter) {
+  const rows = [];
+  let row = [];
+  let field = "";
+  let inQuotes = false;
+  let fieldWasQuoted = false;
+  let rowHasContent = false;
+  const endField = () => {
+    row.push(field);
+    field = "";
+    fieldWasQuoted = false;
+    rowHasContent = true;
+  };
+  const endRow = () => {
+    endField();
+    rows.push(row);
+    row = [];
+    rowHasContent = false;
+  };
+  for (let i = 0;i < text.length; i++) {
+    const char = text[i];
+    if (inQuotes) {
+      if (char === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        field += char;
+      }
+      continue;
+    }
+    if (char === '"' && field.length === 0) {
+      inQuotes = true;
+      fieldWasQuoted = true;
+      continue;
+    }
+    if (char === delimiter) {
+      endField();
+      continue;
+    }
+    if (char === "\r") {
+      if (text[i + 1] === `
+`)
+        i++;
+      endRow();
+      continue;
+    }
+    if (char === `
+`) {
+      endRow();
+      continue;
+    }
+    field += char;
+  }
+  if (field.length > 0 || fieldWasQuoted || rowHasContent) {
+    endRow();
+  }
+  return rows;
+}
+function detectDelimiter(text, candidates = CANDIDATE_DELIMITERS) {
+  const lines = takeSampleLines(text, 20);
+  if (lines.length === 0)
+    return ",";
+  let best;
+  for (const delimiter of candidates) {
+    const counts = lines.map((line) => countOutsideQuotes(line, delimiter));
+    const present = counts.filter((count) => count > 0);
+    if (present.length === 0)
+      continue;
+    const first = counts[0];
+    const consistent = counts.filter((count) => count === first).length;
+    const score = consistent / counts.length + Math.min(first, 50) / 1000;
+    if (!best || score > best.score) {
+      best = { delimiter, score };
+    }
+  }
+  return best?.delimiter ?? ",";
+}
+function takeSampleLines(text, limit) {
+  const lines = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0;i < text.length && lines.length < limit; i++) {
+    const char = text[i];
+    if (char === '"') {
+      if (inQuotes && text[i + 1] === '"') {
+        current += '""';
+        i++;
+        continue;
+      }
+      inQuotes = !inQuotes;
+      current += char;
+      continue;
+    }
+    if (!inQuotes && (char === `
+` || char === "\r")) {
+      if (char === "\r" && text[i + 1] === `
+`)
+        i++;
+      if (current.trim().length > 0)
+        lines.push(current);
+      current = "";
+      continue;
+    }
+    current += char;
+  }
+  if (current.trim().length > 0 && lines.length < limit)
+    lines.push(current);
+  return lines;
+}
+function countOutsideQuotes(line, delimiter) {
+  let count = 0;
+  let inQuotes = false;
+  for (let i = 0;i < line.length; i++) {
+    const char = line[i];
+    if (char === '"') {
+      if (inQuotes && line[i + 1] === '"') {
+        i++;
+        continue;
+      }
+      inQuotes = !inQuotes;
+      continue;
+    }
+    if (!inQuotes && char === delimiter)
+      count++;
+  }
+  return count;
+}
+function stripBom(text) {
+  return text.charCodeAt(0) === 65279 ? text.slice(1) : text;
+}
+
+// src/csv/convert.ts
+var MAX_CELLS = 2000000;
+function convertDelimitedText(text, options = {}) {
+  const {
+    sheetName = "Sheet1",
+    fileName,
+    parseNumbers = true,
+    parseBooleans = true,
+    trimFields = false
+  } = options;
+  const diagnostics = [];
+  const body = stripBom(text);
+  const delimiter = options.delimiter ?? detectDelimiter(body);
+  const rows = parseDelimitedText(body, delimiter);
+  const widest = rows.reduce((max, row) => Math.max(max, row.length), 0);
+  if (rows.length * widest > MAX_CELLS) {
+    throw new Error(`The file has about ${rows.length * widest} cells, more than the ` + `${MAX_CELLS} this importer will load at once.`);
+  }
+  const content = new Map;
+  for (let rowIndex = 0;rowIndex < rows.length; rowIndex++) {
+    const row = rows[rowIndex];
+    for (let colIndex = 0;colIndex < row.length; colIndex++) {
+      const raw = trimFields ? row[colIndex].trim() : row[colIndex];
+      if (raw.length === 0)
+        continue;
+      const reference = `${columnIndexToLabel(colIndex)}${rowIndex + 1}`;
+      content.set(reference, toCellValue(raw, {
+        parseNumbers,
+        parseBooleans,
+        sheetName,
+        reference,
+        diagnostics
+      }));
+    }
+  }
+  const data = {
+    sheets: [{ name: sheetName, content }],
+    workbookMetadata: {
+      fileName,
+      sheetOrder: [sheetName],
+      diagnostics: diagnostics.length > 0 ? diagnostics : undefined
+    }
+  };
+  return { data, diagnostics };
+}
+function toCellValue(raw, context) {
+  if (raw.startsWith("=") && raw.length > 1) {
+    const translated = translateFormula(raw, {
+      sheetName: context.sheetName,
+      cellReference: context.reference
+    });
+    context.diagnostics.push(...translated.diagnostics);
+    return translated.formula;
+  }
+  if (context.parseBooleans) {
+    const upper = raw.toUpperCase();
+    if (upper === "TRUE")
+      return true;
+    if (upper === "FALSE")
+      return false;
+  }
+  if (context.parseNumbers) {
+    const number = toNumber(raw);
+    if (number !== undefined)
+      return number;
+  }
+  return raw;
+}
+function toNumber(raw) {
+  const text = raw.trim();
+  if (text.length === 0)
+    return;
+  if (!/^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$/.test(text)) {
+    return;
+  }
+  const digits = text.replace(/^[+-]/, "");
+  if (/^0\d/.test(digits))
+    return;
+  const value = Number(text);
+  if (!Number.isFinite(value))
+    return;
+  if (!text.includes(".") && !/[eE]/.test(text) && !Number.isSafeInteger(value)) {
+    return;
+  }
+  return value;
+}
+async function readTextFrom(source) {
+  if (typeof source === "string")
+    return source;
+  if (source instanceof Uint8Array || source instanceof ArrayBuffer) {
+    return new TextDecoder("utf-8").decode(source);
+  }
+  return source.text();
+}
+
 // index.ts
 async function excelToFormulaEngine(source, options) {
   const { data } = await convertWorkbook(source, withFileName(source, options));
@@ -1584,5 +1847,46 @@ function withFileName(source, options) {
   const name = typeof File !== "undefined" && source instanceof File ? source.name : undefined;
   return name ? { ...options, fileName: name } : { ...options };
 }
+async function csvToFormulaEngine(source, options) {
+  const { data } = await csvToFormulaEngineDetailed(source, options);
+  return data;
+}
+async function csvToFormulaEngineDetailed(source, options) {
+  const text = await readTextFrom(source);
+  const resolved = withCsvFileName(source, options);
+  return convertDelimitedText(text, resolved);
+}
+async function spreadsheetToFormulaEngine(source, options) {
+  const { data } = await spreadsheetToFormulaEngineDetailed(source, options);
+  return data;
+}
+async function spreadsheetToFormulaEngineDetailed(source, options) {
+  const bytes = await toBytes2(source);
+  if (looksLikeZip(bytes)) {
+    return convertWorkbook(bytes, withFileName(source, options));
+  }
+  return csvToFormulaEngineDetailed(bytes, withCsvFileName(source, options));
+}
+function looksLikeZip(bytes) {
+  return bytes.length >= 4 && bytes[0] === 80 && bytes[1] === 75 && bytes[2] === 3 && bytes[3] === 4;
+}
+async function toBytes2(source) {
+  if (source instanceof Uint8Array)
+    return source;
+  if (source instanceof ArrayBuffer)
+    return new Uint8Array(source);
+  return new Uint8Array(await source.arrayBuffer());
+}
+function withCsvFileName(source, options) {
+  const name = typeof File !== "undefined" && source instanceof File ? source.name : undefined;
+  const fileName = options?.fileName ?? name;
+  if (!fileName)
+    return { ...options };
+  return {
+    ...options,
+    fileName,
+    sheetName: options?.sheetName ?? (fileName.replace(/\.(csv|tsv|tab|txt)$/i, "").trim() || "Sheet1")
+  };
+}
 
-//# debugId=C50C41CAFB8A948064756E2164756E21
+//# debugId=E62FBA8DCEBAE6FE64756E2164756E21