@graphty/graph-io 0.3.25 → 0.3.26

package/dist/chunks/{records-dbRkxwaq.js → records-CQXT__mY.js}

@@ -1,5 +1,5 @@
 import { GraphFormatError } from "@graphty/graph-format";
-import { x as textChunks } from "./report-B1z4WT9e.js";
+import { a4 as textChunks, a5 as MAX_TEXT_LENGTH, a6 as tooLarge } from "./export-BTI4yQgy.js";
 const UNCLOSED_QUOTE_CODE = "E_CSV_UNCLOSED_QUOTE";
 const BAD_QUOTE_CODE = "E_CSV_QUOTE";
 const DELIMITER_CANDIDATES = Object.freeze([",", "	", ";", "|", " "]);
@@ -7,6 +7,9 @@ const PREVIEW_ROWS = 10;
 const PREVIEW_CHARS = 64 * 1024;
 const LF = 10;
 const CR = 13;
+const TAB = 9;
+const SPACE = 32;
+const SEP_DIRECTIVE = /^sep=([^\r\n])(?:\r\n|\n|\r|$)/i;
 const START = 0;
 const UNQUOTED = 1;
 const QUOTED = 2;
@@ -40,108 +43,181 @@ function checkRecordSyntax(syntax) {
 function sniffNewline(text) {
   return !text.includes("\n") && text.includes("\r") ? "\r" : "\n";
 }
-function splitRecords(text, delimiter, quote, maxRows, strictQuotes = false) {
-  const rows = [];
-  const delimiterCode = delimiter.charCodeAt(0);
-  const quoteCode = quote.charCodeAt(0);
-  let cells = [];
-  let state = START;
-  let segment = 0;
-  let field = "";
-  const n = text.length;
-  for (let i = 0; i < n && rows.length < maxRows; i++) {
-    const c = text.charCodeAt(i);
-    if (state === QUOTED) {
-      if (c === quoteCode) {
-        field += text.slice(segment, i);
-        state = CLOSING;
-      }
-      continue;
+function splitRecords(text, delimiter, quote, maxRows, strictQuotes = false, collapse = false) {
+  const delimiterCode = delimiter.codePointAt(0);
+  const splitter = new RecordSplitter(text, quote, delimiterCode, collapse && delimiterCode === SPACE, strictQuotes);
+  let i = 0;
+  while (i < text.length && splitter.rows.length < maxRows) {
+    i = splitter.step(i);
+    if (i < 0) {
+      return null;
     }
-    if (state === CLOSING) {
-      if (c === quoteCode) {
-        field += quote;
-        segment = i + 1;
-        state = QUOTED;
-        continue;
-      }
-      if (strictQuotes && c !== delimiterCode && c !== LF && c !== CR) {
-        return null;
+  }
+  if (splitter.rows.length < maxRows) {
+    splitter.finish();
+  }
+  return splitter.rows;
+}
+class RecordSplitter {
+  /**
+   * Create a splitter.
+   * @param text - the text
+   * @param quote - the quote character
+   * @param delimiterCode - the delimiter's code
+   * @param whitespace - whether runs of spaces and tabs are one separator
+   * @param strictQuotes - whether text after a closing quote gives up
+   */
+  constructor(text, quote, delimiterCode, whitespace, strictQuotes) {
+    this.text = text;
+    this.quote = quote;
+    this.delimiterCode = delimiterCode;
+    this.whitespace = whitespace;
+    this.strictQuotes = strictQuotes;
+    this.rows = [];
+    this.cells = [];
+    this.state = START;
+    this.segment = 0;
+    this.field = "";
+    this.quoteCode = quote.codePointAt(0);
+  }
+  /**
+   * Read one character.
+   * @param i - its index
+   * @returns the index of the next character to read, or -1 to give up (strictQuotes)
+   */
+  step(i) {
+    const c = this.text.codePointAt(i);
+    if (this.state === QUOTED) {
+      if (c === this.quoteCode) {
+        this.field += this.text.slice(this.segment, i);
+        this.state = CLOSING;
       }
-      state = AFTER_QUOTED;
-      segment = i;
+      return i + 1;
     }
-    if (c === delimiterCode || c === LF || c === CR) {
-      if (state === UNQUOTED || state === AFTER_QUOTED) {
-        field += text.slice(segment, i);
+    const isDelimiter = c === this.delimiterCode || this.whitespace && c === TAB;
+    if (this.state === CLOSING) {
+      const next = this.afterQuote(i, c, isDelimiter || c === LF || c === CR);
+      if (next !== null) {
+        return next;
       }
-      if (c === delimiterCode) {
-        cells.push(field);
-        field = "";
-        state = START;
-      } else {
-        if (state !== START || cells.length > 0) {
-          cells.push(field);
-          rows.push(cells);
-          cells = [];
-          field = "";
-        }
-        state = START;
-        if (c === CR && text.charCodeAt(i + 1) === LF) {
-          i++;
-        }
-      }
-      segment = i + 1;
-      continue;
     }
-    if (state === START) {
-      if (c === quoteCode) {
-        state = QUOTED;
-        segment = i + 1;
-      } else {
-        state = UNQUOTED;
-        segment = i;
+    if (isDelimiter || c === LF || c === CR) {
+      return this.separator(i, isDelimiter);
+    }
+    if (this.state === START) {
+      const quoted = c === this.quoteCode;
+      this.state = quoted ? QUOTED : UNQUOTED;
+      this.segment = quoted ? i + 1 : i;
+    }
+    return i + 1;
+  }
+  /**
+   * Read the character after a quote that may close a quoted field: a second quote is a quote
+   * character in the field; anything else closes it.
+   * @param i - its index
+   * @param c - its code
+   * @param separates - whether it is a delimiter or a line break
+   * @returns the index of the next character to read, -1 to give up (strictQuotes), or null to
+   * read this character as the one after the field
+   */
+  afterQuote(i, c, separates) {
+    if (c === this.quoteCode) {
+      this.field += this.quote;
+      this.segment = i + 1;
+      this.state = QUOTED;
+      return i + 1;
+    }
+    if (this.strictQuotes && !separates) {
+      return -1;
+    }
+    this.state = AFTER_QUOTED;
+    this.segment = i;
+    return null;
+  }
+  /**
+   * Read a delimiter or a line break: end the field, and at a line break the row.
+   * @param i - its index
+   * @param isDelimiter - whether it is a delimiter
+   * @returns the index of the next character to read
+   */
+  separator(i, isDelimiter) {
+    if (this.state === UNQUOTED || this.state === AFTER_QUOTED) {
+      this.field += this.text.slice(this.segment, i);
+    }
+    let next = i + 1;
+    if (isDelimiter && this.whitespace && this.state === START) ;
+    else if (isDelimiter) {
+      this.cells.push(this.field);
+      this.field = "";
+      this.state = START;
+    } else {
+      this.endRow();
+      this.state = START;
+      if (this.text.codePointAt(i) === CR && this.text.codePointAt(i + 1) === LF) {
+        next = i + 2;
       }
     }
+    this.segment = next;
+    return next;
   }
-  if (rows.length < maxRows && (state !== START || cells.length > 0)) {
-    if (state === UNQUOTED || state === AFTER_QUOTED || state === QUOTED) {
-      field += text.slice(segment, n);
+  /** End the row at a line break; a blank line is no row. */
+  endRow() {
+    if (this.state === START && this.cells.length === 0) {
+      return;
+    }
+    if (!(this.whitespace && this.state === START)) {
+      this.cells.push(this.field);
     }
-    cells.push(field);
-    rows.push(cells);
+    this.rows.push(this.cells);
+    this.cells = [];
+    this.field = "";
+  }
+  /** End the text: the last row, when it has no line break after it. */
+  finish() {
+    if (this.state === START && this.cells.length === 0) {
+      return;
+    }
+    if (this.state === UNQUOTED || this.state === AFTER_QUOTED || this.state === QUOTED) {
+      this.field += this.text.slice(this.segment);
+    }
+    if (!(this.whitespace && this.state === START)) {
+      this.cells.push(this.field);
+    }
+    this.rows.push(this.cells);
   }
-  return rows;
 }
 function stripLeadingComments(text, comments) {
   if (comments.length === 0) {
     return text;
   }
   let at = 0;
-  while (at < text.length && comments.includes(text[at])) {
+  while (at < text.length) {
+    if (text[at] === "\n" || text[at] === "\r") {
+      at++;
+      continue;
+    }
+    if (!comments.includes(text[at])) {
+      break;
+    }
     const lf = text.indexOf("\n", at);
     const cr = text.indexOf("\r", at);
     const end = Math.min(lf < 0 ? Infinity : lf, cr < 0 ? Infinity : cr);
     if (end === Infinity) {
       return "";
     }
-    at = end + 1;
-    if (text[at - 1] === "\r" && text[at] === "\n") {
-      at++;
-    }
+    at = end;
   }
   return at === 0 ? text : text.slice(at);
 }
-function sniffDelimiter(text, newline = "\n", candidates = DELIMITER_CANDIDATES, quote = '"', skipQuoteErrors = false) {
+function sniffDelimiter(text, newline = "\n", candidates = DELIMITER_CANDIDATES, quote = '"', skipQuoteErrors = false, collapse = false) {
   let best = null;
   let bestDelta = Infinity;
-  let bestAverage = 0;
   const terminated = text.endsWith("\n") || text.endsWith("\r");
   for (const delimiter of candidates) {
     if (delimiter === quote || delimiter === newline) {
       continue;
     }
-    const split = splitRecords(text, delimiter, quote, PREVIEW_ROWS, skipQuoteErrors);
+    const split = splitRecords(text, delimiter, quote, PREVIEW_ROWS, skipQuoteErrors, collapse);
     if (split === null) {
       continue;
     }
@@ -162,10 +238,9 @@ function sniffDelimiter(text, newline = "\n", candidates = DELIMITER_CANDIDATES,
       }
     }
     const average = total / rows.length;
-    if (average > 1.99 && (delta < bestDelta || delta === bestDelta && average > bestAverage)) {
+    if (average > 1.99 && delta < bestDelta) {
       best = delimiter;
       bestDelta = delta;
-      bestAverage = average;
     }
   }
   return best;
@@ -185,6 +260,9 @@ class RecordReader {
     this.cells = [];
     this.quoted = [];
     this.leadingComments = [];
+    this.leadingCommentLines = [];
+    this.strayQuoteLine = 0;
+    this.sepDirective = false;
     this.recordLine = 0;
     this.input = input;
     this.report = report;
@@ -235,12 +313,15 @@ class RecordReader {
   /**
    * Record a text-after-quote error (fatal).
    * @param line - the line the field started on
+   * @param field - the 1-based position of the field in its record
    * @returns never
    */
-  badQuote(line) {
-    return this.report.fail(BAD_QUOTE_CODE, `line ${line}: text after the closing quote of a quoted field`, {
-      line
-    });
+  badQuote(line, field) {
+    return this.report.fail(
+      BAD_QUOTE_CODE,
+      `text after the closing quote of quoted field ${field} (a quote inside a quoted field is written as two quotes)`,
+      { line }
+    );
   }
   /**
    * Record an unclosed-quote error (fatal).
@@ -270,6 +351,10 @@ class RecordReader {
     let breaks = 0;
     let lastWasCr = false;
     let sniffed = false;
+    const commentCodes = (this.syntax.comments ?? []).map((c) => c.codePointAt(0));
+    let leading = commentCodes.length > 0;
+    let lineStart = true;
+    let inComment = false;
     for await (const chunk of source) {
       if (sniffed) {
         yield chunk;
@@ -279,12 +364,18 @@ class RecordReader {
       length += chunk.length;
       for (let i = 0; i < chunk.length && breaks < PREVIEW_ROWS; i++) {
         const c = chunk.charCodeAt(i);
-        if (c === LF) {
-          if (!lastWasCr) {
+        if (c === LF || c === CR) {
+          if (!(c === LF && lastWasCr) && !inComment) {
             breaks++;
           }
-        } else if (c === CR) {
-          breaks++;
+          inComment = false;
+          lineStart = true;
+        } else {
+          if (lineStart && leading) {
+            inComment = commentCodes.includes(c);
+            leading = inComment;
+          }
+          lineStart = false;
         }
         lastWasCr = c === CR;
       }
@@ -310,9 +401,27 @@ class RecordReader {
    * @param text - the preview
    */
   sniff(text) {
+    const preview = text.slice(0, PREVIEW_CHARS);
+    const body = stripLeadingComments(preview, this.syntax.comments ?? []);
+    if (this.syntax.inspect !== void 0) {
+      const skipped = preview.slice(0, preview.length - body.length).match(/\r\n|\r|\n/g)?.length ?? 0;
+      this.syntax.inspect(body, skipped + 1);
+    }
+    const directive = this.syntax.sepDirective === true ? SEP_DIRECTIVE.exec(text) : null;
+    if (directive !== null && directive[1] !== this.syntax.quote) {
+      this.delimiterText = directive[1];
+      this.sepDirective = true;
+      return;
+    }
     const candidates = this.syntax.candidates ?? DELIMITER_CANDIDATES;
-    const body = stripLeadingComments(text.slice(0, PREVIEW_CHARS), this.syntax.comments ?? []);
-    this.delimiterText = sniffDelimiter(body, sniffNewline(body), candidates, this.syntax.quote, this.syntax.skipQuoteErrors) ?? candidates[0];
+    this.delimiterText = sniffDelimiter(
+      body,
+      sniffNewline(body),
+      candidates,
+      this.syntax.quote,
+      this.syntax.skipQuoteErrors,
+      this.syntax.collapseSpaces
+    ) ?? candidates[0];
   }
 }
 class RecordScanner {
@@ -326,6 +435,7 @@ class RecordScanner {
     this.recordLine = 1;
     this.startLine = 1;
     this.delimiterCode = -1;
+    this.collapse = false;
     this.state = START;
     this.field = "";
     this.fieldQuoted = false;
@@ -346,6 +456,7 @@ class RecordScanner {
   setDelimiter(code) {
     if (this.delimiterCode < 0) {
       this.delimiterCode = code;
+      this.collapse = code === SPACE && this.reader.syntax.collapseSpaces === true;
     }
   }
   /**
@@ -363,8 +474,9 @@ class RecordScanner {
       const c = chunk.charCodeAt(i);
       if (this.state === COMMENT) {
         if (c === LF || c === CR) {
-          this.comment += chunk.slice(this.segment, i);
+          this.comment = this.grow(this.comment, chunk.slice(this.segment, i));
           this.reader.leadingComments.push(this.comment);
+          this.reader.leadingCommentLines.push(this.line);
           this.comment = "";
           this.state = START;
           this.endLine(c);
@@ -383,28 +495,29 @@ class RecordScanner {
           i = n;
           break;
         }
-        this.field += chunk.slice(this.segment, q);
+        this.field = this.grow(this.field, chunk.slice(this.segment, q));
         this.state = CLOSING;
         this.lastWasCr = false;
         i = q + 1;
         continue;
       }
+      const isDelimiter = c === delimiterCode || this.collapse && c === TAB;
       if (this.state === CLOSING) {
         if (c === quoteCode) {
-          this.field += this.quoteText;
+          this.field = this.grow(this.field, this.quoteText);
           this.segment = i + 1;
           this.state = QUOTED;
           this.lastWasCr = false;
           i++;
           continue;
         }
-        if (c !== delimiterCode && c !== LF && c !== CR) {
-          this.reader.badQuote(this.startLine);
+        if (!isDelimiter && c !== LF && c !== CR) {
+          this.reader.badQuote(this.startLine, this.count + 1);
         }
         this.state = AFTER_QUOTED;
         this.segment = i;
       }
-      if (c === delimiterCode || c === LF || c === CR) {
+      if (isDelimiter || c === LF || c === CR) {
         if (c === LF && this.lastWasCr) {
           this.lastWasCr = false;
           this.segment = i + 1;
@@ -412,9 +525,15 @@ class RecordScanner {
           continue;
         }
         if (this.state === UNQUOTED) {
-          this.field += chunk.slice(this.segment, i);
+          this.field = this.grow(this.field, chunk.slice(this.segment, i));
         }
-        if (c === delimiterCode) {
+        if (isDelimiter && this.collapse && this.state === START) {
+          this.lastWasCr = false;
+          this.segment = i + 1;
+          i++;
+          continue;
+        }
+        if (isDelimiter) {
           this.push();
           this.state = START;
           this.lastWasCr = false;
@@ -422,9 +541,10 @@ class RecordScanner {
           i++;
           continue;
         }
-        const blank = this.state === START && this.count === 0;
-        if (!blank) {
+        if (!(this.state === START && (this.count === 0 || this.collapse))) {
           this.push();
+        }
+        if (this.count > 0) {
           this.ready = true;
           this.recordLine = this.startLine;
         }
@@ -454,6 +574,8 @@ class RecordScanner {
           this.state = UNQUOTED;
           this.segment = i;
         }
+      } else if (c === quoteCode && this.state === UNQUOTED && this.reader.strayQuoteLine === 0) {
+        this.reader.strayQuoteLine = this.startLine;
       }
       i++;
     }
@@ -469,9 +591,9 @@ class RecordScanner {
       return;
     }
     if (this.state === COMMENT) {
-      this.comment += chunk.slice(this.segment, n);
+      this.comment = this.grow(this.comment, chunk.slice(this.segment, n));
     } else if (this.state === QUOTED || this.state === UNQUOTED) {
-      this.field += chunk.slice(this.segment, n);
+      this.field = this.grow(this.field, chunk.slice(this.segment, n));
     }
     this.segment = n;
   }
@@ -484,13 +606,16 @@ class RecordScanner {
     const { state } = this;
     if (state === COMMENT) {
       this.reader.leadingComments.push(this.comment);
+      this.reader.leadingCommentLines.push(this.line);
       return false;
     }
     if (state === QUOTED) {
       this.reader.unclosedQuote(this.startLine);
     }
     if (state !== START || this.count > 0) {
-      this.push();
+      if (!(state === START && this.collapse)) {
+        this.push();
+      }
       this.recordLine = this.startLine;
       return true;
     }
@@ -503,23 +628,42 @@ class RecordScanner {
    * @param to - the index after the run
    */
   countLineBreaks(chunk, from, to) {
-    let { line, lastWasCr } = this;
-    for (let i = from; i < to; i++) {
-      const c = chunk.charCodeAt(i);
-      if (c === LF) {
-        if (!lastWasCr) {
-          line++;
-        }
-        lastWasCr = false;
-      } else if (c === CR) {
+    const run = chunk.slice(from, to);
+    let { line } = this;
+    let lastCr = this.lastWasCr ? -1 : -2;
+    let lf = run.indexOf("\n");
+    let cr = run.indexOf("\r");
+    while (lf >= 0 || cr >= 0) {
+      if (lf < 0 || cr >= 0 && cr < lf) {
         line++;
-        lastWasCr = true;
+        lastCr = cr;
+        cr = run.indexOf("\r", cr + 1);
       } else {
-        lastWasCr = false;
+        if (lf !== lastCr + 1) {
+          line++;
+        }
+        lf = run.indexOf("\n", lf + 1);
       }
     }
     this.line = line;
-    this.lastWasCr = lastWasCr;
+    this.lastWasCr = lastCr === run.length - 1;
+  }
+  /**
+   * Append to the field or comment in progress, failing with E_TOO_LARGE instead of a RangeError
+   * when the result would be longer than one JavaScript string can hold.
+   * @param text - the text so far
+   * @param piece - the text to append
+   * @returns the joined text
+   */
+  grow(text, piece) {
+    if (text.length + piece.length > MAX_TEXT_LENGTH) {
+      tooLarge(
+        this.reader.report,
+        `a cell is longer than ${MAX_TEXT_LENGTH} characters, the most one JavaScript string holds`,
+        this.startLine
+      );
+    }
+    return text + piece;
   }
   /** Store the field in progress as the next cell. */
   push() {
@@ -557,9 +701,18 @@ class CsvRecordReader {
         delimiter: options.delimiter ?? null,
         quote: '"',
         comments: options.comments,
-        skipQuoteErrors: options.skipQuoteErrors
+        skipQuoteErrors: options.skipQuoteErrors,
+        collapseSpaces: true,
+        sepDirective: true,
+        inspect: options.inspect
       },
-      { signal: options.signal, onProgress: options.onProgress, encoding: options.encoding }
+      {
+        signal: options.signal,
+        onProgress: options.onProgress,
+        encoding: options.encoding,
+        allowEmpty: options.allowEmpty,
+        nulIsBinary: true
+      }
     );
   }
   /**
@@ -569,6 +722,13 @@ class CsvRecordReader {
   get leadingComments() {
     return this.inner.leadingComments;
   }
+  /**
+   * The 1-based line of each leading comment, parallel to `leadingComments`.
+   * @returns the lines
+   */
+  get leadingCommentLines() {
+    return this.inner.leadingCommentLines;
+  }
   /**
    * The 1-based line the most recently yielded row starts on.
    * @returns the line; 0 before the first row
@@ -590,6 +750,20 @@ class CsvRecordReader {
   get quoted() {
     return this.inner.quoted;
   }
+  /**
+   * The line of the first quote inside an unquoted field.
+   * @returns the line, or 0 when there was none
+   */
+  get strayQuoteLine() {
+    return this.inner.strayQuoteLine;
+  }
+  /**
+   * Whether the first record is an Excel `sep=X` line the delimiter was taken from.
+   * @returns true when it is
+   */
+  get sepDirective() {
+    return this.inner.sepDirective;
+  }
   /**
    * Iterate the rows.
    * @yields one row at a time as its cell texts
@@ -598,21 +772,31 @@ class CsvRecordReader {
   async *[Symbol.asyncIterator]() {
     const { inner } = this;
     for await (const count of inner) {
-      if (count === 1 && !inner.quoted[0] && inner.cells[0].trim().length === 0) {
+      if (isBlankRecord(inner.cells, inner.quoted, count)) {
         continue;
       }
       yield inner.cells.slice(0, count);
     }
   }
 }
+function isBlankRecord(cells, quoted, count) {
+  for (let k = 0; k < count; k++) {
+    if (quoted[k] || cells[k].trim().length > 0) {
+      return false;
+    }
+  }
+  return true;
+}
 export {
   BAD_QUOTE_CODE as B,
   CsvRecordReader as C,
   DELIMITER_CANDIDATES as D,
   RecordReader as R,
   UNCLOSED_QUOTE_CODE as U,
-  sniffDelimiter as a,
+  sniffNewline as a,
+  sniffDelimiter as b,
   checkRecordSyntax as c,
-  sniffNewline as s
+  splitRecords as d,
+  stripLeadingComments as s
 };
-//# sourceMappingURL=records-dbRkxwaq.js.map
+//# sourceMappingURL=records-CQXT__mY.js.map