@wix/agent-skills 1.15.1 → 1.16.0

package/skills/wix-replatform/resources/rp-source-csv/lib/csv-parse.js

@@ -0,0 +1,618 @@
+'use strict';
+
+// Shared CSV parse core.
+//
+// This module is the single source of truth for the mechanics a CSV reader must
+// get right: dialect detection, RFC-4180 quoting, line endings, encoding/BOM,
+// the empty-vs-absent decision, and streaming. It is imported by:
+//   - scripts/csv-discovery.js (discovery-time sampling), and
+//   - the bulk-extract reader generated by rp-import-codegen, which vendors a
+//     copy of this file into the migration project so the project stays
+//     self-contained.
+// One parse core is what makes the sampler and the importer agree on what a
+// file contains.
+//
+// Keep this file dependency-free (Node built-ins only) so the vendored copy
+// needs no install step.
+
+const fs = require('node:fs');
+const fsp = require('node:fs/promises');
+const { StringDecoder } = require('node:string_decoder');
+
+const DELIMITER_CANDIDATES = [',', ';', '\t', '|'];
+const DELIMITER_SAMPLE_LINES = 20;
+const DELIMITER_MIN_SCORE = 1.0;
+const DELIMITER_MIN_MARGIN = 1.25;
+const DEFAULT_SAMPLE_BYTES = 256 * 1024;
+const TAIL_SAMPLE_BYTES = 64 * 1024;
+
+// How this writer represents an absent value. Detected per file rather than
+// assumed: exporters that quote every field (Excel, Magento) can never produce
+// an unquoted empty, and exporters that quote nothing unnecessary can never
+// produce a quoted empty. Only a file containing BOTH forms is actually
+// distinguishing them.
+const EMPTY_POLICY = {
+  PRESENT_IF_QUOTED: 'present-if-quoted',
+  ALWAYS_EMPTY: 'always-empty',
+};
+
+const ENCODING_ALIASES = {
+  utf8: 'utf8',
+  'utf-8': 'utf8',
+  ascii: 'ascii',
+  latin1: 'latin1',
+  'iso-8859-1': 'latin1',
+  'windows-1252': 'latin1',
+  cp1252: 'latin1',
+  utf16le: 'utf16le',
+  'utf-16le': 'utf16le',
+};
+
+// Parser states. A field is quoted only when the quote opens at field position
+// 0; a bare `"` anywhere else is a literal character.
+const S_FIELD_START = 0;
+const S_IN_FIELD = 1;
+const S_IN_QUOTED = 2;
+const S_QUOTE_IN_QUOTED = 3;
+
+function nodeEncodingFor(encoding) {
+  const key = String(encoding || 'utf8').toLowerCase();
+  const mapped = ENCODING_ALIASES[key];
+  if (!mapped) {
+    throw new Error(`Unsupported CSV encoding "${encoding}". Set CSV_ENCODING to one of: ${Object.keys(ENCODING_ALIASES).join(', ')}.`);
+  }
+  return mapped;
+}
+
+function stripBom(text) {
+  if (typeof text === 'string' && text.charCodeAt(0) === 0xfeff) {
+    return { text: text.slice(1), bom: 'utf-8' };
+  }
+  return { text, bom: null };
+}
+
+// Byte-order marks decide the encoding before any decoding happens. UTF-16 is
+// reported as unsupported rather than silently decoded as UTF-8 mojibake;
+// full transcoding is deliberately out of scope (see SKILL.md → Encoding).
+function detectEncoding(buffer) {
+  if (!buffer || buffer.length === 0) {
+    return { encoding: 'utf8', bom: null, supported: true };
+  }
+  if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
+    return { encoding: 'utf8', bom: 'utf-8', supported: true };
+  }
+  if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
+    return { encoding: 'utf16le', bom: 'utf-16le', supported: false };
+  }
+  if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) {
+    return { encoding: 'utf16be', bom: 'utf-16be', supported: false };
+  }
+  return { encoding: 'utf8', bom: null, supported: true };
+}
+
+function detectLineEnding(sampleText) {
+  const text = String(sampleText || '');
+  const crlf = (text.match(/\r\n/g) || []).length;
+  const cr = (text.match(/\r(?!\n)/g) || []).length;
+  const lf = (text.match(/(?<!\r)\n/g) || []).length;
+  const ranked = [['\r\n', crlf], ['\n', lf], ['\r', cr]].filter(([, count]) => count > 0);
+  if (ranked.length === 0) {
+    return null;
+  }
+  if (ranked.length > 1) {
+    return 'mixed';
+  }
+  return ranked[0][0];
+}
+
+// The one normalizer. Fingerprint, layout, and fileset all import this rather
+// than rolling their own, so `Body (HTML)`, `body_html`, and `BODY HTML` are
+// the same column everywhere in the adapter.
+function normalizeHeaderName(name) {
+  return String(name === undefined || name === null ? '' : name)
+    .replace(/^\uFEFF/, '')
+    .normalize('NFKC')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '');
+}
+
+// Chunk-fed RFC-4180 state machine.
+//
+// Returns rows as `{ values: string[], quoted?: boolean[], line }`. Values are
+// ALWAYS strings — never null. Whether an empty string means "absent" is a
+// property of the writer's quoting policy, not of the datum, so that decision
+// is made once per file by detectEmptyPolicy and applied by coerceEmpty.
+function createParser({ delimiter = ',', trackQuoted = false, skipEmptyLines = true } = {}) {
+  const delim = String(delimiter);
+  let state = S_FIELD_START;
+  let field = '';
+  let fieldWasQuoted = false;
+  let values = [];
+  let quotedFlags = [];
+  let pendingCR = false;
+  let lineNo = 1;
+  let rowStartLine = 1;
+  let blankLines = 0;
+
+  function endField() {
+    values.push(field);
+    if (trackQuoted) {
+      quotedFlags.push(fieldWasQuoted);
+    }
+    field = '';
+    fieldWasQuoted = false;
+    state = S_FIELD_START;
+  }
+
+  function endRow(rows) {
+    endField();
+    const isBlank = values.length === 1 && values[0] === '' && !(trackQuoted && quotedFlags[0]);
+    if (skipEmptyLines && isBlank) {
+      blankLines += 1;
+    } else {
+      const row = { values, line: rowStartLine };
+      if (trackQuoted) {
+        row.quoted = quotedFlags;
+      }
+      rows.push(row);
+    }
+    values = [];
+    quotedFlags = [];
+    lineNo += 1;
+    rowStartLine = lineNo;
+  }
+
+  function push(text) {
+    const rows = [];
+    const input = String(text);
+    for (let i = 0; i < input.length; i += 1) {
+      const ch = input[i];
+
+      // A lone \r can only be resolved by the next character, which may live in
+      // the next chunk — hence the flag rather than a lookahead.
+      if (pendingCR) {
+        pendingCR = false;
+        if (ch === '\n') {
+          continue;
+        }
+      }
+
+      switch (state) {
+        case S_FIELD_START:
+          if (ch === '"') {
+            state = S_IN_QUOTED;
+            fieldWasQuoted = true;
+          } else if (ch === delim) {
+            endField();
+          } else if (ch === '\n') {
+            endRow(rows);
+          } else if (ch === '\r') {
+            endRow(rows);
+            pendingCR = true;
+          } else {
+            field += ch;
+            state = S_IN_FIELD;
+          }
+          break;
+
+        case S_IN_FIELD:
+          if (ch === delim) {
+            endField();
+          } else if (ch === '\n') {
+            endRow(rows);
+          } else if (ch === '\r') {
+            endRow(rows);
+            pendingCR = true;
+          } else {
+            // A quote inside an unquoted field is a literal character.
+            field += ch;
+          }
+          break;
+
+        case S_IN_QUOTED:
+          if (ch === '"') {
+            state = S_QUOTE_IN_QUOTED;
+          } else {
+            // Newlines inside quotes are data and are preserved verbatim,
+            // including the \r of a \r\n pair.
+            if (ch === '\n') {
+              lineNo += 1;
+            }
+            field += ch;
+          }
+          break;
+
+        case S_QUOTE_IN_QUOTED:
+          if (ch === '"') {
+            field += '"';
+            state = S_IN_QUOTED;
+          } else if (ch === delim) {
+            endField();
+          } else if (ch === '\n') {
+            endRow(rows);
+          } else if (ch === '\r') {
+            endRow(rows);
+            pendingCR = true;
+          } else {
+            field += ch;
+            state = S_IN_FIELD;
+          }
+          break;
+
+        default:
+          break;
+      }
+    }
+    return rows;
+  }
+
+  function end() {
+    const rows = [];
+    const hasPendingContent = values.length > 0
+      || field !== ''
+      || fieldWasQuoted
+      || state === S_IN_QUOTED
+      || state === S_QUOTE_IN_QUOTED;
+    if (hasPendingContent) {
+      endRow(rows);
+    }
+    return rows;
+  }
+
+  function stats() {
+    return { blankLines, unterminatedQuote: state === S_IN_QUOTED };
+  }
+
+  return { push, end, stats };
+}
+
+function parseText(text, { delimiter = ',', trackQuoted = false, skipEmptyLines = true } = {}) {
+  const parser = createParser({ delimiter, trackQuoted, skipEmptyLines });
+  const stripped = stripBom(String(text === undefined || text === null ? '' : text));
+  const rows = parser.push(stripped.text);
+  return rows.concat(parser.end());
+}
+
+function quoteAwareFieldCounts(sampleText, delimiter, maxLines) {
+  const rows = parseText(sampleText, { delimiter });
+  return rows.slice(0, maxLines).map((row) => row.values.length);
+}
+
+// Score each candidate by how consistently it splits the sample AND by how many
+// fields it produces. Consistency alone is not enough: a semicolon file whose
+// text fields each contain one comma splits perfectly consistently on `,` too.
+function detectDelimiter(sampleText, { candidates = DELIMITER_CANDIDATES, maxLines = DELIMITER_SAMPLE_LINES } = {}) {
+  const scored = candidates.map((delimiter) => {
+    const counts = quoteAwareFieldCounts(sampleText, delimiter, maxLines);
+    if (counts.length === 0 || counts[0] < 2) {
+      return { delimiter, fieldCount: counts[0] || 0, consistency: 0, score: 0 };
+    }
+    const consistency = counts.filter((count) => count === counts[0]).length / counts.length;
+    return {
+      delimiter,
+      fieldCount: counts[0],
+      consistency,
+      score: consistency * Math.log2(counts[0]),
+    };
+  });
+
+  const ranked = [...scored].sort((a, b) => b.score - a.score
+    || candidates.indexOf(a.delimiter) - candidates.indexOf(b.delimiter));
+  const winner = ranked[0];
+  const runnerUp = ranked[1];
+  const clearsScore = winner.score >= DELIMITER_MIN_SCORE;
+  const clearsMargin = !runnerUp || runnerUp.score === 0 || winner.score / runnerUp.score >= DELIMITER_MIN_MARGIN;
+  const ambiguous = !(clearsScore && clearsMargin);
+
+  return {
+    delimiter: ambiguous ? (candidates[0] || ',') : winner.delimiter,
+    confidence: ambiguous ? Math.min(0.5, winner.score / 2) : Math.min(1, 0.6 + winner.score / 10),
+    ambiguous,
+    candidates: ranked,
+  };
+}
+
+async function readSample(filePath, { bytes = DEFAULT_SAMPLE_BYTES, encoding = null } = {}) {
+  const handle = await fsp.open(filePath, 'r');
+  try {
+    const stat = await handle.stat();
+    const length = Math.min(bytes, stat.size);
+    const buffer = Buffer.alloc(length);
+    await handle.read(buffer, 0, length, 0);
+    const detected = detectEncoding(buffer);
+    if (!encoding && !detected.supported) {
+      throw new Error(`${filePath} looks like ${detected.encoding} (BOM ${detected.bom}); only UTF-8 family encodings are supported. Set CSV_ENCODING or convert the file to UTF-8.`);
+    }
+    const decoder = new StringDecoder(nodeEncodingFor(encoding || detected.encoding));
+    const decoded = stripBom(decoder.write(buffer) + decoder.end());
+    const truncated = length < stat.size;
+    // A truncated sample almost certainly ends mid-line; dropping the tail keeps
+    // field-count statistics honest.
+    const text = truncated ? decoded.text.slice(0, decoded.text.lastIndexOf('\n') + 1) : decoded.text;
+    return {
+      text,
+      bom: detected.bom,
+      encoding: encoding || detected.encoding,
+      truncated,
+      totalBytes: stat.size,
+    };
+  } finally {
+    await handle.close();
+  }
+}
+
+async function readHeaderRow(filePath, { delimiter = null, encoding = null, sampleBytes = TAIL_SAMPLE_BYTES } = {}) {
+  const sample = await readSample(filePath, { bytes: sampleBytes, encoding });
+  const dialect = delimiter
+    ? { delimiter, confidence: 1, ambiguous: false, candidates: [] }
+    : detectDelimiter(sample.text);
+  const rows = parseText(sample.text, { delimiter: dialect.delimiter });
+  const header = rows.length > 0 ? rows[0].values : [];
+  return {
+    header,
+    delimiter: dialect.delimiter,
+    delimiterAmbiguous: dialect.ambiguous,
+    delimiterCandidates: dialect.candidates,
+    encoding: sample.encoding,
+    bom: sample.bom,
+    lineEnding: detectLineEnding(sample.text),
+    rawHeaderLine: sample.text.split(/\r\n|\n|\r/)[0] || '',
+  };
+}
+
+// Read the last window of the file for tail samples. A quoted field containing
+// a newline can straddle the window boundary and misparse silently, so the
+// sample is only accepted when the quote count is even and every row has the
+// header's width.
+async function readTailSample(filePath, { bytes = TAIL_SAMPLE_BYTES, delimiter = ',', encoding = 'utf8', header = [] } = {}) {
+  const handle = await fsp.open(filePath, 'r');
+  try {
+    const stat = await handle.stat();
+    if (stat.size === 0) {
+      return { rows: [], skipped: false, reason: null };
+    }
+    const length = Math.min(bytes, stat.size);
+    const position = stat.size - length;
+    const buffer = Buffer.alloc(length);
+    await handle.read(buffer, 0, length, position);
+    const decoder = new StringDecoder(nodeEncodingFor(encoding));
+    let text = decoder.write(buffer) + decoder.end();
+    if (position > 0) {
+      const firstBreak = text.indexOf('\n');
+      if (firstBreak === -1) {
+        return { rows: [], skipped: true, reason: 'no-line-break-in-window' };
+      }
+      text = text.slice(firstBreak + 1);
+    } else {
+      text = stripBom(text).text;
+    }
+    const quoteCount = (text.match(/"/g) || []).length;
+    if (position > 0 && quoteCount % 2 !== 0) {
+      return { rows: [], skipped: true, reason: 'odd-quote-count-in-window' };
+    }
+    const rows = parseText(text, { delimiter });
+    const body = position > 0 ? rows : rows.slice(1);
+    if (header.length > 0 && body.some((row) => row.values.length !== header.length)) {
+      return { rows: [], skipped: true, reason: 'ragged-rows-in-window' };
+    }
+    return { rows: body, skipped: false, reason: null };
+  } finally {
+    await handle.close();
+  }
+}
+
+async function* streamRows(filePath, {
+  delimiter = null,
+  encoding = null,
+  trackQuoted = false,
+  skipEmptyLines = true,
+  maxRows = null,
+} = {}) {
+  let resolvedDelimiter = delimiter;
+  let resolvedEncoding = encoding;
+  if (!resolvedDelimiter || !resolvedEncoding) {
+    const sample = await readSample(filePath, { encoding });
+    resolvedEncoding = resolvedEncoding || sample.encoding;
+    resolvedDelimiter = resolvedDelimiter || detectDelimiter(sample.text).delimiter;
+  }
+
+  const decoder = new StringDecoder(nodeEncodingFor(resolvedEncoding));
+  const parser = createParser({ delimiter: resolvedDelimiter, trackQuoted, skipEmptyLines });
+  const stream = fs.createReadStream(filePath);
+  let emitted = 0;
+  let first = true;
+
+  for await (const chunk of stream) {
+    let text = decoder.write(chunk);
+    if (first) {
+      text = stripBom(text).text;
+      first = false;
+    }
+    for (const row of parser.push(text)) {
+      yield row;
+      emitted += 1;
+      if (maxRows !== null && emitted >= maxRows) {
+        stream.destroy();
+        return;
+      }
+    }
+  }
+
+  const tail = decoder.end();
+  const pending = tail ? parser.push(tail) : [];
+  for (const row of pending.concat(parser.end())) {
+    yield row;
+    emitted += 1;
+    if (maxRows !== null && emitted >= maxRows) {
+      return;
+    }
+  }
+}
+
+async function* streamRecords(filePath, options = {}) {
+  let header = options.header || null;
+  for await (const row of streamRows(filePath, options)) {
+    if (!header) {
+      header = row.values;
+      continue;
+    }
+    yield { record: toRecord(header, row.values), row };
+  }
+}
+
+function toRecord(header, values) {
+  const record = {};
+  for (let i = 0; i < header.length; i += 1) {
+    record[header[i]] = i < values.length ? values[i] : '';
+  }
+  return record;
+}
+
+function rowWidthReport(header, values) {
+  return {
+    expected: header.length,
+    actual: values.length,
+    padded: values.length < header.length,
+    truncated: values.length > header.length,
+    ragged: values.length !== header.length,
+  };
+}
+
+// Requires rows parsed with `trackQuoted: true`.
+function detectEmptyPolicy(rows) {
+  let sawQuotedEmpty = false;
+  let sawUnquotedEmpty = false;
+  for (const row of rows) {
+    const quoted = row.quoted || [];
+    for (let i = 0; i < row.values.length; i += 1) {
+      if (row.values[i] !== '') {
+        continue;
+      }
+      if (quoted[i]) {
+        sawQuotedEmpty = true;
+      } else {
+        sawUnquotedEmpty = true;
+      }
+      if (sawQuotedEmpty && sawUnquotedEmpty) {
+        return EMPTY_POLICY.PRESENT_IF_QUOTED;
+      }
+    }
+  }
+  return EMPTY_POLICY.ALWAYS_EMPTY;
+}
+
+// The one place empty-vs-absent is decided. Generated readers call this so a
+// required Wix field is never fed an empty string the source did not have.
+function coerceEmpty(value, isQuoted, policy) {
+  if (value !== '') {
+    return value;
+  }
+  if (policy === EMPTY_POLICY.PRESENT_IF_QUOTED) {
+    return isQuoted ? '' : null;
+  }
+  return '';
+}
+
+const TYPE_TESTS = [
+  ['integer', (value) => /^-?\d+$/.test(value)],
+  ['number', (value) => /^-?(\d+\.\d*|\.\d+|\d+)$/.test(value)],
+  ['date-time', (value) => /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(value)],
+  ['date', (value) => /^\d{4}-\d{2}-\d{2}$/.test(value)],
+  ['url', (value) => /^https?:\/\/\S+$/i.test(value)],
+  ['email', (value) => /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(value)],
+];
+
+const BOOLEAN_VALUES = new Set(['true', 'false', 'yes', 'no']);
+
+function inferValueType(values) {
+  const present = values.filter((value) => typeof value === 'string' && value.trim() !== '');
+  if (present.length === 0) {
+    return 'unknown';
+  }
+  if (present.every((value) => BOOLEAN_VALUES.has(value.trim().toLowerCase()))) {
+    return 'boolean';
+  }
+  for (const [type, test] of TYPE_TESTS) {
+    if (present.every((value) => test(value.trim()))) {
+      return type;
+    }
+  }
+  return 'string';
+}
+
+function summarizeColumn(name, samples, { maxExamples = 3, distinctCap = 5000 } = {}) {
+  const distinct = new Set();
+  let blankCount = 0;
+  let quotedEmptyCount = 0;
+  let maxLength = 0;
+  let multiValueHint = false;
+  const examples = [];
+  const present = [];
+
+  for (const sample of samples) {
+    const value = typeof sample === 'string' ? sample : sample.value;
+    const isQuoted = typeof sample === 'string' ? false : Boolean(sample.quoted);
+    if (value === '' || value === null || value === undefined) {
+      blankCount += 1;
+      if (isQuoted) {
+        quotedEmptyCount += 1;
+      }
+      continue;
+    }
+    present.push(value);
+    if (distinct.size < distinctCap) {
+      distinct.add(value);
+    }
+    maxLength = Math.max(maxLength, value.length);
+    if (!multiValueHint && /[,;|]/.test(value) && value.length < 200) {
+      multiValueHint = true;
+    }
+    if (examples.length < maxExamples) {
+      examples.push(value);
+    }
+  }
+
+  const total = samples.length;
+  return {
+    name,
+    type: inferValueType(present),
+    required: total > 0 && blankCount === 0,
+    blankCount,
+    quotedEmptyCount,
+    sampled: total,
+    distinctCount: distinct.size,
+    distinctCapped: distinct.size >= distinctCap,
+    unique: present.length > 0 && distinct.size === present.length && distinct.size < distinctCap,
+    maxLength,
+    multiValueHint,
+    examples,
+  };
+}
+
+module.exports = {
+  DELIMITER_CANDIDATES,
+  DELIMITER_SAMPLE_LINES,
+  DELIMITER_MIN_SCORE,
+  DELIMITER_MIN_MARGIN,
+  DEFAULT_SAMPLE_BYTES,
+  TAIL_SAMPLE_BYTES,
+  EMPTY_POLICY,
+  createParser,
+  parseText,
+  detectDelimiter,
+  detectEncoding,
+  detectLineEnding,
+  stripBom,
+  normalizeHeaderName,
+  readSample,
+  readHeaderRow,
+  readTailSample,
+  streamRows,
+  streamRecords,
+  toRecord,
+  rowWidthReport,
+  detectEmptyPolicy,
+  coerceEmpty,
+  inferValueType,
+  summarizeColumn,
+};