@wix/agent-skills 1.35.1 → 1.36.1

package/skills/wix-replatform/resources/rp-source-csv/lib/csv-parse.js

@@ -1,618 +0,0 @@
-'use strict';
-
-// Shared CSV parse core.
-//
-// This module is the single source of truth for the mechanics a CSV reader must
-// get right: dialect detection, RFC-4180 quoting, line endings, encoding/BOM,
-// the empty-vs-absent decision, and streaming. It is imported by:
-//   - scripts/csv-discovery.js (discovery-time sampling), and
-//   - the bulk-extract reader generated by rp-import-codegen, which vendors a
-//     copy of this file into the migration project so the project stays
-//     self-contained.
-// One parse core is what makes the sampler and the importer agree on what a
-// file contains.
-//
-// Keep this file dependency-free (Node built-ins only) so the vendored copy
-// needs no install step.
-
-const fs = require('node:fs');
-const fsp = require('node:fs/promises');
-const { StringDecoder } = require('node:string_decoder');
-
-const DELIMITER_CANDIDATES = [',', ';', '\t', '|'];
-const DELIMITER_SAMPLE_LINES = 20;
-const DELIMITER_MIN_SCORE = 1.0;
-const DELIMITER_MIN_MARGIN = 1.25;
-const DEFAULT_SAMPLE_BYTES = 256 * 1024;
-const TAIL_SAMPLE_BYTES = 64 * 1024;
-
-// How this writer represents an absent value. Detected per file rather than
-// assumed: exporters that quote every field (Excel, Magento) can never produce
-// an unquoted empty, and exporters that quote nothing unnecessary can never
-// produce a quoted empty. Only a file containing BOTH forms is actually
-// distinguishing them.
-const EMPTY_POLICY = {
-  PRESENT_IF_QUOTED: 'present-if-quoted',
-  ALWAYS_EMPTY: 'always-empty',
-};
-
-const ENCODING_ALIASES = {
-  utf8: 'utf8',
-  'utf-8': 'utf8',
-  ascii: 'ascii',
-  latin1: 'latin1',
-  'iso-8859-1': 'latin1',
-  'windows-1252': 'latin1',
-  cp1252: 'latin1',
-  utf16le: 'utf16le',
-  'utf-16le': 'utf16le',
-};
-
-// Parser states. A field is quoted only when the quote opens at field position
-// 0; a bare `"` anywhere else is a literal character.
-const S_FIELD_START = 0;
-const S_IN_FIELD = 1;
-const S_IN_QUOTED = 2;
-const S_QUOTE_IN_QUOTED = 3;
-
-function nodeEncodingFor(encoding) {
-  const key = String(encoding || 'utf8').toLowerCase();
-  const mapped = ENCODING_ALIASES[key];
-  if (!mapped) {
-    throw new Error(`Unsupported CSV encoding "${encoding}". Set CSV_ENCODING to one of: ${Object.keys(ENCODING_ALIASES).join(', ')}.`);
-  }
-  return mapped;
-}
-
-function stripBom(text) {
-  if (typeof text === 'string' && text.charCodeAt(0) === 0xfeff) {
-    return { text: text.slice(1), bom: 'utf-8' };
-  }
-  return { text, bom: null };
-}
-
-// Byte-order marks decide the encoding before any decoding happens. UTF-16 is
-// reported as unsupported rather than silently decoded as UTF-8 mojibake;
-// full transcoding is deliberately out of scope (see SKILL.md → Encoding).
-function detectEncoding(buffer) {
-  if (!buffer || buffer.length === 0) {
-    return { encoding: 'utf8', bom: null, supported: true };
-  }
-  if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
-    return { encoding: 'utf8', bom: 'utf-8', supported: true };
-  }
-  if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
-    return { encoding: 'utf16le', bom: 'utf-16le', supported: false };
-  }
-  if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) {
-    return { encoding: 'utf16be', bom: 'utf-16be', supported: false };
-  }
-  return { encoding: 'utf8', bom: null, supported: true };
-}
-
-function detectLineEnding(sampleText) {
-  const text = String(sampleText || '');
-  const crlf = (text.match(/\r\n/g) || []).length;
-  const cr = (text.match(/\r(?!\n)/g) || []).length;
-  const lf = (text.match(/(?<!\r)\n/g) || []).length;
-  const ranked = [['\r\n', crlf], ['\n', lf], ['\r', cr]].filter(([, count]) => count > 0);
-  if (ranked.length === 0) {
-    return null;
-  }
-  if (ranked.length > 1) {
-    return 'mixed';
-  }
-  return ranked[0][0];
-}
-
-// The one normalizer. Fingerprint, layout, and fileset all import this rather
-// than rolling their own, so `Body (HTML)`, `body_html`, and `BODY HTML` are
-// the same column everywhere in the adapter.
-function normalizeHeaderName(name) {
-  return String(name === undefined || name === null ? '' : name)
-    .replace(/^\uFEFF/, '')
-    .normalize('NFKC')
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, '');
-}
-
-// Chunk-fed RFC-4180 state machine.
-//
-// Returns rows as `{ values: string[], quoted?: boolean[], line }`. Values are
-// ALWAYS strings — never null. Whether an empty string means "absent" is a
-// property of the writer's quoting policy, not of the datum, so that decision
-// is made once per file by detectEmptyPolicy and applied by coerceEmpty.
-function createParser({ delimiter = ',', trackQuoted = false, skipEmptyLines = true } = {}) {
-  const delim = String(delimiter);
-  let state = S_FIELD_START;
-  let field = '';
-  let fieldWasQuoted = false;
-  let values = [];
-  let quotedFlags = [];
-  let pendingCR = false;
-  let lineNo = 1;
-  let rowStartLine = 1;
-  let blankLines = 0;
-
-  function endField() {
-    values.push(field);
-    if (trackQuoted) {
-      quotedFlags.push(fieldWasQuoted);
-    }
-    field = '';
-    fieldWasQuoted = false;
-    state = S_FIELD_START;
-  }
-
-  function endRow(rows) {
-    endField();
-    const isBlank = values.length === 1 && values[0] === '' && !(trackQuoted && quotedFlags[0]);
-    if (skipEmptyLines && isBlank) {
-      blankLines += 1;
-    } else {
-      const row = { values, line: rowStartLine };
-      if (trackQuoted) {
-        row.quoted = quotedFlags;
-      }
-      rows.push(row);
-    }
-    values = [];
-    quotedFlags = [];
-    lineNo += 1;
-    rowStartLine = lineNo;
-  }
-
-  function push(text) {
-    const rows = [];
-    const input = String(text);
-    for (let i = 0; i < input.length; i += 1) {
-      const ch = input[i];
-
-      // A lone \r can only be resolved by the next character, which may live in
-      // the next chunk — hence the flag rather than a lookahead.
-      if (pendingCR) {
-        pendingCR = false;
-        if (ch === '\n') {
-          continue;
-        }
-      }
-
-      switch (state) {
-        case S_FIELD_START:
-          if (ch === '"') {
-            state = S_IN_QUOTED;
-            fieldWasQuoted = true;
-          } else if (ch === delim) {
-            endField();
-          } else if (ch === '\n') {
-            endRow(rows);
-          } else if (ch === '\r') {
-            endRow(rows);
-            pendingCR = true;
-          } else {
-            field += ch;
-            state = S_IN_FIELD;
-          }
-          break;
-
-        case S_IN_FIELD:
-          if (ch === delim) {
-            endField();
-          } else if (ch === '\n') {
-            endRow(rows);
-          } else if (ch === '\r') {
-            endRow(rows);
-            pendingCR = true;
-          } else {
-            // A quote inside an unquoted field is a literal character.
-            field += ch;
-          }
-          break;
-
-        case S_IN_QUOTED:
-          if (ch === '"') {
-            state = S_QUOTE_IN_QUOTED;
-          } else {
-            // Newlines inside quotes are data and are preserved verbatim,
-            // including the \r of a \r\n pair.
-            if (ch === '\n') {
-              lineNo += 1;
-            }
-            field += ch;
-          }
-          break;
-
-        case S_QUOTE_IN_QUOTED:
-          if (ch === '"') {
-            field += '"';
-            state = S_IN_QUOTED;
-          } else if (ch === delim) {
-            endField();
-          } else if (ch === '\n') {
-            endRow(rows);
-          } else if (ch === '\r') {
-            endRow(rows);
-            pendingCR = true;
-          } else {
-            field += ch;
-            state = S_IN_FIELD;
-          }
-          break;
-
-        default:
-          break;
-      }
-    }
-    return rows;
-  }
-
-  function end() {
-    const rows = [];
-    const hasPendingContent = values.length > 0
-      || field !== ''
-      || fieldWasQuoted
-      || state === S_IN_QUOTED
-      || state === S_QUOTE_IN_QUOTED;
-    if (hasPendingContent) {
-      endRow(rows);
-    }
-    return rows;
-  }
-
-  function stats() {
-    return { blankLines, unterminatedQuote: state === S_IN_QUOTED };
-  }
-
-  return { push, end, stats };
-}
-
-function parseText(text, { delimiter = ',', trackQuoted = false, skipEmptyLines = true } = {}) {
-  const parser = createParser({ delimiter, trackQuoted, skipEmptyLines });
-  const stripped = stripBom(String(text === undefined || text === null ? '' : text));
-  const rows = parser.push(stripped.text);
-  return rows.concat(parser.end());
-}
-
-function quoteAwareFieldCounts(sampleText, delimiter, maxLines) {
-  const rows = parseText(sampleText, { delimiter });
-  return rows.slice(0, maxLines).map((row) => row.values.length);
-}
-
-// Score each candidate by how consistently it splits the sample AND by how many
-// fields it produces. Consistency alone is not enough: a semicolon file whose
-// text fields each contain one comma splits perfectly consistently on `,` too.
-function detectDelimiter(sampleText, { candidates = DELIMITER_CANDIDATES, maxLines = DELIMITER_SAMPLE_LINES } = {}) {
-  const scored = candidates.map((delimiter) => {
-    const counts = quoteAwareFieldCounts(sampleText, delimiter, maxLines);
-    if (counts.length === 0 || counts[0] < 2) {
-      return { delimiter, fieldCount: counts[0] || 0, consistency: 0, score: 0 };
-    }
-    const consistency = counts.filter((count) => count === counts[0]).length / counts.length;
-    return {
-      delimiter,
-      fieldCount: counts[0],
-      consistency,
-      score: consistency * Math.log2(counts[0]),
-    };
-  });
-
-  const ranked = [...scored].sort((a, b) => b.score - a.score
-    || candidates.indexOf(a.delimiter) - candidates.indexOf(b.delimiter));
-  const winner = ranked[0];
-  const runnerUp = ranked[1];
-  const clearsScore = winner.score >= DELIMITER_MIN_SCORE;
-  const clearsMargin = !runnerUp || runnerUp.score === 0 || winner.score / runnerUp.score >= DELIMITER_MIN_MARGIN;
-  const ambiguous = !(clearsScore && clearsMargin);
-
-  return {
-    delimiter: ambiguous ? (candidates[0] || ',') : winner.delimiter,
-    confidence: ambiguous ? Math.min(0.5, winner.score / 2) : Math.min(1, 0.6 + winner.score / 10),
-    ambiguous,
-    candidates: ranked,
-  };
-}
-
-async function readSample(filePath, { bytes = DEFAULT_SAMPLE_BYTES, encoding = null } = {}) {
-  const handle = await fsp.open(filePath, 'r');
-  try {
-    const stat = await handle.stat();
-    const length = Math.min(bytes, stat.size);
-    const buffer = Buffer.alloc(length);
-    await handle.read(buffer, 0, length, 0);
-    const detected = detectEncoding(buffer);
-    if (!encoding && !detected.supported) {
-      throw new Error(`${filePath} looks like ${detected.encoding} (BOM ${detected.bom}); only UTF-8 family encodings are supported. Set CSV_ENCODING or convert the file to UTF-8.`);
-    }
-    const decoder = new StringDecoder(nodeEncodingFor(encoding || detected.encoding));
-    const decoded = stripBom(decoder.write(buffer) + decoder.end());
-    const truncated = length < stat.size;
-    // A truncated sample almost certainly ends mid-line; dropping the tail keeps
-    // field-count statistics honest.
-    const text = truncated ? decoded.text.slice(0, decoded.text.lastIndexOf('\n') + 1) : decoded.text;
-    return {
-      text,
-      bom: detected.bom,
-      encoding: encoding || detected.encoding,
-      truncated,
-      totalBytes: stat.size,
-    };
-  } finally {
-    await handle.close();
-  }
-}
-
-async function readHeaderRow(filePath, { delimiter = null, encoding = null, sampleBytes = TAIL_SAMPLE_BYTES } = {}) {
-  const sample = await readSample(filePath, { bytes: sampleBytes, encoding });
-  const dialect = delimiter
-    ? { delimiter, confidence: 1, ambiguous: false, candidates: [] }
-    : detectDelimiter(sample.text);
-  const rows = parseText(sample.text, { delimiter: dialect.delimiter });
-  const header = rows.length > 0 ? rows[0].values : [];
-  return {
-    header,
-    delimiter: dialect.delimiter,
-    delimiterAmbiguous: dialect.ambiguous,
-    delimiterCandidates: dialect.candidates,
-    encoding: sample.encoding,
-    bom: sample.bom,
-    lineEnding: detectLineEnding(sample.text),
-    rawHeaderLine: sample.text.split(/\r\n|\n|\r/)[0] || '',
-  };
-}
-
-// Read the last window of the file for tail samples. A quoted field containing
-// a newline can straddle the window boundary and misparse silently, so the
-// sample is only accepted when the quote count is even and every row has the
-// header's width.
-async function readTailSample(filePath, { bytes = TAIL_SAMPLE_BYTES, delimiter = ',', encoding = 'utf8', header = [] } = {}) {
-  const handle = await fsp.open(filePath, 'r');
-  try {
-    const stat = await handle.stat();
-    if (stat.size === 0) {
-      return { rows: [], skipped: false, reason: null };
-    }
-    const length = Math.min(bytes, stat.size);
-    const position = stat.size - length;
-    const buffer = Buffer.alloc(length);
-    await handle.read(buffer, 0, length, position);
-    const decoder = new StringDecoder(nodeEncodingFor(encoding));
-    let text = decoder.write(buffer) + decoder.end();
-    if (position > 0) {
-      const firstBreak = text.indexOf('\n');
-      if (firstBreak === -1) {
-        return { rows: [], skipped: true, reason: 'no-line-break-in-window' };
-      }
-      text = text.slice(firstBreak + 1);
-    } else {
-      text = stripBom(text).text;
-    }
-    const quoteCount = (text.match(/"/g) || []).length;
-    if (position > 0 && quoteCount % 2 !== 0) {
-      return { rows: [], skipped: true, reason: 'odd-quote-count-in-window' };
-    }
-    const rows = parseText(text, { delimiter });
-    const body = position > 0 ? rows : rows.slice(1);
-    if (header.length > 0 && body.some((row) => row.values.length !== header.length)) {
-      return { rows: [], skipped: true, reason: 'ragged-rows-in-window' };
-    }
-    return { rows: body, skipped: false, reason: null };
-  } finally {
-    await handle.close();
-  }
-}
-
-async function* streamRows(filePath, {
-  delimiter = null,
-  encoding = null,
-  trackQuoted = false,
-  skipEmptyLines = true,
-  maxRows = null,
-} = {}) {
-  let resolvedDelimiter = delimiter;
-  let resolvedEncoding = encoding;
-  if (!resolvedDelimiter || !resolvedEncoding) {
-    const sample = await readSample(filePath, { encoding });
-    resolvedEncoding = resolvedEncoding || sample.encoding;
-    resolvedDelimiter = resolvedDelimiter || detectDelimiter(sample.text).delimiter;
-  }
-
-  const decoder = new StringDecoder(nodeEncodingFor(resolvedEncoding));
-  const parser = createParser({ delimiter: resolvedDelimiter, trackQuoted, skipEmptyLines });
-  const stream = fs.createReadStream(filePath);
-  let emitted = 0;
-  let first = true;
-
-  for await (const chunk of stream) {
-    let text = decoder.write(chunk);
-    if (first) {
-      text = stripBom(text).text;
-      first = false;
-    }
-    for (const row of parser.push(text)) {
-      yield row;
-      emitted += 1;
-      if (maxRows !== null && emitted >= maxRows) {
-        stream.destroy();
-        return;
-      }
-    }
-  }
-
-  const tail = decoder.end();
-  const pending = tail ? parser.push(tail) : [];
-  for (const row of pending.concat(parser.end())) {
-    yield row;
-    emitted += 1;
-    if (maxRows !== null && emitted >= maxRows) {
-      return;
-    }
-  }
-}
-
-async function* streamRecords(filePath, options = {}) {
-  let header = options.header || null;
-  for await (const row of streamRows(filePath, options)) {
-    if (!header) {
-      header = row.values;
-      continue;
-    }
-    yield { record: toRecord(header, row.values), row };
-  }
-}
-
-function toRecord(header, values) {
-  const record = {};
-  for (let i = 0; i < header.length; i += 1) {
-    record[header[i]] = i < values.length ? values[i] : '';
-  }
-  return record;
-}
-
-function rowWidthReport(header, values) {
-  return {
-    expected: header.length,
-    actual: values.length,
-    padded: values.length < header.length,
-    truncated: values.length > header.length,
-    ragged: values.length !== header.length,
-  };
-}
-
-// Requires rows parsed with `trackQuoted: true`.
-function detectEmptyPolicy(rows) {
-  let sawQuotedEmpty = false;
-  let sawUnquotedEmpty = false;
-  for (const row of rows) {
-    const quoted = row.quoted || [];
-    for (let i = 0; i < row.values.length; i += 1) {
-      if (row.values[i] !== '') {
-        continue;
-      }
-      if (quoted[i]) {
-        sawQuotedEmpty = true;
-      } else {
-        sawUnquotedEmpty = true;
-      }
-      if (sawQuotedEmpty && sawUnquotedEmpty) {
-        return EMPTY_POLICY.PRESENT_IF_QUOTED;
-      }
-    }
-  }
-  return EMPTY_POLICY.ALWAYS_EMPTY;
-}
-
-// The one place empty-vs-absent is decided. Generated readers call this so a
-// required Wix field is never fed an empty string the source did not have.
-function coerceEmpty(value, isQuoted, policy) {
-  if (value !== '') {
-    return value;
-  }
-  if (policy === EMPTY_POLICY.PRESENT_IF_QUOTED) {
-    return isQuoted ? '' : null;
-  }
-  return '';
-}
-
-const TYPE_TESTS = [
-  ['integer', (value) => /^-?\d+$/.test(value)],
-  ['number', (value) => /^-?(\d+\.\d*|\.\d+|\d+)$/.test(value)],
-  ['date-time', (value) => /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(value)],
-  ['date', (value) => /^\d{4}-\d{2}-\d{2}$/.test(value)],
-  ['url', (value) => /^https?:\/\/\S+$/i.test(value)],
-  ['email', (value) => /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(value)],
-];
-
-const BOOLEAN_VALUES = new Set(['true', 'false', 'yes', 'no']);
-
-function inferValueType(values) {
-  const present = values.filter((value) => typeof value === 'string' && value.trim() !== '');
-  if (present.length === 0) {
-    return 'unknown';
-  }
-  if (present.every((value) => BOOLEAN_VALUES.has(value.trim().toLowerCase()))) {
-    return 'boolean';
-  }
-  for (const [type, test] of TYPE_TESTS) {
-    if (present.every((value) => test(value.trim()))) {
-      return type;
-    }
-  }
-  return 'string';
-}
-
-function summarizeColumn(name, samples, { maxExamples = 3, distinctCap = 5000 } = {}) {
-  const distinct = new Set();
-  let blankCount = 0;
-  let quotedEmptyCount = 0;
-  let maxLength = 0;
-  let multiValueHint = false;
-  const examples = [];
-  const present = [];
-
-  for (const sample of samples) {
-    const value = typeof sample === 'string' ? sample : sample.value;
-    const isQuoted = typeof sample === 'string' ? false : Boolean(sample.quoted);
-    if (value === '' || value === null || value === undefined) {
-      blankCount += 1;
-      if (isQuoted) {
-        quotedEmptyCount += 1;
-      }
-      continue;
-    }
-    present.push(value);
-    if (distinct.size < distinctCap) {
-      distinct.add(value);
-    }
-    maxLength = Math.max(maxLength, value.length);
-    if (!multiValueHint && /[,;|]/.test(value) && value.length < 200) {
-      multiValueHint = true;
-    }
-    if (examples.length < maxExamples) {
-      examples.push(value);
-    }
-  }
-
-  const total = samples.length;
-  return {
-    name,
-    type: inferValueType(present),
-    required: total > 0 && blankCount === 0,
-    blankCount,
-    quotedEmptyCount,
-    sampled: total,
-    distinctCount: distinct.size,
-    distinctCapped: distinct.size >= distinctCap,
-    unique: present.length > 0 && distinct.size === present.length && distinct.size < distinctCap,
-    maxLength,
-    multiValueHint,
-    examples,
-  };
-}
-
-module.exports = {
-  DELIMITER_CANDIDATES,
-  DELIMITER_SAMPLE_LINES,
-  DELIMITER_MIN_SCORE,
-  DELIMITER_MIN_MARGIN,
-  DEFAULT_SAMPLE_BYTES,
-  TAIL_SAMPLE_BYTES,
-  EMPTY_POLICY,
-  createParser,
-  parseText,
-  detectDelimiter,
-  detectEncoding,
-  detectLineEnding,
-  stripBom,
-  normalizeHeaderName,
-  readSample,
-  readHeaderRow,
-  readTailSample,
-  streamRows,
-  streamRecords,
-  toRecord,
-  rowWidthReport,
-  detectEmptyPolicy,
-  coerceEmpty,
-  inferValueType,
-  summarizeColumn,
-};