mevebu 1.11.0 → 2.1.0

package/mevebu/libs/js/tools/csv-viewer.js

@@ -1,56 +1,6 @@
 // Reads CSV or delimited text as a table, with row and column counts.
 import { el, bench, stack, panel, textArea, selectInput, fileInput, notice, copyButton, button, actions, toolcard } from "../core.js";
-
-function parseDelimited(text, delimiter) {
-  const rows = [];
-  let row = [];
-  let field = "";
-  let quoted = false;
-  for (let i = 0; i < text.length; i++) {
-    const char = text[i];
-    if (quoted) {
-      if (char === '"') {
-        if (text[i + 1] === '"') { field += '"'; i++; }
-        else quoted = false;
-      } else field += char;
-    } else if (char === '"') {
-      quoted = true;
-    } else if (char === delimiter) {
-      row.push(field);
-      field = "";
-    } else if (char === "\n") {
-      row.push(field);
-      rows.push(row);
-      row = [];
-      field = "";
-    } else if (char !== "\r") {
-      field += char;
-    }
-  }
-  if (field !== "" || row.length) {
-    row.push(field);
-    rows.push(row);
-  }
-  return rows;
-}
-
-function detectDelimiter(text) {
-  const candidates = [",", ";", "\t", "|"];
-  const first = text.split(/\r?\n/)[0] ?? "";
-  const counts = { ",": 0, ";": 0, "\t": 0, "|": 0 };
-  let quoted = false;
-  for (let i = 0; i < first.length; i++) {
-    const char = first[i];
-    if (char === '"') quoted = !quoted;
-    else if (!quoted && char in counts) counts[char]++;
-  }
-  let best = ",";
-  let bestCount = 0;
-  for (const candidate of candidates) {
-    if (counts[candidate] > bestCount) { bestCount = counts[candidate]; best = candidate; }
-  }
-  return best;
-}
+import { parseDelimited, detectDelimiter } from "../csv.js";
 
 function tile(label) {
   const value = el("span", { class: "wrap-anywhere text-xl font-bold tracking-[-0.02em] tabular-nums" }, "0");

package/mevebu/libs/js/tools/data-dictionary.js

@@ -0,0 +1,204 @@
+// The document a dataset needs before anyone uses it: one row per column, with
+// the type, the null count, the distinct count, the range and a sample.
+import {
+  el, toast, download, bench, stack, panel, button, actions, toolcard, blank,
+  notice, output,
+} from "../core.js";
+import { dropzone, baseName, formatBytes } from "../files.js";
+import { parseDelimited, detectDelimiter, headerKeys, toCsv } from "../csv.js";
+
+const TYPE_LABELS = {
+  number: "number",
+  date: "date",
+  boolean: "boolean",
+  text: "text",
+  empty: "empty",
+};
+
+function guessType(values) {
+  const sample = values.filter((value) => value !== "");
+  if (!sample.length) return "empty";
+  if (sample.every((value) => /^-?\d+(\.\d+)?$/.test(value))) return "number";
+  if (sample.every((value) => /^\d{4}-\d{2}-\d{2}$/.test(value))) return "date";
+  if (sample.every((value) => /^(true|false)$/i.test(value))) return "boolean";
+  return "text";
+}
+
+function summarise(name, values) {
+  const filled = values.filter((value) => value !== "");
+  const nulls = values.length - filled.length;
+  const distinct = new Set(filled).size;
+  const type = guessType(filled);
+
+  let min = "";
+  let max = "";
+  if (type === "number") {
+    const numbers = filled.map(Number);
+    min = String(Math.min(...numbers));
+    max = String(Math.max(...numbers));
+  } else if (type === "date" || type === "text") {
+    const sorted = [...filled].sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
+    min = sorted[0] ?? "";
+    max = sorted[sorted.length - 1] ?? "";
+  }
+
+  const counts = new Map();
+  for (const value of filled) counts.set(value, (counts.get(value) ?? 0) + 1);
+  const top = [...counts].sort((a, b) => b[1] - a[1])[0];
+
+  return {
+    name,
+    type,
+    nulls,
+    distinct,
+    min,
+    max,
+    sample: filled[0] ?? "",
+    top: top ? `${top[0]} (${top[1]})` : "",
+  };
+}
+
+export function mount(slot) {
+  let file = null;
+  let header = [];
+  let rows = [];
+  let summaryRows = [];
+
+  const error = notice("warn");
+  const summary = el("p", { class: "text-sm text-ink-2" });
+  const table = el("div", { class: "overflow-x-auto rounded-r2 border border-orbit" });
+  const markdown = output({ rows: 12 });
+  const empty = blank("table", "Drop a CSV to describe it.");
+
+  const drop = dropzone({ label: "Drop a CSV here", accept: ".csv,text/csv,text/plain", iconName: "table" });
+  const clearButton = button("Clear", { icon: "trash", kind: "quiet" });
+  const copyButton = button("Copy Markdown", {
+    icon: "copy",
+    onclick: async () => {
+      if (!markdown.textContent) return;
+      try {
+        await navigator.clipboard.writeText(markdown.textContent);
+        toast("Dictionary copied");
+      } catch {
+        error.write("The clipboard is blocked here. Download the file instead.");
+      }
+    },
+  });
+  const downloadMarkdown = button("Download .md", {
+    icon: "download",
+    onclick: () => {
+      if (!markdown.textContent) return;
+      download(`${baseName(file?.name || "data")}-dictionary.md`, markdown.textContent, "text/markdown");
+    },
+  });
+  const downloadCsv = button("Download CSV", { icon: "download", kind: "fill" });
+
+  function renderTable() {
+    empty.hidden = summaryRows.length > 0;
+    table.hidden = summaryRows.length === 0;
+    markdown.hidden = summaryRows.length === 0;
+    if (!summaryRows.length) {
+      table.replaceChildren();
+      markdown.write("");
+      summary.textContent = "";
+      return;
+    }
+
+    const columns = ["Column", "Type", "Nulls", "Distinct", "Min", "Max", "Sample"];
+    const head = el("div", {
+      class: "grid gap-3 border-b border-orbit bg-deck-2 px-3 py-2 text-2xs font-bold tracking-[0.06em] text-ink-3 uppercase [grid-template-columns:minmax(90px,1fr)_70px_52px_64px_minmax(70px,.9fr)_minmax(70px,.9fr)_minmax(90px,1.1fr)]",
+    }, ...columns.map((cell) => el("span", {}, cell)));
+
+    const body = summaryRows.map((row) =>
+      el("div", {
+        class: "grid gap-3 border-b border-orbit px-3 py-2 text-sm last:border-b-0 [grid-template-columns:minmax(90px,1fr)_70px_52px_64px_minmax(70px,.9fr)_minmax(70px,.9fr)_minmax(90px,1.1fr)]",
+      },
+        el("span", { class: "truncate font-semibold" }, row.name),
+        el("span", { class: "text-2xs text-ink-3" }, TYPE_LABELS[row.type] ?? row.type),
+        el("span", { class: "font-mono text-2xs tabular-nums" }, String(row.nulls)),
+        el("span", { class: "font-mono text-2xs tabular-nums" }, String(row.distinct)),
+        el("span", { class: "truncate font-mono text-2xs" }, row.min),
+        el("span", { class: "truncate font-mono text-2xs" }, row.max),
+        el("span", { class: "truncate font-mono text-2xs text-ink-3" }, row.sample))
+    );
+
+    table.replaceChildren(head, ...body);
+  }
+
+  function renderMarkdown() {
+    const lines = [
+      `| Column | Type | Nulls | Distinct | Min | Max | Sample |`,
+      `| --- | --- | ---: | ---: | --- | --- | --- |`,
+      ...summaryRows.map((row) =>
+        `| ${row.name} | ${row.type} | ${row.nulls} | ${row.distinct} | ${row.min} | ${row.max} | ${row.sample} |`),
+    ];
+    markdown.write(lines.join("\n"));
+  }
+
+  function run() {
+    summaryRows = header.map((name, index) => summarise(name, rows.map((row) => row[index] ?? "")));
+    const nulls = summaryRows.reduce((total, row) => total + row.nulls, 0);
+    summary.textContent = `${rows.length} row${rows.length === 1 ? "" : "s"} · ${header.length} column${header.length === 1 ? "" : "s"} · ${nulls} empty cell${nulls === 1 ? "" : "s"}`;
+    renderTable();
+    renderMarkdown();
+  }
+
+  async function open(files) {
+    const picked = Array.from(files).find((entry) => /\.(csv|tsv|txt)$/i.test(entry.name) || entry.type.startsWith("text/"));
+    if (!picked) {
+      error.write("That file is not a CSV.");
+      return;
+    }
+    try {
+      const text = await picked.text();
+      const parsed = parseDelimited(text, detectDelimiter(text));
+      if (parsed.length < 2) throw new Error("A CSV needs a header row and at least one data row.");
+      file = picked;
+      header = headerKeys(parsed[0]);
+      rows = parsed.slice(1).filter((row) => row.some((cell) => cell !== ""));
+      error.write("");
+      run();
+    } catch (caught) {
+      error.write(caught?.message || "This file could not be read.");
+    }
+  }
+
+  drop.onFiles(open);
+  clearButton.addEventListener("click", () => {
+    file = null;
+    header = [];
+    rows = [];
+    summaryRows = [];
+    error.write("");
+    renderTable();
+  });
+  downloadCsv.addEventListener("click", () => {
+    if (!summaryRows.length) return;
+    const csv = toCsv([
+      ["column", "type", "nulls", "distinct", "min", "max", "sample", "top"],
+      ...summaryRows.map((row) => [row.name, row.type, row.nulls, row.distinct, row.min, row.max, row.sample, row.top]),
+    ]);
+    download(`${baseName(file?.name || "data")}-dictionary.csv`, csv, "text/csv");
+  });
+
+  renderTable();
+
+  const shell = toolcard({
+    body: bench(
+      stack(
+        drop,
+        panel("What it reports", el("p", { class: "m-0 text-sm leading-[1.6] text-ink-3" },
+          "The type it detects, empty cells, distinct values, the smallest and largest value, and one sample. A type can be wrong on a mixed column, so check it before you publish.")),
+        error
+      ),
+      stack(summary, empty, table, markdown)
+    ),
+    foot: actions(clearButton, copyButton, downloadMarkdown, downloadCsv),
+  });
+
+  slot.append(shell.node);
+
+  return {};
+}
+
+export default { mount };

package/mevebu/libs/js/tools/dataset-splitter.js

@@ -0,0 +1,181 @@
+// Split a CSV into train, validation and test sets. Seeded, so the same seed
+// and the same file always give the same split, and a row never lands in two
+// sets.
+import {
+  el, toast, download, bench, stack, panel, numberInput, textInput, selectInput,
+  button, actions, toolcard, blank, notice,
+} from "../core.js";
+import { dropzone, baseName, formatBytes } from "../files.js";
+import { parseDelimited, detectDelimiter, headerKeys, toCsv } from "../csv.js";
+import { seeded, shuffle } from "../random.js";
+import { zip } from "../zip.js";
+
+export function mount(slot) {
+  let file = null;
+  let header = [];
+  let rows = [];
+  let split = null;
+
+  const train = numberInput({ min: "0", max: "100", step: "5", value: "70", "aria-label": "Train percent" });
+  const validation = numberInput({ min: "0", max: "100", step: "5", value: "15", "aria-label": "Validation percent" });
+  const seed = textInput({ value: "mevebu", "aria-label": "Seed", class: "font-mono" });
+  const stratify = selectInput([{ value: "", label: "Do not stratify" }], { value: "", "aria-label": "Stratify by" });
+
+  const error = notice("warn");
+  const summary = el("p", { class: "text-sm text-ink-2" });
+  const tiles = el("div", { class: "grid grid-cols-3 gap-2" });
+  const empty = blank("table", "Drop a CSV to split it.");
+
+  const drop = dropzone({ label: "Drop a CSV here", accept: ".csv,text/csv,text/plain", iconName: "table" });
+  const clearButton = button("Clear", { icon: "trash", kind: "quiet" });
+  const splitButton = button("Split and download", { icon: "download", kind: "fill" });
+
+  function tile(label) {
+    const value = el("span", { class: "text-xl font-bold tracking-[-0.02em] tabular-nums" }, "0");
+    const rows = el("span", { class: "text-2xs text-ink-3" }, "");
+    const node = el("div", { class: "grid gap-0.5 rounded-r1 border border-orbit bg-deck-2 px-3 py-2.5" },
+      el("span", { class: "text-2xs font-semibold tracking-[0.06em] text-ink-3 uppercase" }, label),
+      value, rows);
+    node.set = (count, percent) => {
+      value.textContent = String(count);
+      rows.textContent = `${percent}% of rows`;
+    };
+    return node;
+  }
+
+  const trainTile = tile("Train");
+  const validationTile = tile("Validation");
+  const testTile = tile("Test");
+  tiles.append(trainTile, validationTile, testTile);
+
+  function weights() {
+    const a = Math.max(0, Number(train.value) || 0);
+    const b = Math.max(0, Number(validation.value) || 0);
+    const c = Math.max(0, 100 - a - b);
+    return { train: a, validation: b, test: c };
+  }
+
+  function compute() {
+    const w = weights();
+    const random = seeded(seed.value || "mevebu");
+    const key = stratify.value;
+    const keyIndex = key ? header.indexOf(key) : -1;
+
+    const buckets = new Map();
+    for (const row of rows) {
+      const group = keyIndex >= 0 ? row[keyIndex] ?? "" : "__all__";
+      if (!buckets.has(group)) buckets.set(group, []);
+      buckets.get(group).push(row);
+    }
+
+    const out = { train: [], validation: [], test: [] };
+    for (const group of buckets.values()) {
+      const list = shuffle(group, random);
+      const n = list.length;
+      const trainEnd = Math.round((w.train / 100) * n);
+      const validationEnd = trainEnd + Math.round((w.validation / 100) * n);
+      out.train.push(...list.slice(0, trainEnd));
+      out.validation.push(...list.slice(trainEnd, validationEnd));
+      out.test.push(...list.slice(validationEnd));
+    }
+
+    // Shuffle the final sets so a stratified split is not grouped by class.
+    for (const name of Object.keys(out)) out[name] = shuffle(out[name], random);
+    return out;
+  }
+
+  function render() {
+    empty.hidden = rows.length > 0;
+    tiles.hidden = rows.length === 0;
+    splitButton.disabled = rows.length === 0;
+    clearButton.disabled = rows.length === 0;
+    if (!rows.length) {
+      summary.textContent = "";
+      return;
+    }
+    const w = weights();
+    split = compute();
+    const total = rows.length;
+    trainTile.set(split.train.length, Math.round((split.train.length / total) * 100));
+    validationTile.set(split.validation.length, Math.round((split.validation.length / total) * 100));
+    testTile.set(split.test.length, Math.round((split.test.length / total) * 100));
+    summary.textContent = `${total} rows · ${header.length} columns · seed ${seed.value || "mevebu"}${w.test < 0 ? " · the percentages add to more than 100, so test is empty" : ""}`;
+    if (w.test < 0) error.write("Train and validation add to more than 100 percent. Test gets nothing.");
+    else error.write("");
+  }
+
+  async function open(files) {
+    const picked = Array.from(files).find((entry) => /\.(csv|tsv|txt)$/i.test(entry.name) || entry.type.startsWith("text/"));
+    if (!picked) {
+      error.write("That file is not a CSV.");
+      return;
+    }
+    try {
+      const text = await picked.text();
+      const parsed = parseDelimited(text, detectDelimiter(text));
+      if (parsed.length < 2) throw new Error("A CSV needs a header row and at least one data row.");
+      file = picked;
+      header = headerKeys(parsed[0]);
+      rows = parsed.slice(1).filter((row) => row.some((cell) => cell !== ""));
+      stratify.replaceChildren(
+        el("option", { value: "" }, "Do not stratify"),
+        ...header.map((name) => el("option", { value: name }, name))
+      );
+      stratify.value = "";
+      error.write("");
+      render();
+    } catch (caught) {
+      error.write(caught?.message || "This file could not be read.");
+    }
+  }
+
+  drop.onFiles(open);
+  train.addEventListener("input", render);
+  validation.addEventListener("input", render);
+  seed.addEventListener("input", render);
+  stratify.addEventListener("change", render);
+
+  splitButton.addEventListener("click", () => {
+    if (!split) return;
+    const base = baseName(file?.name || "data");
+    const files = ["train", "validation", "test"].map((name) => ({
+      name: `${base}-${name}.csv`,
+      bytes: new TextEncoder().encode(toCsv([header, ...split[name]])),
+    }));
+    download(`${base}-split.zip`, zip(files), "application/zip");
+    toast("Three files ready");
+  });
+
+  clearButton.addEventListener("click", () => {
+    file = null;
+    header = [];
+    rows = [];
+    split = null;
+    error.write("");
+    render();
+  });
+
+  render();
+
+  const shell = toolcard({
+    body: bench(
+      stack(
+        drop,
+        el("div", { class: "grid grid-cols-2 gap-3" }, panel("Train %", train), panel("Validation %", validation)),
+        panel("Seed", seed, "The same seed and file give the same split."),
+        panel("Stratify by", stratify, "Keeps each class proportional across the three sets."),
+        error
+      ),
+      stack(summary, empty, tiles,
+        el("p", { class: "m-0 text-2xs leading-[1.6] text-ink-3" },
+          "Every row goes to exactly one set. The test share is whatever is left after train and validation."))
+    ),
+    foot: actions(clearButton, splitButton),
+  });
+
+  slot.append(shell.node);
+
+  return {};
+}
+
+export default { mount };