mevebu 2.0.0 → 2.2.0

package/mevebu/libs/js/tools/dataset-splitter.js

@@ -0,0 +1,181 @@
+// Split a CSV into train, validation and test sets. Seeded, so the same seed
+// and the same file always give the same split, and a row never lands in two
+// sets.
+import {
+  el, toast, download, bench, stack, panel, numberInput, textInput, selectInput,
+  button, actions, toolcard, blank, notice,
+} from "../core.js";
+import { dropzone, baseName, formatBytes } from "../files.js";
+import { parseDelimited, detectDelimiter, headerKeys, toCsv } from "../csv.js";
+import { seeded, shuffle } from "../random.js";
+import { zip } from "../zip.js";
+
+export function mount(slot) {
+  let file = null;
+  let header = [];
+  let rows = [];
+  let split = null;
+
+  const train = numberInput({ min: "0", max: "100", step: "5", value: "70", "aria-label": "Train percent" });
+  const validation = numberInput({ min: "0", max: "100", step: "5", value: "15", "aria-label": "Validation percent" });
+  const seed = textInput({ value: "mevebu", "aria-label": "Seed", class: "font-mono" });
+  const stratify = selectInput([{ value: "", label: "Do not stratify" }], { value: "", "aria-label": "Stratify by" });
+
+  const error = notice("warn");
+  const summary = el("p", { class: "text-sm text-ink-2" });
+  const tiles = el("div", { class: "grid grid-cols-3 gap-2" });
+  const empty = blank("table", "Drop a CSV to split it.");
+
+  const drop = dropzone({ label: "Drop a CSV here", accept: ".csv,text/csv,text/plain", iconName: "table" });
+  const clearButton = button("Clear", { icon: "trash", kind: "quiet" });
+  const splitButton = button("Split and download", { icon: "download", kind: "fill" });
+
+  function tile(label) {
+    const value = el("span", { class: "text-xl font-bold tracking-[-0.02em] tabular-nums" }, "0");
+    const rows = el("span", { class: "text-2xs text-ink-3" }, "");
+    const node = el("div", { class: "grid gap-0.5 rounded-r1 border border-orbit bg-deck-2 px-3 py-2.5" },
+      el("span", { class: "text-2xs font-semibold tracking-[0.06em] text-ink-3 uppercase" }, label),
+      value, rows);
+    node.set = (count, percent) => {
+      value.textContent = String(count);
+      rows.textContent = `${percent}% of rows`;
+    };
+    return node;
+  }
+
+  const trainTile = tile("Train");
+  const validationTile = tile("Validation");
+  const testTile = tile("Test");
+  tiles.append(trainTile, validationTile, testTile);
+
+  function weights() {
+    const a = Math.max(0, Number(train.value) || 0);
+    const b = Math.max(0, Number(validation.value) || 0);
+    const c = Math.max(0, 100 - a - b);
+    return { train: a, validation: b, test: c };
+  }
+
+  function compute() {
+    const w = weights();
+    const random = seeded(seed.value || "mevebu");
+    const key = stratify.value;
+    const keyIndex = key ? header.indexOf(key) : -1;
+
+    const buckets = new Map();
+    for (const row of rows) {
+      const group = keyIndex >= 0 ? row[keyIndex] ?? "" : "__all__";
+      if (!buckets.has(group)) buckets.set(group, []);
+      buckets.get(group).push(row);
+    }
+
+    const out = { train: [], validation: [], test: [] };
+    for (const group of buckets.values()) {
+      const list = shuffle(group, random);
+      const n = list.length;
+      const trainEnd = Math.round((w.train / 100) * n);
+      const validationEnd = trainEnd + Math.round((w.validation / 100) * n);
+      out.train.push(...list.slice(0, trainEnd));
+      out.validation.push(...list.slice(trainEnd, validationEnd));
+      out.test.push(...list.slice(validationEnd));
+    }
+
+    // Shuffle the final sets so a stratified split is not grouped by class.
+    for (const name of Object.keys(out)) out[name] = shuffle(out[name], random);
+    return out;
+  }
+
+  function render() {
+    empty.hidden = rows.length > 0;
+    tiles.hidden = rows.length === 0;
+    splitButton.disabled = rows.length === 0;
+    clearButton.disabled = rows.length === 0;
+    if (!rows.length) {
+      summary.textContent = "";
+      return;
+    }
+    const w = weights();
+    split = compute();
+    const total = rows.length;
+    trainTile.set(split.train.length, Math.round((split.train.length / total) * 100));
+    validationTile.set(split.validation.length, Math.round((split.validation.length / total) * 100));
+    testTile.set(split.test.length, Math.round((split.test.length / total) * 100));
+    summary.textContent = `${total} rows · ${header.length} columns · seed ${seed.value || "mevebu"}${w.test < 0 ? " · the percentages add to more than 100, so test is empty" : ""}`;
+    if (w.test < 0) error.write("Train and validation add to more than 100 percent. Test gets nothing.");
+    else error.write("");
+  }
+
+  async function open(files) {
+    const picked = Array.from(files).find((entry) => /\.(csv|tsv|txt)$/i.test(entry.name) || entry.type.startsWith("text/"));
+    if (!picked) {
+      error.write("That file is not a CSV.");
+      return;
+    }
+    try {
+      const text = await picked.text();
+      const parsed = parseDelimited(text, detectDelimiter(text));
+      if (parsed.length < 2) throw new Error("A CSV needs a header row and at least one data row.");
+      file = picked;
+      header = headerKeys(parsed[0]);
+      rows = parsed.slice(1).filter((row) => row.some((cell) => cell !== ""));
+      stratify.replaceChildren(
+        el("option", { value: "" }, "Do not stratify"),
+        ...header.map((name) => el("option", { value: name }, name))
+      );
+      stratify.value = "";
+      error.write("");
+      render();
+    } catch (caught) {
+      error.write(caught?.message || "This file could not be read.");
+    }
+  }
+
+  drop.onFiles(open);
+  train.addEventListener("input", render);
+  validation.addEventListener("input", render);
+  seed.addEventListener("input", render);
+  stratify.addEventListener("change", render);
+
+  splitButton.addEventListener("click", () => {
+    if (!split) return;
+    const base = baseName(file?.name || "data");
+    const files = ["train", "validation", "test"].map((name) => ({
+      name: `${base}-${name}.csv`,
+      bytes: new TextEncoder().encode(toCsv([header, ...split[name]])),
+    }));
+    download(`${base}-split.zip`, zip(files), "application/zip");
+    toast("Three files ready");
+  });
+
+  clearButton.addEventListener("click", () => {
+    file = null;
+    header = [];
+    rows = [];
+    split = null;
+    error.write("");
+    render();
+  });
+
+  render();
+
+  const shell = toolcard({
+    body: bench(
+      stack(
+        drop,
+        el("div", { class: "grid grid-cols-2 gap-3" }, panel("Train %", train), panel("Validation %", validation)),
+        panel("Seed", seed, "The same seed and file give the same split."),
+        panel("Stratify by", stratify, "Keeps each class proportional across the three sets."),
+        error
+      ),
+      stack(summary, empty, tiles,
+        el("p", { class: "m-0 text-2xs leading-[1.6] text-ink-3" },
+          "Every row goes to exactly one set. The test share is whatever is left after train and validation."))
+    ),
+    foot: actions(clearButton, splitButton),
+  });
+
+  slot.append(shell.node);
+
+  return {};
+}
+
+export default { mount };

package/mevebu/libs/js/tools/file-hash.js

@@ -0,0 +1,154 @@
+// The hash of a file, to compare against a published checksum. The file is read
+// here with Web Crypto, so a download can be verified before it is opened.
+import {
+  el, icon, toast, bench, stack, panel, textInput, button, actions, toolcard,
+  blank, notice, copyButton, output,
+} from "../core.js";
+import { dropzone, formatBytes } from "../files.js";
+
+const ALGORITHMS = [
+  { id: "SHA-1", label: "SHA-1" },
+  { id: "SHA-256", label: "SHA-256" },
+  { id: "SHA-512", label: "SHA-512" },
+];
+
+function hex(bytes) {
+  return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+export function mount(slot) {
+  let file = null;
+  let digests = {};
+
+  const expected = textInput({
+    placeholder: "Paste the published checksum",
+    "aria-label": "Expected checksum",
+    class: "font-mono text-sm",
+    autocomplete: "off",
+    spellcheck: "false",
+  });
+
+  const error = notice("warn");
+  const summary = el("p", { class: "text-sm text-ink-2" });
+  const verdictTitle = el("p", { class: "m-0 text-lg font-bold" }, "Waiting");
+  const verdictBody = el("p", { class: "m-0 text-sm text-ink-2" }, "Drop a file, then paste the checksum you were given.");
+  const verdict = el("div", { class: "grid gap-1 rounded-r2 border border-orbit bg-deck-2 p-4" }, verdictTitle, verdictBody);
+  const rows = el("div", { class: "grid gap-2" });
+  const empty = blank("file-key", "Drop a file to hash it.");
+
+  const drop = dropzone({ label: "Drop a file here", accept: "", iconName: "file-key" });
+  const clearButton = button("Clear", { icon: "trash", kind: "quiet" });
+
+  function paint(state, title, body) {
+    const tone = state === "ok"
+      ? "border-jade bg-jade-soft"
+      : state === "bad"
+        ? "border-danger/40 bg-danger/10"
+        : "border-orbit bg-deck-2";
+    verdict.className = `grid gap-1 rounded-r2 border p-4 ${tone}`;
+    verdictTitle.textContent = title;
+    verdictTitle.className = `m-0 text-lg font-bold ${state === "ok" ? "text-jade" : state === "bad" ? "text-danger" : ""}`;
+    verdictBody.textContent = body;
+    verdictBody.className = state === "ok" || state === "bad" ? "m-0 text-sm" : "m-0 text-sm text-ink-2";
+  }
+
+  function compare() {
+    if (!file) {
+      paint("idle", "Waiting", "Drop a file, then paste the checksum you were given.");
+      return;
+    }
+    const wanted = expected.value.replace(/\s+/g, "").toLowerCase();
+    if (!wanted) {
+      paint("idle", "Ready", "Paste a checksum to compare, or copy one below.");
+      return;
+    }
+    const match = Object.entries(digests).find(([, value]) => value === wanted);
+    if (match) {
+      paint("ok", "Match", `The file matches the ${match[0]} you pasted.`);
+    } else {
+      const length = wanted.length;
+      const sameLength = Object.entries(digests).find(([, value]) => value.length === length);
+      paint(
+        "bad",
+        "No match",
+        sameLength
+          ? `That is not the ${sameLength[0]} of this file. Either the file or the checksum is wrong.`
+          : `A checksum of ${length} characters is not a SHA-1, SHA-256 or SHA-512 digest.`
+      );
+    }
+  }
+
+  function renderRows() {
+    rows.replaceChildren(
+      ...ALGORITHMS.map((algorithm) =>
+        el("div", { class: "grid gap-1.5 rounded-r1 border border-orbit bg-deck-2 p-2.5" },
+          el("div", { class: "flex items-center justify-between gap-2" },
+            el("span", { class: "text-2xs font-bold tracking-[0.06em] text-ink-3 uppercase" }, algorithm.label),
+            copyButton(() => digests[algorithm.id] ?? "", "Copy")),
+          el("p", { class: "m-0 wrap-anywhere font-mono text-2xs leading-[1.6] text-ink" }, digests[algorithm.id] ?? "")))
+    );
+  }
+
+  async function open(files) {
+    const picked = Array.from(files)[0];
+    if (!picked) return;
+    error.write("");
+    summary.textContent = `Reading ${picked.name}`;
+    try {
+      const bytes = await picked.arrayBuffer();
+      const next = {};
+      for (const algorithm of ALGORITHMS) {
+        const digest = await crypto.subtle.digest(algorithm.id, bytes);
+        next[algorithm.id] = hex(new Uint8Array(digest));
+      }
+      file = picked;
+      digests = next;
+      summary.textContent = `${picked.name} · ${formatBytes(picked.size)}`;
+      empty.hidden = true;
+      rows.hidden = false;
+      renderRows();
+      compare();
+    } catch (caught) {
+      summary.textContent = "";
+      error.write(caught?.message || "This file could not be read.");
+    }
+  }
+
+  drop.onFiles(open);
+  expected.addEventListener("input", compare);
+  clearButton.addEventListener("click", () => {
+    file = null;
+    digests = {};
+    expected.value = "";
+    summary.textContent = "";
+    error.write("");
+    rows.replaceChildren();
+    rows.hidden = true;
+    empty.hidden = false;
+    paint("idle", "Waiting", "Drop a file, then paste the checksum you were given.");
+  });
+
+  paint("idle", "Waiting", "Drop a file, then paste the checksum you were given.");
+  rows.hidden = true;
+
+  const shell = toolcard({
+    body: bench(
+      stack(
+        drop,
+        panel("Expected checksum", expected, "SHA-1, SHA-256 or SHA-512. Case and spaces are ignored."),
+        verdict,
+        error,
+        el("p", { class: "m-0 text-2xs leading-[1.6] text-ink-3" },
+          "The file is read in this tab. Nothing is uploaded, and the size limit is whatever the browser can hold in memory.")
+      ),
+      stack(summary, empty, rows)
+    ),
+    foot: actions(clearButton),
+  });
+
+  slot.append(shell.node);
+
+  return {};
+}
+
+export default { mount };

package/mevebu/libs/js/tools/json-schema-sample.js

@@ -0,0 +1,254 @@
+// Sample documents from a JSON Schema. Covers the keywords that decide whether
+// a document validates, and reports the ones it could not honour, so a passing
+// sample is never mistaken for a complete implementation.
+import {
+  el, toast, download, bench, stack, panel, textArea, numberInput, textInput,
+  button, actions, toolcard, blank, notice, output,
+} from "../core.js";
+import { seeded, pick, uuid, gaussian } from "../random.js";
+import { zip } from "../zip.js";
+
+const SUPPORTED = new Set([
+  "$schema", "$id", "title", "description", "type", "properties", "required",
+  "items", "enum", "const", "default", "examples", "minimum", "maximum",
+  "exclusiveMinimum", "exclusiveMaximum", "multipleOf", "minLength", "maxLength",
+  "pattern", "format", "minItems", "maxItems", "uniqueItems", "additionalProperties",
+  "minProperties", "maxProperties", "anyOf", "oneOf", "allOf", "not", "nullable",
+]);
+
+const FORMATS = {
+  "date-time": () => new Date(Date.UTC(2025, 0, 1) + Math.floor(Math.random() * 3e10)).toISOString(),
+  date: (random) => new Date(Date.UTC(2025, 0, 1) + Math.floor(random() * 700) * 86400000).toISOString().slice(0, 10),
+  time: () => `${String(Math.floor(Math.random() * 24)).padStart(2, "0")}:00:00Z`,
+  email: (random) => `${pick(random, ["ani", "budi", "citra", "dewi"])}${Math.floor(random() * 99)}@example.com`,
+  uri: () => "https://example.com/resource",
+  uuid: (random) => uuid(random),
+  ipv4: () => `192.168.${Math.floor(Math.random() * 255)}.${Math.floor(Math.random() * 255)}`,
+  hostname: () => "example.com",
+};
+
+function regexFrom(pattern) {
+  try {
+    return new RegExp(pattern);
+  } catch {
+    return null;
+  }
+}
+
+function stringValue(schema, random) {
+  if (schema.format && FORMATS[schema.format]) return FORMATS[schema.format](random);
+  if (schema.enum) return pick(random, schema.enum);
+  if (schema.const !== undefined) return schema.const;
+
+  const min = Math.max(0, schema.minLength ?? 0);
+  const max = Math.max(min, schema.maxLength ?? Math.max(min, 12));
+  const regex = schema.pattern ? regexFrom(schema.pattern) : null;
+  const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789";
+
+  // Try a few random strings against the pattern before falling back, so a
+  // pattern like ^[A-Z]{3}$ still gets a plausible value.
+  for (let attempt = 0; attempt < 60; attempt++) {
+    const length = min + Math.floor(random() * Math.max(1, Math.min(max, 24) - min + 1));
+    let out = "";
+    for (let i = 0; i < length; i++) out += alphabet[Math.floor(random() * alphabet.length)];
+    if (!regex || regex.test(out)) return out;
+  }
+  const literal = schema.pattern?.replace(/[\^$.*+?()[\]{}|\\]/g, "") ?? "";
+  if (literal.length >= min) return literal.slice(0, Math.max(min, Math.min(max, literal.length)));
+  return "sample".padEnd(min, "x").slice(0, Math.max(min, 1));
+}
+
+function numberValue(schema, random, integer) {
+  if (schema.enum) return pick(random, schema.enum);
+  if (schema.const !== undefined) return schema.const;
+  const min = schema.minimum ?? schema.exclusiveMinimum ?? 0;
+  const max = schema.maximum ?? schema.exclusiveMaximum ?? (min + 1000);
+  let value = min + random() * (max - min);
+  if (integer) value = Math.round(value);
+  if (schema.multipleOf) value = Math.round(value / schema.multipleOf) * schema.multipleOf;
+  return value;
+}
+
+function sample(schema, random, path, unknown) {
+  if (!schema || typeof schema !== "object") return null;
+
+  if (schema.const !== undefined) return schema.const;
+  if (schema.enum) return pick(random, schema.enum);
+  if (schema.default !== undefined) return schema.default;
+  if (schema.examples?.length) return schema.examples[0];
+  if (schema.anyOf?.length) return sample(schema.anyOf[0], random, path, unknown);
+  if (schema.oneOf?.length) return sample(schema.oneOf[0], random, path, unknown);
+  if (schema.allOf?.length) {
+    // Merge the object branches, which is the common shape.
+    const merged = schema.allOf.reduce(
+      (acc, branch) => ({
+        ...acc,
+        ...branch,
+        properties: { ...(acc.properties ?? {}), ...(branch.properties ?? {}) },
+        required: [...new Set([...(acc.required ?? []), ...(branch.required ?? [])])],
+      }),
+      { ...schema, allOf: undefined }
+    );
+    return sample(merged, random, path, unknown);
+  }
+
+  const type = Array.isArray(schema.type)
+    ? schema.type.find((entry) => entry !== "null") ?? "null"
+    : schema.type ?? (schema.properties ? "object" : schema.items ? "array" : "string");
+
+  if (schema.nullable && random() < 0.1) return null;
+
+  switch (type) {
+    case "object": {
+      const out = {};
+      const properties = schema.properties ?? {};
+      for (const [key, child] of Object.entries(properties)) {
+        out[key] = sample(child, random, `${path}.${key}`, unknown);
+      }
+      for (const key of schema.required ?? []) {
+        if (!(key in out)) out[key] = sample({}, random, `${path}.${key}`, unknown);
+      }
+      if (schema.additionalProperties && typeof schema.additionalProperties === "object") {
+        out.extra = sample(schema.additionalProperties, random, `${path}.extra`, unknown);
+      }
+      return out;
+    }
+    case "array": {
+      const min = Math.max(0, schema.minItems ?? 1);
+      const max = Math.max(min, schema.maxItems ?? min + 2);
+      const count = min + Math.floor(random() * (Math.min(max, 8) - min + 1));
+      const items = Array.from({ length: count }, (_, index) =>
+        sample(schema.items ?? {}, random, `${path}[${index}]`, unknown)
+      );
+      return schema.uniqueItems ? [...new Set(items.map((item) => JSON.stringify(item)))].map((text) => JSON.parse(text)) : items;
+    }
+    case "integer":
+      return numberValue(schema, random, true);
+    case "number":
+      return Number(numberValue(schema, random, false).toFixed(3));
+    case "boolean":
+      return random() > 0.5;
+    case "null":
+      return null;
+    default:
+      return stringValue(schema, random);
+  }
+}
+
+function collectUnknown(schema, found, path = "$") {
+  if (!schema || typeof schema !== "object") return;
+  for (const [key, value] of Object.entries(schema)) {
+    if (["properties", "items", "anyOf", "oneOf", "allOf"].includes(key)) continue;
+    if (key.startsWith("$") || key.startsWith("x-")) continue;
+    if (!SUPPORTED.has(key) && !found.includes(key)) found.push(key);
+  }
+  for (const [name, child] of Object.entries(schema.properties ?? {})) {
+    collectUnknown(child, found, `${path}.${name}`);
+  }
+  if (schema.items && typeof schema.items === "object") collectUnknown(schema.items, found, `${path}[]`);
+}
+
+export function mount(slot) {
+  let documents = [];
+  let unknown = [];
+
+  const schema = textArea({
+    rows: 12,
+    value: JSON.stringify(
+      {
+        $schema: "https://json-schema.org/draft/2020-12/schema",
+        type: "object",
+        required: ["id", "name", "email"],
+        properties: {
+          id: { type: "string", format: "uuid" },
+          name: { type: "string", minLength: 3, maxLength: 24 },
+          email: { type: "string", format: "email" },
+          age: { type: "integer", minimum: 18, maximum: 90 },
+          tags: { type: "array", items: { type: "string" }, minItems: 1, maxItems: 4 },
+        },
+      },
+      null,
+      2
+    ),
+    "aria-label": "JSON Schema",
+  });
+
+  const count = numberInput({ min: "1", max: "200", step: "1", value: "3", "aria-label": "Documents" });
+  const seed = textInput({ value: "mevebu", "aria-label": "Seed", class: "font-mono" });
+  const error = notice("warn");
+  const summary = el("p", { class: "text-sm text-ink-2" });
+  const result = output({ rows: 16 });
+  const empty = blank("braces", "Paste a schema to generate documents.");
+
+  const generate = button("Generate", { icon: "refresh-cw", kind: "fill" });
+  const downloadJson = button("Download JSON", { icon: "download" });
+  const downloadZip = button("Download ZIP", { icon: "download" });
+
+  function run() {
+    error.write("");
+    try {
+      const parsed = JSON.parse(schema.value);
+      const random = seeded(seed.value || "mevebu");
+      unknown = [];
+      collectUnknown(parsed, unknown);
+      const total = Math.max(1, Math.min(200, Number(count.value) || 1));
+      documents = Array.from({ length: total }, () => sample(parsed, random, "$", unknown));
+      result.write(JSON.stringify(documents, null, 2));
+      summary.textContent = `${total} document${total === 1 ? "" : "s"} · seed ${seed.value || "mevebu"}`;
+      empty.hidden = true;
+      result.hidden = false;
+      if (unknown.length) {
+        error.write(`Not implemented, so left out: ${unknown.join(", ")}. Everything else follows the schema.`);
+      }
+    } catch (caught) {
+      documents = [];
+      result.write("");
+      result.hidden = true;
+      empty.hidden = false;
+      summary.textContent = "";
+      error.write(caught?.message || "That schema is not valid JSON.");
+    }
+  }
+
+  generate.addEventListener("click", () => {
+    run();
+    if (documents.length) toast("Documents ready");
+  });
+  schema.addEventListener("input", run);
+  count.addEventListener("input", run);
+  seed.addEventListener("input", run);
+
+  downloadJson.addEventListener("click", () => {
+    if (!documents.length) return;
+    download("samples.json", JSON.stringify(documents, null, 2), "application/json");
+  });
+  downloadZip.addEventListener("click", () => {
+    if (!documents.length) return;
+    const files = documents.map((doc, index) => ({
+      name: `sample-${String(index + 1).padStart(3, "0")}.json`,
+      bytes: new TextEncoder().encode(JSON.stringify(doc, null, 2)),
+    }));
+    download("samples.zip", zip(files), "application/zip");
+  });
+
+  run();
+
+  const shell = toolcard({
+    body: bench(
+      stack(
+        panel("Schema", schema),
+        panel("Documents", count),
+        panel("Seed", seed, "The same seed and schema give the same documents."),
+        error
+      ),
+      stack(summary, empty, result)
+    ),
+    foot: actions(downloadJson, downloadZip, generate),
+  });
+
+  slot.append(shell.node);
+
+  return {};
+}
+
+export default { mount };