@qe-mcp/ena 0.0.0-stage → 0.1.18

package/webena.js

@@ -0,0 +1,651 @@
+/**
+ * webena.js — the ENA Web Tool's conventions, so a model fit here comes out,
+ * plots and exports as the same model does in the web tool (rENA-webtool).
+ *
+ * Everything here is copied from the web tool and must be kept in step with it:
+ *   group order, colours   js/ena.ww.organize-data.js, js/global-ena.js DEF_COLORS / NA
+ *   default rotation       js/ena.controllers.js updateUnits (which flags turn on)
+ *                          js/WasmWrapper.js generate(), step 5 (the GMR it builds)
+ *   default window         js/ena.controllers.js (a new set takes the CCD window)
+ *   points, groups         js/WasmWrapper.js _formatResult (scaleUnits, group means,
+ *                          confidence intervals, group line weights)
+ *   plots                  js/ena.qeviz.js buildPlot (the comparison plot)
+ *   set file               js/ENAmodel.js constructor (the stored set's defaults)
+ */
+
+import { deflateRawSync } from 'zlib';
+
+// ── Groups ────────────────────────────────────────────────────────────────────
+
+// js/global-ena.js ENA.constants.DEF_COLORS: [color, secondaryColor] per group.
+export const COLORS = [
+    ['#cc423a', '#56bd7c'],
+    ['#218ebf', '#ef691b'],
+    ['#9d5dbb', '#fbc848'],
+    ['#56bd7c', '#d0386c'],
+    ['#f18e9f', '#9a9eab'],
+    ['#ff8c39', '#346b88'],
+];
+export const NODE_COLOR = '#4D4D4D';
+const SHOW_MEAN_INDEX = 20;              // ENA.constants.SHOW_MEAN_INDEX
+const NA = new Set(['#N/A', 'NA', '']); // ENA.constants.NA
+
+export const isNA = (v) => v == null || NA.has(String(v));
+
+// The web tool groups units by the values of their first unit column, in the
+// order a JS object keeps its keys (organize-data's `for (key in grouped)`):
+// integer-like values first, ascending, then the rest in order of appearance.
+export function groupOrder(rows, col) {
+    const seen = new Set();
+    for (const r of rows) if (!isNA(r[col])) seen.add(String(r[col]));
+    const isIndex = (s) => /^(0|[1-9]\d*)$/.test(s) && Number(s) < 4294967295;
+    const all = [...seen];
+    return [...all.filter(isIndex).sort((a, b) => a - b), ...all.filter(s => !isIndex(s))];
+}
+
+// Colour and mean flags of the i-th top-level group (organize-data).
+export function groupStyle(i) {
+    const [color, secondaryColor] = COLORS[i % COLORS.length];
+    return { color, secondaryColor, showMean: i < SHOW_MEAN_INDEX };
+}
+
+// The groups the web tool makes of these units: the first unit column's values
+// when there are two or more unit columns; with one column every unit sits in
+// a single group named after it (organize-data's singleConditionWrapper).
+export function unitGroups(rows, unitCols) {
+    if (unitCols.length < 2) return { col: null, order: [unitCols[0]], wrapper: true };
+    return { col: unitCols[0], order: groupOrder(rows, unitCols[0]), wrapper: false };
+}
+
+// ── Default rotation ──────────────────────────────────────────────────────────
+
+// With two or more groups the web tool switches regressionRotated on (and
+// meansRotated with exactly two), seeds meansRotatedBy with the first two
+// groups, and fits a generalized means rotation (GMR) through them.
+export function defaultRotation(groups) {
+    if (groups.wrapper || groups.order.length < 2) return null;
+    return { col: groups.col, pair: groups.order.slice(0, 2), nGroups: groups.order.length };
+}
+
+// rena-wasm GMR parameters exactly as WasmWrapper builds them for a regression
+// rotation: one categorical target (each unit's value on its first row),
+// levels = the pair then the other values in order of appearance, treatment
+// coding against the first of the pair, GMR1 fit on the pair's units.
+export function gmrParams(rows, unitCols, unitLabels, targetCol, pair) {
+    const firstRow = new Map();
+    for (const r of rows) {
+        const key = unitCols.map(c => String(r[c])).join('__');
+        if (!firstRow.has(key)) firstRow.set(key, r);
+    }
+    const n = unitLabels.length;
+    const vals = unitLabels.map(l => String(firstRow.get(l)?.[targetCol] ?? '').trim());
+    const clean = pair.map(g => String(g).trim());
+    for (const g of clean)
+        if (!vals.includes(g))
+            throw new Error(`No modeled unit has ${targetCol}='${g}'.`);
+    const levels = [...clean, ...[...new Set(vals)].filter(v => !clean.includes(v))];
+    const nDummies = levels.length - 1;
+
+    const xTarget = new Float64Array(n);
+    const xModelMatrix = new Float64Array(n * nDummies);
+    const subset = [];
+    for (let u = 0; u < n; u++) {
+        const lvl = levels.indexOf(vals[u]);
+        xTarget[u] = lvl >= 0 ? lvl : levels.length - 1;
+        if (lvl > 0) xModelMatrix[u * nDummies + (lvl - 1)] = 1;
+        if (clean.includes(vals[u])) subset.push(u);
+    }
+    if (!subset.length) for (let u = 0; u < n; u++) subset.push(u);
+
+    return {
+        levels,
+        subsetSize: subset.length,
+        gParams: {
+            xModelMatrix, xmRows: n, xmCols: nDummies,
+            xTarget,
+            x1Cols:       new Int32Array(nDummies).map((_, i) => i),
+            xCategorical: true,
+            xNGroups:     levels.length,
+            xSubset:      new Int32Array(subset),
+            hasY:         false,
+            // Stub Y params, as WasmWrapper passes them (ignored when hasY=false).
+            yModelMatrix: new Float64Array(n), ymRows: n, ymCols: 1,
+            yTarget:      new Float64Array(n),
+            y1Cols:       new Int32Array([0]),
+            yCategorical: false,
+            yNGroups:     0,
+        },
+    };
+}
+
+// A new set in the web tool takes the window that Cross-Covariance Decay
+// recommends (WasmWrapper ccdWindow: maxWindow 20, minOverlap 10).
+export const CCD_OPTS = { maxWindow: 20, minOverlap: 10 };
+export const FALLBACK_WINDOW = 4;
+
+// ── Points and groups ─────────────────────────────────────────────────────────
+
+const CI_LEVEL = 0.95;
+
+// WasmWrapper's scaleUnits factor: on the first two dimensions, the ratio of
+// the code nodes' extent to the points' (largest |value|, and smallest
+// non-zero |value|), s = min(max(min-ratios), max(max-ratios)).
+export function pointScale(points, nodes, nUnits, nCodes, stride) {
+    const dimsToScale = Math.min(2, stride);
+    const maxRatios = [], minRatios = [];
+    for (let d = 0; d < dimsToScale; d++) {
+        const pt = Array.from({ length: nUnits }, (_, i) => Math.abs(points[i * stride + d]));
+        const nd = Array.from({ length: nCodes }, (_, i) => Math.abs(nodes[i * stride + d]));
+        const ptNZ = pt.filter(v => v > 0), ndNZ = nd.filter(v => v > 0);
+        const pMax = Math.max(...pt, 0) || 1;
+        const nMax = Math.max(...nd, 0) || 1;
+        const pMin = (ptNZ.length ? Math.min(...ptNZ) : pMax) || 1;
+        const nMin = (ndNZ.length ? Math.min(...ndNZ) : nMax) || 1;
+        maxRatios.push(Math.abs(nMax / pMax));
+        minRatios.push(Math.abs(nMin / pMin));
+    }
+    const fin = (a) => a.filter(v => Number.isFinite(v));
+    const mx = fin(maxRatios), mn = fin(minRatios);
+    if (!mx.length || !mn.length) return 1;
+    const s = Math.min(Math.max(...mn), Math.max(...mx));
+    return Number.isFinite(s) && s > 0 ? s : 1;
+}
+
+// WasmWrapper._scale: non-zero values rescaled linearly into [lo, hi], zeros
+// kept at 0; a constant non-zero array becomes all `hi`.
+export function rescale(arr, lo, hi) {
+    const nz = arr.filter(v => v !== 0);
+    if (!nz.length) return arr.map(() => 0);
+    const min = Math.min(...nz), max = Math.max(...nz);
+    if (max === min) return arr.map(v => (v === 0 ? 0 : hi));
+    return arr.map(v => (v === 0 ? 0 : lo + (v - min) * (hi - lo) / (max - min)));
+}
+
+function colMeans(flat, rowIdx, nCols) {
+    const out = new Array(nCols).fill(0);
+    if (!rowIdx.length) return out;
+    for (const i of rowIdx) for (let j = 0; j < nCols; j++) out[j] += flat[i * nCols + j];
+    return out.map(v => v / rowIdx.length);
+}
+
+/**
+ * What the web tool computes for its plots after a fit:
+ *   scale    the scaleUnits factor
+ *   points   unit points × scale (nUnits × stride, flat)
+ *   groups   per group: unit indices, mean of the scaled points, 95%
+ *            confidence bounds and outlier half-spans (Tukey fences, 1.5 × IQR)
+ *            per dimension, the mean line weights rescaled
+ *            to [0.1, 1] (meanOrSum = mean), colour and mean flags
+ * `members(i)` gives a unit's group name; `names` the groups, in order.
+ */
+export function webSet(ena, model, unitLabels, names, members) {
+    const stride = model.dims ?? 2;
+    const nUnits = unitLabels.length;
+    const nCodes = model.positions.length / stride;
+    const scale = pointScale(model.centroids, model.positions, nUnits, nCodes, stride);
+    const points = Float64Array.from(model.centroids, v => v * scale);
+
+    const groups = names.map((name, gi) => {
+        const idx = unitLabels.flatMap((_, i) => (members(i) === name ? [i] : []));
+        const mean = colMeans(points, idx, stride);
+        let ci = null, oi = null;
+        if (idx.length > 1) {
+            const flat = new Float64Array(idx.length * stride);
+            idx.forEach((u, r) => { for (let d = 0; d < stride; d++) flat[r * stride + d] = points[u * stride + d]; });
+            try {
+                const res = ena.confInts(flat, idx.length, stride, CI_LEVEL);
+                ci = Array.from({ length: stride }, (_, d) => [res.data[d * 3 + 1], res.data[d * 3 + 2]]);
+            } catch { ci = null; }
+            // js/WasmWrapper.js: outlier.ints[d] = (upper fence − lower fence) / 2,
+            // drawn as ± that around the group mean.
+            try {
+                const res = ena.outlierInts(flat, idx.length, stride, 1.5);
+                oi = Array.from({ length: stride }, (_, d) => (res.data[d * 2 + 1] - res.data[d * 2]) / 2);
+            } catch { oi = null; }
+        }
+        const lineWeights = rescale(colMeans(model.networks, idx, model.nConnections), 0.1, 1);
+        return { name, idx, mean, ci, oi, lineWeights, n: idx.length, ...groupStyle(gi) };
+    });
+    return { scale, points, stride, groups, byName: new Map(groups.map(g => [g.name, g])) };
+}
+
+// ── Plots (js/ena.qeviz.js buildPlot, the comparison plot) ────────────────────
+
+const RANGE_MODIFIER = 1.2;
+const EDGE_PX = 7.5;              // a weight-1 edge, px (× "Scale edge weights")
+const SUBTRACTION_MAX_PX = 4;     // a subtraction's strongest edge, px
+const NODE_PX = 4.75;             // code-node radius per unit of Σ|weight|, px
+export const TEXT_SIZE = 12;
+const LABEL_COLOR = '#000000';
+const LABEL_FONT = 'arial';
+const UNIT_SET = 'units';
+
+const finite = (v) => typeof v === 'number' && Number.isFinite(v);
+
+function frame(columns, types, rows) {
+    return {
+        data: rows.map(r => Object.fromEntries(columns.map(c => [c, r[c] === undefined ? null : r[c]]))),
+        types: Object.fromEntries(columns.map(c => [c, types[c] || 'numeric'])),
+    };
+}
+
+// The web tool shows unit paths with '::'; rena-wasm labels join with '__'.
+export const unitPath = (label) => label.split('__').join('::');
+
+/**
+ * Build one plot as the web tool's comparison plot draws it.
+ *   entry  the cached model: model, codes, unitLabels, web (webSet), dimNames, variance
+ *   view   { first, second, overlay, scale, axes, flip, style }: first/second
+ *          are { kind: 'group' | 'unit' | 'all', name }; with a second the
+ *          plot draws first − second, or (overlay: true) both networks.
+ *   style  the web tool's Plot Settings: { minWeight (minimumThreshold),
+ *          hideUnconnected (!showUnconnectedCodes), textSize, colors
+ *          ({ group: colour }), ci (confidenceIntervals, default on),
+ *          outliers (outlierIntervals, default off) }.
+ * Returns { data, layers, attrs, edgePx, title, maxWeight, scale, subtract };
+ * the renderer
+ * sets edge-width = edgePx ÷ the plot's shorter side in px. Group rows carry
+ * both the CI and the outlier bounds whichever is shown, so a viewer can
+ * toggle them.
+ */
+export function buildPlot(entry, view = {}) {
+    const { model, codes, unitLabels, web } = entry;
+    const stride = web.stride;
+    const ordered = model.ordered === true;
+    const scale = Number(view.scale) > 0 ? Number(view.scale) : 1;
+    const conn = model.connectionNames.map(n => n.replace(' & ', '.'));
+    const pairs = model.connectionNames.map(n => n.split(' & '));
+    const style = view.style ?? {};
+    const colorOf = (g) => style.colors?.[g.name] ?? g.color;
+
+    const all = () => {
+        const idx = unitLabels.map((_, i) => i);
+        return rescale(colMeans(model.networks, idx, model.nConnections), 0.1, 1);
+    };
+    const network = (sel) => {
+        if (!sel) return null;
+        if (sel.kind === 'unit') {
+            const i = unitLabels.indexOf(sel.name);
+            if (i < 0) throw new Error(`Unit '${sel.name}' not found.`);
+            const g = web.groups.find(gr => gr.idx.includes(i));
+            return {
+                name: unitPath(sel.name),
+                w: Array.from(model.networks.slice(i * model.nConnections, (i + 1) * model.nConnections)),
+                color: g ? colorOf(g) : NODE_COLOR, secondaryColor: g?.secondaryColor ?? NODE_COLOR,
+            };
+        }
+        if (sel.kind === 'all') {
+            const only = web.groups.length === 1 ? web.groups[0] : null;
+            return { name: only?.name ?? 'All units', w: only?.lineWeights ?? all(),
+                color: only ? colorOf(only) : NODE_COLOR, secondaryColor: only?.secondaryColor ?? NODE_COLOR };
+        }
+        const g = web.byName.get(String(sel.name));
+        if (!g) throw new Error(`Group '${sel.name}' not found. Groups: ${web.groups.map(x => x.name).join(', ')}`);
+        return { name: g.name, w: g.lineWeights, color: colorOf(g), secondaryColor: g.secondaryColor };
+    };
+
+    // Axes: any two of the model's dimensions, either flipped (the web tool's
+    // Plot Settings dimension choice and Plot Tools Flip X / Flip Y).
+    const [ax, ay] = view.axes ?? [0, 1];
+    const [sx, sy] = (view.flip ?? [false, false]).map(f => (f ? -1 : 1));
+    const box = ([lo, hi], sgn) => (sgn > 0 ? [lo, hi] : [-hi, -lo]);
+
+    const first = network(view.first);
+    const second = network(view.second);
+    const subtract = !!(first && second && !view.overlay);
+    const secondColor = second && first && second.color === first.color ? second.secondaryColor : second?.color;
+
+    // ── networks ──
+    const networks = [];
+    const weightsOf = (net, minus) => Object.fromEntries(conn.map((c, j) => [c, net.w[j] - (minus ? minus.w[j] : 0)]));
+    let primaryWeights = null;
+    if (first) {
+        if (subtract) {
+            const name = `${first.name} - ${second.name}`;
+            primaryWeights = weightsOf(first, second);
+            networks.push({ name, weights: primaryWeights, colors: [first.color, secondColor] });
+        } else {
+            primaryWeights = weightsOf(first);
+            networks.push({ name: first.name, weights: primaryWeights, colors: [first.color] });
+            if (second) networks.push({ name: second.name, weights: weightsOf(second), colors: [secondColor] });
+        }
+    }
+
+    // ── range: unit points and CI boxes on the plotted dims ──
+    let extent = 0;
+    const grow = (v) => { if (finite(v)) extent = Math.max(extent, Math.abs(v)); };
+    for (let i = 0; i < unitLabels.length; i++) { grow(web.points[i * stride + ax]); grow(web.points[i * stride + ay]); }
+    for (const g of web.groups) if (g.ci && g.ci[ax] && g.ci[ay]) [...g.ci[ax], ...g.ci[ay]].forEach(grow);
+    if (!extent) for (let c = 0; c < codes.length; c++) { grow(model.positions[c * stride + ax]); grow(model.positions[c * stride + ay]); }
+
+    // ── nodes: size = 2 × (4.75 × Σ|incident weight| × scale + 1) px ──
+    const nodeRows = codes.map((code, c) => {
+        const row = { code, label: code, x: sx * model.positions[c * stride + ax], y: sy * model.positions[c * stride + ay] };
+        if (primaryWeights) {
+            let sum = 0;
+            pairs.forEach(([src, tgt], j) => {
+                if ((src === code && !ordered) || tgt === code) sum += Math.abs(primaryWeights[conn[j]] || 0);
+            });
+            row.size = 2 * (NODE_PX * sum * scale + 1);
+        }
+        return row;
+    });
+
+    // ── unit points, coloured by group ──
+    const groupOfUnit = new Map();
+    web.groups.forEach(g => g.idx.forEach(i => groupOfUnit.set(i, g)));
+    const pinned = new Set([view.first, view.second].filter(s => s?.kind === 'unit').map(s => unitPath(s.name)));
+    const pointRows = unitLabels.map((lbl, i) => ({
+        set: UNIT_SET, id: unitPath(lbl),
+        x: sx * web.points[i * stride + ax], y: sy * web.points[i * stride + ay],
+        color: groupOfUnit.has(i) ? colorOf(groupOfUnit.get(i)) : NODE_COLOR,
+        label: unitPath(lbl), 'label.pinned': pinned.has(unitPath(lbl)),
+    }));
+
+    // ── group means with 95% CI boxes ──
+    const groupRows = web.groups.filter(g => g.showMean && g.n > 0).map(g => {
+        const x = sx * g.mean[ax], y = sy * g.mean[ay];
+        const row = { group: g.name, x, y, color: colorOf(g), intervals: 'none', outlier: false, label: g.name };
+        const ci = g.ci;
+        if (ci && ci[ax] && ci[ay] && [ci[ax][0], ci[ax][1], ci[ay][0], ci[ay][1]].every(finite) && g.n > 1) {
+            const [xl, xh] = box(ci[ax], sx), [yl, yh] = box(ci[ay], sy);
+            Object.assign(row, { intervals: style.ci === false ? 'none' : 'box', 'x.low': xl, 'x.high': xh, 'y.low': yl, 'y.high': yh });
+        }
+        const oi = g.oi;
+        if (oi && finite(oi[ax]) && finite(oi[ay])) {
+            const ox = Math.abs(oi[ax]), oy = Math.abs(oi[ay]);
+            Object.assign(row, { outlier: style.outliers === true, 'x.olow': x - ox, 'x.ohigh': x + ox, 'y.olow': y - oy, 'y.ohigh': y + oy });
+        }
+        return row;
+    });
+
+    const LABELS = ['label', 'label.dx', 'label.dy'];
+    const data = {
+        directed: ordered,
+        updated: Date.now(),
+        nodes: frame(['code', 'x', 'y', 'size', ...LABELS], { code: 'character', label: 'character' }, nodeRows),
+        node_id_col: 'code',
+        x_col: 'x',
+        y_col: 'y',
+        pointsets: frame(['set', 'id', 'x', 'y', 'color', ...LABELS, 'label.pinned'],
+            { set: 'character', id: 'character', color: 'character', label: 'character', 'label.pinned': 'logical' }, pointRows),
+    };
+    if (groupRows.length)
+        data.groups = frame(['group', 'x', 'y', 'color', 'intervals', 'outlier', 'x.low', 'x.high', 'y.low', 'y.high',
+            'x.olow', 'x.ohigh', 'y.olow', 'y.ohigh', ...LABELS],
+            { group: 'character', color: 'character', intervals: 'character', outlier: 'logical', label: 'character' }, groupRows);
+    if (networks.length) {
+        const cols = ['name', ...conn];
+        data.networks = frame(cols, { name: 'character' },
+            networks.map(n => ({ name: n.name, ...n.weights })));
+    }
+
+    // ── layers ──
+    const axisLabel = (d) => {
+        const name = entry.dimNames?.[d];
+        const v = entry.variance?.[d];
+        if (!name) return null;
+        return finite(v) ? `${name} (${(v * 100).toFixed(2)}%)` : name;
+    };
+    const layers = [['qe-axes', { 'fixed-size': '', x: axisLabel(ax), y: axisLabel(ay) }]];
+    if (networks.length) {
+        layers.push(['qe-nodes', { label: 'on', 'label-font-weight': 'bold',
+            unconnected: style.hideUnconnected ? 'hide' : null, 'fixed-size': '' }]);
+        for (const n of networks)
+            layers.push(['qe-edges', {
+                network: n.name, name: n.name, colors: n.colors.join(','),
+                threshold: style.minWeight > 0 ? String(style.minWeight) : null,
+                'self-size': ordered ? 'node' : null, 'color-scale': 'plot', 'fixed-size': '',
+            }]);
+    }
+    layers.push(['qe-points', { set: UNIT_SET, label: 'auto', 'fixed-size': '' }]);
+    if (groupRows.length) layers.push(['qe-means', { 'fixed-size': '', label: 'on' }]);
+
+    let edgePx = EDGE_PX * scale;
+    if (subtract) {
+        const maxDiff = Math.max(0, ...Object.values(primaryWeights).map(Math.abs));
+        if (maxDiff > 0) edgePx = SUBTRACTION_MAX_PX / maxDiff;
+    }
+
+    const title = !first ? 'All units'
+        : (subtract ? `${first.name} – ${second.name}`
+            : second ? `${first.name} + ${second.name}` : first.name) + ` (scaled ${scale.toFixed(1)}x)`;
+
+    const maxWeight = Math.max(0, ...networks.flatMap(n => Object.values(n.weights).map(Math.abs).filter(finite)));
+
+    return {
+        data, layers, edgePx, title, maxWeight, scale, subtract,
+        attrs: {
+            range: extent > 0 ? String(extent * RANGE_MODIFIER) : null,
+            'view-padding': '1',
+            'zoom-step': '1.5', 'zoom-min': String(1 / 15), 'zoom-max': '100',
+            'point-scale': '1',
+            'label-font-size': String(style.textSize > 0 ? style.textSize : TEXT_SIZE),
+            'label-font-color': LABEL_COLOR,
+            'label-font-family': LABEL_FONT,
+        },
+    };
+}
+
+// ── Set file (js/ENAmodel.js constructor defaults) ────────────────────────────
+
+export const SET_FORMAT_VERSION = 1;
+
+const MODEL_SETTINGS = {
+    stanza: 'moving', stanzaSize: 4, nodePositions: 'optimal', trajectoryBy: 'Activity',
+    rotationMatrixSet: null, weightModel: false, weightModelBy: null,
+    autoUpdateRotations: true, meansRotated: 1, regressionRotated: 1,
+    customRegression: false, customRegressionFormula: '', yRegressionFormula: null,
+    meansRotatedBy: [], sphereNorm: true, codeVariablesBinary: false, ordered: false,
+    pria: false, priaThreshold: 0.95, alwaysAskNAs: true, createGroupDialog: true,
+    priaDialog: true, codeMask: null, codeMaskLabels: null, centerZero: true,
+    hoo_flexible: false, derivedCodes: [], derivedStrategy: 'prime',
+};
+const TOOL_SETTINGS = {
+    dimensionLabels: true, xAxis: null, yAxis: null, varianceExplained: true,
+    scaleUnits: true, unitLabels: false, groupLabels: true, confidenceIntervals: true,
+    outlierIntervals: false, groups: true, comparisonNetwork: true, codeLabels: true,
+    showUnconnectedCodes: true, connectionWeights: true, minimumThreshold: 0,
+    scaleLineWeights: 1, yFlip: false, xFlip: false, dimensionX: null, dimensionY: null,
+    displayBy: null, intensityBy: 'set', intensityMinimum: 0, intensityMaximum: 1,
+    textSize: TEXT_SIZE, meanOrSum: true,
+};
+const STATS_SETTINGS = {
+    significantDigits: 2, type: null, comparison: { first: null, second: null },
+    results: {}, inputText: null, Correlation1: null, Correlation2: null,
+};
+
+/**
+ * The stored set the web tool imports: its own unified set format, with the
+ * web tool's defaults and this model's choices applied.
+ *   s = { setName, csvName, units, conversation, codes, groups: [{ name, color,
+ *         secondaryColor, showMean }], groupCol, window (number | Infinity),
+ *         weightModel, sphereNorm, ordered, rotation: { meansRotated,
+ *         regressionRotated, meansRotatedBy }, plotted: { primary, secondary },
+ *         source, unitTree?, convoTree?, order? }
+ *   unitTree: [{ values: [unit column values...], include }] for every unit in
+ *   the data, when some are excluded (units_used). The set then stores the
+ *   whole units tree with each node's include flag: the web tool rebuilds the
+ *   tree from the data on load and copies flags from stored nodes by path
+ *   (js/ena.ww.organize-data.js), and models only included units (unitsUsed,
+ *   js/WasmWrapper.js generate()). Every level is stored so each rebuilt node
+ *   has an exact match — a missing intermediate level would take its flag
+ *   from the first stored node below it.
+ */
+function unitNodes(unitTree, unitCols, groups, depth = 0, prefix = []) {
+    const byValue = new Map();
+    for (const u of unitTree) {
+        const v = String(u.values[depth]);
+        if (!byValue.has(v)) byValue.set(v, []);
+        byValue.get(v).push(u);
+    }
+    const leaf = depth === unitCols.length - 1;
+    return [...byValue.entries()].map(([name, members], index) => {
+        const pathArr = [...prefix, name];
+        const items = leaf ? [] : unitNodes(members, unitCols, groups, depth + 1, pathArr);
+        const g = depth === 0 ? groups.find(x => x.name === name) : null;
+        return {
+            name, path: pathArr.join('::'), pathArr, condition: unitCols[depth], leaf, items, index,
+            include: members.some(m => m.include), show: true, showLabel: true,
+            isCustomGroup: false, singleConditionWrapper: false,
+            ...(g && { color: g.color, secondaryColor: g.secondaryColor, showMean: g.showMean,
+                       showConfidence: true, showOutline: false, showHiddenPoints: false }),
+        };
+    });
+}
+
+export function setFile(s) {
+    const now = Date.now();
+    const infinite = s.window === Infinity;
+    return {
+        formatVersion: SET_FORMAT_VERSION,
+        source: s.source,
+        setName: s.setName,
+        sampleSet: false,
+        description: s.description ?? null,
+        parentSet: { projectKey: null, setID: null },
+        created: now,
+        modified: now,
+        changeText: 'Set Created',
+        changeLocation: 'Projects',
+        deleted: false,
+        dataRef: s.csvName,
+        enasetRef: null,
+        set: null,
+        rotationSet: null,
+        model: {
+            units: [...s.units], codes: [...s.codes], conversation: [...s.conversation],
+            hoo: {}, group: null, groups: [], columns: {}, utterance: null, rotationSet: null,
+        },
+        selectionsRef: null,
+        selections: {
+            // Top-level groups only: the web tool builds the units under them
+            // from the data, keeping these groups' colours and flags. With
+            // excluded units, the whole tree (see unitNodes).
+            units: s.unitTree ? unitNodes(s.unitTree, s.units, s.groups) : s.groups.map((g, i) => ({
+                name: g.name, path: g.name, pathArr: [g.name],
+                condition: s.groupCol, leaf: false, items: [], index: i,
+                include: true, show: true, isCustomGroup: false, singleConditionWrapper: false,
+                color: g.color, secondaryColor: g.secondaryColor,
+                showMean: g.showMean, showConfidence: true, showOutline: false,
+                showHiddenPoints: false, showLabel: true,
+            })),
+            // Excluded horizons (exclude_conversations): the whole conversation
+            // tree with include flags, which WasmWrapper._conversationsUsed()
+            // reads (last-column nodes with include === true).
+            conversation: s.convoTree ? unitNodes(s.convoTree, s.conversation, []) : [],
+            codes: s.codes.map((label, index) => ({ label, label_r: label, show: true, showLabel: true, include: true, index })),
+            groups: [],
+            stanzas: [],
+        },
+        settings: {
+            model: {
+                ...structuredClone(MODEL_SETTINGS),
+                stanza: infinite ? 'infinite' : 'moving',
+                stanzaSize: infinite ? FALLBACK_WINDOW : s.window,
+                windowType: 'lines',
+                weightModel: !!s.weightModel,
+                weightModelBy: s.weightModel ?? null,
+                sphereNorm: s.sphereNorm !== false,
+                ordered: !!s.ordered,
+                meansRotated: s.rotation.meansRotated,
+                regressionRotated: s.rotation.regressionRotated,
+                meansRotatedBy: [...s.rotation.meansRotatedBy],
+                // Window order (WasmWrapper._orderColumn): a column, or the default.
+                ...(s.order && { order: s.order }),
+                // The Code Masking Matrix, positional over codeMaskLabels
+                // (WasmWrapper._alignCodeMask passes it to rena-wasm as is).
+                ...(s.codeMask && { codeMask: s.codeMask.map(r => [...r]), codeMaskLabels: [...s.codes] }),
+            },
+            tools: structuredClone(TOOL_SETTINGS),
+            stats: {
+                ...structuredClone(STATS_SETTINGS),
+                comparison: { first: s.plotted.primary ?? null, second: s.plotted.secondary ?? null },
+            },
+            view: {
+                plottedNodes: { primary: s.plotted.primary ?? null, secondary: s.plotted.secondary ?? null },
+                selectedCodes: [],
+                dataViewPlot: 'primary',
+            },
+        },
+    };
+}
+
+// ── ZIP (stored entries deflated; no dependency) ──────────────────────────────
+
+const CRC_TABLE = (() => {
+    const t = new Uint32Array(256);
+    for (let n = 0; n < 256; n++) {
+        let c = n;
+        for (let k = 0; k < 8; k++) c = c & 1 ? 0xEDB88320 ^ (c >>> 1) : c >>> 1;
+        t[n] = c >>> 0;
+    }
+    return t;
+})();
+
+function crc32(buf) {
+    let c = 0xFFFFFFFF;
+    for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]) & 0xFF] ^ (c >>> 8);
+    return (c ^ 0xFFFFFFFF) >>> 0;
+}
+
+function dosTime(d) {
+    return {
+        time: (d.getHours() << 11) | (d.getMinutes() << 5) | (d.getSeconds() >> 1),
+        date: ((d.getFullYear() - 1980) << 9) | ((d.getMonth() + 1) << 5) | d.getDate(),
+    };
+}
+
+/** entries: [{ name, data: Buffer | string }] (a name ending in '/' is a folder). */
+export function zip(entries) {
+    const { time, date } = dosTime(new Date());
+    const locals = [], centrals = [];
+    let offset = 0;
+    for (const e of entries) {
+        const name = Buffer.from(e.name, 'utf8');
+        const raw = e.name.endsWith('/') ? Buffer.alloc(0) : Buffer.isBuffer(e.data) ? e.data : Buffer.from(e.data ?? '', 'utf8');
+        const deflated = raw.length ? deflateRawSync(raw) : raw;
+        const method = raw.length ? 8 : 0;
+        const crc = crc32(raw);
+
+        const local = Buffer.alloc(30);
+        local.writeUInt32LE(0x04034b50, 0);
+        local.writeUInt16LE(20, 4);
+        local.writeUInt16LE(0x0800, 6);          // UTF-8 names
+        local.writeUInt16LE(method, 8);
+        local.writeUInt16LE(time, 10);
+        local.writeUInt16LE(date, 12);
+        local.writeUInt32LE(crc, 14);
+        local.writeUInt32LE(deflated.length, 18);
+        local.writeUInt32LE(raw.length, 22);
+        local.writeUInt16LE(name.length, 26);
+        local.writeUInt16LE(0, 28);
+        locals.push(local, name, deflated);
+
+        const central = Buffer.alloc(46);
+        central.writeUInt32LE(0x02014b50, 0);
+        central.writeUInt16LE(20, 4);
+        central.writeUInt16LE(20, 6);
+        central.writeUInt16LE(0x0800, 8);
+        central.writeUInt16LE(method, 10);
+        central.writeUInt16LE(time, 12);
+        central.writeUInt16LE(date, 14);
+        central.writeUInt32LE(crc, 16);
+        central.writeUInt32LE(deflated.length, 20);
+        central.writeUInt32LE(raw.length, 24);
+        central.writeUInt16LE(name.length, 28);
+        central.writeUInt32LE(e.name.endsWith('/') ? 0x10 : 0, 38);   // directory attribute
+        central.writeUInt32LE(offset, 42);
+        centrals.push(central, name);
+
+        offset += local.length + name.length + deflated.length;
+    }
+    const centralSize = centrals.reduce((n, b) => n + b.length, 0);
+    const end = Buffer.alloc(22);
+    end.writeUInt32LE(0x06054b50, 0);
+    end.writeUInt16LE(entries.length, 8);
+    end.writeUInt16LE(entries.length, 10);
+    end.writeUInt32LE(centralSize, 12);
+    end.writeUInt32LE(offset, 16);
+    return Buffer.concat([...locals, ...centrals, end]);
+}