@qe-mcp/ena 0.0.0-stage → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +612 -2
- package/fonts/Inter-Bold.ttf +0 -0
- package/fonts/Inter-Regular.ttf +0 -0
- package/fonts/OFL.txt +93 -0
- package/fonts/README.md +4 -0
- package/hooks/ena-privacy-gate.mjs +31 -0
- package/lines.js +349 -0
- package/manifest.json +120 -0
- package/package.json +42 -4
- package/server.js +3449 -0
- package/skill/ena-analysis/SKILL.md +348 -0
- package/webena.js +651 -0
- package/xlsx.js +130 -0
package/server.js
ADDED
|
@@ -0,0 +1,3449 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* ena-mcp-wasm — MCP server for Epistemic Network Analysis
|
|
4
|
+
*
|
|
5
|
+
* Zero-Python WASM backend using @qe-libs/rena-wasm.
|
|
6
|
+
* Tools:
|
|
7
|
+
* ena_profile — privacy-safe column metadata for a local CSV
|
|
8
|
+
* ena_inspect — triage tabular data for ENA suitability
|
|
9
|
+
* ena_fit — full pipeline; returns reusable model_id
|
|
10
|
+
* ena_accumulate — accumulation step only
|
|
11
|
+
* ena_window — window-size estimate (Cross-Covariance Decay)
|
|
12
|
+
* ena_compare_groups — mean-rotation comparison of two groups; returns model_id
|
|
13
|
+
* tma_fit — directed TMA / ONA model (context tensor); returns model_id
|
|
14
|
+
* tma_accumulate — TMA / ONA accumulation step only
|
|
15
|
+
* ena_plot — render any view from a cached model_id
|
|
16
|
+
* ena_export — write a cached model as an ENA Web Tool project ZIP
|
|
17
|
+
*
|
|
18
|
+
* Defaults, plots and exports follow the ENA Web Tool (see webena.js), so a
|
|
19
|
+
* model fit here is the same model, drawn the same way, once imported there.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { readFileSync, mkdirSync, writeFileSync, statSync, existsSync } from 'fs';
|
|
23
|
+
import { tmpdir } from 'os';
|
|
24
|
+
import { createHash } from 'crypto';
|
|
25
|
+
import { join, basename, dirname, extname } from 'path';
|
|
26
|
+
import * as web from './webena.js';
|
|
27
|
+
import { xlsxToCSV, isXlsx } from './xlsx.js';
|
|
28
|
+
import { connectionWindows, viewerHtml } from './lines.js';
|
|
29
|
+
import { defaultTensor } from '@qe-libs/rena-wasm/src/tensor.js';
|
|
30
|
+
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
31
|
+
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
32
|
+
import {
|
|
33
|
+
CallToolRequestSchema,
|
|
34
|
+
ListToolsRequestSchema,
|
|
35
|
+
ListPromptsRequestSchema,
|
|
36
|
+
GetPromptRequestSchema,
|
|
37
|
+
} from '@modelcontextprotocol/sdk/types.js';
|
|
38
|
+
import loadENA, { detectParams, isBinary } from '@qe-libs/rena-wasm';
|
|
39
|
+
|
|
40
|
+
// Path to the qeviz UMD bundle (self-contained, no external deps)
|
|
41
|
+
import { createRequire } from 'module';
|
|
42
|
+
const _require = createRequire(import.meta.url);
|
|
43
|
+
const QEVIZ_BUNDLE = _require.resolve('@qe-libs/qeviz');
|
|
44
|
+
let _qevizJS = null;
|
|
45
|
+
function qevizJS() {
|
|
46
|
+
// The bundle is inlined into pages and jsdom, so drop its source-map comment
|
|
47
|
+
// (the .map file is not next to the inlined copy).
|
|
48
|
+
if (!_qevizJS) _qevizJS = readFileSync(QEVIZ_BUNDLE, 'utf8')
|
|
49
|
+
.replace(/\n\/\/# sourceMappingURL=\S*\s*$/, '\n');
|
|
50
|
+
return _qevizJS;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// ── CSV parser ────────────────────────────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
// RFC 4180: quoted fields may hold commas, doubled quotes and line breaks
|
|
56
|
+
// (common in discourse text), so the text is scanned as a whole rather than
|
|
57
|
+
// split into lines first. A UTF-8 BOM (Excel exports) is dropped so the first
|
|
58
|
+
// header matches its column name.
|
|
59
|
+
function parseCSV(text) {
|
|
60
|
+
if (text.charCodeAt(0) === 0xFEFF) text = text.slice(1);
|
|
61
|
+
|
|
62
|
+
const records = [];
|
|
63
|
+
let record = [], field = '', inQ = false;
|
|
64
|
+
for (let i = 0; i < text.length; i++) {
|
|
65
|
+
const ch = text[i];
|
|
66
|
+
if (inQ) {
|
|
67
|
+
if (ch === '"') {
|
|
68
|
+
if (text[i + 1] === '"') { field += '"'; i++; }
|
|
69
|
+
else inQ = false;
|
|
70
|
+
} else field += ch;
|
|
71
|
+
} else if (ch === '"') inQ = true;
|
|
72
|
+
else if (ch === ',') { record.push(field); field = ''; }
|
|
73
|
+
else if (ch === '\n' || ch === '\r') {
|
|
74
|
+
if (ch === '\r' && text[i + 1] === '\n') i++;
|
|
75
|
+
record.push(field); records.push(record);
|
|
76
|
+
record = []; field = '';
|
|
77
|
+
} else field += ch;
|
|
78
|
+
}
|
|
79
|
+
if (field !== '' || record.length) { record.push(field); records.push(record); }
|
|
80
|
+
|
|
81
|
+
// Blank lines (a lone empty field) are not rows.
|
|
82
|
+
const nonBlank = records.filter(r => r.length > 1 || r[0].trim() !== '');
|
|
83
|
+
if (!nonBlank.length) return [];
|
|
84
|
+
const headers = nonBlank[0].map(h => h.trim());
|
|
85
|
+
return nonBlank.slice(1).map(vals => {
|
|
86
|
+
const row = {};
|
|
87
|
+
headers.forEach((h, i) => { row[h] = vals[i] ?? ''; });
|
|
88
|
+
return row;
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// CSV text of a data file: a .csv as is, an .xlsx's first worksheet (as the
|
|
93
|
+
// ENA Web Tool reads one).
|
|
94
|
+
function dataFileText(dataPath) {
|
|
95
|
+
if (/\.xls$/i.test(dataPath))
|
|
96
|
+
throw new Error('Old-format .xls files are not supported. Save the sheet as .xlsx or .csv.');
|
|
97
|
+
if (isXlsx(dataPath)) {
|
|
98
|
+
try { return xlsxToCSV(readFileSync(dataPath)); }
|
|
99
|
+
catch (e) { throw new Error(`Could not read ${basename(dataPath)} as an Excel workbook: ${e.message}`); }
|
|
100
|
+
}
|
|
101
|
+
return readFileSync(dataPath, 'utf8');
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function loadData(dataPath, dataCsv) {
|
|
105
|
+
if (dataPath) return parseCSV(dataFileText(dataPath));
|
|
106
|
+
if (dataCsv) return parseCSV(dataCsv);
|
|
107
|
+
throw new Error('Provide either data_path (path to a .csv or .xlsx file) or data_csv (raw CSV string).');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const toArray = (v) => (Array.isArray(v) ? v : v == null ? [] : [v]);
|
|
111
|
+
|
|
112
|
+
// ── Result helpers ────────────────────────────────────────────────────────────
|
|
113
|
+
|
|
114
|
+
const R = (v, places = 6) => Math.round(v * 10 ** places) / 10 ** places;
|
|
115
|
+
|
|
116
|
+
// stride: the row width of fa when it holds more columns than are reported.
|
|
117
|
+
function toRecords(fa, nRows, nCols, labels, prefix = 'dim', stride = nCols) {
|
|
118
|
+
return Array.from({ length: nRows }, (_, i) => {
|
|
119
|
+
const rec = { label: labels[i] };
|
|
120
|
+
for (let j = 0; j < nCols; j++)
|
|
121
|
+
rec[`${prefix}${j + 1}`] = R(fa[i * stride + j]);
|
|
122
|
+
return rec;
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Share of total variance per dimension (= R's set$model$variance), or null
|
|
127
|
+
// when the backend doesn't report it.
|
|
128
|
+
function varianceExplained(model, dims) {
|
|
129
|
+
const v = model.variance;
|
|
130
|
+
if (!v?.length) return null;
|
|
131
|
+
return Object.fromEntries(Array.from({ length: Math.min(dims, v.length) },
|
|
132
|
+
(_, i) => [`dim${i + 1}`, R(v[i], 4)]));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Goodness of fit per dimension: how well the distances between unit points
|
|
136
|
+
// match the distances between their network centroids — Pearson r and Spearman
|
|
137
|
+
// rho over all pairwise differences. Ported from the ENA Web Tool
|
|
138
|
+
// (js/WasmWrapper.js, "correlations (Goodness of Fit)"), which flags < 0.9.
|
|
139
|
+
// Both are unaffected by rescaling, so the raw points and centroids are used.
|
|
140
|
+
const GOF_THRESHOLD = 0.9;
|
|
141
|
+
|
|
142
|
+
// Dimensions every fit keeps, so ena_plot can put any of them on an axis — as
|
|
143
|
+
// many as the ENA Web Tool computes (config/default.json sets.dimensions).
|
|
144
|
+
const PLOT_DIMS = 6;
|
|
145
|
+
function goodnessOfFit(model, dims) {
|
|
146
|
+
const pts = model.centroids, cts = model.networkCentroids, n = model.nUnits;
|
|
147
|
+
const stride = model.dims ?? dims;
|
|
148
|
+
if (!pts || !cts || n < 3) return null;
|
|
149
|
+
const ranks = (arr) => {
|
|
150
|
+
const s = arr.map((v, i) => ({ v, i })).sort((a, b) => a.v - b.v);
|
|
151
|
+
const r = new Float64Array(arr.length);
|
|
152
|
+
for (let i = 0; i < s.length;) {
|
|
153
|
+
let j = i;
|
|
154
|
+
while (j < s.length && s[j].v === s[i].v) j++;
|
|
155
|
+
for (let k = i; k < j; k++) r[s[k].i] = (i + 1 + j) / 2;
|
|
156
|
+
i = j;
|
|
157
|
+
}
|
|
158
|
+
return r;
|
|
159
|
+
};
|
|
160
|
+
const pearson = (x, y) => {
|
|
161
|
+
const m = x.length;
|
|
162
|
+
let sx = 0, sy = 0;
|
|
163
|
+
for (let i = 0; i < m; i++) { sx += x[i]; sy += y[i]; }
|
|
164
|
+
const mx = sx / m, my = sy / m;
|
|
165
|
+
let num = 0, dx2 = 0, dy2 = 0;
|
|
166
|
+
for (let i = 0; i < m; i++) {
|
|
167
|
+
const dx = x[i] - mx, dy = y[i] - my;
|
|
168
|
+
num += dx * dy; dx2 += dx * dx; dy2 += dy * dy;
|
|
169
|
+
}
|
|
170
|
+
const den = Math.sqrt(dx2 * dy2);
|
|
171
|
+
return den === 0 ? 0 : Math.max(-1, Math.min(1, num / den));
|
|
172
|
+
};
|
|
173
|
+
const out = {};
|
|
174
|
+
const nPairs = n * (n - 1) / 2;
|
|
175
|
+
for (let d = 0; d < Math.min(dims, stride); d++) {
|
|
176
|
+
const pd = new Float64Array(nPairs), cd = new Float64Array(nPairs);
|
|
177
|
+
let k = 0;
|
|
178
|
+
for (let i = 0; i < n - 1; i++)
|
|
179
|
+
for (let j = i + 1; j < n; j++, k++) {
|
|
180
|
+
pd[k] = pts[i * stride + d] - pts[j * stride + d];
|
|
181
|
+
cd[k] = cts[i * stride + d] - cts[j * stride + d];
|
|
182
|
+
}
|
|
183
|
+
out[`dim${d + 1}`] = { pearson: R(pearson(pd, cd), 4), spearman: R(pearson(ranks(Array.from(pd)), ranks(Array.from(cd))), 4) };
|
|
184
|
+
}
|
|
185
|
+
return out;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const fitLine = (gof) => gof
|
|
189
|
+
? [` Goodness of fit (Pearson / Spearman): ` +
|
|
190
|
+
Object.entries(gof).map(([d, g]) => `${d}=${g.pearson.toFixed(2)} / ${g.spearman.toFixed(2)}` +
|
|
191
|
+
(Math.min(g.pearson, g.spearman) < GOF_THRESHOLD ? ' ⚠ below 0.9' : '')).join(', ')]
|
|
192
|
+
: [];
|
|
193
|
+
|
|
194
|
+
const varianceLine = (ve) => ve
|
|
195
|
+
? [` Variance explained: ${Object.entries(ve).map(([d, v]) => `${d}=${(v * 100).toFixed(1)}%`).join(', ')}`]
|
|
196
|
+
: [];
|
|
197
|
+
|
|
198
|
+
// Matches rena-wasm's rowKey() — separator must stay '__'
|
|
199
|
+
function rowKey(row, cols) {
|
|
200
|
+
return cols.map(c => row[c]).join('__');
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function buildUnitGroupMap(rows, unitCols, groupCol) {
|
|
204
|
+
const map = new Map();
|
|
205
|
+
for (const row of rows) {
|
|
206
|
+
const key = rowKey(row, unitCols);
|
|
207
|
+
if (!map.has(key)) map.set(key, String(row[groupCol]));
|
|
208
|
+
}
|
|
209
|
+
return map;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// ── Generalized means rotation (GMR) parameters ─────────────────────────────
|
|
213
|
+
// Mirrors rENA's ena.rotate.by.generalized(): x_var / y_var name unit-level
|
|
214
|
+
// metadata columns, the first being the target and the rest covariates that
|
|
215
|
+
// libqe controls for via Lasso. The model matrix is R's
|
|
216
|
+
// model.matrix(~ . or ~ .^2)[, -1]: numeric columns as-is, categorical ones
|
|
217
|
+
// treatment-coded against the first sorted level, pairwise interactions
|
|
218
|
+
// (first term varying fastest) when there are covariates and interactions=TRUE.
|
|
219
|
+
|
|
220
|
+
const MISSING = /^(|na|nan|null)$/i;
|
|
221
|
+
|
|
222
|
+
// One value per modeled unit, in unitLabels order.
|
|
223
|
+
function unitLevelValues(rows, unitCols, unitLabels, col) {
|
|
224
|
+
if (!rows.length || !(col in rows[0]))
|
|
225
|
+
throw new Error(`Generalized rotation: column '${col}' is not in the data.`);
|
|
226
|
+
const vals = new Map(), varies = new Set();
|
|
227
|
+
for (const row of rows) {
|
|
228
|
+
const key = rowKey(row, unitCols);
|
|
229
|
+
const v = String(row[col] ?? '').trim();
|
|
230
|
+
if (!vals.has(key)) vals.set(key, v);
|
|
231
|
+
else if (vals.get(key) !== v) varies.add(key);
|
|
232
|
+
}
|
|
233
|
+
const out = unitLabels.map(l => vals.get(l) ?? '');
|
|
234
|
+
const variesHere = unitLabels.filter(l => varies.has(l));
|
|
235
|
+
if (variesHere.length)
|
|
236
|
+
throw new Error(`Generalized rotation: '${col}' takes more than one value within ${variesHere.length} ` +
|
|
237
|
+
`of ${unitLabels.length} unit(s). x_var / y_var must be unit-level attributes — one value per unit.`);
|
|
238
|
+
const missing = unitLabels.filter((_, i) => MISSING.test(out[i]));
|
|
239
|
+
if (missing.length)
|
|
240
|
+
throw new Error(`Generalized rotation: '${col}' is missing for ${missing.length} of ${unitLabels.length} ` +
|
|
241
|
+
'unit(s). Exclude those units with units_used (their rows still count as context; the units list ' +
|
|
242
|
+
'in an ena_accumulate result gives the labels) or fill in the values. (rENA fails on missing values here too.)');
|
|
243
|
+
return out;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function designTerm(name, values) {
|
|
247
|
+
const nums = values.map(Number);
|
|
248
|
+
if (nums.every(Number.isFinite)) return { name, numeric: true, cols: [{ name, vals: nums }] };
|
|
249
|
+
const levels = [...new Set(values)].sort((a, b) => a.localeCompare(b)); // R's factor() order
|
|
250
|
+
return {
|
|
251
|
+
name, numeric: false, levels,
|
|
252
|
+
cols: levels.slice(1).map(l => ({ name: `${name}${l}`, vals: values.map(v => (v === l ? 1 : 0)) })),
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function gmrAxis(rows, unitCols, unitLabels, vars, interactions, select2) {
|
|
257
|
+
const names = Array.isArray(vars) ? vars : [vars];
|
|
258
|
+
const values = names.map(c => unitLevelValues(rows, unitCols, unitLabels, c));
|
|
259
|
+
const terms = names.map((c, i) => designTerm(c, values[i]));
|
|
260
|
+
|
|
261
|
+
const cols = terms.flatMap(t => t.cols);
|
|
262
|
+
if (names.length > 1 && interactions)
|
|
263
|
+
for (let i = 0; i < terms.length; i++)
|
|
264
|
+
for (let j = i + 1; j < terms.length; j++)
|
|
265
|
+
for (const b of terms[j].cols)
|
|
266
|
+
for (const a of terms[i].cols)
|
|
267
|
+
cols.push({ name: `${a.name}:${b.name}`, vals: a.vals.map((v, r) => v * b.vals[r]) });
|
|
268
|
+
|
|
269
|
+
const n = unitLabels.length, p = cols.length;
|
|
270
|
+
const target = terms[0];
|
|
271
|
+
if (!target.cols.length)
|
|
272
|
+
throw new Error(`Generalized rotation: '${target.name}' has a single value across the modeled units.`);
|
|
273
|
+
const mm = new Float64Array(n * p);
|
|
274
|
+
for (let r = 0; r < n; r++)
|
|
275
|
+
for (let c = 0; c < p; c++) mm[r * p + c] = cols[c].vals[r];
|
|
276
|
+
|
|
277
|
+
// Target encoding: numeric as-is; categorical as 0-based codes in order of
|
|
278
|
+
// appearance, with the select_2_groups pair first (libqe uses codes 0 / 1).
|
|
279
|
+
let enc, nGroups = 0, subset = [];
|
|
280
|
+
if (target.numeric) {
|
|
281
|
+
if (select2) throw new Error(`select_2_groups needs a categorical target; '${target.name}' is numeric.`);
|
|
282
|
+
enc = target.cols[0].vals;
|
|
283
|
+
} else {
|
|
284
|
+
const seen = [...new Set(values[0])];
|
|
285
|
+
if (select2) {
|
|
286
|
+
if (select2.length !== 2) throw new Error('select_2_groups takes exactly two values.');
|
|
287
|
+
for (const g of select2)
|
|
288
|
+
if (!seen.includes(String(g)))
|
|
289
|
+
throw new Error(`select_2_groups: no modeled unit has ${target.name}='${g}' ` +
|
|
290
|
+
`(${target.name} has ${seen.length} distinct values; ena_profile lists them when there are 10 or fewer).`);
|
|
291
|
+
}
|
|
292
|
+
const pair = select2?.map(String) ?? [];
|
|
293
|
+
const levels = [...pair, ...seen.filter(v => !pair.includes(v))];
|
|
294
|
+
enc = values[0].map(v => levels.indexOf(v));
|
|
295
|
+
nGroups = levels.length;
|
|
296
|
+
if (select2) subset = values[0].flatMap((v, i) => (pair.includes(v) ? [i] : []));
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
return {
|
|
300
|
+
names, terms, nCols: p,
|
|
301
|
+
modelMatrix: mm,
|
|
302
|
+
target: Float64Array.from(enc),
|
|
303
|
+
oneCols: Int32Array.from(target.cols.map((_, i) => i)), // target main effects come first
|
|
304
|
+
categorical: !target.numeric,
|
|
305
|
+
nGroups,
|
|
306
|
+
subset: Int32Array.from(subset),
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function buildGmrParams(rows, unitCols, unitLabels, p) {
|
|
311
|
+
if (!p.x_var || (Array.isArray(p.x_var) && !p.x_var.length))
|
|
312
|
+
throw new Error("rotation='generalized' requires x_var: the target column, optionally followed by covariate columns.");
|
|
313
|
+
const interactions = p.interactions ?? true;
|
|
314
|
+
const x = gmrAxis(rows, unitCols, unitLabels, p.x_var, interactions, p.select_2_groups ?? null);
|
|
315
|
+
const y = p.y_var ? gmrAxis(rows, unitCols, unitLabels, p.y_var, interactions, null) : null;
|
|
316
|
+
const n = unitLabels.length;
|
|
317
|
+
|
|
318
|
+
const gParams = {
|
|
319
|
+
xModelMatrix: x.modelMatrix, xmRows: n, xmCols: x.nCols,
|
|
320
|
+
xTarget: x.target, x1Cols: x.oneCols,
|
|
321
|
+
xCategorical: x.categorical, xNGroups: x.nGroups, xSubset: x.subset,
|
|
322
|
+
hasY: !!y,
|
|
323
|
+
...(y && {
|
|
324
|
+
yModelMatrix: y.modelMatrix, ymRows: n, ymCols: y.nCols,
|
|
325
|
+
yTarget: y.target, y1Cols: y.oneCols,
|
|
326
|
+
yCategorical: y.categorical, yNGroups: y.nGroups,
|
|
327
|
+
}),
|
|
328
|
+
};
|
|
329
|
+
|
|
330
|
+
const describe = (a, axis) => {
|
|
331
|
+
const t = a.terms[0];
|
|
332
|
+
const kind = t.numeric ? 'numeric' : `categorical, ${t.levels.length} levels`;
|
|
333
|
+
const cov = a.names.slice(1);
|
|
334
|
+
return ` ${axis} target: ${t.name} (${kind})` +
|
|
335
|
+
(cov.length ? `; controlling for ${cov.join(', ')}${interactions ? ' with pairwise interactions' : ''} (Lasso)` : '');
|
|
336
|
+
};
|
|
337
|
+
const lines = [
|
|
338
|
+
describe(x, 'GMR1'),
|
|
339
|
+
...(p.select_2_groups ? [` GMR1 fit on ${p.select_2_groups.join(' vs ')} (${x.subset.length} units)`] : []),
|
|
340
|
+
...(y ? [describe(y, 'GMR2')] : []),
|
|
341
|
+
];
|
|
342
|
+
return { gParams, lines };
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// Rotation options for rena-wasm fit(), from the tool's rotation parameters.
|
|
346
|
+
// rotation_from: project into another fitted model's space — the ENA Web
|
|
347
|
+
// Tool's "Use Rotation Matrix From" (WasmWrapper._rotationSetFor), rena-wasm's
|
|
348
|
+
// rotationSet (= R's rotation.set): centred on the source's centerVec, rotated
|
|
349
|
+
// by its rotation matrix, with its node positions.
|
|
350
|
+
function rotationSetFrom(sourceId, codes, ordered) {
|
|
351
|
+
const src = cacheGet(modelCache, sourceId);
|
|
352
|
+
if (!src)
|
|
353
|
+
throw new Error(`rotation_from: unknown model_id '${sourceId}'. Fit the source model first ` +
|
|
354
|
+
`(the ${MAX_MODELS} most recently used models are kept).`);
|
|
355
|
+
const m = src.model;
|
|
356
|
+
const srcOrdered = m.ordered === true;
|
|
357
|
+
if (srcOrdered !== ordered)
|
|
358
|
+
throw new Error(`rotation_from: model ${sourceId} is ${srcOrdered ? 'ordered (ONA/TMA)' : 'undirected (ENA)'}; ` +
|
|
359
|
+
`this model is ${ordered ? 'ordered' : 'undirected'}. Project only between models of the same kind, as the web tool does.`);
|
|
360
|
+
const same = src.codes.length === codes.length && [...src.codes].sort().every((c, i) => c === [...codes].sort()[i]);
|
|
361
|
+
if (!same)
|
|
362
|
+
throw new Error(`rotation_from: model ${sourceId} has codes ${src.codes.join(', ')}; this model needs the same codes.`);
|
|
363
|
+
if (!m.rotationMatrix || !m.centerVec)
|
|
364
|
+
throw new Error(`rotation_from: model ${sourceId} has no rotation to project into. Refit it with this server version.`);
|
|
365
|
+
return {
|
|
366
|
+
rotationSet: {
|
|
367
|
+
rotationMatrix: m.rotationMatrix, rotationCols: m.dims, nodes: m.positions, centerVec: m.centerVec,
|
|
368
|
+
codes: src.codes, columnNames: m.columnNames, eigenvalues: m.eigenvalues,
|
|
369
|
+
},
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function rotationOpts(rows, unitCols, unitLabels, params, uMap) {
|
|
374
|
+
if (params.rotation_from != null) {
|
|
375
|
+
const ordered = params._ordered === true;
|
|
376
|
+
return { opts: rotationSetFrom(String(params.rotation_from), toArray(params.codes), ordered),
|
|
377
|
+
lines: [` Projected into model ${params.rotation_from}'s space (its rotation, centring and node positions)`] };
|
|
378
|
+
}
|
|
379
|
+
const { rotation = 'svd', group_col, group1_value, group2_value } = params;
|
|
380
|
+
if (params._web_gmr) {
|
|
381
|
+
// The web tool's rotation by groups (resolveDefaults): its own GMR.
|
|
382
|
+
const { col, pair } = params._web_gmr;
|
|
383
|
+
const g = web.gmrParams(rows, unitCols, unitLabels, col, pair);
|
|
384
|
+
return {
|
|
385
|
+
opts: { rotation: 'generalized', gParams: g.gParams },
|
|
386
|
+
lines: [` GMR1 through ${pair.join(' vs ')} on ${col} (${g.subsetSize} units), as the ENA Web Tool rotates grouped units`],
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
if (rotation === 'svd') return { opts: { rotation: 'svd' }, lines: [] };
|
|
390
|
+
if (rotation === 'mean') {
|
|
391
|
+
if (!group_col || !group1_value || !group2_value)
|
|
392
|
+
throw new Error("rotation='mean' requires group_col, group1_value, and group2_value.");
|
|
393
|
+
const groupA = unitLabels.flatMap((lbl, i) => uMap.get(lbl) === String(group1_value) ? [i] : []);
|
|
394
|
+
const groupB = unitLabels.flatMap((lbl, i) => uMap.get(lbl) === String(group2_value) ? [i] : []);
|
|
395
|
+
if (!groupA.length) throw new Error(`No units found for ${group_col}=${group1_value}`);
|
|
396
|
+
if (!groupB.length) throw new Error(`No units found for ${group_col}=${group2_value}`);
|
|
397
|
+
return { opts: { rotation: 'mean', groupA, groupB }, lines: [] };
|
|
398
|
+
}
|
|
399
|
+
if (rotation === 'generalized') {
|
|
400
|
+
const { gParams, lines } = buildGmrParams(rows, unitCols, unitLabels, params);
|
|
401
|
+
return { opts: { rotation: 'generalized', gParams }, lines };
|
|
402
|
+
}
|
|
403
|
+
throw new Error(`Unknown rotation '${rotation}'. Supported: svd, mean, generalized.`);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// A generalized rotation on two selected groups colours the plot and runs the
|
|
407
|
+
// group tests on those groups, as a means rotation would, unless the caller
|
|
408
|
+
// named its own groups.
|
|
409
|
+
function withGmrGroups(params) {
|
|
410
|
+
if (params.rotation !== 'generalized' || !params.select_2_groups || params.group_col) return params;
|
|
411
|
+
const target = Array.isArray(params.x_var) ? params.x_var[0] : params.x_var;
|
|
412
|
+
return {
|
|
413
|
+
...params,
|
|
414
|
+
group_col: target,
|
|
415
|
+
group1_value: params.group1_value ?? String(params.select_2_groups[0]),
|
|
416
|
+
group2_value: params.group2_value ?? String(params.select_2_groups[1]),
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// ── Column-role detection ─────────────────────────────────────────────────────
|
|
421
|
+
// rena-wasm's detectParams: token-based name hints and nesting-aware unit /
|
|
422
|
+
// conversation suggestions. It classes a column as metadata when its name has
|
|
423
|
+
// an excluded token (text, time, content, pre, ...) before testing for 0/1
|
|
424
|
+
// values, which drops codes such as "Content Knowledge". A multi-word binary
|
|
425
|
+
// column is kept as a code here; single-word names ("Pre", "Post", "Time")
|
|
426
|
+
// stay metadata, as those are usually indicator flags.
|
|
427
|
+
|
|
428
|
+
const nameTokens = (s) =>
|
|
429
|
+
s.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
|
430
|
+
|
|
431
|
+
function detectColumns(rows) {
|
|
432
|
+
const det = detectParams(rows);
|
|
433
|
+
for (const c of det.columns) {
|
|
434
|
+
if (c.role === 'metadata' && nameTokens(c.name).length > 1 && isBinary(rows, c.name))
|
|
435
|
+
c.role = 'code';
|
|
436
|
+
}
|
|
437
|
+
det.codes = det.columns.filter(c => c.role === 'code').map(c => c.name);
|
|
438
|
+
return det;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// ── qeviz viz helpers ─────────────────────────────────────────────────────────
|
|
442
|
+
// Plots are built by webena.buildPlot as the ENA Web Tool's comparison plot:
|
|
443
|
+
// { data (qeviz ModelData), layers ([tag, attrs] child elements of
|
|
444
|
+
// <qe-graph>), attrs (<qe-graph> attributes), edgePx, title }.
|
|
445
|
+
|
|
446
|
+
// Group values and titles come from the data and from tool arguments; escape
|
|
447
|
+
// them before they go into markup (renderSVG parses it with scripts enabled).
|
|
448
|
+
const escHtml = (v) => String(v).replace(/[&<>"']/g, ch =>
|
|
449
|
+
({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[ch]);
|
|
450
|
+
|
|
451
|
+
// JSON inside <script>: escape '<' so a value such as "</script>" cannot close it.
|
|
452
|
+
const scriptJSON = (v) => JSON.stringify(v).replace(/</g, '\\u003c');
|
|
453
|
+
|
|
454
|
+
const attrString = (attrs) => Object.entries(attrs)
|
|
455
|
+
.filter(([, v]) => v != null)
|
|
456
|
+
.map(([k, v]) => (v === '' ? ` ${k}` : ` ${k}="${escHtml(v)}"`))
|
|
457
|
+
.join('');
|
|
458
|
+
|
|
459
|
+
const layerMarkup = (layers) => layers.map(([tag, a]) => `<${tag}${attrString(a)}></${tag}>`).join('');
|
|
460
|
+
|
|
461
|
+
// The web tool sizes edges in px: edge-width is a weight-1 edge as a fraction
|
|
462
|
+
// of the plot's shorter side.
|
|
463
|
+
const edgeWidth = (edgePx, viewPx) => String(Math.min(1, edgePx / viewPx));
|
|
464
|
+
|
|
465
|
+
// The interactive plot: the same <qe-graph> as the SVG, plus a control bar
|
|
466
|
+
// for the web tool's Plot Settings, applied live in the page (qeviz watches
|
|
467
|
+
// its layer elements; interval and node-size changes reload the model data).
|
|
468
|
+
function generateHTML(plot, title = plot.title) {
|
|
469
|
+
const hasNet = plot.layers.some(([t]) => t === 'qe-edges');
|
|
470
|
+
const hasPoints = plot.layers.some(([t]) => t === 'qe-points');
|
|
471
|
+
const groupRows = plot.data.groups?.data ?? [];
|
|
472
|
+
const hasCI = groupRows.some(r => r['x.low'] != null);
|
|
473
|
+
const hasOut = groupRows.some(r => r['x.olow'] != null);
|
|
474
|
+
const style = Object.fromEntries(plot.layers);
|
|
475
|
+
const textSize = Number(plot.attrs['label-font-size']) || 12;
|
|
476
|
+
const maxW = plot.maxWeight > 0 ? plot.maxWeight : 1;
|
|
477
|
+
const minW = Number(style['qe-edges']?.threshold) || 0;
|
|
478
|
+
const check = (id, label, on) =>
|
|
479
|
+
`<label><input type="checkbox" id="${id}"${on ? ' checked' : ''}> ${label}</label>`;
|
|
480
|
+
const range = (id, label, min, max, step, value) =>
|
|
481
|
+
`<label class="rng">${label} <input type="range" id="${id}" min="${min}" max="${max}" step="${step}" value="${value}"><output for="${id}"></output></label>`;
|
|
482
|
+
const controls = [
|
|
483
|
+
hasNet && range('minw', 'Min edge weight', 0, maxW, maxW / 100, minW),
|
|
484
|
+
hasNet && range('mag', 'Scale edges', 0.5, 5, 0.1, plot.scale ?? 1),
|
|
485
|
+
range('txt', 'Text size', 8, 32, 1, textSize),
|
|
486
|
+
hasNet && check('unconn', 'Unconnected codes', style['qe-nodes']?.unconnected !== 'hide'),
|
|
487
|
+
hasNet && check('codelab', 'Code labels', true),
|
|
488
|
+
hasPoints && check('pts', 'Unit points', true),
|
|
489
|
+
hasPoints && check('ptlab', 'Unit labels', false),
|
|
490
|
+
groupRows.length && check('grplab', 'Group labels', true),
|
|
491
|
+
hasCI && check('ci', 'Confidence intervals', groupRows.some(r => r.intervals === 'box')),
|
|
492
|
+
hasOut && check('out', 'Outlier boxes', groupRows.some(r => r.outlier)),
|
|
493
|
+
].filter(Boolean).join('\n ');
|
|
494
|
+
return `<!DOCTYPE html>
|
|
495
|
+
<html lang="en">
|
|
496
|
+
<head>
|
|
497
|
+
<meta charset="UTF-8">
|
|
498
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
499
|
+
<title>${escHtml(title)}</title>
|
|
500
|
+
<style>
|
|
501
|
+
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
502
|
+
body { font-family: system-ui, sans-serif; background: #f0f0f0; padding: 1rem; color: #333; }
|
|
503
|
+
h1 { font-size: 1rem; margin-bottom: .75rem; font-weight: 500; }
|
|
504
|
+
.controls { display: flex; flex-wrap: wrap; gap: .4rem 1.1rem; align-items: center; font-size: .8rem;
|
|
505
|
+
background: #fff; border: 1px solid #ddd; border-radius: 6px; padding: .55rem .75rem; margin-bottom: .6rem; }
|
|
506
|
+
.controls label { display: inline-flex; align-items: center; gap: .3rem; white-space: nowrap; cursor: pointer; }
|
|
507
|
+
.controls input[type=range] { width: 110px; }
|
|
508
|
+
.controls output { min-width: 2.6em; font-variant-numeric: tabular-nums; color: #666; }
|
|
509
|
+
.controls button { font: inherit; padding: .15rem .6rem; border: 1px solid #ccc; border-radius: 4px; background: #f7f7f7; cursor: pointer; }
|
|
510
|
+
.viz-wrap { background: #fff; border: 1px solid #ddd; border-radius: 6px; overflow: hidden; }
|
|
511
|
+
.provenance { margin-top: .5rem; font-size: .72rem; color: #777; }
|
|
512
|
+
qe-visual { display: block; width: 100%; height: 620px; }
|
|
513
|
+
qe-graph { display: block; width: 100%; height: 100%; }
|
|
514
|
+
</style>
|
|
515
|
+
</head>
|
|
516
|
+
<body>
|
|
517
|
+
<h1>${escHtml(title)}</h1>
|
|
518
|
+
<div class="controls">
|
|
519
|
+
${controls}
|
|
520
|
+
<button id="reset" type="button">Reset</button>
|
|
521
|
+
</div>
|
|
522
|
+
<div class="viz-wrap">
|
|
523
|
+
<qe-visual id="vis">
|
|
524
|
+
<qe-graph id="g" width="100%" height="620"${attrString(plot.attrs)}>
|
|
525
|
+
${layerMarkup(plot.layers)}
|
|
526
|
+
</qe-graph>
|
|
527
|
+
</qe-visual>
|
|
528
|
+
</div>
|
|
529
|
+
<p class="provenance">Drawn by ${escHtml(PROVENANCE)}</p>
|
|
530
|
+
<script>
|
|
531
|
+
${qevizJS()}
|
|
532
|
+
</script>
|
|
533
|
+
<script>
|
|
534
|
+
const base = ${scriptJSON(plot.data)};
|
|
535
|
+
const EDGE_PX = ${plot.edgePx}, SCALE0 = ${plot.scale ?? 1}, SUBTRACT = ${!!plot.subtract};
|
|
536
|
+
const TITLE = ${scriptJSON(String(title))};
|
|
537
|
+
const vis = document.getElementById('vis');
|
|
538
|
+
const graph = document.getElementById('g');
|
|
539
|
+
const $ = id => document.getElementById(id);
|
|
540
|
+
const layer = tag => graph.querySelector(tag);
|
|
541
|
+
const pointsLayer = layer('qe-points');
|
|
542
|
+
const viewPx = () => Math.min(graph.clientWidth || 620, graph.clientHeight || 620);
|
|
543
|
+
const initial = {};
|
|
544
|
+
document.querySelectorAll('.controls input').forEach(i => initial[i.id] = i.type === 'checkbox' ? i.checked : i.value);
|
|
545
|
+
|
|
546
|
+
function setAttr(el, name, value) {
|
|
547
|
+
if (!el) return;
|
|
548
|
+
if (value == null) el.removeAttribute(name); else el.setAttribute(name, value);
|
|
549
|
+
}
|
|
550
|
+
function show(id) {
|
|
551
|
+
const i = $(id), o = i && i.parentNode.querySelector('output');
|
|
552
|
+
if (o) o.textContent = id === 'mag' ? Number(i.value).toFixed(1) + 'x' : id === 'txt' ? i.value : Number(i.value).toPrecision(2);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// Interval toggles and the node sizes ("Scale edges") change the model data.
|
|
556
|
+
function update() {
|
|
557
|
+
const data = JSON.parse(JSON.stringify(base));
|
|
558
|
+
data.updated = Date.now();
|
|
559
|
+
const m = $('mag') ? Number($('mag').value) / SCALE0 : 1;
|
|
560
|
+
if (data.nodes && m !== 1)
|
|
561
|
+
data.nodes.data.forEach(r => { if (r.size != null) r.size = 2 * ((r.size / 2 - 1) * m + 1); });
|
|
562
|
+
if (data.groups)
|
|
563
|
+
data.groups.data.forEach(r => {
|
|
564
|
+
if ($('ci') && r['x.low'] != null) r.intervals = $('ci').checked ? 'box' : 'none';
|
|
565
|
+
if ($('out') && r['x.olow'] != null) r.outlier = $('out').checked;
|
|
566
|
+
});
|
|
567
|
+
graph.setAttribute('edge-width', String(Math.min(1, EDGE_PX * (SUBTRACT ? 1 : m) / viewPx())));
|
|
568
|
+
vis.setModelData(data);
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
function apply() {
|
|
572
|
+
document.querySelectorAll('.controls input[type=range]').forEach(i => show(i.id));
|
|
573
|
+
graph.querySelectorAll('qe-edges').forEach(e =>
|
|
574
|
+
setAttr(e, 'threshold', $('minw') && Number($('minw').value) > 0 ? $('minw').value : null));
|
|
575
|
+
if ($('unconn')) setAttr(layer('qe-nodes'), 'unconnected', $('unconn').checked ? null : 'hide');
|
|
576
|
+
if ($('codelab')) setAttr(layer('qe-nodes'), 'label', $('codelab').checked ? 'on' : 'off');
|
|
577
|
+
if ($('grplab')) setAttr(layer('qe-means'), 'label', $('grplab').checked ? 'on' : 'off');
|
|
578
|
+
if ($('ptlab')) setAttr(pointsLayer, 'label', $('ptlab').checked ? 'on' : 'auto');
|
|
579
|
+
if ($('pts')) {
|
|
580
|
+
if ($('pts').checked && !pointsLayer.isConnected) graph.appendChild(pointsLayer);
|
|
581
|
+
if (!$('pts').checked && pointsLayer.isConnected) pointsLayer.remove();
|
|
582
|
+
}
|
|
583
|
+
graph.setAttribute('label-font-size', $('txt').value);
|
|
584
|
+
if ($('mag')) {
|
|
585
|
+
const t = TITLE.replace(/\\(scaled [\\d.]+x\\)/, '(scaled ' + Number($('mag').value).toFixed(1) + 'x)');
|
|
586
|
+
document.querySelector('h1').textContent = t;
|
|
587
|
+
document.title = t;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
document.querySelectorAll('.controls input').forEach(i => i.addEventListener('input', () => {
|
|
592
|
+
apply();
|
|
593
|
+
if (['mag', 'ci', 'out'].includes(i.id)) update();
|
|
594
|
+
}));
|
|
595
|
+
$('reset').addEventListener('click', () => {
|
|
596
|
+
document.querySelectorAll('.controls input').forEach(i => {
|
|
597
|
+
if (i.type === 'checkbox') i.checked = initial[i.id]; else i.value = initial[i.id];
|
|
598
|
+
});
|
|
599
|
+
apply();
|
|
600
|
+
update();
|
|
601
|
+
});
|
|
602
|
+
apply();
|
|
603
|
+
update();
|
|
604
|
+
// Render again once the plot has its size, and whenever it is resized: qeviz
|
|
605
|
+
// measures the plot to size points and edges in px, and the first render (the
|
|
606
|
+
// graph has no size until it has data) measures 0 and draws them at data scale.
|
|
607
|
+
let pending = 0;
|
|
608
|
+
new ResizeObserver(() => {
|
|
609
|
+
clearTimeout(pending);
|
|
610
|
+
pending = setTimeout(() => { if (graph.clientWidth > 0) update(); }, 30);
|
|
611
|
+
}).observe(graph);
|
|
612
|
+
</script>
|
|
613
|
+
</body>
|
|
614
|
+
</html>`;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// model_ids restart at m1 in every server process, so each process writes to
|
|
618
|
+
// its own folder — otherwise concurrent clients (e.g. Claude Code and Claude
|
|
619
|
+
// Desktop) overwrite each other's m1.svg.
|
|
620
|
+
const VIZ_DIR = join(tmpdir(), 'ena-viz', String(process.pid));
|
|
621
|
+
|
|
622
|
+
function writeVizFile(content, name, ext = 'html') {
|
|
623
|
+
const dir = VIZ_DIR;
|
|
624
|
+
mkdirSync(dir, { recursive: true });
|
|
625
|
+
const path = join(dir, `${name}.${ext}`);
|
|
626
|
+
writeFileSync(path, content);
|
|
627
|
+
return path;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// File name for a plot: model_id + plot_type + whatever selects the view
|
|
631
|
+
// (group, unit, group pair), so e.g. the FirstGame and SecondGame group plots
|
|
632
|
+
// don't both land on m1_group.svg and overwrite each other.
|
|
633
|
+
function vizSlug(result, args = {}) {
|
|
634
|
+
const safe = s => String(s).replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '');
|
|
635
|
+
const selectors = {
|
|
636
|
+
group: [args.group_value],
|
|
637
|
+
unit: [args.unit_label],
|
|
638
|
+
comparison: [args.group1, args.group2],
|
|
639
|
+
subtraction: [args.group1, args.group2],
|
|
640
|
+
}[result.plot_type] ?? [];
|
|
641
|
+
return [result.model_id, result.plot_type, ...selectors]
|
|
642
|
+
.filter(v => v != null && v !== '')
|
|
643
|
+
.map(safe)
|
|
644
|
+
.filter(Boolean)
|
|
645
|
+
.join('_');
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// ── jsdom SVG renderer ────────────────────────────────────────────────────────
|
|
649
|
+
|
|
650
|
+
import { JSDOM } from 'jsdom';
|
|
651
|
+
|
|
652
|
+
// PNG output (the inline image block and the png_plot file) prefers the native
|
|
653
|
+
// @resvg/resvg-js, an OPTIONAL dependency: fastest, and it reads system fonts.
|
|
654
|
+
// Where it is absent — the cross-platform .mcpb bundle, or a platform without a
|
|
655
|
+
// prebuilt binary — @resvg/resvg-wasm renders the same image. It cannot see
|
|
656
|
+
// system fonts, so plot labels are drawn with the bundled Inter (fonts/, OFL).
|
|
657
|
+
// Loaded lazily; if neither loads, SVG and interactive HTML are still produced.
|
|
658
|
+
let _Resvg; // undefined = not yet attempted; null = unavailable
|
|
659
|
+
async function getResvg() {
|
|
660
|
+
if (_Resvg !== undefined) return _Resvg;
|
|
661
|
+
try {
|
|
662
|
+
({ Resvg: _Resvg } = await import('@resvg/resvg-js'));
|
|
663
|
+
return _Resvg;
|
|
664
|
+
} catch { /* fall back to WASM */ }
|
|
665
|
+
try {
|
|
666
|
+
const { initWasm, Resvg } = await import('@resvg/resvg-wasm');
|
|
667
|
+
await initWasm(readFileSync(_require.resolve('@resvg/resvg-wasm/index_bg.wasm')));
|
|
668
|
+
const fontBuffers = ['Inter-Regular.ttf', 'Inter-Bold.ttf']
|
|
669
|
+
.map(f => readFileSync(new URL(`./fonts/${f}`, import.meta.url)));
|
|
670
|
+
const font = { fontBuffers, defaultFontFamily: 'Inter', sansSerifFamily: 'Inter' };
|
|
671
|
+
// Same shape as the native class — new Resvg(svg, opts).render().asPng()
|
|
672
|
+
// — but asPng() gives a Uint8Array here, so hand back a Buffer as native does.
|
|
673
|
+
_Resvg = function WasmResvg(svg, opts = {}) {
|
|
674
|
+
const r = new Resvg(svg, { ...opts, font });
|
|
675
|
+
return { render: () => { const img = r.render(); return { asPng: () => Buffer.from(img.asPng()) }; } };
|
|
676
|
+
};
|
|
677
|
+
} catch (e) {
|
|
678
|
+
_Resvg = null;
|
|
679
|
+
process.stderr.write(
|
|
680
|
+
`[ena-mcp] no PNG renderer (${e.message}) — inline PNG and png_plot ` +
|
|
681
|
+
'disabled; svg_plot and interactive_plot are still produced.\n'
|
|
682
|
+
);
|
|
683
|
+
}
|
|
684
|
+
return _Resvg;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
const SVG_SIZE = 600;
|
|
688
|
+
|
|
689
|
+
async function renderSVG(plot) {
|
|
690
|
+
const dom = new JSDOM(
|
|
691
|
+
`<!DOCTYPE html><body><qe-visual id="vis"><qe-graph id="g" width="${SVG_SIZE}" height="${SVG_SIZE}"` +
|
|
692
|
+
`${attrString({ ...plot.attrs, 'edge-width': edgeWidth(plot.edgePx, SVG_SIZE) })}>${layerMarkup(plot.layers)}</qe-graph></qe-visual></body>`,
|
|
693
|
+
{ runScripts: 'dangerously', pretendToBeVisual: true }
|
|
694
|
+
);
|
|
695
|
+
const { window } = dom;
|
|
696
|
+
|
|
697
|
+
window.Element.prototype.getBoundingClientRect = () =>
|
|
698
|
+
({ x:0, y:0, width:SVG_SIZE, height:SVG_SIZE, top:0, left:0, right:SVG_SIZE, bottom:SVG_SIZE });
|
|
699
|
+
window.requestAnimationFrame = cb => setTimeout(cb, 0);
|
|
700
|
+
|
|
701
|
+
const toStderr = (...a) => process.stderr.write(a.map(String).join(' ') + '\n');
|
|
702
|
+
window.console = { log: toStderr, info: toStderr, warn: toStderr, error: toStderr, debug: () => {} };
|
|
703
|
+
|
|
704
|
+
const s = window.document.createElement('script');
|
|
705
|
+
s.textContent = qevizJS();
|
|
706
|
+
window.document.head.appendChild(s);
|
|
707
|
+
|
|
708
|
+
window.document.getElementById('vis').setModelData(plot.data);
|
|
709
|
+
await new Promise(r => setTimeout(r, 200));
|
|
710
|
+
|
|
711
|
+
const svgEl = window.document.getElementById('g')?.svg_element;
|
|
712
|
+
if (!svgEl) return null;
|
|
713
|
+
|
|
714
|
+
// XML serialization: outerHTML (HTML rules) leaves '<' unescaped inside
|
|
715
|
+
// attribute values, which resvg rejects when a label contains one.
|
|
716
|
+
let svgHtml = new window.XMLSerializer().serializeToString(svgEl);
|
|
717
|
+
// resvg requires the SVG namespace declaration to parse the document
|
|
718
|
+
if (!svgHtml.includes('xmlns=')) {
|
|
719
|
+
svgHtml = svgHtml.replace('<svg', '<svg xmlns="http://www.w3.org/2000/svg"');
|
|
720
|
+
}
|
|
721
|
+
return withProvenance(withWhiteBackground(withTitle(fitLabels(scaleFixedStrokes(svgHtml, SVG_SIZE)), plot.title)));
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// qeviz draws some outlines at a fixed on-screen width with
|
|
725
|
+
// vector-effect="non-scaling-stroke" (directed edges' 1 px outline, the
|
|
726
|
+
// chevrons' white halo). resvg ignores vector-effect, so it read those widths
|
|
727
|
+
// in viewBox units -- about 75 px each on a typical model -- and directed (TMA /
|
|
728
|
+
// ONA) edges rasterised as fat capsules. Convert each to viewBox units for the
|
|
729
|
+
// size the plot is rendered at, and drop the attribute.
|
|
730
|
+
function scaleFixedStrokes(svgHtml, renderedPx) {
|
|
731
|
+
const vb = svgHtml.match(/<svg[^>]*\sviewBox\s*=\s*"([^"]+)"/)?.[1].trim().split(/\s+/).map(Number);
|
|
732
|
+
if (!vb || !(vb[2] > 0)) return svgHtml;
|
|
733
|
+
const unitsPerPx = vb[2] / renderedPx;
|
|
734
|
+
return svgHtml.replace(/<[a-zA-Z]+\b[^>]*\svector-effect="non-scaling-stroke"[^>]*>/g, tag =>
|
|
735
|
+
tag.replace(/\svector-effect="non-scaling-stroke"/, '')
|
|
736
|
+
.replace(/(\sstroke-width=")([-\d.eE]+)(")/, (_, a, w, b) => `${a}${Number(w) * unitsPerPx}${b}`));
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// qeviz sizes the square viewBox to the node and point extent, so long node
|
|
740
|
+
// labels at the left / right edge ("Performance Parameters") run past it and
|
|
741
|
+
// are cut off in the SVG / PNG. Widen the viewBox to the labels' estimated
|
|
742
|
+
// extent (average glyph ≈ 0.6 em); it only ever grows, and the plot itself
|
|
743
|
+
// stays where it is.
|
|
744
|
+
function fitLabels(svgHtml) {
|
|
745
|
+
const openTag = svgHtml.match(/<svg[^>]*>/)?.[0];
|
|
746
|
+
const vb = openTag?.match(/viewBox\s*=\s*"([^"]+)"/);
|
|
747
|
+
if (!vb) return svgHtml;
|
|
748
|
+
let [x0, y0, w, h] = vb[1].trim().split(/\s+/).map(Number);
|
|
749
|
+
if (![x0, y0, w, h].every(Number.isFinite)) return svgHtml;
|
|
750
|
+
|
|
751
|
+
let minX = x0, maxX = x0 + w, minY = y0, maxY = y0 + h;
|
|
752
|
+
for (const m of svgHtml.matchAll(/<text\b([^>]*)>([^<]*)</g)) {
|
|
753
|
+
const attr = (name) => m[1].match(new RegExp(`\\s${name}="([^"]*)"`))?.[1];
|
|
754
|
+
if (attr('display') === 'none') continue;
|
|
755
|
+
const x = Number(attr('x')), y = Number(attr('y')), fs = Number(attr('font-size'));
|
|
756
|
+
if (![x, y, fs].every(Number.isFinite)) continue;
|
|
757
|
+
const width = m[2].length * fs * 0.6;
|
|
758
|
+
const anchor = attr('text-anchor') ?? 'start';
|
|
759
|
+
const left = anchor === 'end' ? x - width : anchor === 'middle' ? x - width / 2 : x;
|
|
760
|
+
minX = Math.min(minX, left - fs * 0.5);
|
|
761
|
+
maxX = Math.max(maxX, left + width + fs * 0.5);
|
|
762
|
+
minY = Math.min(minY, y - fs * 1.2);
|
|
763
|
+
maxY = Math.max(maxY, y + fs * 0.5);
|
|
764
|
+
}
|
|
765
|
+
if (minX === x0 && maxX === x0 + w && minY === y0 && maxY === y0 + h) return svgHtml;
|
|
766
|
+
const fitted = `${minX} ${minY} ${maxX - minX} ${maxY - minY}`;
|
|
767
|
+
return svgHtml.replace(openTag, openTag.replace(vb[0], `viewBox="${fitted}"`));
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// The web tool names the comparison plot above it ("FirstGame – SecondGame
|
|
771
|
+
// (scaled 1.0x)"). Add that title in a band over the top of the viewBox.
|
|
772
|
+
// Invisible provenance in the SVG (and its <desc>), so the file says what drew it.
|
|
773
|
+
function withProvenance(svgHtml) {
|
|
774
|
+
const openTag = svgHtml.match(/<svg[^>]*>/)?.[0];
|
|
775
|
+
if (!openTag) return svgHtml;
|
|
776
|
+
return svgHtml.replace(openTag, `${openTag}<desc>ENA plot drawn by ${escHtml(PROVENANCE)}</desc>`);
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
function withTitle(svgHtml, title) {
|
|
780
|
+
if (!title) return svgHtml;
|
|
781
|
+
const openTag = svgHtml.match(/<svg[^>]*>/)?.[0];
|
|
782
|
+
const vb = openTag?.match(/viewBox\s*=\s*"([^"]+)"/);
|
|
783
|
+
if (!vb) return svgHtml;
|
|
784
|
+
const [x0, y0, w, h] = vb[1].trim().split(/\s+/).map(Number);
|
|
785
|
+
if (![x0, y0, w, h].every(Number.isFinite)) return svgHtml;
|
|
786
|
+
const fs = Math.max(w, h) * 0.028, band = fs * 2.2;
|
|
787
|
+
const fitted = `${x0} ${y0 - band} ${w} ${h + band}`;
|
|
788
|
+
const text = `<text x="${x0 + fs * 0.6}" y="${y0 - band + fs * 1.4}" font-size="${fs}" ` +
|
|
789
|
+
`font-family="arial, sans-serif" fill="#444444">${escHtml(title)}</text>`;
|
|
790
|
+
return svgHtml.replace(openTag, openTag.replace(vb[0], `viewBox="${fitted}"`) + text);
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
// The qeviz SVG has a transparent background, which makes the plot invisible in
|
|
794
|
+
// dark-mode viewers. Insert an opaque white rect as the first child (behind all
|
|
795
|
+
// content), sized to the SVG's viewBox so it covers the full canvas.
|
|
796
|
+
function withWhiteBackground(svgHtml) {
|
|
797
|
+
const openTagMatch = svgHtml.match(/<svg[^>]*>/);
|
|
798
|
+
if (!openTagMatch) return svgHtml;
|
|
799
|
+
const openTag = openTagMatch[0];
|
|
800
|
+
|
|
801
|
+
const vb = openTag.match(/viewBox\s*=\s*["']\s*([-\d.]+)\s+([-\d.]+)\s+([-\d.]+)\s+([-\d.]+)\s*["']/);
|
|
802
|
+
const rect = vb
|
|
803
|
+
? `<rect x="${vb[1]}" y="${vb[2]}" width="${vb[3]}" height="${vb[4]}" fill="#ffffff"/>`
|
|
804
|
+
: `<rect x="0" y="0" width="100%" height="100%" fill="#ffffff"/>`;
|
|
805
|
+
|
|
806
|
+
return svgHtml.replace(openTag, openTag + rect);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
// ── Model / accumulation cache ────────────────────────────────────────────────
|
|
810
|
+
|
|
811
|
+
const accumCache = new Map(); // accumKey → { acc, rows }
|
|
812
|
+
const modelCache = new Map(); // model_id → cache entry
|
|
813
|
+
const fitCache = new Map(); // fitKey → model_id
|
|
814
|
+
let modelSeq = 0;
|
|
815
|
+
|
|
816
|
+
// Bounded LRU caches: accumulations hold the parsed rows, so keep only a few;
|
|
817
|
+
// models are small but would otherwise grow for the life of the server.
|
|
818
|
+
const MAX_ACCUM = 8;
|
|
819
|
+
const MAX_MODELS = 50;
|
|
820
|
+
|
|
821
|
+
function cacheGet(map, key) {
|
|
822
|
+
if (!map.has(key)) return undefined;
|
|
823
|
+
const v = map.get(key);
|
|
824
|
+
map.delete(key); map.set(key, v); // most recently used last
|
|
825
|
+
return v;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
function cacheSet(map, key, value, max) {
|
|
829
|
+
map.delete(key); map.set(key, value);
|
|
830
|
+
while (map.size > max) map.delete(map.keys().next().value);
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// Identifies the input data. For a file, include its mtime and size so an
|
|
834
|
+
// edited CSV is re-read instead of serving a model fitted on the old contents.
|
|
835
|
+
function dataStamp(p) {
|
|
836
|
+
if (p.data_path) {
|
|
837
|
+
const st = statSync(p.data_path);
|
|
838
|
+
return `${p.data_path}@${st.mtimeMs}:${st.size}`;
|
|
839
|
+
}
|
|
840
|
+
// Hash pasted CSV so the cache keys don't hold a copy of the whole dataset.
|
|
841
|
+
return 'csv:' + createHash('sha1').update(p.data_csv ?? '').digest('hex');
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
function accumCacheKey(p) {
|
|
845
|
+
return JSON.stringify({
|
|
846
|
+
data: dataStamp(p),
|
|
847
|
+
units: p.units,
|
|
848
|
+
conversations: p.conversations,
|
|
849
|
+
codes: p.codes, // order matters: it fixes the connection order
|
|
850
|
+
window_size: p.window_size ?? 4,
|
|
851
|
+
binary: p.binary ?? true,
|
|
852
|
+
weight_model: p.weight_model ?? null,
|
|
853
|
+
units_used: p.units_used ?? null,
|
|
854
|
+
mask_connections: p.mask_connections ?? null,
|
|
855
|
+
exclude_conversations: p.exclude_conversations ?? null,
|
|
856
|
+
order_by: p.order_by ?? null,
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
// kind keeps tools apart: ena_fit and ena_compare_groups can fit the same
|
|
861
|
+
// model but return different results.
|
|
862
|
+
function fitCacheKey(p, kind = 'fit') {
|
|
863
|
+
return JSON.stringify({
|
|
864
|
+
kind,
|
|
865
|
+
data: dataStamp(p),
|
|
866
|
+
units: p.units,
|
|
867
|
+
conversations: p.conversations,
|
|
868
|
+
codes: [...p.codes].sort(),
|
|
869
|
+
window_size: p.window_size ?? 4,
|
|
870
|
+
binary: p.binary ?? true,
|
|
871
|
+
dims: p.dims ?? 2,
|
|
872
|
+
rotation: p.rotation ?? 'svd',
|
|
873
|
+
group_col: p.group_col ?? null,
|
|
874
|
+
group1_value: p.group1_value ?? null,
|
|
875
|
+
group2_value: p.group2_value ?? null,
|
|
876
|
+
weight_model: p.weight_model ?? null,
|
|
877
|
+
units_used: p.units_used ?? null,
|
|
878
|
+
mask_connections: p.mask_connections ?? null,
|
|
879
|
+
exclude_conversations: p.exclude_conversations ?? null,
|
|
880
|
+
order_by: p.order_by ?? null,
|
|
881
|
+
sphere_norm: p.sphere_norm ?? true,
|
|
882
|
+
center_align_to_origin: p.center_align_to_origin ?? true,
|
|
883
|
+
x_var: p.x_var ?? null,
|
|
884
|
+
y_var: p.y_var ?? null,
|
|
885
|
+
select_2_groups: p.select_2_groups ?? null,
|
|
886
|
+
interactions: p.interactions ?? true,
|
|
887
|
+
web_gmr: p._web_gmr ?? null,
|
|
888
|
+
rotation_from: p.rotation_from ?? null,
|
|
889
|
+
});
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
// Accumulation options shared by every tool (rena-wasm accumulate() and fit()).
|
|
893
|
+
// units_used must reach accumulate() too, so its unitLabels match the fit's.
|
|
894
|
+
function accumOpts(p, ordered = false) {
|
|
895
|
+
const codeMask = codeMaskOf(p, ordered);
|
|
896
|
+
return {
|
|
897
|
+
...(p.weight_model && { weightModel: p.weight_model }),
|
|
898
|
+
...(p.units_used?.length && { unitsUsed: p.units_used }),
|
|
899
|
+
...(codeMask && { codeMask }),
|
|
900
|
+
};
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
// mask_connections → rena-wasm's codeMask: an nCodes × nCodes 0/1 matrix over
|
|
904
|
+
// the model's codes (the ENA Web Tool's Code Masking Matrix, settings.model
|
|
905
|
+
// codeMask / codeMaskLabels). Unordered: "A & B" drops the pair either way
|
|
906
|
+
// round. Ordered: "A & B" drops A → B only (mask[from][to]).
|
|
907
|
+
function codeMaskOf(p, ordered) {
|
|
908
|
+
const names = toArray(p.mask_connections);
|
|
909
|
+
if (!names.length) return null;
|
|
910
|
+
const codes = toArray(p.codes);
|
|
911
|
+
const at = new Map(codes.map((c, i) => [c, i]));
|
|
912
|
+
const mask = codes.map(() => codes.map(() => 1));
|
|
913
|
+
for (const name of names) {
|
|
914
|
+
const parts = String(name).split(' & ');
|
|
915
|
+
const [a, b] = parts.map(x => at.get(x));
|
|
916
|
+
if (parts.length !== 2 || a === undefined || b === undefined)
|
|
917
|
+
throw new Error(`mask_connections: ${name} must name two of the model's codes as A & B (codes: ${codes.join(', ')}).`);
|
|
918
|
+
if (a === b && !ordered)
|
|
919
|
+
throw new Error(`mask_connections: ${name} — an undirected model has no self-connections.`);
|
|
920
|
+
mask[a][b] = 0;
|
|
921
|
+
if (!ordered) mask[b][a] = 0;
|
|
922
|
+
}
|
|
923
|
+
return mask;
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
// Is connection "A & B" (as the model names it) masked? Unordered either way round.
|
|
927
|
+
function isMasked(p, ordered, connName) {
|
|
928
|
+
const names = new Set(toArray(p.mask_connections));
|
|
929
|
+
if (names.has(connName)) return true;
|
|
930
|
+
if (ordered) return false;
|
|
931
|
+
const [a, b] = connName.split(' & ');
|
|
932
|
+
return names.has(`${b} & ${a}`);
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
// Fit-only options on top of accumOpts.
|
|
936
|
+
function fitOpts(p, ordered = false) {
|
|
937
|
+
return {
|
|
938
|
+
...accumOpts(p, ordered),
|
|
939
|
+
sphereNorm: p.sphere_norm ?? true,
|
|
940
|
+
centerAlignToOrigin: p.center_align_to_origin ?? true,
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
// JSON has no Infinity, so an infinite window (= rENA window.size.back = Inf)
|
|
945
|
+
// arrives as the string "Inf".
|
|
946
|
+
function windowValue(w) {
|
|
947
|
+
if (typeof w === 'string') {
|
|
948
|
+
if (/^(inf|infinity)$/i.test(w.trim())) return Infinity;
|
|
949
|
+
const n = Number(w);
|
|
950
|
+
if (Number.isFinite(n)) return n;
|
|
951
|
+
throw new Error(`window_size must be a number or "Inf", got '${w}'`);
|
|
952
|
+
}
|
|
953
|
+
return w;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
// TMA-only options: flexible horizons (= tma contexts() with HOO rules).
|
|
957
|
+
// hoo_flexible was never read by rena-wasm, so reject it rather than ignore it.
|
|
958
|
+
function tmaOpts(p) {
|
|
959
|
+
if (p.hoo_flexible)
|
|
960
|
+
throw new Error('hoo_flexible is no longer supported (it had no effect). Use horizons: ' +
|
|
961
|
+
'{ by: <column>, rules: { <value>: [<columns>], ... } } for flexible horizons of observation.');
|
|
962
|
+
const h = p.horizons;
|
|
963
|
+
if (!h) return {};
|
|
964
|
+
if (typeof h.by !== 'string' || !h.rules || typeof h.rules !== 'object' || Array.isArray(h.rules))
|
|
965
|
+
throw new Error('horizons must be { by: <column>, rules: { <value of by>: [<columns>], ... } }.');
|
|
966
|
+
return { horizons: h };
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
// Summary lines for the non-default options, so the model's setup is visible.
|
|
970
|
+
function optsLines(p, nUnits) {
|
|
971
|
+
return [
|
|
972
|
+
...(p.weight_model ? [` Weight model: ${p.weight_model}`] : []),
|
|
973
|
+
...(p.units_used?.length ? [` Units modeled: ${nUnits} of ${p.units_used.length} listed (other units' rows kept as context)`] : []),
|
|
974
|
+
...(p.sphere_norm === false ? [' Sphere normalization: off (networks scaled by the longest)'] : []),
|
|
975
|
+
...(p.center_align_to_origin === false ? [' Centering: on every unit, including zero-network units'] : []),
|
|
976
|
+
...(toArray(p.mask_connections).length ? [` Masked connections: ${toArray(p.mask_connections).join(', ')}`] : []),
|
|
977
|
+
...(toArray(p.exclude_conversations).length ? [` Excluded conversations: ${toArray(p.exclude_conversations).join(', ')} (their lines removed)`] : []),
|
|
978
|
+
...(p.order_by ? [` Lines ordered by ${p.order_by}`] : []),
|
|
979
|
+
...(p.horizons ? [` Horizons: by ${p.horizons.by} — ` +
|
|
980
|
+
Object.entries(p.horizons.rules).map(([v, cols]) => `${v}: ${[].concat(cols).join(' + ')}`).join('; ')] : []),
|
|
981
|
+
];
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
function nextModelId() { return `m${++modelSeq}`; }
|
|
985
|
+
|
|
986
|
+
// ── Web tool defaults ─────────────────────────────────────────────────────────
|
|
987
|
+
|
|
988
|
+
// Parsed rows by input, so resolving defaults and accumulating parse once.
|
|
989
|
+
const rowsCache = new Map();
|
|
990
|
+
const MAX_ROWS = 4;
|
|
991
|
+
function loadRows(p) {
|
|
992
|
+
const key = dataStamp(p);
|
|
993
|
+
let rows;
|
|
994
|
+
if (rowsCache.has(key)) rows = cacheGet(rowsCache, key);
|
|
995
|
+
else { rows = loadData(p.data_path, p.data_csv); cacheSet(rowsCache, key, rows, MAX_ROWS); }
|
|
996
|
+
if (!toArray(p.exclude_conversations).length && !p.order_by) return rows;
|
|
997
|
+
|
|
998
|
+
const vKey = JSON.stringify([key, toArray(p.conversations), toArray(p.exclude_conversations), p.order_by ?? null]);
|
|
999
|
+
if (rowsCache.has(vKey)) return cacheGet(rowsCache, vKey);
|
|
1000
|
+
const view = prepareRows(rows, p);
|
|
1001
|
+
cacheSet(rowsCache, vKey, view, MAX_ROWS);
|
|
1002
|
+
return view;
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// Missing values (rENA-api / R: NA, data.table sorts them first).
|
|
1006
|
+
const isMissing = (v) => v == null || /^\s*(|NA|NaN|#N\/A)\s*$/.test(String(v));
|
|
1007
|
+
|
|
1008
|
+
// exclude_conversations and order_by, as the ENA Web Tool's former R backend
|
|
1009
|
+
// applied them (rENA-api ena.generate: conversations.used, then
|
|
1010
|
+
// data.table::setorderv(file, order)). Excluded conversations' rows are
|
|
1011
|
+
// dropped before anything else — they are neither lines nor context. order_by
|
|
1012
|
+
// sorts every row by that column, ascending and stable (ties keep file order,
|
|
1013
|
+
// missing first; numeric when every value is a number, else C-locale text),
|
|
1014
|
+
// which orders the lines within each conversation. The result keeps each row's
|
|
1015
|
+
// position in the file as rows.fileRow, so row numbers still refer to the file.
|
|
1016
|
+
function prepareRows(rows, p) {
|
|
1017
|
+
const convoCols = toArray(p.conversations);
|
|
1018
|
+
const allCols = rows.length ? Object.keys(rows[0]) : [];
|
|
1019
|
+
let idx = rows.map((_, i) => i);
|
|
1020
|
+
|
|
1021
|
+
const excl = toArray(p.exclude_conversations).map(String);
|
|
1022
|
+
if (excl.length) {
|
|
1023
|
+
if (!convoCols.length) throw new Error('exclude_conversations needs conversations.');
|
|
1024
|
+
const keyOf = (r) => convoCols.map(c => String(r[c])).join('::');
|
|
1025
|
+
const known = new Set(rows.map(keyOf));
|
|
1026
|
+
const want = new Set(excl.map(e => e.split('__').join('::')));
|
|
1027
|
+
for (const e of want)
|
|
1028
|
+
if (!known.has(e))
|
|
1029
|
+
throw new Error(`exclude_conversations: no conversation ${e}. Conversations are named by their ` +
|
|
1030
|
+
`${convoCols.join(' + ')} values joined by ::, e.g. ${[...known][0]}.`);
|
|
1031
|
+
idx = idx.filter(i => !want.has(keyOf(rows[i])));
|
|
1032
|
+
if (!idx.length) throw new Error('exclude_conversations leaves no rows.');
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
if (p.order_by) {
|
|
1036
|
+
const col = String(p.order_by);
|
|
1037
|
+
if (!allCols.includes(col)) throw new Error(`order_by: column ${col} is not in the data.`);
|
|
1038
|
+
const vals = idx.map(i => rows[i][col]);
|
|
1039
|
+
const numeric = vals.every(v => isMissing(v) || Number.isFinite(Number(v)));
|
|
1040
|
+
const key = (v) => (isMissing(v) ? null : numeric ? Number(v) : String(v));
|
|
1041
|
+
const decorated = idx.map((i, pos) => ({ i, pos, k: key(rows[i][col]) }));
|
|
1042
|
+
decorated.sort((a, b) => {
|
|
1043
|
+
if (a.k === null || b.k === null) return a.k === b.k ? a.pos - b.pos : (a.k === null ? -1 : 1);
|
|
1044
|
+
if (a.k < b.k) return -1;
|
|
1045
|
+
if (a.k > b.k) return 1;
|
|
1046
|
+
return a.pos - b.pos;
|
|
1047
|
+
});
|
|
1048
|
+
idx = decorated.map(d => d.i);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
const view = idx.map(i => rows[i]);
|
|
1052
|
+
Object.defineProperty(view, 'fileRow', { value: Int32Array.from(idx), enumerable: false });
|
|
1053
|
+
return view;
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
// A row's position in the data file (0-based), through exclude_conversations / order_by.
|
|
1057
|
+
const fileRowOf = (rows, i) => (rows.fileRow ? rows.fileRow[i] : i);
|
|
1058
|
+
|
|
1059
|
+
/**
|
|
1060
|
+
* Fill in what a call leaves out as the ENA Web Tool does for a new set:
|
|
1061
|
+
* window the Cross-Covariance Decay estimate (line windows only)
|
|
1062
|
+
* rotation with two or more groups in the first unit column, its GMR
|
|
1063
|
+
* through the first two (or group1/group2, when named on that column)
|
|
1064
|
+
* groups units are grouped (and coloured) by the first unit column,
|
|
1065
|
+
* unless group_col names another column
|
|
1066
|
+
* kind: 'fit' | 'compare' | 'accumulate' (window only). ena_compare_groups
|
|
1067
|
+
* always takes the web tool's GMR through its two groups.
|
|
1068
|
+
*/
|
|
1069
|
+
function resolveDefaults(ena, params, kind = 'fit') {
|
|
1070
|
+
const p = { ...params };
|
|
1071
|
+
const unitCols = toArray(p.units), convoCols = toArray(p.conversations);
|
|
1072
|
+
const rows = loadRows(p);
|
|
1073
|
+
if (p.window_size == null && (p.window_type ?? 'lines') === 'lines') {
|
|
1074
|
+
let w = null;
|
|
1075
|
+
try { w = ena.ccdWindow(rows, { codes: p.codes, conversations: convoCols, ...web.CCD_OPTS }); }
|
|
1076
|
+
catch { /* too little data for an estimate: the fallback below */ }
|
|
1077
|
+
p.window_size = Number.isFinite(w) && w > 0 ? w : web.FALLBACK_WINDOW;
|
|
1078
|
+
p._window_auto = true;
|
|
1079
|
+
}
|
|
1080
|
+
if (kind === 'accumulate') return p;
|
|
1081
|
+
|
|
1082
|
+
const groups = web.unitGroups(rows, unitCols);
|
|
1083
|
+
if (kind === 'compare') {
|
|
1084
|
+
p.rotation = 'generalized';
|
|
1085
|
+
p._web_gmr = { col: p.group_col, pair: [String(p.group1_value), String(p.group2_value)] };
|
|
1086
|
+
} else if (p.rotation_from != null) {
|
|
1087
|
+
p.rotation = 'projected'; // rotation_from replaces any rotation
|
|
1088
|
+
} else if (p.rotation == null) {
|
|
1089
|
+
const def = web.defaultRotation(groups);
|
|
1090
|
+
if (def) {
|
|
1091
|
+
const named = p.group_col === def.col && p.group1_value != null && p.group2_value != null;
|
|
1092
|
+
const pair = named ? [String(p.group1_value), String(p.group2_value)] : def.pair;
|
|
1093
|
+
p.rotation = 'generalized';
|
|
1094
|
+
p._web_gmr = { col: def.col, pair };
|
|
1095
|
+
if (!p.group_col) Object.assign(p, { group_col: def.col, group1_value: pair[0], group2_value: pair[1] });
|
|
1096
|
+
} else {
|
|
1097
|
+
p.rotation = 'svd';
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
if (!p.group_col && !groups.wrapper) p.group_col = groups.col;
|
|
1101
|
+
return p;
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
const windowLine = (p) => ` Window: back=${p.window_size}` +
|
|
1105
|
+
(p._window_auto ? ' (Cross-Covariance Decay estimate — the web tool\'s default for a new set)' : '');
|
|
1106
|
+
|
|
1107
|
+
const rotationLabel = (p) => p.rotation_from != null ? `projected into model ${p.rotation_from}'s space`
|
|
1108
|
+
: p._web_gmr ? 'generalized (GMR, the web tool\'s rotation for grouped units)' : p.rotation;
|
|
1109
|
+
|
|
1110
|
+
// Plot groups: group_col's values in the web tool's order, or one group of
|
|
1111
|
+
// every unit for a single unit column (the web tool's wrapper group).
|
|
1112
|
+
function plotGroups(rows, unitCols, unitLabels, groupCol) {
|
|
1113
|
+
if (groupCol) {
|
|
1114
|
+
const uMap = buildUnitGroupMap(rows, unitCols, groupCol);
|
|
1115
|
+
return { names: web.groupOrder(rows, groupCol), members: (i) => uMap.get(unitLabels[i]), uMap };
|
|
1116
|
+
}
|
|
1117
|
+
const name = unitCols.length === 1 ? unitCols[0] : 'All units';
|
|
1118
|
+
return { names: [name], members: () => name, uMap: null };
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// What the plots and the export need, computed as the web tool does.
|
|
1122
|
+
function webEntry(ena, model, unitLabels, rows, unitCols, p) {
|
|
1123
|
+
const pg = plotGroups(rows, unitCols, unitLabels, p.group_col ?? null);
|
|
1124
|
+
return {
|
|
1125
|
+
web: web.webSet(ena, model, unitLabels, pg.names, pg.members),
|
|
1126
|
+
dimNames: model.columnNames ? Array.from(model.columnNames) : null,
|
|
1127
|
+
variance: model.variance ? Array.from(model.variance) : null,
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
// The plot a fit returns, and the one an exported set opens with: the two
|
|
1132
|
+
// named groups as first − second, else the first group's mean network.
|
|
1133
|
+
function defaultView(p, w) {
|
|
1134
|
+
const g1 = p.group1_value != null ? String(p.group1_value) : null;
|
|
1135
|
+
const g2 = p.group2_value != null ? String(p.group2_value) : null;
|
|
1136
|
+
if (g1 && g2 && w.byName.has(g1) && w.byName.has(g2))
|
|
1137
|
+
return { first: { kind: 'group', name: g1 }, second: { kind: 'group', name: g2 } };
|
|
1138
|
+
const g = w.groups.find(x => x.n > 0);
|
|
1139
|
+
return g ? { first: { kind: 'group', name: g.name } } : { first: { kind: 'all' } };
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
function computeGroupMean(networks, nConnections, idx) {
|
|
1143
|
+
const mean = new Float64Array(nConnections);
|
|
1144
|
+
for (const i of idx)
|
|
1145
|
+
for (let j = 0; j < nConnections; j++)
|
|
1146
|
+
mean[j] += networks[i * nConnections + j];
|
|
1147
|
+
if (idx.length > 0)
|
|
1148
|
+
for (let j = 0; j < nConnections; j++) mean[j] /= idx.length;
|
|
1149
|
+
return mean;
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
// Per-dimension two-group tests on the projected unit points, computed by
|
|
1153
|
+
// libqe::group_stats (rena-wasm compareGroups). Values are passed through as
|
|
1154
|
+
// libqe reports them: the Mann-Whitney p uses the normal approximation with
|
|
1155
|
+
// continuity correction, and effect_r = 1 − 2U/(n1·n2) (negative when group 1
|
|
1156
|
+
// tends higher).
|
|
1157
|
+
const P = (v) => Number(v.toPrecision(4)); // p-values: significant figures, not places
|
|
1158
|
+
|
|
1159
|
+
function groupStats(ena, points, dims, g1Idx, g2Idx, group1, group2, stride = dims) {
|
|
1160
|
+
const pick = (idx) => Float64Array.from(
|
|
1161
|
+
idx.flatMap(i => Array.from({ length: dims }, (_, j) => points[i * stride + j])));
|
|
1162
|
+
const s = ena.compareGroups(pick(g1Idx), g1Idx.length, pick(g2Idx), g2Idx.length, dims);
|
|
1163
|
+
|
|
1164
|
+
// means/sds/medians are row-major {data, rows: 2 groups, cols: dims}
|
|
1165
|
+
const cell = (m, g, d) => m.data[g * m.cols + d];
|
|
1166
|
+
const dimensions = Array.from({ length: dims }, (_, d) => ({
|
|
1167
|
+
dim: `dim${d + 1}`,
|
|
1168
|
+
group1: { name: group1, n: s.n1, mean: R(cell(s.means, 0, d)), sd: R(cell(s.sds, 0, d)), median: R(cell(s.medians, 0, d)) },
|
|
1169
|
+
group2: { name: group2, n: s.n2, mean: R(cell(s.means, 1, d)), sd: R(cell(s.sds, 1, d)), median: R(cell(s.medians, 1, d)) },
|
|
1170
|
+
welch_t: { t: R(s.t[d], 4), df: R(s.df[d], 2), p: P(s.pvalue_t[d]), cohens_d: R(s.cohens_d[d], 4) },
|
|
1171
|
+
mann_whitney: { U: s.U[d], p: P(s.pvalue_u[d]), effect_r: R(s.effect_r[d], 4) },
|
|
1172
|
+
}));
|
|
1173
|
+
|
|
1174
|
+
const lines = [
|
|
1175
|
+
`Group tests (${group1} n=${s.n1} vs ${group2} n=${s.n2}):`,
|
|
1176
|
+
...dimensions.map(x =>
|
|
1177
|
+
` ${x.dim}: Welch t(${x.welch_t.df})=${x.welch_t.t}, p=${x.welch_t.p}, d=${x.welch_t.cohens_d}` +
|
|
1178
|
+
` Mann-Whitney U=${x.mann_whitney.U}, p=${x.mann_whitney.p}, r=${x.mann_whitney.effect_r}`),
|
|
1179
|
+
' (Mann-Whitney p: normal approximation with continuity correction; r = 1 − 2U/(n1·n2), negative when group 1 is higher.' +
|
|
1180
|
+
' Under rotation=mean, dim1 is constructed to separate these groups.)',
|
|
1181
|
+
];
|
|
1182
|
+
|
|
1183
|
+
return {
|
|
1184
|
+
stats: {
|
|
1185
|
+
method: 'libqe group_stats: Welch two-sample t-test; Mann-Whitney U (normal approximation, ' +
|
|
1186
|
+
'continuity correction); effect_r = 1 − 2U/(n1·n2), negative when group 1 tends higher',
|
|
1187
|
+
dimensions,
|
|
1188
|
+
},
|
|
1189
|
+
lines,
|
|
1190
|
+
};
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
function buildContextTensor(rows, params) {
|
|
1194
|
+
const {
|
|
1195
|
+
stanza_ground = [],
|
|
1196
|
+
stanza_response = [],
|
|
1197
|
+
stanza_modality = [],
|
|
1198
|
+
window_type = 'lines',
|
|
1199
|
+
time_column = null,
|
|
1200
|
+
time_unit = null,
|
|
1201
|
+
stanza_overrides = null,
|
|
1202
|
+
} = params;
|
|
1203
|
+
const window_size = windowValue(params.window_size ?? 4);
|
|
1204
|
+
|
|
1205
|
+
const stanzaModality = Array.isArray(stanza_modality)
|
|
1206
|
+
? stanza_modality
|
|
1207
|
+
: (stanza_modality ? [stanza_modality] : []);
|
|
1208
|
+
const stanzaGround = Array.isArray(stanza_ground) ? stanza_ground : [stanza_ground];
|
|
1209
|
+
const stanzaResponse = Array.isArray(stanza_response) ? stanza_response : [stanza_response];
|
|
1210
|
+
|
|
1211
|
+
const ml = stanzaModality.length;
|
|
1212
|
+
const gl = stanzaGround.length;
|
|
1213
|
+
const rl = stanzaResponse.length;
|
|
1214
|
+
|
|
1215
|
+
// Axis order: [mode..., ground..., response...]
|
|
1216
|
+
const factors = [...stanzaModality, ...stanzaGround, ...stanzaResponse];
|
|
1217
|
+
const dimsSender = Array.from({ length: gl }, (_, i) => ml + i);
|
|
1218
|
+
const dimsReceiver = Array.from({ length: rl }, (_, i) => ml + gl + i);
|
|
1219
|
+
const dimsMode = Array.from({ length: ml }, (_, i) => i);
|
|
1220
|
+
|
|
1221
|
+
// infer unique values per factor from the full dataset
|
|
1222
|
+
const factorLevels = {};
|
|
1223
|
+
for (const col of factors) {
|
|
1224
|
+
factorLevels[col] = {};
|
|
1225
|
+
let idx = 0;
|
|
1226
|
+
for (const row of rows) {
|
|
1227
|
+
const v = row[col];
|
|
1228
|
+
if (v != null && v !== '' && !(v in factorLevels[col])) {
|
|
1229
|
+
factorLevels[col][v] = idx++;
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
// tensor shape & flat data array
|
|
1235
|
+
const dimsShape = factors.map(col => Math.max(1, Object.keys(factorLevels[col]).length));
|
|
1236
|
+
const dims = [...dimsShape, 2];
|
|
1237
|
+
const nCombinations = dimsShape.reduce((a, b) => a * b, 1);
|
|
1238
|
+
const data = new Float64Array(nCombinations * 2);
|
|
1239
|
+
|
|
1240
|
+
// Default: every combination -> weight=1, window=stanzaSize (window_size)
|
|
1241
|
+
for (let c = 0; c < nCombinations; c++) {
|
|
1242
|
+
data[c] = 1;
|
|
1243
|
+
data[c + nCombinations] = window_size;
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
// apply overrides if flat array
|
|
1247
|
+
if (stanza_overrides && Array.isArray(stanza_overrides)) {
|
|
1248
|
+
for (const ov of stanza_overrides) {
|
|
1249
|
+
if (!ov.path) continue;
|
|
1250
|
+
const w = parseFloat(ov.weight ?? 1);
|
|
1251
|
+
const win = windowValue(ov.window ?? window_size);
|
|
1252
|
+
|
|
1253
|
+
// A wrong path must not fall back to some other combination.
|
|
1254
|
+
const order = factors.join(', ');
|
|
1255
|
+
if (!factors.length)
|
|
1256
|
+
throw new Error('stanza_overrides needs stanza_modality, stanza_ground or stanza_response columns to override.');
|
|
1257
|
+
if (ov.path.length !== factors.length)
|
|
1258
|
+
throw new Error(`stanza_overrides path ${JSON.stringify(ov.path)} has ${ov.path.length} value(s); ` +
|
|
1259
|
+
`it needs one per factor, in this order: ${order} (modality, then ground, then response).`);
|
|
1260
|
+
|
|
1261
|
+
// Column-major flat index into data
|
|
1262
|
+
let c = 0, stride = 1;
|
|
1263
|
+
for (let f = 0; f < factors.length; f++) {
|
|
1264
|
+
const val = ov.path[f];
|
|
1265
|
+
const fIdx = factorLevels[factors[f]][val];
|
|
1266
|
+
if (fIdx === undefined)
|
|
1267
|
+
throw new Error(`stanza_overrides path ${JSON.stringify(ov.path)}: '${val}' is not a value of ` +
|
|
1268
|
+
`${factors[f]} (position ${f + 1} of ${order}). ${factors[f]} has ` +
|
|
1269
|
+
`${Object.keys(factorLevels[factors[f]]).length} distinct values; ena_profile lists them when there are 10 or fewer.`);
|
|
1270
|
+
c += fIdx * stride;
|
|
1271
|
+
stride *= dimsShape[f];
|
|
1272
|
+
}
|
|
1273
|
+
data[c] = w;
|
|
1274
|
+
data[c + nCombinations] = win;
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
const tensorDef = { dims, dimsSender, dimsReceiver, dimsMode, factors, factorLevels, data };
|
|
1279
|
+
if ((window_type === 'time' || window_type === 'time_duration') && time_column) {
|
|
1280
|
+
tensorDef.timesCol = time_column;
|
|
1281
|
+
if (time_unit) tensorDef.timeUnit = time_unit;
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
return tensorDef;
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
async function toolTmaFit(ena, params) {
|
|
1288
|
+
params = resolveDefaults(ena, withGmrGroups(params), 'fit');
|
|
1289
|
+
const {
|
|
1290
|
+
units, conversations, codes,
|
|
1291
|
+
window_size = 4, window_type = 'lines', time_column = null,
|
|
1292
|
+
stanza_overrides = null,
|
|
1293
|
+
ordered = true, dims: requestedDims = 2, rotation,
|
|
1294
|
+
group_col, group1_value, group2_value,
|
|
1295
|
+
} = params;
|
|
1296
|
+
|
|
1297
|
+
const unitCols = toArray(units);
|
|
1298
|
+
const convoCols = toArray(conversations);
|
|
1299
|
+
|
|
1300
|
+
// Load data and build tensor definition
|
|
1301
|
+
const rows = loadRows(params);
|
|
1302
|
+
const tensorDef = buildContextTensor(rows, params);
|
|
1303
|
+
|
|
1304
|
+
const acc = ena.accumulate(rows, {
|
|
1305
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1306
|
+
ordered, tensor: tensorDef, ...tmaOpts(params), ...accumOpts(params, ordered),
|
|
1307
|
+
});
|
|
1308
|
+
const { unitLabels, nConnections, nUnits } = acc;
|
|
1309
|
+
|
|
1310
|
+
let uMap;
|
|
1311
|
+
if (group_col) uMap = buildUnitGroupMap(rows, unitCols, group_col);
|
|
1312
|
+
const rot = rotationOpts(rows, unitCols, unitLabels, { ...params, _ordered: ordered }, uMap);
|
|
1313
|
+
|
|
1314
|
+
const fitArgs = {
|
|
1315
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1316
|
+
dims: Math.max(requestedDims, PLOT_DIMS), ...rot.opts,
|
|
1317
|
+
ordered, tensor: tensorDef, ...tmaOpts(params), ...fitOpts(params, ordered),
|
|
1318
|
+
};
|
|
1319
|
+
const rawModel = ena.fit(rows, fitArgs);
|
|
1320
|
+
|
|
1321
|
+
const model = adaptModel(rawModel);
|
|
1322
|
+
// Fits keep PLOT_DIMS dimensions for ena_plot's axis choice; results report
|
|
1323
|
+
// the requested ones. rena-wasm caps both at the rotation's columns.
|
|
1324
|
+
const stride = model.dims ?? requestedDims;
|
|
1325
|
+
const dims = Math.min(requestedDims, stride);
|
|
1326
|
+
|
|
1327
|
+
const groupMap = uMap ?? null;
|
|
1328
|
+
|
|
1329
|
+
// Pre-compute per-group mean networks for ena_plot use
|
|
1330
|
+
let groupNetworks = null;
|
|
1331
|
+
if (groupMap) {
|
|
1332
|
+
const allGroups = [...new Set(groupMap.values())];
|
|
1333
|
+
groupNetworks = {};
|
|
1334
|
+
for (const gv of allGroups) {
|
|
1335
|
+
const idx = unitLabels.flatMap((lbl, i) => groupMap.get(lbl) === gv ? [i] : []);
|
|
1336
|
+
groupNetworks[gv] = computeGroupMean(model.networks, model.nConnections, idx);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
const dimCols = Array.from({ length: dims }, (_, i) => `dim${i + 1}`);
|
|
1341
|
+
const variance = varianceExplained(model, dims);
|
|
1342
|
+
const fit = goodnessOfFit(model, dims);
|
|
1343
|
+
const centroids = toRecords(model.centroids, model.nUnits, dims, model.unitLabels, 'dim', stride);
|
|
1344
|
+
const nodePos = toRecords(model.positions, codes.length, dims, codes, 'dim', stride);
|
|
1345
|
+
|
|
1346
|
+
// Group tests whenever two groups are named, whatever the rotation.
|
|
1347
|
+
let tests = null;
|
|
1348
|
+
if (groupMap && group1_value && group2_value) {
|
|
1349
|
+
const idxOf = (gv) => unitLabels.flatMap((lbl, i) => groupMap.get(lbl) === String(gv) ? [i] : []);
|
|
1350
|
+
const [i1, i2] = [idxOf(group1_value), idxOf(group2_value)];
|
|
1351
|
+
if (i1.length && i2.length)
|
|
1352
|
+
tests = groupStats(ena, model.centroids, dims, i1, i2, String(group1_value), String(group2_value), stride);
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
const nodeLines = nodePos.map(n =>
|
|
1356
|
+
` ${n.label.padEnd(32)}` + dimCols.map(d =>
|
|
1357
|
+
` ${(n[d] ?? 0) >= 0 ? '+' : ''}${(n[d] ?? 0).toFixed(4)}`).join('')
|
|
1358
|
+
);
|
|
1359
|
+
|
|
1360
|
+
const model_id = nextModelId();
|
|
1361
|
+
const summary = [
|
|
1362
|
+
`TMA model fitted. model_id: ${model_id}`,
|
|
1363
|
+
` Units: ${model.nUnits} Codes: ${codes.length} Connections: ${model.nConnections} Dims: ${dims}`,
|
|
1364
|
+
` Rotation: ${rotationLabel(params)}` + (rotation === 'generalized' ? ` Axes: ${model.columnNames?.slice(0, dims).join(', ')}` : ''),
|
|
1365
|
+
...rot.lines,
|
|
1366
|
+
...varianceLine(variance),
|
|
1367
|
+
...fitLine(fit),
|
|
1368
|
+
` Window: type=${window_type}, size=${window_size}${params._window_auto ? ' (Cross-Covariance Decay estimate)' : ''}, ordered=${ordered}`,
|
|
1369
|
+
...optsLines(params, model.nUnits),
|
|
1370
|
+
'',
|
|
1371
|
+
`Node positions (${dimCols.join(', ')}):`,
|
|
1372
|
+
...nodeLines,
|
|
1373
|
+
...(tests ? ['', ...tests.lines] : []),
|
|
1374
|
+
'',
|
|
1375
|
+
`Use ena_plot with model_id="${model_id}" for additional views (group, unit, subtraction, comparison).`,
|
|
1376
|
+
].join('\n');
|
|
1377
|
+
|
|
1378
|
+
const result = {
|
|
1379
|
+
model_id,
|
|
1380
|
+
centroids,
|
|
1381
|
+
node_positions: nodePos,
|
|
1382
|
+
connection_names: model.connectionNames,
|
|
1383
|
+
units: model.unitLabels,
|
|
1384
|
+
codes,
|
|
1385
|
+
n_units: model.nUnits,
|
|
1386
|
+
n_codes: codes.length,
|
|
1387
|
+
n_connections: model.nConnections,
|
|
1388
|
+
window_size,
|
|
1389
|
+
rotation: rotationLabel(params),
|
|
1390
|
+
variance_explained: variance,
|
|
1391
|
+
goodness_of_fit: fit,
|
|
1392
|
+
...(tests && { stats: tests.stats }),
|
|
1393
|
+
summary,
|
|
1394
|
+
};
|
|
1395
|
+
|
|
1396
|
+
const entry = {
|
|
1397
|
+
kind: 'tma', params, fitArgs,
|
|
1398
|
+
model, codes, unitLabels: model.unitLabels,
|
|
1399
|
+
groupMap, groupCol: group_col ?? null,
|
|
1400
|
+
groupNetworks, connectionNames: model.connectionNames, nConnections: model.nConnections, dims,
|
|
1401
|
+
...webEntry(ena, model, model.unitLabels, rows, unitCols, params),
|
|
1402
|
+
result,
|
|
1403
|
+
};
|
|
1404
|
+
entry.view = defaultView(params, entry.web);
|
|
1405
|
+
cacheSet(modelCache, model_id, entry, MAX_MODELS);
|
|
1406
|
+
|
|
1407
|
+
return { ...result, _plot: web.buildPlot(entry, entry.view) };
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
async function toolTmaAccumulate(ena, params) {
|
|
1411
|
+
params = resolveDefaults(ena, params, 'accumulate');
|
|
1412
|
+
const {
|
|
1413
|
+
units, conversations, codes,
|
|
1414
|
+
window_size = 4, window_type = 'lines', time_column = null,
|
|
1415
|
+
stanza_overrides = null,
|
|
1416
|
+
ordered = true,
|
|
1417
|
+
} = params;
|
|
1418
|
+
|
|
1419
|
+
const unitCols = toArray(units);
|
|
1420
|
+
const convoCols = toArray(conversations);
|
|
1421
|
+
const rows = loadRows(params);
|
|
1422
|
+
const tensorDef = buildContextTensor(rows, params);
|
|
1423
|
+
|
|
1424
|
+
const result = ena.accumulate(rows, {
|
|
1425
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1426
|
+
ordered, tensor: tensorDef, ...tmaOpts(params), ...accumOpts(params, ordered),
|
|
1427
|
+
});
|
|
1428
|
+
|
|
1429
|
+
const adapted = adaptAccum(result);
|
|
1430
|
+
const { networks: connectionCounts, unitLabels, connectionNames, nUnits, nConnections } = adapted;
|
|
1431
|
+
|
|
1432
|
+
const networks = unitLabels.map((lbl, i) => {
|
|
1433
|
+
const counts = Array.from(connectionCounts.subarray(i * nConnections, (i + 1) * nConnections));
|
|
1434
|
+
const nonZero = counts.flatMap((val, j) => val > 0 ? [{ connection: connectionNames[j], co_occurrences: val }] : []);
|
|
1435
|
+
return {
|
|
1436
|
+
unit: lbl,
|
|
1437
|
+
total_co_occurrences: counts.reduce((a, b) => a + b, 0),
|
|
1438
|
+
connections: nonZero.sort((a, b) => b.co_occurrences - a.co_occurrences).slice(0, 15),
|
|
1439
|
+
};
|
|
1440
|
+
});
|
|
1441
|
+
|
|
1442
|
+
return {
|
|
1443
|
+
n_units: nUnits,
|
|
1444
|
+
n_connections: nConnections,
|
|
1445
|
+
networks,
|
|
1446
|
+
summary: [
|
|
1447
|
+
'TMA context tensor accumulated.',
|
|
1448
|
+
` Units: ${nUnits} Directed Connections: ${nConnections}`,
|
|
1449
|
+
` Window: size=${window_size}, type=${window_type}`,
|
|
1450
|
+
...optsLines(params, nUnits),
|
|
1451
|
+
].join('\n'),
|
|
1452
|
+
};
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
// ── Tool implementations ──────────────────────────────────────────────────────
|
|
1456
|
+
|
|
1457
|
+
/**
|
|
1458
|
+
* Returns lightweight column metadata only — no raw data reaches Claude.
|
|
1459
|
+
* Used for privacy-sensitive datasets where the user wants to keep data local.
|
|
1460
|
+
*/
|
|
1461
|
+
const PROFILE_MAX_LEVELS = 10;
|
|
1462
|
+
|
|
1463
|
+
function toolProfile({ data_path }) {
|
|
1464
|
+
if (!data_path)
|
|
1465
|
+
throw new Error('ena_profile requires data_path. Raw CSV is never needed — the file stays on your machine.');
|
|
1466
|
+
|
|
1467
|
+
const rows = loadData(data_path, null);
|
|
1468
|
+
const nRows = rows.length;
|
|
1469
|
+
const headers = nRows > 0 ? Object.keys(rows[0]) : [];
|
|
1470
|
+
|
|
1471
|
+
const columns = headers.map(col => {
|
|
1472
|
+
const vals = rows.map(r => r[col]).filter(v => v !== '' && v != null);
|
|
1473
|
+
const uniq = [...new Set(vals)];
|
|
1474
|
+
const nUnique = uniq.length;
|
|
1475
|
+
const numericCount = vals.filter(v => !isNaN(Number(v))).length;
|
|
1476
|
+
const isBin = nUnique <= 2 && numericCount === vals.length &&
|
|
1477
|
+
uniq.every(v => v === '0' || v === '1');
|
|
1478
|
+
const info = {
|
|
1479
|
+
name: col,
|
|
1480
|
+
n_unique: nUnique,
|
|
1481
|
+
n_rows: vals.length,
|
|
1482
|
+
is_binary: isBin,
|
|
1483
|
+
};
|
|
1484
|
+
// Category labels (e.g. condition names) are needed to choose group
|
|
1485
|
+
// values; anything higher-cardinality — utterance text, names, IDs,
|
|
1486
|
+
// timestamps — is row-level data and is never returned.
|
|
1487
|
+
if (nUnique <= PROFILE_MAX_LEVELS) info.values = uniq.slice().sort();
|
|
1488
|
+
else info.values_withheld = true;
|
|
1489
|
+
return info;
|
|
1490
|
+
});
|
|
1491
|
+
|
|
1492
|
+
const binaryCols = columns.filter(c => c.is_binary).map(c => c.name);
|
|
1493
|
+
const summary = [
|
|
1494
|
+
`Dataset profile (no raw data transmitted): ${nRows} rows × ${headers.length} columns`,
|
|
1495
|
+
`Binary code columns detected (${binaryCols.length}): ${binaryCols.join(', ') || 'none'}`,
|
|
1496
|
+
`All columns: ${columns.map(c => `${c.name} (${c.n_unique} unique${c.is_binary ? ', binary' : ''})`).join('; ')}`,
|
|
1497
|
+
].join('\n');
|
|
1498
|
+
|
|
1499
|
+
return {
|
|
1500
|
+
privacy_note: `PRIVACY SAFE — only column metadata was computed locally: names, counts, and the distinct ` +
|
|
1501
|
+
`labels of columns with ≤${PROFILE_MAX_LEVELS} values (e.g. condition names). No row-level values ` +
|
|
1502
|
+
`(text, names, IDs, timestamps) were sent to Claude.`,
|
|
1503
|
+
n_rows: nRows,
|
|
1504
|
+
n_cols: headers.length,
|
|
1505
|
+
columns,
|
|
1506
|
+
binary_columns: binaryCols,
|
|
1507
|
+
summary,
|
|
1508
|
+
};
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
function toolInspect({ data_path, data_csv }) {
|
|
1512
|
+
const isPrivacySafe = !!data_path && !data_csv;
|
|
1513
|
+
const rows = loadData(data_path, data_csv);
|
|
1514
|
+
const det = detectColumns(rows);
|
|
1515
|
+
const nRows = det.nRows;
|
|
1516
|
+
const headers = nRows > 0 ? Object.keys(rows[0]) : [];
|
|
1517
|
+
|
|
1518
|
+
const codeCols = det.codes;
|
|
1519
|
+
const toSuggestion = ({ column, nUnique, score }) => ({ column, n_unique: nUnique, score });
|
|
1520
|
+
const suggestedUnits = det.units.map(toSuggestion);
|
|
1521
|
+
const suggestedConvos = det.conversations.map(toSuggestion);
|
|
1522
|
+
const suggestedGroups = det.groups;
|
|
1523
|
+
const colSummary = det.columns.map(c => ({ name: c.name, n_unique: c.nUnique, likely_role: c.role }));
|
|
1524
|
+
|
|
1525
|
+
const nCodes = codeCols.length;
|
|
1526
|
+
const topUnit = suggestedUnits[0];
|
|
1527
|
+
const topConvo = suggestedConvos[0];
|
|
1528
|
+
const hasUnits = !!topUnit && topUnit.score >= 0.3;
|
|
1529
|
+
const hasConvo = !!topConvo && topConvo.score >= 0.2;
|
|
1530
|
+
|
|
1531
|
+
let recommended, confidence, reason;
|
|
1532
|
+
if (nCodes >= 3 && hasUnits && hasConvo) {
|
|
1533
|
+
recommended = true;
|
|
1534
|
+
confidence = nCodes >= 5 && topUnit.score >= 0.7 ? 'high' : 'medium';
|
|
1535
|
+
reason = `Found ${nCodes} binary code columns, a likely unit column ` +
|
|
1536
|
+
`('${topUnit.column}', ${topUnit.n_unique} unique values), and a ` +
|
|
1537
|
+
`likely conversation column ('${topConvo.column}', ${topConvo.n_unique} unique values). ` +
|
|
1538
|
+
`This matches the ENA data signature: row-per-utterance with binary ` +
|
|
1539
|
+
`presence/absence codes grouped by unit and conversation.`;
|
|
1540
|
+
} else if (nCodes >= 2 && (hasUnits || hasConvo)) {
|
|
1541
|
+
recommended = true;
|
|
1542
|
+
confidence = 'low';
|
|
1543
|
+
reason = `Found ${nCodes} binary code column(s) and partial structure. ENA may be applicable.`;
|
|
1544
|
+
} else if (nCodes >= 2) {
|
|
1545
|
+
recommended = true;
|
|
1546
|
+
confidence = 'low';
|
|
1547
|
+
reason = `Found ${nCodes} binary code columns but no clear unit/conversation columns by name.`;
|
|
1548
|
+
} else if (nCodes === 1) {
|
|
1549
|
+
recommended = false;
|
|
1550
|
+
confidence = 'low';
|
|
1551
|
+
reason = `Only 1 binary code column was detected. ENA models co-occurrence ` +
|
|
1552
|
+
`*between* codes, so it needs at least 2 (ideally 4+) binary presence/absence columns.`;
|
|
1553
|
+
} else {
|
|
1554
|
+
recommended = false;
|
|
1555
|
+
confidence = 'low';
|
|
1556
|
+
reason = `No binary (0/1) code columns were detected in this dataset. ENA models the ` +
|
|
1557
|
+
`co-occurrence of *codes* — columns that mark, for each row, whether a concept, ` +
|
|
1558
|
+
`behavior, or feature is present (1) or absent (0). This dataset has none, so ENA ` +
|
|
1559
|
+
`cannot be run on it as-is.`;
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
// Actionable guidance for the not-applicable cases, so Claude can explain
|
|
1563
|
+
// to the user *why* ENA doesn't fit and what would be needed.
|
|
1564
|
+
let guidance = null;
|
|
1565
|
+
if (!recommended) {
|
|
1566
|
+
const needs =
|
|
1567
|
+
'For a dataset to be analyzable with ENA it needs:\n' +
|
|
1568
|
+
' • 2+ binary (0/1) CODE columns — each marking presence/absence of a concept or behavior per row\n' +
|
|
1569
|
+
' • a UNIT column identifying who or what each row belongs to\n' +
|
|
1570
|
+
' • a CONVERSATION column grouping rows into episodes';
|
|
1571
|
+
guidance = nCodes === 0
|
|
1572
|
+
? `${needs}\n\nThis dataset appears to contain no coded columns. If it is uncoded text or ` +
|
|
1573
|
+
`raw events, each concept of interest must first be turned into a 0/1 column (a ` +
|
|
1574
|
+
`qualitative-coding step) before ENA can be applied. If you expected coded columns, ` +
|
|
1575
|
+
`check that they use 0/1 values rather than text labels.`
|
|
1576
|
+
: `${needs}\n\nAdd more binary code columns, or use a standard statistical analysis instead.`;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
let nextStep = {};
|
|
1580
|
+
if (recommended && suggestedUnits.length && suggestedConvos.length) {
|
|
1581
|
+
// detectParams' combined selections: nesting unit columns (broadest
|
|
1582
|
+
// first) and up to two crossed conversation columns.
|
|
1583
|
+
const pick = (cols, fallback) =>
|
|
1584
|
+
cols?.length > 1 ? cols : (cols?.[0] ?? fallback);
|
|
1585
|
+
const bestUnit = pick(det.suggestedUnits, suggestedUnits[0].column);
|
|
1586
|
+
const bestConvo = pick(det.suggestedConversations, suggestedConvos[0].column);
|
|
1587
|
+
nextStep = {
|
|
1588
|
+
tool: 'ena_fit',
|
|
1589
|
+
parameters: {
|
|
1590
|
+
data_path: data_path ?? '<save CSV to a path first>',
|
|
1591
|
+
units: bestUnit,
|
|
1592
|
+
conversations: bestConvo,
|
|
1593
|
+
codes: codeCols,
|
|
1594
|
+
window_size: 4,
|
|
1595
|
+
rotation: 'svd',
|
|
1596
|
+
},
|
|
1597
|
+
};
|
|
1598
|
+
const grpCol = Object.keys(suggestedGroups)[0];
|
|
1599
|
+
if (grpCol && suggestedGroups[grpCol].length === 2) {
|
|
1600
|
+
nextStep.alternative_tool = 'ena_compare_groups';
|
|
1601
|
+
nextStep.alternative_parameters = {
|
|
1602
|
+
...nextStep.parameters,
|
|
1603
|
+
group_col: grpCol,
|
|
1604
|
+
group1_value: suggestedGroups[grpCol][0],
|
|
1605
|
+
group2_value: suggestedGroups[grpCol][1],
|
|
1606
|
+
};
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
const summaryLines = [
|
|
1611
|
+
`ENA inspection: ${recommended ? 'RECOMMENDED' : 'NOT RECOMMENDED'} (confidence: ${confidence})`,
|
|
1612
|
+
` Dataset: ${nRows} rows × ${headers.length} columns`,
|
|
1613
|
+
` ${reason}`,
|
|
1614
|
+
'',
|
|
1615
|
+
`Binary code columns (${nCodes}): ${codeCols.join(', ') || 'none detected'}`,
|
|
1616
|
+
];
|
|
1617
|
+
if (suggestedUnits.length)
|
|
1618
|
+
summaryLines.push(`Top unit candidates: ` +
|
|
1619
|
+
suggestedUnits.slice(0,3).map(u => `${u.column} (n=${u.n_unique}, score=${u.score})`).join(', '));
|
|
1620
|
+
if (suggestedConvos.length)
|
|
1621
|
+
summaryLines.push(`Top conversation candidates: ` +
|
|
1622
|
+
suggestedConvos.slice(0,3).map(c => `${c.column} (n=${c.n_unique}, score=${c.score})`).join(', '));
|
|
1623
|
+
for (const [col, vals] of Object.entries(suggestedGroups))
|
|
1624
|
+
summaryLines.push(`Likely group column: '${col}' → ${JSON.stringify(vals)}`);
|
|
1625
|
+
if (guidance)
|
|
1626
|
+
summaryLines.push('', guidance);
|
|
1627
|
+
|
|
1628
|
+
const privacyNote = isPrivacySafe
|
|
1629
|
+
? 'PRIVACY SAFE — data was read from a local file path. No raw row data was sent to Claude; only this analysis result was transmitted.'
|
|
1630
|
+
: 'RAW DATA TRANSMITTED — data_csv was used, so row data passed through this conversation. For sensitive datasets, ask the user to save the file and provide data_path instead.';
|
|
1631
|
+
|
|
1632
|
+
return {
|
|
1633
|
+
privacy_note: privacyNote,
|
|
1634
|
+
ena_recommended: recommended,
|
|
1635
|
+
confidence,
|
|
1636
|
+
reason,
|
|
1637
|
+
guidance,
|
|
1638
|
+
n_rows: nRows,
|
|
1639
|
+
n_cols: headers.length,
|
|
1640
|
+
suggested_codes: codeCols,
|
|
1641
|
+
suggested_units: suggestedUnits,
|
|
1642
|
+
suggested_conversations: suggestedConvos,
|
|
1643
|
+
suggested_groups: suggestedGroups,
|
|
1644
|
+
column_summary: colSummary,
|
|
1645
|
+
next_step: nextStep,
|
|
1646
|
+
summary: summaryLines.join('\n'),
|
|
1647
|
+
};
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
async function toolFit(ena, params) {
|
|
1651
|
+
params = resolveDefaults(ena, withGmrGroups(params), 'fit');
|
|
1652
|
+
const {
|
|
1653
|
+
units, conversations, codes,
|
|
1654
|
+
window_size, binary = true,
|
|
1655
|
+
dims: requestedDims = 2, rotation,
|
|
1656
|
+
group_col, group1_value, group2_value,
|
|
1657
|
+
} = params;
|
|
1658
|
+
|
|
1659
|
+
// Normalize units/conversations to arrays (rena-wasm joins with '__')
|
|
1660
|
+
const unitCols = toArray(units);
|
|
1661
|
+
const convoCols = toArray(conversations);
|
|
1662
|
+
|
|
1663
|
+
// Return cached model if params are identical
|
|
1664
|
+
const fKey = fitCacheKey(params);
|
|
1665
|
+
const cachedId = fitCache.get(fKey);
|
|
1666
|
+
if (cachedId && modelCache.has(cachedId)) { // the model may have been evicted
|
|
1667
|
+
const model_id = cachedId;
|
|
1668
|
+
const entry = cacheGet(modelCache, model_id);
|
|
1669
|
+
return {
|
|
1670
|
+
model_id,
|
|
1671
|
+
...entry.result,
|
|
1672
|
+
summary: `Using cached model ${model_id}.\n${entry.result.summary}`,
|
|
1673
|
+
_plot: web.buildPlot(entry, entry.view),
|
|
1674
|
+
};
|
|
1675
|
+
}
|
|
1676
|
+
|
|
1677
|
+
// Load data; reuse parsed rows if already cached for this accumulation key
|
|
1678
|
+
const aKey = accumCacheKey(params);
|
|
1679
|
+
let rows, acc;
|
|
1680
|
+
if (accumCache.has(aKey)) {
|
|
1681
|
+
({ rows, acc } = cacheGet(accumCache, aKey));
|
|
1682
|
+
} else {
|
|
1683
|
+
rows = loadRows(params);
|
|
1684
|
+
acc = ena.accumulate(rows, {
|
|
1685
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1686
|
+
window: windowValue(window_size), binary, ...accumOpts(params),
|
|
1687
|
+
});
|
|
1688
|
+
cacheSet(accumCache, aKey, { rows, acc }, MAX_ACCUM);
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
const { unitLabels, nConnections, connectionNames, nUnits } = acc;
|
|
1692
|
+
|
|
1693
|
+
let uMap;
|
|
1694
|
+
if (group_col) uMap = buildUnitGroupMap(rows, unitCols, group_col);
|
|
1695
|
+
const rot = rotationOpts(rows, unitCols, unitLabels, params, uMap);
|
|
1696
|
+
|
|
1697
|
+
const fitArgs = {
|
|
1698
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1699
|
+
window: windowValue(window_size), binary, dims: Math.max(requestedDims, PLOT_DIMS),
|
|
1700
|
+
...rot.opts, ...fitOpts(params),
|
|
1701
|
+
};
|
|
1702
|
+
const model = ena.fit(rows, fitArgs);
|
|
1703
|
+
// Fits keep PLOT_DIMS dimensions for ena_plot's axis choice; results report
|
|
1704
|
+
// the requested ones. rena-wasm caps both at the rotation's columns.
|
|
1705
|
+
const stride = model.dims ?? requestedDims;
|
|
1706
|
+
const dims = Math.min(requestedDims, stride);
|
|
1707
|
+
|
|
1708
|
+
const groupMap = uMap ?? null;
|
|
1709
|
+
|
|
1710
|
+
// Per-group mean networks (the reported numbers) — from the sphere-normed
|
|
1711
|
+
// line weights (as rENA does), not raw counts, so they share the plotted space
|
|
1712
|
+
let groupNetworks = null;
|
|
1713
|
+
if (groupMap) {
|
|
1714
|
+
const allGroups = [...new Set(groupMap.values())];
|
|
1715
|
+
groupNetworks = {};
|
|
1716
|
+
for (const gv of allGroups) {
|
|
1717
|
+
const idx = unitLabels.flatMap((lbl, i) => groupMap.get(lbl) === gv ? [i] : []);
|
|
1718
|
+
groupNetworks[gv] = computeGroupMean(model.networks, nConnections, idx);
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
const dimCols = Array.from({ length: dims }, (_, i) => `dim${i + 1}`);
|
|
1723
|
+
const variance = varianceExplained(model, dims);
|
|
1724
|
+
const fit = goodnessOfFit(model, dims);
|
|
1725
|
+
const centroids = toRecords(model.centroids, nUnits, dims, unitLabels, 'dim', stride);
|
|
1726
|
+
const nodePos = toRecords(model.positions, codes.length, dims, codes, 'dim', stride);
|
|
1727
|
+
|
|
1728
|
+
// Group tests whenever two groups are named, whatever the rotation.
|
|
1729
|
+
let tests = null;
|
|
1730
|
+
if (groupMap && group1_value && group2_value) {
|
|
1731
|
+
const idxOf = (gv) => unitLabels.flatMap((lbl, i) => groupMap.get(lbl) === String(gv) ? [i] : []);
|
|
1732
|
+
const [i1, i2] = [idxOf(group1_value), idxOf(group2_value)];
|
|
1733
|
+
if (i1.length && i2.length)
|
|
1734
|
+
tests = groupStats(ena, model.centroids, dims, i1, i2, String(group1_value), String(group2_value), stride);
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
const nodeLines = nodePos.map(n =>
|
|
1738
|
+
` ${n.label.padEnd(32)}` + dimCols.map(d =>
|
|
1739
|
+
` ${(n[d] ?? 0) >= 0 ? '+' : ''}${(n[d] ?? 0).toFixed(4)}`).join('')
|
|
1740
|
+
);
|
|
1741
|
+
|
|
1742
|
+
const model_id = nextModelId();
|
|
1743
|
+
const summary = [
|
|
1744
|
+
`ENA model fitted. model_id: ${model_id}`,
|
|
1745
|
+
` Units: ${nUnits} Codes: ${codes.length} Connections: ${nConnections} Dims: ${dims}`,
|
|
1746
|
+
` Rotation: ${rotationLabel(params)}` + (rotation === 'generalized' ? ` Axes: ${model.columnNames?.slice(0, dims).join(', ')}` : ''),
|
|
1747
|
+
...rot.lines,
|
|
1748
|
+
...varianceLine(variance),
|
|
1749
|
+
...fitLine(fit),
|
|
1750
|
+
`${windowLine(params)}, binary=${binary}`,
|
|
1751
|
+
...optsLines(params, nUnits),
|
|
1752
|
+
'',
|
|
1753
|
+
`Node positions (${dimCols.join(', ')}):`,
|
|
1754
|
+
...nodeLines,
|
|
1755
|
+
...(tests ? ['', ...tests.lines] : []),
|
|
1756
|
+
'',
|
|
1757
|
+
`Use ena_plot with model_id="${model_id}" for additional views (group, unit, subtraction, comparison), ` +
|
|
1758
|
+
`or ena_export to open the model in the ENA Web Tool.`,
|
|
1759
|
+
].join('\n');
|
|
1760
|
+
|
|
1761
|
+
const result = {
|
|
1762
|
+
model_id,
|
|
1763
|
+
centroids,
|
|
1764
|
+
node_positions: nodePos,
|
|
1765
|
+
connection_names: model.connectionNames,
|
|
1766
|
+
units: unitLabels,
|
|
1767
|
+
codes,
|
|
1768
|
+
n_units: nUnits,
|
|
1769
|
+
n_codes: codes.length,
|
|
1770
|
+
n_connections: nConnections,
|
|
1771
|
+
window_size,
|
|
1772
|
+
rotation: rotationLabel(params),
|
|
1773
|
+
variance_explained: variance,
|
|
1774
|
+
goodness_of_fit: fit,
|
|
1775
|
+
...(tests && { stats: tests.stats }),
|
|
1776
|
+
summary,
|
|
1777
|
+
};
|
|
1778
|
+
|
|
1779
|
+
const entry = {
|
|
1780
|
+
kind: 'ena', params, fitArgs,
|
|
1781
|
+
model, codes, unitLabels,
|
|
1782
|
+
groupMap, groupCol: group_col ?? null,
|
|
1783
|
+
groupNetworks, connectionNames, nConnections, dims,
|
|
1784
|
+
...webEntry(ena, model, unitLabels, rows, unitCols, params),
|
|
1785
|
+
result,
|
|
1786
|
+
};
|
|
1787
|
+
entry.view = defaultView(params, entry.web);
|
|
1788
|
+
cacheSet(modelCache, model_id, entry, MAX_MODELS);
|
|
1789
|
+
cacheSet(fitCache, fKey, model_id, MAX_MODELS);
|
|
1790
|
+
|
|
1791
|
+
return { ...result, _plot: web.buildPlot(entry, entry.view) };
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
async function toolAccumulate(ena, params) {
|
|
1795
|
+
params = resolveDefaults(ena, params, 'accumulate');
|
|
1796
|
+
const {
|
|
1797
|
+
units, conversations, codes,
|
|
1798
|
+
window_size, binary = true,
|
|
1799
|
+
} = params;
|
|
1800
|
+
const unitCols = toArray(units);
|
|
1801
|
+
const convoCols = toArray(conversations);
|
|
1802
|
+
const rows = loadRows(params);
|
|
1803
|
+
const result = ena.accumulate(rows, {
|
|
1804
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1805
|
+
window: windowValue(window_size), binary, ...accumOpts(params),
|
|
1806
|
+
});
|
|
1807
|
+
|
|
1808
|
+
const { networks: connectionCounts, unitLabels, connectionNames, nUnits, nConnections } = result;
|
|
1809
|
+
|
|
1810
|
+
const networks = unitLabels.map((lbl, i) => {
|
|
1811
|
+
const rec = { unit: lbl };
|
|
1812
|
+
for (let j = 0; j < nConnections; j++)
|
|
1813
|
+
rec[connectionNames[j]] = R(connectionCounts[i * nConnections + j], 4);
|
|
1814
|
+
return rec;
|
|
1815
|
+
});
|
|
1816
|
+
|
|
1817
|
+
const totals = new Float64Array(nConnections);
|
|
1818
|
+
for (let i = 0; i < nUnits; i++)
|
|
1819
|
+
for (let j = 0; j < nConnections; j++)
|
|
1820
|
+
totals[j] += connectionCounts[i * nConnections + j];
|
|
1821
|
+
|
|
1822
|
+
const topIdx = Array.from({ length: nConnections }, (_, i) => i)
|
|
1823
|
+
.sort((a, b) => totals[b] - totals[a])
|
|
1824
|
+
.slice(0, 10)
|
|
1825
|
+
.filter(i => totals[i] > 0);
|
|
1826
|
+
|
|
1827
|
+
const topConnections = topIdx.map(i => ({
|
|
1828
|
+
connection: connectionNames[i],
|
|
1829
|
+
total_weight: R(totals[i], 4),
|
|
1830
|
+
}));
|
|
1831
|
+
|
|
1832
|
+
const summary = [
|
|
1833
|
+
'ENA accumulation complete.',
|
|
1834
|
+
` Units: ${nUnits} Connections: ${nConnections}`,
|
|
1835
|
+
`${windowLine(params)}, binary=${binary}`,
|
|
1836
|
+
...optsLines(params, nUnits),
|
|
1837
|
+
'',
|
|
1838
|
+
'Top 10 connections by total weight:',
|
|
1839
|
+
...topConnections.map(t => ` ${t.connection.padEnd(40)} ${t.total_weight.toFixed(1)}`),
|
|
1840
|
+
].join('\n');
|
|
1841
|
+
|
|
1842
|
+
return {
|
|
1843
|
+
networks,
|
|
1844
|
+
connection_names: connectionNames,
|
|
1845
|
+
units: unitLabels,
|
|
1846
|
+
codes: [...codes],
|
|
1847
|
+
n_units: nUnits,
|
|
1848
|
+
n_codes: codes.length,
|
|
1849
|
+
n_connections: nConnections,
|
|
1850
|
+
window_size,
|
|
1851
|
+
top_connections: topConnections,
|
|
1852
|
+
summary,
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
// Window-size estimate by Cross-Covariance Decay (= rENA ena.ccd). Runs on the
|
|
1857
|
+
// raw code matrix per conversation; no accumulation or rotation.
|
|
1858
|
+
async function toolWindow(ena, {
|
|
1859
|
+
conversations, codes,
|
|
1860
|
+
data_path, data_csv, exclude_conversations, order_by,
|
|
1861
|
+
max_window = 20, min_overlap = 10,
|
|
1862
|
+
}) {
|
|
1863
|
+
const convoCols = toArray(conversations);
|
|
1864
|
+
const rows = loadRows({ data_path, data_csv, conversations, exclude_conversations, order_by });
|
|
1865
|
+
const r = ena.ccd(rows, { codes, conversations: convoCols, maxWindow: max_window, minOverlap: min_overlap });
|
|
1866
|
+
|
|
1867
|
+
const curve = Array.from(r.lag, (lag, i) => ({
|
|
1868
|
+
lag,
|
|
1869
|
+
cross_covariance: R(r.frob_unbiased_signed[i], 4),
|
|
1870
|
+
weight: r.total_weight[i],
|
|
1871
|
+
}));
|
|
1872
|
+
const peak = Math.max(...curve.map(c => Math.abs(c.cross_covariance))) || 1;
|
|
1873
|
+
|
|
1874
|
+
const summary = [
|
|
1875
|
+
`Estimated window size: ${r.window_size} (Cross-Covariance Decay, = rENA ena.ccd)`,
|
|
1876
|
+
` Peak lag: ${r.peak_lag} Lags evaluated: 0–${curve.at(-1)?.lag ?? 0}`,
|
|
1877
|
+
'',
|
|
1878
|
+
'Noise-corrected cross-covariance by lag:',
|
|
1879
|
+
...curve.map(c => ` ${String(c.lag).padStart(3)} ${c.cross_covariance.toFixed(4).padStart(8)} ` +
|
|
1880
|
+
'#'.repeat(Math.round(30 * Math.abs(c.cross_covariance) / peak))),
|
|
1881
|
+
'',
|
|
1882
|
+
`Pass window_size=${r.window_size} to ena_fit / tma_fit, or compare against a nearby value.`,
|
|
1883
|
+
].join('\n');
|
|
1884
|
+
|
|
1885
|
+
return {
|
|
1886
|
+
window_size: r.window_size,
|
|
1887
|
+
peak_lag: r.peak_lag,
|
|
1888
|
+
curve,
|
|
1889
|
+
summary,
|
|
1890
|
+
};
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
async function toolCompareGroups(ena, params) {
|
|
1894
|
+
if (!params.group_col || params.group1_value == null || params.group2_value == null)
|
|
1895
|
+
throw new Error('ena_compare_groups requires group_col, group1_value and group2_value.');
|
|
1896
|
+
if (params.rotation_from != null)
|
|
1897
|
+
throw new Error('ena_compare_groups is defined by its rotation through the two groups; use ena_fit with rotation_from to project into another model.');
|
|
1898
|
+
params = resolveDefaults(ena, params, 'compare');
|
|
1899
|
+
const {
|
|
1900
|
+
units, conversations, codes,
|
|
1901
|
+
group_col, group1_value, group2_value,
|
|
1902
|
+
window_size, binary = true,
|
|
1903
|
+
dims: requestedDims = 2,
|
|
1904
|
+
} = params;
|
|
1905
|
+
|
|
1906
|
+
// Return cached model if params are identical
|
|
1907
|
+
const fKey = fitCacheKey(params, 'compare');
|
|
1908
|
+
const cachedId = fitCache.get(fKey);
|
|
1909
|
+
if (cachedId && modelCache.has(cachedId)) { // the model may have been evicted
|
|
1910
|
+
const model_id = cachedId;
|
|
1911
|
+
const entry = cacheGet(modelCache, model_id);
|
|
1912
|
+
return {
|
|
1913
|
+
model_id,
|
|
1914
|
+
...entry.result,
|
|
1915
|
+
summary: `Using cached model ${model_id}.\n${entry.result.summary}`,
|
|
1916
|
+
_plot: web.buildPlot(entry, entry.view),
|
|
1917
|
+
};
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
// Normalize units/conversations to arrays
|
|
1921
|
+
const unitCols = toArray(units);
|
|
1922
|
+
const convoCols = toArray(conversations);
|
|
1923
|
+
|
|
1924
|
+
// Accumulate (or use cache)
|
|
1925
|
+
const aKey = accumCacheKey(params);
|
|
1926
|
+
let rows, acc;
|
|
1927
|
+
if (accumCache.has(aKey)) {
|
|
1928
|
+
({ rows, acc } = cacheGet(accumCache, aKey));
|
|
1929
|
+
} else {
|
|
1930
|
+
rows = loadRows(params);
|
|
1931
|
+
acc = ena.accumulate(rows, {
|
|
1932
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1933
|
+
window: windowValue(window_size), binary, ...accumOpts(params),
|
|
1934
|
+
});
|
|
1935
|
+
cacheSet(accumCache, aKey, { rows, acc }, MAX_ACCUM);
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
const { unitLabels, connectionNames, nUnits, nConnections } = acc;
|
|
1939
|
+
|
|
1940
|
+
const uMap = buildUnitGroupMap(rows, unitCols, group_col);
|
|
1941
|
+
const g1Idx = unitLabels.flatMap((lbl, i) => uMap.get(lbl) === String(group1_value) ? [i] : []);
|
|
1942
|
+
const g2Idx = unitLabels.flatMap((lbl, i) => uMap.get(lbl) === String(group2_value) ? [i] : []);
|
|
1943
|
+
|
|
1944
|
+
if (!g1Idx.length) throw new Error(`No units found for ${group_col}=${group1_value}`);
|
|
1945
|
+
if (!g2Idx.length) throw new Error(`No units found for ${group_col}=${group2_value}`);
|
|
1946
|
+
|
|
1947
|
+
const rot = rotationOpts(rows, unitCols, unitLabels, params, uMap);
|
|
1948
|
+
const fitArgs = {
|
|
1949
|
+
codes, units: unitCols, conversations: convoCols,
|
|
1950
|
+
window: windowValue(window_size), binary, dims: Math.max(requestedDims, PLOT_DIMS),
|
|
1951
|
+
...rot.opts, ...fitOpts(params),
|
|
1952
|
+
};
|
|
1953
|
+
const model = ena.fit(rows, fitArgs);
|
|
1954
|
+
// Fits keep PLOT_DIMS dimensions for ena_plot's axis choice; results report
|
|
1955
|
+
// the requested ones. rena-wasm caps both at the rotation's columns.
|
|
1956
|
+
const stride = model.dims ?? requestedDims;
|
|
1957
|
+
const dims = Math.min(requestedDims, stride);
|
|
1958
|
+
|
|
1959
|
+
const dimCols = Array.from({ length: dims }, (_, i) => `dim${i + 1}`);
|
|
1960
|
+
const variance = varianceExplained(model, dims);
|
|
1961
|
+
const fit = goodnessOfFit(model, dims);
|
|
1962
|
+
const allCentroids = toRecords(model.centroids, nUnits, dims, unitLabels, 'dim', stride);
|
|
1963
|
+
const g1Set = new Set(g1Idx);
|
|
1964
|
+
const g2Set = new Set(g2Idx);
|
|
1965
|
+
const g1Centroids = allCentroids.filter((_, i) => g1Set.has(i));
|
|
1966
|
+
const g2Centroids = allCentroids.filter((_, i) => g2Set.has(i));
|
|
1967
|
+
const nodePos = toRecords(model.positions, codes.length, dims, codes, 'dim', stride);
|
|
1968
|
+
|
|
1969
|
+
const groupMeanPos = (centroids) => Object.fromEntries(
|
|
1970
|
+
dimCols.map(d => [d, R(centroids.reduce((s, c) => s + (c[d] ?? 0), 0) / centroids.length)])
|
|
1971
|
+
);
|
|
1972
|
+
const g1Mean = groupMeanPos(g1Centroids);
|
|
1973
|
+
const g2Mean = groupMeanPos(g2Centroids);
|
|
1974
|
+
const diff = Object.fromEntries(dimCols.map(d => [d, R(g1Mean[d] - g2Mean[d])]));
|
|
1975
|
+
const tests = groupStats(ena, model.centroids, dims, g1Idx, g2Idx, String(group1_value), String(group2_value), stride);
|
|
1976
|
+
|
|
1977
|
+
const g1NetMean = computeGroupMean(model.networks, nConnections, g1Idx);
|
|
1978
|
+
const g2NetMean = computeGroupMean(model.networks, nConnections, g2Idx);
|
|
1979
|
+
const subNet = g1NetMean.map((v, j) => v - g2NetMean[j]);
|
|
1980
|
+
|
|
1981
|
+
const groupNetworks = {
|
|
1982
|
+
[String(group1_value)]: g1NetMean,
|
|
1983
|
+
[String(group2_value)]: g2NetMean,
|
|
1984
|
+
};
|
|
1985
|
+
|
|
1986
|
+
const topIdx = Array.from({ length: nConnections }, (_, i) => i)
|
|
1987
|
+
.sort((a, b) => Math.abs(subNet[b]) - Math.abs(subNet[a]))
|
|
1988
|
+
.slice(0, 10);
|
|
1989
|
+
|
|
1990
|
+
const topDiff = topIdx.map(i => ({
|
|
1991
|
+
connection: connectionNames[i],
|
|
1992
|
+
[`${group1_value}_mean`]: R(g1NetMean[i], 4),
|
|
1993
|
+
[`${group2_value}_mean`]: R(g2NetMean[i], 4),
|
|
1994
|
+
difference: R(subNet[i], 4),
|
|
1995
|
+
}));
|
|
1996
|
+
|
|
1997
|
+
const netDict = (arr) => Object.fromEntries(
|
|
1998
|
+
connectionNames.map((n, i) => [n, R(arr[i], 4)])
|
|
1999
|
+
);
|
|
2000
|
+
|
|
2001
|
+
const groupMap = uMap;
|
|
2002
|
+
const model_id = nextModelId();
|
|
2003
|
+
|
|
2004
|
+
const summary = [
|
|
2005
|
+
`ENA group comparison: '${group1_value}' vs '${group2_value}' model_id: ${model_id}`,
|
|
2006
|
+
` Rotation: ${rotationLabel(params)} Axes: ${model.columnNames?.slice(0, dims).join(', ')}`,
|
|
2007
|
+
...rot.lines,
|
|
2008
|
+
...varianceLine(variance),
|
|
2009
|
+
...fitLine(fit),
|
|
2010
|
+
windowLine(params),
|
|
2011
|
+
...optsLines(params, nUnits),
|
|
2012
|
+
` ${group1_value}: n=${g1Idx.length} centroid = ` +
|
|
2013
|
+
dimCols.map(d => `${d}=${g1Mean[d] >= 0 ? '+' : ''}${g1Mean[d].toFixed(4)}`).join(', '),
|
|
2014
|
+
` ${group2_value}: n=${g2Idx.length} centroid = ` +
|
|
2015
|
+
dimCols.map(d => `${d}=${g2Mean[d] >= 0 ? '+' : ''}${g2Mean[d].toFixed(4)}`).join(', '),
|
|
2016
|
+
` Difference (g1−g2): ` +
|
|
2017
|
+
dimCols.map(d => `${d}=${diff[d] >= 0 ? '+' : ''}${diff[d].toFixed(4)}`).join(', '),
|
|
2018
|
+
'',
|
|
2019
|
+
...tests.lines,
|
|
2020
|
+
'',
|
|
2021
|
+
'Top differential connections (difference of group-mean line weights):',
|
|
2022
|
+
...topDiff.map(t =>
|
|
2023
|
+
` ${t.connection.padEnd(40)} ` +
|
|
2024
|
+
`${group1_value}=${t[`${group1_value}_mean`] >= 0 ? '+' : ''}${t[`${group1_value}_mean`].toFixed(3)} ` +
|
|
2025
|
+
`${group2_value}=${t[`${group2_value}_mean`] >= 0 ? '+' : ''}${t[`${group2_value}_mean`].toFixed(3)} ` +
|
|
2026
|
+
`Δ=${t.difference >= 0 ? '+' : ''}${t.difference.toFixed(3)}`
|
|
2027
|
+
),
|
|
2028
|
+
'',
|
|
2029
|
+
`Use ena_plot with model_id="${model_id}" for group, subtraction, comparison, or unit views, ` +
|
|
2030
|
+
`or ena_export to open the model in the ENA Web Tool.`,
|
|
2031
|
+
].join('\n');
|
|
2032
|
+
|
|
2033
|
+
const result = {
|
|
2034
|
+
model_id,
|
|
2035
|
+
group1_centroids: g1Centroids,
|
|
2036
|
+
group2_centroids: g2Centroids,
|
|
2037
|
+
group1_mean: g1Mean,
|
|
2038
|
+
group2_mean: g2Mean,
|
|
2039
|
+
difference: diff,
|
|
2040
|
+
stats: tests.stats,
|
|
2041
|
+
window_size,
|
|
2042
|
+
rotation: rotationLabel(params),
|
|
2043
|
+
node_positions: nodePos,
|
|
2044
|
+
variance_explained: variance,
|
|
2045
|
+
goodness_of_fit: fit,
|
|
2046
|
+
connection_names: [...connectionNames],
|
|
2047
|
+
group1_networks: netDict(g1NetMean),
|
|
2048
|
+
group2_networks: netDict(g2NetMean),
|
|
2049
|
+
subtraction_network: netDict(subNet),
|
|
2050
|
+
top_differential: topDiff,
|
|
2051
|
+
summary,
|
|
2052
|
+
};
|
|
2053
|
+
|
|
2054
|
+
const entry = {
|
|
2055
|
+
kind: 'compare', params, fitArgs,
|
|
2056
|
+
model, codes, unitLabels,
|
|
2057
|
+
groupMap, groupCol: group_col,
|
|
2058
|
+
groupNetworks, connectionNames, nConnections, dims,
|
|
2059
|
+
// Extra fields for compare models
|
|
2060
|
+
subNet, g1Idx, g2Idx,
|
|
2061
|
+
group1_value: String(group1_value),
|
|
2062
|
+
group2_value: String(group2_value),
|
|
2063
|
+
...webEntry(ena, model, unitLabels, rows, unitCols, params),
|
|
2064
|
+
result,
|
|
2065
|
+
};
|
|
2066
|
+
entry.view = defaultView(params, entry.web);
|
|
2067
|
+
cacheSet(modelCache, model_id, entry, MAX_MODELS);
|
|
2068
|
+
cacheSet(fitCache, fKey, model_id, MAX_MODELS);
|
|
2069
|
+
|
|
2070
|
+
return { ...result, _plot: web.buildPlot(entry, entry.view) };
|
|
2071
|
+
}
|
|
2072
|
+
|
|
2073
|
+
// ena_plot's display options → buildPlot's view.style (the web tool's Plot
|
|
2074
|
+
// Settings). Validated here so a bad colour or group fails with a message
|
|
2075
|
+
// instead of a blank plot.
|
|
2076
|
+
// qeviz resolves colour names through a canvas, which the SVG renderer
|
|
2077
|
+
// (jsdom) lacks, so names are turned into hex here: the CSS named colours.
|
|
2078
|
+
const CSS_NAMED = Object.fromEntries((
|
|
2079
|
+
'aliceblue:f0f8ff antiquewhite:faebd7 aqua:00ffff aquamarine:7fffd4 azure:f0ffff beige:f5f5dc ' +
|
|
2080
|
+
'bisque:ffe4c4 black:000000 blanchedalmond:ffebcd blue:0000ff blueviolet:8a2be2 brown:a52a2a ' +
|
|
2081
|
+
'burlywood:deb887 cadetblue:5f9ea0 chartreuse:7fff00 chocolate:d2691e coral:ff7f50 ' +
|
|
2082
|
+
'cornflowerblue:6495ed cornsilk:fff8dc crimson:dc143c cyan:00ffff darkblue:00008b darkcyan:008b8b ' +
|
|
2083
|
+
'darkgoldenrod:b8860b darkgray:a9a9a9 darkgreen:006400 darkgrey:a9a9a9 darkkhaki:bdb76b ' +
|
|
2084
|
+
'darkmagenta:8b008b darkolivegreen:556b2f darkorange:ff8c00 darkorchid:9932cc darkred:8b0000 ' +
|
|
2085
|
+
'darksalmon:e9967a darkseagreen:8fbc8f darkslateblue:483d8b darkslategray:2f4f4f darkslategrey:2f4f4f ' +
|
|
2086
|
+
'darkturquoise:00ced1 darkviolet:9400d3 deeppink:ff1493 deepskyblue:00bfff dimgray:696969 ' +
|
|
2087
|
+
'dimgrey:696969 dodgerblue:1e90ff firebrick:b22222 floralwhite:fffaf0 forestgreen:228b22 ' +
|
|
2088
|
+
'fuchsia:ff00ff gainsboro:dcdcdc ghostwhite:f8f8ff gold:ffd700 goldenrod:daa520 gray:808080 ' +
|
|
2089
|
+
'green:008000 greenyellow:adff2f grey:808080 honeydew:f0fff0 hotpink:ff69b4 indianred:cd5c5c ' +
|
|
2090
|
+
'indigo:4b0082 ivory:fffff0 khaki:f0e68c lavender:e6e6fa lavenderblush:fff0f5 lawngreen:7cfc00 ' +
|
|
2091
|
+
'lemonchiffon:fffacd lightblue:add8e6 lightcoral:f08080 lightcyan:e0ffff lightgoldenrodyellow:fafad2 ' +
|
|
2092
|
+
'lightgray:d3d3d3 lightgreen:90ee90 lightgrey:d3d3d3 lightpink:ffb6c1 lightsalmon:ffa07a ' +
|
|
2093
|
+
'lightseagreen:20b2aa lightskyblue:87cefa lightslategray:778899 lightslategrey:778899 ' +
|
|
2094
|
+
'lightsteelblue:b0c4de lightyellow:ffffe0 lime:00ff00 limegreen:32cd32 linen:faf0e6 magenta:ff00ff ' +
|
|
2095
|
+
'maroon:800000 mediumaquamarine:66cdaa mediumblue:0000cd mediumorchid:ba55d3 mediumpurple:9370db ' +
|
|
2096
|
+
'mediumseagreen:3cb371 mediumslateblue:7b68ee mediumspringgreen:00fa9a mediumturquoise:48d1cc ' +
|
|
2097
|
+
'mediumvioletred:c71585 midnightblue:191970 mintcream:f5fffa mistyrose:ffe4e1 moccasin:ffe4b5 ' +
|
|
2098
|
+
'navajowhite:ffdead navy:000080 oldlace:fdf5e6 olive:808000 olivedrab:6b8e23 orange:ffa500 ' +
|
|
2099
|
+
'orangered:ff4500 orchid:da70d6 palegoldenrod:eee8aa palegreen:98fb98 paleturquoise:afeeee ' +
|
|
2100
|
+
'palevioletred:db7093 papayawhip:ffefd5 peachpuff:ffdab9 peru:cd853f pink:ffc0cb plum:dda0dd ' +
|
|
2101
|
+
'powderblue:b0e0e6 purple:800080 rebeccapurple:663399 red:ff0000 rosybrown:bc8f8f royalblue:4169e1 ' +
|
|
2102
|
+
'saddlebrown:8b4513 salmon:fa8072 sandybrown:f4a460 seagreen:2e8b57 seashell:fff5ee sienna:a0522d ' +
|
|
2103
|
+
'silver:c0c0c0 skyblue:87ceeb slateblue:6a5acd slategray:708090 slategrey:708090 snow:fffafa ' +
|
|
2104
|
+
'springgreen:00ff7f steelblue:4682b4 tan:d2b48c teal:008080 thistle:d8bfd8 tomato:ff6347 ' +
|
|
2105
|
+
'turquoise:40e0d0 violet:ee82ee wheat:f5deb3 white:ffffff whitesmoke:f5f5f5 yellow:ffff00 ' +
|
|
2106
|
+
'yellowgreen:9acd32'
|
|
2107
|
+
).trim().split(/\s+/).map(p => p.split(':')));
|
|
2108
|
+
function cssColor(c) {
|
|
2109
|
+
const v = String(c).trim().toLowerCase();
|
|
2110
|
+
if (/^#[0-9a-f]{6}$/.test(v)) return v;
|
|
2111
|
+
if (/^#[0-9a-f]{3}$/.test(v)) return '#' + [...v.slice(1)].map(h => h + h).join('');
|
|
2112
|
+
return CSS_NAMED[v] ? `#${CSS_NAMED[v]}` : null;
|
|
2113
|
+
}
|
|
2114
|
+
function plotStyle(w, o) {
|
|
2115
|
+
const style = {};
|
|
2116
|
+
if (o.min_edge_weight != null) {
|
|
2117
|
+
const v = Number(o.min_edge_weight);
|
|
2118
|
+
if (!(v >= 0)) throw new Error('min_edge_weight must be a number ≥ 0');
|
|
2119
|
+
style.minWeight = v;
|
|
2120
|
+
}
|
|
2121
|
+
if (o.hide_unconnected_codes) style.hideUnconnected = true;
|
|
2122
|
+
if (o.text_size != null) {
|
|
2123
|
+
const v = Number(o.text_size);
|
|
2124
|
+
if (!(v >= 6 && v <= 48)) throw new Error('text_size must be between 6 and 48 (px)');
|
|
2125
|
+
style.textSize = v;
|
|
2126
|
+
}
|
|
2127
|
+
if (o.colors != null) {
|
|
2128
|
+
if (typeof o.colors !== 'object' || Array.isArray(o.colors))
|
|
2129
|
+
throw new Error('colors must be an object: { "<group>": "<colour>" }');
|
|
2130
|
+
const groups = w.groups.map(g => g.name);
|
|
2131
|
+
for (const [g, c] of Object.entries(o.colors)) {
|
|
2132
|
+
if (!w.byName.has(g)) throw new Error(`colors: no group '${g}'. Groups: ${groups.join(', ')}`);
|
|
2133
|
+
if (!cssColor(c)) throw new Error(`colors: '${c}' is not a colour. Use #rgb, #rrggbb or a CSS colour name such as orange or steelblue.`);
|
|
2134
|
+
}
|
|
2135
|
+
style.colors = Object.fromEntries(Object.entries(o.colors).map(([g, c]) => [g, cssColor(c)]));
|
|
2136
|
+
}
|
|
2137
|
+
if (o.confidence_intervals === false) style.ci = false;
|
|
2138
|
+
if (o.outlier_intervals === true) style.outliers = true;
|
|
2139
|
+
return style;
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
async function toolPlot({ model_id, type = 'all', group_value, group1, group2, unit_label, magnify,
|
|
2143
|
+
x_dim = 1, y_dim = 2, flip_x = false, flip_y = false,
|
|
2144
|
+
min_edge_weight, hide_unconnected_codes, text_size, colors,
|
|
2145
|
+
confidence_intervals, outlier_intervals }) {
|
|
2146
|
+
const entry = cacheGet(modelCache, model_id);
|
|
2147
|
+
if (!entry)
|
|
2148
|
+
throw new Error(`Unknown model_id '${model_id}'. Call ena_fit, ena_compare_groups or tma_fit first ` +
|
|
2149
|
+
`(the ${MAX_MODELS} most recently used models are kept; refitting is cheap).`);
|
|
2150
|
+
|
|
2151
|
+
const { codes, unitLabels, groupCol, web: w } = entry;
|
|
2152
|
+
const groupNames = w.groups.map(g => g.name);
|
|
2153
|
+
const needGroup = (g, what) => {
|
|
2154
|
+
if (!groupCol && w.groups.length < 2)
|
|
2155
|
+
throw new Error(`Model ${model_id} has no groups. Refit with group_col (or two or more unit columns).`);
|
|
2156
|
+
if (!w.byName.has(String(g)))
|
|
2157
|
+
throw new Error(`${what}: no group '${g}'. Groups: ${groupNames.join(', ')}`);
|
|
2158
|
+
return { kind: 'group', name: String(g) };
|
|
2159
|
+
};
|
|
2160
|
+
const pair = () => {
|
|
2161
|
+
const g1 = group1 ?? entry.group1_value ?? entry.params?.group1_value;
|
|
2162
|
+
const g2 = group2 ?? entry.group2_value ?? entry.params?.group2_value;
|
|
2163
|
+
if (g1 == null || g2 == null) throw new Error(`type='${type}' requires group1 and group2`);
|
|
2164
|
+
return [needGroup(g1, 'group1'), needGroup(g2, 'group2')];
|
|
2165
|
+
};
|
|
2166
|
+
|
|
2167
|
+
let view, label;
|
|
2168
|
+
switch (type) {
|
|
2169
|
+
case 'all':
|
|
2170
|
+
view = { first: { kind: 'all' } };
|
|
2171
|
+
label = `ENA model ${model_id} — all units`;
|
|
2172
|
+
break;
|
|
2173
|
+
case 'group': {
|
|
2174
|
+
if (group_value == null) throw new Error("type='group' requires group_value");
|
|
2175
|
+
view = { first: needGroup(group_value, 'group_value') };
|
|
2176
|
+
label = `ENA mean network — ${group_value}`;
|
|
2177
|
+
break;
|
|
2178
|
+
}
|
|
2179
|
+
case 'comparison': {
|
|
2180
|
+
const [a, b] = pair();
|
|
2181
|
+
view = { first: a, second: b, overlay: true };
|
|
2182
|
+
label = `ENA comparison — ${a.name} vs ${b.name}`;
|
|
2183
|
+
break;
|
|
2184
|
+
}
|
|
2185
|
+
case 'subtraction': {
|
|
2186
|
+
const [a, b] = pair();
|
|
2187
|
+
view = { first: a, second: b };
|
|
2188
|
+
label = `ENA subtraction — ${a.name} − ${b.name}`;
|
|
2189
|
+
break;
|
|
2190
|
+
}
|
|
2191
|
+
case 'unit': {
|
|
2192
|
+
if (unit_label == null) throw new Error("type='unit' requires unit_label");
|
|
2193
|
+
if (!unitLabels.includes(String(unit_label)))
|
|
2194
|
+
throw new Error(`Unit '${unit_label}' not found. Available: ${unitLabels.slice(0, 20).join(', ')}`);
|
|
2195
|
+
view = { first: { kind: 'unit', name: String(unit_label) } };
|
|
2196
|
+
label = `ENA network — ${unit_label}`;
|
|
2197
|
+
break;
|
|
2198
|
+
}
|
|
2199
|
+
default:
|
|
2200
|
+
throw new Error(`Unknown type '${type}'. Options: all, group, comparison, subtraction, unit`);
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
if (magnify != null) {
|
|
2204
|
+
if (!(Number(magnify) > 0)) throw new Error('magnify must be a positive number');
|
|
2205
|
+
view.scale = Number(magnify);
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
// Axes: dimensions are 1-based here, as in the results (dim1, dim2, ...).
|
|
2209
|
+
const nDims = entry.model.dims ?? 2;
|
|
2210
|
+
const names = entry.dimNames ?? [];
|
|
2211
|
+
for (const [k, v] of [['x_dim', x_dim], ['y_dim', y_dim]])
|
|
2212
|
+
if (!Number.isInteger(Number(v)) || v < 1 || v > nDims)
|
|
2213
|
+
throw new Error(`${k} must be a dimension from 1 to ${nDims} (this model has ${nDims}: ` +
|
|
2214
|
+
`${Array.from({ length: nDims }, (_, i) => names[i] ?? `dim${i + 1}`).join(', ')}).`);
|
|
2215
|
+
if (Number(x_dim) === Number(y_dim)) throw new Error('x_dim and y_dim must be different dimensions.');
|
|
2216
|
+
view.axes = [Number(x_dim) - 1, Number(y_dim) - 1];
|
|
2217
|
+
view.flip = [!!flip_x, !!flip_y];
|
|
2218
|
+
const axesNote = (x_dim != 1 || y_dim != 2 || flip_x || flip_y)
|
|
2219
|
+
? ` [x: ${flip_x ? '−' : ''}${names[x_dim - 1] ?? `dim${x_dim}`}, y: ${flip_y ? '−' : ''}${names[y_dim - 1] ?? `dim${y_dim}`}]` : '';
|
|
2220
|
+
|
|
2221
|
+
view.style = plotStyle(w, { min_edge_weight, hide_unconnected_codes, text_size, colors,
|
|
2222
|
+
confidence_intervals, outlier_intervals });
|
|
2223
|
+
|
|
2224
|
+
const plot = web.buildPlot(entry, view);
|
|
2225
|
+
label += ` (scaled ${(view.scale ?? 1).toFixed(1)}x)${axesNote}`;
|
|
2226
|
+
const st = view.style;
|
|
2227
|
+
const settings = {
|
|
2228
|
+
...(st.minWeight > 0 && { min_edge_weight: st.minWeight }),
|
|
2229
|
+
...(st.hideUnconnected && { hide_unconnected_codes: true }),
|
|
2230
|
+
...(st.textSize && { text_size: st.textSize }),
|
|
2231
|
+
...(st.colors && { colors: st.colors }),
|
|
2232
|
+
...(st.ci === false && { confidence_intervals: false }),
|
|
2233
|
+
...(st.outliers && { outlier_intervals: true }),
|
|
2234
|
+
};
|
|
2235
|
+
|
|
2236
|
+
return {
|
|
2237
|
+
model_id,
|
|
2238
|
+
plot_type: type,
|
|
2239
|
+
magnify: view.scale ?? 1,
|
|
2240
|
+
axes: { x: names[x_dim - 1] ?? `dim${x_dim}`, y: names[y_dim - 1] ?? `dim${y_dim}`, flip_x: !!flip_x, flip_y: !!flip_y },
|
|
2241
|
+
label,
|
|
2242
|
+
title: plot.title,
|
|
2243
|
+
...(plot.maxWeight > 0 && { max_edge_weight: Number(plot.maxWeight.toPrecision(4)) }),
|
|
2244
|
+
...(Object.keys(settings).length && { plot_settings: settings }),
|
|
2245
|
+
n_codes: codes.length,
|
|
2246
|
+
n_units: unitLabels.length,
|
|
2247
|
+
summary: label,
|
|
2248
|
+
_plot: plot,
|
|
2249
|
+
};
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// ── Export to the ENA Web Tool ────────────────────────────────────────────────
|
|
2253
|
+
|
|
2254
|
+
// Choices the web tool has no setting for: a model that uses them would open
|
|
2255
|
+
// there as a different model, so it is not exported.
|
|
2256
|
+
function exportBlockers(entry) {
|
|
2257
|
+
const p = entry.params ?? {};
|
|
2258
|
+
const unitCols = toArray(p.units);
|
|
2259
|
+
const out = [];
|
|
2260
|
+
if (p.center_align_to_origin === false) out.push('center_align_to_origin=false (the web tool always keeps zero-network units at the origin)');
|
|
2261
|
+
if (p.binary === false && !p.weight_model) out.push('binary=false (weight_model="product" keeps raw counts and exports)');
|
|
2262
|
+
if (entry.kind === 'tma') {
|
|
2263
|
+
if (toArray(p.stanza_modality).length || toArray(p.stanza_ground).length || toArray(p.stanza_response).length ||
|
|
2264
|
+
p.stanza_overrides || p.horizons)
|
|
2265
|
+
out.push('transmodal windows (stanza_modality / stanza_ground / stanza_response / stanza_overrides / horizons)');
|
|
2266
|
+
if ((p.window_type ?? 'lines') !== 'lines') out.push('time-based windows');
|
|
2267
|
+
if (p.ordered === false) out.push('an unordered tma_fit model (use ena_fit)');
|
|
2268
|
+
}
|
|
2269
|
+
if (p.rotation_from != null)
|
|
2270
|
+
out.push(`a projection into model ${p.rotation_from}'s space (in the web tool, "Use Rotation Matrix From" refers to another set in the same project; export the source and set it there)`);
|
|
2271
|
+
const rot = p.rotation ?? 'svd';
|
|
2272
|
+
if (p.rotation_from != null) { /* reported above */ }
|
|
2273
|
+
else if (p._web_gmr) {
|
|
2274
|
+
if (p._web_gmr.col !== unitCols[0] || unitCols.length < 2)
|
|
2275
|
+
out.push(`a rotation by ${p._web_gmr.col} (the web tool rotates by the groups of the first of two or more unit columns` +
|
|
2276
|
+
`${unitCols.length > 1 ? `, here ${unitCols[0]}` : ''})`);
|
|
2277
|
+
} else if (rot === 'generalized') {
|
|
2278
|
+
out.push('a generalized rotation on x_var (the web tool rotates only by its unit groups)');
|
|
2279
|
+
} else if (rot === 'mean' && (p.group_col !== unitCols[0] || unitCols.length < 2)) {
|
|
2280
|
+
out.push(`a means rotation on ${p.group_col} (the web tool rotates by the groups of its first unit column)`);
|
|
2281
|
+
}
|
|
2282
|
+
return out;
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
// The web tool's rotation switches for this model.
|
|
2286
|
+
function exportRotation(entry) {
|
|
2287
|
+
const p = entry.params;
|
|
2288
|
+
if (p._web_gmr) {
|
|
2289
|
+
const nGroups = entry.web.groups.length;
|
|
2290
|
+
return { meansRotated: nGroups === 2 ? 1 : null, regressionRotated: 1, meansRotatedBy: [...p._web_gmr.pair] };
|
|
2291
|
+
}
|
|
2292
|
+
if (p.rotation === 'mean')
|
|
2293
|
+
return { meansRotated: 1, regressionRotated: null, meansRotatedBy: [String(p.group1_value), String(p.group2_value)] };
|
|
2294
|
+
return { meansRotated: null, regressionRotated: null, meansRotatedBy: [] };
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
const safeFileName = (s) => String(s).replace(/[\/\\:*?"<>|\u0000-\u001f]+/g, '-').trim() || 'ENA model';
|
|
2298
|
+
|
|
2299
|
+
function freePath(path) {
|
|
2300
|
+
if (!existsSync(path)) return path;
|
|
2301
|
+
const ext = extname(path), stem = ext ? path.slice(0, -ext.length) : path;
|
|
2302
|
+
for (let i = 2; ; i++) if (!existsSync(`${stem} (${i})${ext}`)) return `${stem} (${i})${ext}`;
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
// ── ena_tables: the model's data tables as CSV ──────────────────────────────
|
|
2306
|
+
// The ENA Web Tool's "Download Model" tables (js/ena.controllers.js export()):
|
|
2307
|
+
// nodes.csv and points.csv as it writes them (code / ENA_UNIT, then one column
|
|
2308
|
+
// per dimension, named as the dimensions are), and line-weights.csv and
|
|
2309
|
+
// connection-counts.csv with one "A & B" column per connection. Those two lead
|
|
2310
|
+
// with ENA_UNIT and the unit columns, as rENA's meta.data did — the web tool's
|
|
2311
|
+
// WASM sets no longer carry meta.data, which leaves its rows unlabelled.
|
|
2312
|
+
function csvCell(v) {
|
|
2313
|
+
if (v == null) return '';
|
|
2314
|
+
const raw = typeof v === 'number' ? (Number.isFinite(v) ? String(v) : '') : String(v);
|
|
2315
|
+
return /[",\n]/.test(raw) ? `"${raw.replace(/"/g, '""')}"` : raw;
|
|
2316
|
+
}
|
|
2317
|
+
const csvText = (header, rows) => [header, ...rows].map(r => r.map(csvCell).join(',')).join('\n') + '\n';
|
|
2318
|
+
|
|
2319
|
+
async function toolTables(ena, { model_id, out_path }) {
|
|
2320
|
+
const entry = cacheGet(modelCache, model_id);
|
|
2321
|
+
if (!entry)
|
|
2322
|
+
throw new Error(`Unknown model_id '${model_id}'. Fit the model first (ena_fit, ena_compare_groups or tma_fit).`);
|
|
2323
|
+
const { model, codes, unitLabels } = entry;
|
|
2324
|
+
const p = entry.params ?? {};
|
|
2325
|
+
const unitCols = toArray(p.units);
|
|
2326
|
+
const stride = model.dims ?? 2;
|
|
2327
|
+
const dimNames = Array.from({ length: stride }, (_, d) => entry.dimNames?.[d] ?? `dimension_${d + 1}`);
|
|
2328
|
+
const nConn = model.nConnections;
|
|
2329
|
+
|
|
2330
|
+
const rowsOf = (flat, n, width) => Array.from({ length: n }, (_, i) => Array.from(flat.subarray
|
|
2331
|
+
? flat.subarray(i * width, (i + 1) * width) : flat.slice(i * width, (i + 1) * width)));
|
|
2332
|
+
const unitMeta = unitLabels.map(lbl => {
|
|
2333
|
+
const parts = lbl.split('__');
|
|
2334
|
+
return [web.unitPath(lbl), ...unitCols.map((_, j) => parts[j] ?? '')];
|
|
2335
|
+
});
|
|
2336
|
+
const metaHeader = ['ENA_UNIT', ...unitCols];
|
|
2337
|
+
|
|
2338
|
+
const tables = {
|
|
2339
|
+
'nodes.csv': csvText(['code', ...dimNames], rowsOf(model.positions, codes.length, stride).map((r, c) => [codes[c], ...r])),
|
|
2340
|
+
'points.csv': csvText(['ENA_UNIT', ...dimNames], rowsOf(model.centroids, unitLabels.length, stride).map((r, i) => [unitMeta[i][0], ...r])),
|
|
2341
|
+
'line-weights.csv': csvText([...metaHeader, ...model.connectionNames],
|
|
2342
|
+
rowsOf(model.networks, unitLabels.length, nConn).map((r, i) => [...unitMeta[i], ...r])),
|
|
2343
|
+
};
|
|
2344
|
+
if (model.connectionCounts)
|
|
2345
|
+
tables['connection-counts.csv'] = csvText([...metaHeader, ...model.connectionNames],
|
|
2346
|
+
rowsOf(model.connectionCounts, unitLabels.length, nConn).map((r, i) => [...unitMeta[i], ...r]));
|
|
2347
|
+
|
|
2348
|
+
const base = safeFileName(`ENA model ${model_id} tables`);
|
|
2349
|
+
const dir = freePath(out_path ?? join(p.data_path ? dirname(p.data_path) : VIZ_DIR, base));
|
|
2350
|
+
mkdirSync(dir, { recursive: true });
|
|
2351
|
+
const files = Object.entries(tables).map(([name, text]) => {
|
|
2352
|
+
const path = join(dir, name);
|
|
2353
|
+
writeFileSync(path, text);
|
|
2354
|
+
return { file: name, path, rows: text.split('\n').length - 2 };
|
|
2355
|
+
});
|
|
2356
|
+
|
|
2357
|
+
const summary = [
|
|
2358
|
+
`Wrote model ${model_id}'s tables to ${dir}`,
|
|
2359
|
+
...files.map(f => ` ${f.file.padEnd(22)} ${f.rows} rows`),
|
|
2360
|
+
` Dimensions: ${dimNames.join(', ')} Connections: ${nConn}${model.ordered ? ' (directed: "A & B" is A → B)' : ''}`,
|
|
2361
|
+
' points.csv holds the unit points as fitted (unscaled); line-weights.csv the normalized networks;',
|
|
2362
|
+
' connection-counts.csv the raw accumulated counts. No rows of the data file are included.',
|
|
2363
|
+
].join('\n');
|
|
2364
|
+
return { model_id, folder: dir, files, dimensions: dimNames, summary };
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2367
|
+
// ── ena_simplify: PRIA, the ENA Web Tool's Simplify Model ───────────────────
|
|
2368
|
+
// rena-wasm pria() (= R's PRIA::pria) finds the largest set of codes, up to
|
|
2369
|
+
// remove_num and never leaving fewer than 3, whose removal keeps the model
|
|
2370
|
+
// within threshold of the full one. Run with the fit's own options, as the web
|
|
2371
|
+
// tool does (WasmWrapper: ena.pria(rows, { ...fitOpts, removeNum, threshold })),
|
|
2372
|
+
// then, as the web tool does, refit without the removed codes.
|
|
2373
|
+
const PRIA_MIN = 0.8, PRIA_MAX = 0.99;
|
|
2374
|
+
|
|
2375
|
+
async function toolSimplify(ena, { model_id, threshold = 0.95, remove_num = 3 }) {
|
|
2376
|
+
const entry = cacheGet(modelCache, model_id);
|
|
2377
|
+
if (!entry)
|
|
2378
|
+
throw new Error(`Unknown model_id '${model_id}'. Fit the model first (ena_fit, ena_compare_groups or tma_fit).`);
|
|
2379
|
+
if (!entry.fitArgs) throw new Error(`Model ${model_id} can't be simplified. Refit it with this server version.`);
|
|
2380
|
+
const p = entry.params ?? {};
|
|
2381
|
+
const codes = entry.codes;
|
|
2382
|
+
if (codes.length < 4)
|
|
2383
|
+
throw new Error(`Model ${model_id} has ${codes.length} codes; PRIA needs more than 3 (it never leaves fewer than 3).`);
|
|
2384
|
+
if (!(threshold >= PRIA_MIN && threshold <= PRIA_MAX))
|
|
2385
|
+
throw new Error(`threshold must be between ${PRIA_MIN} and ${PRIA_MAX}, as in the ENA Web Tool (default 0.95).`);
|
|
2386
|
+
if (!(Number.isInteger(remove_num) && remove_num >= 1))
|
|
2387
|
+
throw new Error('remove_num must be a whole number of codes, 1 or more (default 3).');
|
|
2388
|
+
|
|
2389
|
+
const rows = loadRows(p);
|
|
2390
|
+
const pr = ena.pria(rows, { ...entry.fitArgs, removeNum: remove_num, threshold });
|
|
2391
|
+
const removed = pr.removed ?? [];
|
|
2392
|
+
const header = `PRIA (threshold ${threshold}, up to ${Math.min(remove_num, codes.length - 3)} codes) on model ${model_id}: `;
|
|
2393
|
+
if (!removed.length) {
|
|
2394
|
+
return {
|
|
2395
|
+
model_id, removed_codes: [], kept_codes: codes, threshold,
|
|
2396
|
+
summary: header + 'no codes can be removed — every code is needed to keep the model within the threshold.',
|
|
2397
|
+
};
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
// Refit without them, as the web tool excludes them and regenerates.
|
|
2401
|
+
const gone = new Set(removed);
|
|
2402
|
+
const kept = codes.filter(c => !gone.has(c));
|
|
2403
|
+
const mask = toArray(p.mask_connections).filter(n => !String(n).split(' & ').some(c => gone.has(c)));
|
|
2404
|
+
const next = { ...p, codes: kept, mask_connections: mask.length ? mask : undefined };
|
|
2405
|
+
const refit = entry.kind === 'tma' ? await toolTmaFit(ena, next)
|
|
2406
|
+
: entry.kind === 'compare' ? await toolCompareGroups(ena, next)
|
|
2407
|
+
: await toolFit(ena, next);
|
|
2408
|
+
return {
|
|
2409
|
+
...refit,
|
|
2410
|
+
simplified_from: model_id,
|
|
2411
|
+
removed_codes: removed,
|
|
2412
|
+
kept_codes: kept,
|
|
2413
|
+
threshold,
|
|
2414
|
+
summary: header + `removed ${removed.join(', ')} (${removed.length} of ${codes.length} codes). ` +
|
|
2415
|
+
`Refit with the remaining ${kept.length} as model ${refit.model_id}; the window and every other setting are unchanged.\n\n` +
|
|
2416
|
+
refit.summary,
|
|
2417
|
+
};
|
|
2418
|
+
}
|
|
2419
|
+
|
|
2420
|
+
// ── ena_lines: the lines behind a connection ────────────────────────────────
|
|
2421
|
+
// A local HTML viewer of the lines and windows that produced a connection
|
|
2422
|
+
// (the ENA Web Tool's Data View), from the model's own rowConnectionCounts and
|
|
2423
|
+
// libtma's window rule (lines.js). Returns counts and row numbers, never text.
|
|
2424
|
+
const DEFAULT_MAX_WINDOWS = 500;
|
|
2425
|
+
|
|
2426
|
+
async function toolLines(ena, { model_id, connection, units, group, columns, max_windows, out_path }) {
|
|
2427
|
+
const entry = cacheGet(modelCache, model_id);
|
|
2428
|
+
if (!entry)
|
|
2429
|
+
throw new Error(`Unknown model_id '${model_id}'. Fit the model first (ena_fit, ena_compare_groups or tma_fit).`);
|
|
2430
|
+
const { model, codes, unitLabels } = entry;
|
|
2431
|
+
const p = entry.params ?? {};
|
|
2432
|
+
if (!model.rowConnectionCounts)
|
|
2433
|
+
throw new Error(`Model ${model_id} has no per-line counts. Refit it with this server version.`);
|
|
2434
|
+
const rows = loadRows(p);
|
|
2435
|
+
const unitCols = toArray(p.units), convoCols = toArray(p.conversations);
|
|
2436
|
+
const isTma = entry.kind === 'tma';
|
|
2437
|
+
const ordered = isTma ? (p.ordered ?? true) : false;
|
|
2438
|
+
|
|
2439
|
+
// Connection: as named in connection_names; unordered pairs either way round.
|
|
2440
|
+
let conn = null;
|
|
2441
|
+
if (connection != null) {
|
|
2442
|
+
const names = model.connectionNames;
|
|
2443
|
+
conn = names.indexOf(String(connection));
|
|
2444
|
+
if (conn < 0 && !ordered) {
|
|
2445
|
+
const [a, b] = String(connection).split(' & ');
|
|
2446
|
+
conn = names.indexOf(`${b} & ${a}`);
|
|
2447
|
+
}
|
|
2448
|
+
if (conn < 0)
|
|
2449
|
+
throw new Error(`Model ${model_id} has no connection '${connection}'. Connections: ${names.join(', ')}` +
|
|
2450
|
+
(ordered ? ' (directed: "A & B" is A → B).' : '.'));
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
// Selection: units by label ("a::b" or "a__b"), or a group.
|
|
2454
|
+
const norm = (l) => String(l).split('::').join('__');
|
|
2455
|
+
let selected;
|
|
2456
|
+
if (units?.length) {
|
|
2457
|
+
selected = new Set(toArray(units).map(u => {
|
|
2458
|
+
const i = unitLabels.indexOf(norm(u));
|
|
2459
|
+
if (i < 0) throw new Error(`Unit '${u}' is not in model ${model_id}.`);
|
|
2460
|
+
return i;
|
|
2461
|
+
}));
|
|
2462
|
+
} else if (group != null) {
|
|
2463
|
+
const g = entry.web?.byName.get(String(group));
|
|
2464
|
+
if (!g) throw new Error(`Model ${model_id} has no group '${group}'. Groups: ${(entry.web?.groups ?? []).map(x => x.name).join(', ')}`);
|
|
2465
|
+
selected = new Set(g.idx);
|
|
2466
|
+
} else if (unitLabels.length === 1) {
|
|
2467
|
+
selected = new Set([0]);
|
|
2468
|
+
} else {
|
|
2469
|
+
throw new Error('Choose the lines to show with units (unit labels) or group.');
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2472
|
+
const allCols = rows.length ? Object.keys(rows[0]) : [];
|
|
2473
|
+
for (const c of toArray(columns))
|
|
2474
|
+
if (!allCols.includes(c)) throw new Error(`Column '${c}' is not in the data.`);
|
|
2475
|
+
|
|
2476
|
+
if (conn != null && isMasked(p, ordered, model.connectionNames[conn]))
|
|
2477
|
+
throw new Error(`${model.connectionNames[conn]} is masked in model ${model_id} (mask_connections), so no lines count toward it. ` +
|
|
2478
|
+
'Fit the model without the mask to see its lines.');
|
|
2479
|
+
const tensorDef = isTma ? buildContextTensor(rows, p) : defaultTensor(windowValue(p.window_size ?? 4));
|
|
2480
|
+
const res = connectionWindows({
|
|
2481
|
+
rows, codes, unitCols, convoCols, unitsUsed: p.units_used ?? null,
|
|
2482
|
+
tensorDef, horizons: isTma ? (p.horizons ?? null) : null, ordered,
|
|
2483
|
+
unitLabels, rowCounts: model.rowConnectionCounts, connectionNames: model.connectionNames,
|
|
2484
|
+
conn, selected,
|
|
2485
|
+
});
|
|
2486
|
+
|
|
2487
|
+
const connName = conn == null ? null : model.connectionNames[conn];
|
|
2488
|
+
const highlight = connName ? [...new Set(connName.split(' & '))] : [];
|
|
2489
|
+
const stanzaCols = isTma ? [...toArray(p.stanza_modality), ...toArray(p.stanza_ground), ...toArray(p.stanza_response)] : [];
|
|
2490
|
+
// The web tool's Data View: units, conversations, codes (the connection's
|
|
2491
|
+
// when one is shown); plus what the user adds, and TMA's stanza columns.
|
|
2492
|
+
const shown = [...new Set([...unitCols, ...convoCols, ...stanzaCols, ...toArray(columns),
|
|
2493
|
+
...(highlight.length ? highlight : codes)])];
|
|
2494
|
+
const maxWindows = Number.isInteger(max_windows) && max_windows > 0 ? max_windows : DEFAULT_MAX_WINDOWS;
|
|
2495
|
+
|
|
2496
|
+
const unitPath = (i) => unitLabels[i].split('__').join('::');
|
|
2497
|
+
const selLabel = group != null ? `${group} (${selected.size} units)`
|
|
2498
|
+
: selected.size === 1 ? unitPath([...selected][0]) : `${selected.size} units`;
|
|
2499
|
+
const dataFile = p.data_path ? basename(p.data_path) : 'pasted data';
|
|
2500
|
+
const html = viewerHtml({
|
|
2501
|
+
title: connName ? `${connName}${ordered ? ' (→)' : ''} — ${selLabel}` : `Lines — ${selLabel}`,
|
|
2502
|
+
subtitle: `Model ${model_id}: units ${unitCols.join(' + ')} · conversations ${convoCols.join(' + ')} · ` +
|
|
2503
|
+
`window ${p.window_size ?? 4}${isTma ? ' (TMA)' : ''} · ${res.windows.length} windows` +
|
|
2504
|
+
(conn != null ? ` · check: ${res.check.mismatches.length ? `${res.check.mismatches.length} MISMATCHES` : 'every line matches the model'}` : ''),
|
|
2505
|
+
note: `This file contains rows of ${dataFile}. Row numbers are data rows (the header is not counted). ` +
|
|
2506
|
+
(ordered && connName ? `Directed: ground lines carrying ${highlight[0]} in the window, response line carrying ${highlight[highlight.length - 1]}.` : ''),
|
|
2507
|
+
dataFile, columns: allCols,
|
|
2508
|
+
roles: { units: unitCols, conversations: convoCols, codes, shown },
|
|
2509
|
+
highlight, ordered, rows, result: res, unitLabels, maxWindows,
|
|
2510
|
+
rowNumber: (i) => fileRowOf(rows, i) + 1,
|
|
2511
|
+
});
|
|
2512
|
+
|
|
2513
|
+
const base = safeFileName(`ENA lines ${model_id} ${connName ?? 'all'} ${selLabel}`).slice(0, 120);
|
|
2514
|
+
const target = freePath(out_path
|
|
2515
|
+
? (extname(out_path).toLowerCase() === '.html' ? out_path : join(out_path, `${base}.html`))
|
|
2516
|
+
: join(p.data_path ? dirname(p.data_path) : VIZ_DIR, `${base}.html`));
|
|
2517
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
2518
|
+
writeFileSync(target, html);
|
|
2519
|
+
|
|
2520
|
+
const byUnit = new Map();
|
|
2521
|
+
for (const w of res.windows) {
|
|
2522
|
+
const b = byUnit.get(w.unit) ?? { unit: unitPath(w.unit), windows: 0, count: 0 };
|
|
2523
|
+
b.windows++; b.count += w.count ?? 0; byUnit.set(w.unit, b);
|
|
2524
|
+
}
|
|
2525
|
+
const ctxUsed = new Set(res.windows.map(w => w.context));
|
|
2526
|
+
let modelTotal = 0;
|
|
2527
|
+
if (conn != null) for (const u of selected) modelTotal += model.connectionCounts[u * model.nConnections + conn];
|
|
2528
|
+
const linesTotal = res.windows.reduce((s, w) => s + (w.count ?? 0), 0);
|
|
2529
|
+
const match = conn == null ? null : (res.check.mismatches.length === 0 && Math.abs(linesTotal - modelTotal) < 1e-9);
|
|
2530
|
+
|
|
2531
|
+
const summary = [
|
|
2532
|
+
connName
|
|
2533
|
+
? `${res.windows.length} windows behind ${connName}${ordered ? ' (directed)' : ''} for ${selLabel}, in ${ctxUsed.size} conversation(s).`
|
|
2534
|
+
: `${res.windows.length} lines for ${selLabel}, in ${ctxUsed.size} conversation(s).`,
|
|
2535
|
+
...(conn != null ? [match
|
|
2536
|
+
? ` Check: every one of the selection's ${res.check.checked} lines agrees with the model, and the windows' counts sum to the model's (${R(modelTotal, 4)}).`
|
|
2537
|
+
: ` CHECK FAILED: ${res.check.mismatches.length} line(s) disagree with the model — please report this.`] : []),
|
|
2538
|
+
` Viewer: ${target}`,
|
|
2539
|
+
` Open it in a browser. It contains rows of ${dataFile}, so treat it like the data file. Row numbers are data rows; add 1 for the spreadsheet row (header).`,
|
|
2540
|
+
...(res.windows.length > maxWindows ? [` The viewer shows the first ${maxWindows} windows; window_rows lists the same.`] : []),
|
|
2541
|
+
].join('\n');
|
|
2542
|
+
|
|
2543
|
+
return {
|
|
2544
|
+
model_id,
|
|
2545
|
+
connection: connName,
|
|
2546
|
+
units: selected.size,
|
|
2547
|
+
windows: res.windows.length,
|
|
2548
|
+
conversations: ctxUsed.size,
|
|
2549
|
+
...(conn != null && { count_check: { lines_checked: res.check.checked, mismatches: res.check.mismatches.length,
|
|
2550
|
+
lines_total: R(linesTotal, 6), model_total: R(modelTotal, 6), match } }),
|
|
2551
|
+
by_unit: [...byUnit.values()].map(b => ({ ...b, count: R(b.count, 6) })),
|
|
2552
|
+
window_rows: res.windows.slice(0, maxWindows).map(w => ({
|
|
2553
|
+
unit: unitPath(w.unit), response_row: fileRowOf(rows, w.response) + 1,
|
|
2554
|
+
context_rows: w.members.map(r => fileRowOf(rows, r) + 1), ...(w.count != null && { count: R(w.count, 6) }),
|
|
2555
|
+
})),
|
|
2556
|
+
viewer: target,
|
|
2557
|
+
summary,
|
|
2558
|
+
};
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
async function toolExport(ena, { model_id, set_name, out_path }) {
|
|
2562
|
+
const entry = cacheGet(modelCache, model_id);
|
|
2563
|
+
if (!entry)
|
|
2564
|
+
throw new Error(`Unknown model_id '${model_id}'. Fit the model first (ena_fit, ena_compare_groups or tma_fit).`);
|
|
2565
|
+
const blockers = exportBlockers(entry);
|
|
2566
|
+
if (blockers.length)
|
|
2567
|
+
throw new Error(`Model ${model_id} can't be exported to the ENA Web Tool, which has no setting for: ${blockers.join('; ')}. ` +
|
|
2568
|
+
'Refit without these to export it.');
|
|
2569
|
+
|
|
2570
|
+
const p = entry.params;
|
|
2571
|
+
const unitCols = toArray(p.units);
|
|
2572
|
+
const warnings = [];
|
|
2573
|
+
|
|
2574
|
+
// The web tool groups units by their first unit column; take that column's
|
|
2575
|
+
// groups (order, colours) for the stored set.
|
|
2576
|
+
let groups = [];
|
|
2577
|
+
let groupCol = null;
|
|
2578
|
+
if (unitCols.length > 1) {
|
|
2579
|
+
groupCol = unitCols[0];
|
|
2580
|
+
if (entry.groupCol === groupCol) {
|
|
2581
|
+
groups = entry.web.groups.map(g => ({ name: g.name, color: g.color, secondaryColor: g.secondaryColor, showMean: g.showMean }));
|
|
2582
|
+
} else {
|
|
2583
|
+
const rows = loadRows(p);
|
|
2584
|
+
groups = web.groupOrder(rows, groupCol).map((name, i) => ({ name, ...web.groupStyle(i) }));
|
|
2585
|
+
if (entry.groupCol)
|
|
2586
|
+
warnings.push(`The web tool groups units by ${groupCol}, so its plots show ${groupCol} groups, not ${entry.groupCol}.`);
|
|
2587
|
+
}
|
|
2588
|
+
}
|
|
2589
|
+
|
|
2590
|
+
// What the set opens with: the model's default view, when it names groups
|
|
2591
|
+
// the web tool has.
|
|
2592
|
+
const names = new Set(groups.map(g => g.name));
|
|
2593
|
+
const asNode = (sel) => (sel?.kind === 'group' && names.has(sel.name) ? sel.name : null);
|
|
2594
|
+
const plotted = { primary: asNode(entry.view?.first), secondary: asNode(entry.view?.second) };
|
|
2595
|
+
if (!plotted.primary) plotted.secondary = null;
|
|
2596
|
+
|
|
2597
|
+
// The web tool's project holds a CSV; an .xlsx goes in as its first sheet.
|
|
2598
|
+
const csvName = p.data_path ? basename(p.data_path).replace(/\.xlsx$/i, '.csv') : 'data.csv';
|
|
2599
|
+
const csv = p.data_path
|
|
2600
|
+
? (isXlsx(p.data_path) ? Buffer.from(dataFileText(p.data_path), 'utf8') : readFileSync(p.data_path))
|
|
2601
|
+
: Buffer.from(p.data_csv ?? '', 'utf8');
|
|
2602
|
+
const setName = safeFileName(set_name ??
|
|
2603
|
+
(plotted.secondary ? `${plotted.primary} vs ${plotted.secondary}` : `ENA model ${model_id}`));
|
|
2604
|
+
|
|
2605
|
+
// Excluded units (units_used): the web tool's Units tab excludes units the
|
|
2606
|
+
// same way — their lines stay as context — so the set stores which.
|
|
2607
|
+
// Built from the whole file: the web tool applies exclude_conversations
|
|
2608
|
+
// and order_by itself (WasmWrapper._modelRowIndices), on the data file the
|
|
2609
|
+
// ZIP carries unchanged.
|
|
2610
|
+
const fileRows = loadRows({ data_path: p.data_path, data_csv: p.data_csv });
|
|
2611
|
+
const exclConvos = toArray(p.exclude_conversations).map(e => String(e).split('__').join('::'));
|
|
2612
|
+
let unitTree = null, excluded = 0;
|
|
2613
|
+
if (p.units_used?.length || exclConvos.length) {
|
|
2614
|
+
// With excluded horizons the full units tree goes in too: a partial one
|
|
2615
|
+
// fails the web tool's unitsTreeMatchesModel(), and rebuilding the units
|
|
2616
|
+
// rebuilds the conversation tree, which drops its include flags.
|
|
2617
|
+
const used = p.units_used?.length ? new Set(p.units_used.map(String)) : null;
|
|
2618
|
+
const seen = new Map();
|
|
2619
|
+
for (const r of fileRows) {
|
|
2620
|
+
const values = unitCols.map(c => String(r[c]));
|
|
2621
|
+
const key = values.join('__');
|
|
2622
|
+
if (!seen.has(key)) seen.set(key, { values, include: used ? used.has(key) : true });
|
|
2623
|
+
}
|
|
2624
|
+
unitTree = [...seen.values()];
|
|
2625
|
+
excluded = unitTree.filter(u => !u.include).length;
|
|
2626
|
+
}
|
|
2627
|
+
let convoTree = null;
|
|
2628
|
+
if (exclConvos.length) {
|
|
2629
|
+
const convoCols = toArray(p.conversations);
|
|
2630
|
+
const out = new Set(exclConvos);
|
|
2631
|
+
const seen = new Map();
|
|
2632
|
+
for (const r of fileRows) {
|
|
2633
|
+
const values = convoCols.map(c => String(r[c]));
|
|
2634
|
+
const key = values.join('::');
|
|
2635
|
+
if (!seen.has(key)) seen.set(key, { values, include: !out.has(key) });
|
|
2636
|
+
}
|
|
2637
|
+
convoTree = [...seen.values()];
|
|
2638
|
+
}
|
|
2639
|
+
|
|
2640
|
+
const set = web.setFile({
|
|
2641
|
+
unitTree, convoTree, order: p.order_by ?? null,
|
|
2642
|
+
codeMask: codeMaskOf(p, entry.kind === 'tma' ? (p.ordered ?? true) : false),
|
|
2643
|
+
setName, csvName,
|
|
2644
|
+
units: unitCols, conversation: toArray(p.conversations), codes: entry.codes,
|
|
2645
|
+
groups, groupCol,
|
|
2646
|
+
window: windowValue(p.window_size),
|
|
2647
|
+
weightModel: p.weight_model ?? null,
|
|
2648
|
+
sphereNorm: p.sphere_norm !== false,
|
|
2649
|
+
ordered: entry.kind === 'tma' ? (p.ordered ?? true) : false,
|
|
2650
|
+
rotation: exportRotation(entry),
|
|
2651
|
+
plotted,
|
|
2652
|
+
description: `Exported from ${PKG.name} ${PKG.version} (model ${model_id}).`,
|
|
2653
|
+
source: { tool: PKG.name, version: PKG.version, model_id, exported: new Date().toISOString() },
|
|
2654
|
+
});
|
|
2655
|
+
|
|
2656
|
+
const zipBuf = web.zip([
|
|
2657
|
+
{ name: csvName, data: csv },
|
|
2658
|
+
{ name: 'Folder/' },
|
|
2659
|
+
{ name: `Folder/${setName}.json`, data: JSON.stringify(set, null, 2) },
|
|
2660
|
+
]);
|
|
2661
|
+
|
|
2662
|
+
let target = out_path
|
|
2663
|
+
? (extname(out_path).toLowerCase() === '.zip' ? out_path : join(out_path, `${setName}.zip`))
|
|
2664
|
+
: join(p.data_path ? dirname(p.data_path) : VIZ_DIR, `${setName}.zip`);
|
|
2665
|
+
target = freePath(target);
|
|
2666
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
2667
|
+
writeFileSync(target, zipBuf);
|
|
2668
|
+
|
|
2669
|
+
const rot = set.settings.model;
|
|
2670
|
+
const settings = {
|
|
2671
|
+
units: unitCols, conversations: toArray(p.conversations), codes: entry.codes,
|
|
2672
|
+
window: rot.stanza === 'infinite' ? 'infinite' : rot.stanzaSize,
|
|
2673
|
+
rotation: rotationLabel(p),
|
|
2674
|
+
means_rotated_by: rot.meansRotatedBy,
|
|
2675
|
+
weight_model: rot.weightModelBy, sphere_norm: rot.sphereNorm, ordered: rot.ordered,
|
|
2676
|
+
groups: groups.map(g => `${g.name} ${g.color}`),
|
|
2677
|
+
opens_with: plotted.primary ? (plotted.secondary ? `${plotted.primary} – ${plotted.secondary}` : plotted.primary) : 'nothing plotted',
|
|
2678
|
+
...(p.units_used?.length && { excluded_units: excluded }),
|
|
2679
|
+
...(convoTree && { excluded_conversations: exclConvos }),
|
|
2680
|
+
...(p.order_by && { order_by: p.order_by }),
|
|
2681
|
+
...(toArray(p.mask_connections).length && { masked_connections: toArray(p.mask_connections) }),
|
|
2682
|
+
};
|
|
2683
|
+
const summary = [
|
|
2684
|
+
`Exported model ${model_id} as an ENA Web Tool project: ${target}`,
|
|
2685
|
+
` Set: Folder / ${setName} Data: ${csvName}`,
|
|
2686
|
+
` Opens with: ${settings.opens_with}`,
|
|
2687
|
+
...warnings.map(w => ` Note: ${w}`),
|
|
2688
|
+
'',
|
|
2689
|
+
'To open it: start a local session in the ENA Web Tool ("Run Offline"), click Import in the Sets panel,',
|
|
2690
|
+
`choose the ZIP, then open Folder › ${setName}. (Project import works in local sessions only.)`,
|
|
2691
|
+
].join('\n');
|
|
2692
|
+
|
|
2693
|
+
return {
|
|
2694
|
+
zip_path: target,
|
|
2695
|
+
set_name: setName,
|
|
2696
|
+
data_file: csvName,
|
|
2697
|
+
settings,
|
|
2698
|
+
warnings,
|
|
2699
|
+
privacy_note: 'The ZIP contains your data file. It was written on this machine; no rows were sent to Claude.',
|
|
2700
|
+
summary,
|
|
2701
|
+
};
|
|
2702
|
+
}
|
|
2703
|
+
|
|
2704
|
+
// ── Tool schemas ──────────────────────────────────────────────────────────────
|
|
2705
|
+
|
|
2706
|
+
const DATA_PROPS = {
|
|
2707
|
+
data_path: {
|
|
2708
|
+
type: 'string',
|
|
2709
|
+
description:
|
|
2710
|
+
'Absolute path to a .csv or .xlsx file on the user\'s machine (for .xlsx, the first worksheet, as in the ENA Web Tool). ' +
|
|
2711
|
+
'PREFERRED for sensitive data — the file is read locally by the WASM server; ' +
|
|
2712
|
+
'no raw row data is transmitted to Claude.',
|
|
2713
|
+
},
|
|
2714
|
+
data_csv: {
|
|
2715
|
+
type: 'string',
|
|
2716
|
+
description:
|
|
2717
|
+
'Raw CSV text (use this OR data_path). ' +
|
|
2718
|
+
'WARNING: the full dataset passes through this conversation. ' +
|
|
2719
|
+
'Ask the user whether their data is sensitive before using this parameter.',
|
|
2720
|
+
},
|
|
2721
|
+
};
|
|
2722
|
+
const CORE_PROPS = {
|
|
2723
|
+
...DATA_PROPS,
|
|
2724
|
+
units: { type: ['string', 'array'], items: { type: 'string' }, description: 'Column name(s) identifying units of analysis. Pass a string for a single column (e.g. "student_id") or an array for compound units (e.g. ["UserName","Condition"]). Compound units are joined with "__" to form labels like "alice__FirstGame".' },
|
|
2725
|
+
conversations: { type: ['string', 'array'], items: { type: 'string' }, description: 'Column name(s) segmenting conversations — accumulation resets between conversations. Pass a string or array (e.g. ["Condition","GameDay"]).' },
|
|
2726
|
+
codes: { type: 'array', items: { type: 'string' }, description: 'List of binary code column names' },
|
|
2727
|
+
window_size: { type: ['number', 'string'], description: 'Stanza window — rows to look back. Omitted: the Cross-Covariance Decay estimate (as ena_window gives, and as a new set in the ENA Web Tool takes), or 4 when the data is too sparse for one. Pass "Inf" for an infinite window (= rENA window.size.back = Inf): each line connects to every earlier line in its conversation. Use ena_window to estimate a window size from the data.' },
|
|
2728
|
+
binary: { type: 'boolean', description: 'Binarise co-occurrences (default true)' },
|
|
2729
|
+
weight_model: { type: 'string', enum: ['product', 'sqrt', 'log'], description: 'Weight each line\'s co-occurrence counts before they are summed per unit (= rENA weight.by / tma weight_by): "product" keeps raw counts, "sqrt" square root, "log" log(1 + x). Omit for the default (binary per line; raw directed counts for ordered models). Overrides binary.' },
|
|
2730
|
+
units_used: { type: 'array', items: { type: 'string' }, description: 'Model only these units (= rENA units.used), as unit labels — the unit column values joined with "__", as returned in units. Other units\' rows stay in the data as context in the modeled units\' windows but get no network. Unlike dropping rows, this keeps e.g. teacher talk as context for student units.' },
|
|
2731
|
+
mask_connections: { type: 'array', items: { type: 'string' }, description: 'Connections to switch off, as in connection_names, e.g. ["Data & Technical Constraints"] — the ENA Web Tool\'s Code Masking Matrix. A masked connection counts as 0 for every unit before normalization and rotation. Undirected models: either order of the two codes. Ordered (tma_fit) models: "A & B" masks A → B only.' },
|
|
2732
|
+
exclude_conversations: { type: 'array', items: { type: 'string' }, description: 'Conversations to leave out, named by their conversation column values joined by :: (e.g. "FirstGame::Electric"). Their lines are removed from the data before anything else — not modeled and not context — as the ENA Web Tool\'s former R backend did for excluded horizons.' },
|
|
2733
|
+
order_by: { type: 'string', description: 'Column to order the lines by before windowing (the web tool\'s Order setting): rows are sorted by it, ascending and stable, so ties keep file order; numeric if every value is a number. Default: file order. Row numbers in results still refer to the file.' },
|
|
2734
|
+
};
|
|
2735
|
+
|
|
2736
|
+
// Fit-only options (ena_fit, ena_compare_groups, tma_fit).
|
|
2737
|
+
const FIT_PROPS = {
|
|
2738
|
+
sphere_norm: { type: 'boolean', description: 'Sphere-normalize each unit network (default true). false scales all networks by the longest one instead (= rENA fun_skip_sphere_norm; ona normalize = skip_sphere_norm), keeping differences in network size.' },
|
|
2739
|
+
center_align_to_origin: { type: 'boolean', description: 'Keep units with an all-zero network at the origin and out of the centering mean (default true, = rENA center.align.to.origin). false centers on every unit. Unordered models only; ordered models always center as ona::model() does.' },
|
|
2740
|
+
};
|
|
2741
|
+
|
|
2742
|
+
// Rotation parameters shared by ena_fit and tma_fit.
|
|
2743
|
+
const ROTATION_PROPS = {
|
|
2744
|
+
rotation_from: { type: 'string', description: 'model_id of a fitted model to project this one into — the ENA Web Tool\'s "Use Rotation Matrix From" (= rENA rotation.set): this model\'s units are centred and rotated by the source\'s rotation and drawn with its node positions, so both models share one space (e.g. a second cohort, a post-test). Needs the same codes and the same kind of network (ordered or not). Replaces rotation.' },
|
|
2745
|
+
rotation: { type: 'string', enum: ['svd', 'mean', 'generalized'], description: '"svd", "mean" (requires group_col, group1_value, group2_value) or "generalized" — generalized means rotation, GMR (= rENA ena.rotate.by.generalized; requires x_var). Omitted: as the ENA Web Tool rotates — with two or more unit columns and two or more values in the first, a GMR through that column\'s first two groups in data order (or group1_value / group2_value when group_col is that column); otherwise "svd".' },
|
|
2746
|
+
x_var: { type: ['string', 'array'], items: { type: 'string' }, description: 'For rotation="generalized": unit-level metadata column(s) — one value per unit, none missing. The first is the target that dimension 1 (GMR1) is built to represent: categorical (axis through the group means) or numeric (axis along the variable). Further columns are covariates, controlled for by Lasso.' },
|
|
2747
|
+
y_var: { type: ['string', 'array'], items: { type: 'string' }, description: 'For rotation="generalized", optional: same format as x_var; builds dimension 2 (GMR2) for a second target instead of the next SVD dimension.' },
|
|
2748
|
+
select_2_groups: { type: 'array', items: { type: 'string' }, minItems: 2, maxItems: 2, description: 'For rotation="generalized" with a categorical target: fit GMR1 on these two target values only (other units are still projected). Also colours the plot and runs the group tests on these two groups unless group_col is given.' },
|
|
2749
|
+
interactions: { type: 'boolean', description: 'For rotation="generalized" with covariates: include pairwise interaction terms in the Lasso model (default true, as in rENA). false for main effects only.' },
|
|
2750
|
+
};
|
|
2751
|
+
|
|
2752
|
+
// ── Analysis guide ────────────────────────────────────────────────────────────
|
|
2753
|
+
// The ena-analysis skill ships inside the package and is served by ena_guide,
|
|
2754
|
+
// so every MCP client gets the workflow without installing the skill
|
|
2755
|
+
// separately. Sections are the skill's "## " headings.
|
|
2756
|
+
|
|
2757
|
+
const GUIDE = (() => {
|
|
2758
|
+
let md;
|
|
2759
|
+
try { md = readFileSync(new URL('./skill/ena-analysis/SKILL.md', import.meta.url), 'utf8'); }
|
|
2760
|
+
catch { return null; }
|
|
2761
|
+
md = md.replace(/^---\n[\s\S]*?\n---\n+/, ''); // frontmatter
|
|
2762
|
+
const sections = [];
|
|
2763
|
+
for (const part of md.split(/^(?=## )/m).slice(1)) {
|
|
2764
|
+
const heading = part.match(/^## (.*)/)[1].trim();
|
|
2765
|
+
const topic = heading.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
2766
|
+
sections.push({ topic, heading, text: part.trim() });
|
|
2767
|
+
}
|
|
2768
|
+
return { text: md.trim(), sections };
|
|
2769
|
+
})();
|
|
2770
|
+
const GUIDE_TOPICS = GUIDE?.sections.map(s => s.topic) ?? [];
|
|
2771
|
+
|
|
2772
|
+
function toolGuide({ topic } = {}) {
|
|
2773
|
+
if (!GUIDE) throw new Error('The analysis guide is missing from this installation (skill/ena-analysis/SKILL.md).');
|
|
2774
|
+
const footer = `\n\n---\nTopics: ${GUIDE_TOPICS.join(', ')}`;
|
|
2775
|
+
if (!topic) return GUIDE.text + footer;
|
|
2776
|
+
const t = String(topic).toLowerCase().trim();
|
|
2777
|
+
const hit = GUIDE.sections.find(s => s.topic === t)
|
|
2778
|
+
?? GUIDE.sections.find(s => s.topic.includes(t.replace(/[^a-z0-9]+/g, '-')));
|
|
2779
|
+
if (!hit) throw new Error(`Unknown topic. Topics: ${GUIDE_TOPICS.join(', ')}`);
|
|
2780
|
+
return hit.text + footer;
|
|
2781
|
+
}
|
|
2782
|
+
|
|
2783
|
+
const TOOLS = [
|
|
2784
|
+
{
|
|
2785
|
+
name: 'ena_guide',
|
|
2786
|
+
description:
|
|
2787
|
+
'The ENA analysis guide: how to run an ENA / ONA / TMA analysis with these tools — the privacy gate, triage, choosing a window, ' +
|
|
2788
|
+
'fitting and comparing groups, rotations (means, generalized, projection), masking, simplifying, ONA/TMA, tables, the lines behind a ' +
|
|
2789
|
+
'connection, export, and reading the results. Call it once with no arguments before the first analysis in a conversation; ' +
|
|
2790
|
+
'pass topic for one section.',
|
|
2791
|
+
inputSchema: {
|
|
2792
|
+
type: 'object',
|
|
2793
|
+
properties: {
|
|
2794
|
+
topic: { type: 'string', enum: GUIDE_TOPICS.length ? GUIDE_TOPICS : undefined, description: 'One section of the guide (default: the whole guide).' },
|
|
2795
|
+
},
|
|
2796
|
+
},
|
|
2797
|
+
},
|
|
2798
|
+
{
|
|
2799
|
+
name: 'ena_profile',
|
|
2800
|
+
description:
|
|
2801
|
+
'PRIVACY-SAFE dataset profiler: reads a CSV or .xlsx file from the local file system and returns only column metadata ' +
|
|
2802
|
+
'(column names, unique-value counts, binary detection, and the distinct labels of low-cardinality columns ' +
|
|
2803
|
+
'— at most 10 values, e.g. condition names — so group values can be chosen). ' +
|
|
2804
|
+
'No row-level values (text, names, IDs, timestamps) are ever sent to Claude — use this when the user has sensitive data and wants to keep it local. ' +
|
|
2805
|
+
'Returns enough information to determine ENA parameters (units, conversations, codes) without exposing the data itself. ' +
|
|
2806
|
+
'WHEN TO USE: before ena_inspect or ena_fit, if the user says their data is sensitive, confidential, or private, ' +
|
|
2807
|
+
'or if they ask whether their data will be sent to Claude.',
|
|
2808
|
+
inputSchema: {
|
|
2809
|
+
type: 'object',
|
|
2810
|
+
properties: {
|
|
2811
|
+
data_path: {
|
|
2812
|
+
type: 'string',
|
|
2813
|
+
description: 'Absolute path to the .csv or .xlsx file on the user\'s machine (.xlsx: the first worksheet).',
|
|
2814
|
+
},
|
|
2815
|
+
},
|
|
2816
|
+
required: ['data_path'],
|
|
2817
|
+
},
|
|
2818
|
+
},
|
|
2819
|
+
{
|
|
2820
|
+
name: 'ena_inspect',
|
|
2821
|
+
description:
|
|
2822
|
+
'Inspect a dataset and recommend whether ENA is an appropriate analysis. ' +
|
|
2823
|
+
'Call this FIRST whenever a user provides tabular data and asks for analysis. ' +
|
|
2824
|
+
'Returns ena_recommended, confidence, suggested codes/units/conversations, and a ready-to-use next_step call. ' +
|
|
2825
|
+
'When ena_recommended is false, the result includes a "guidance" field explaining why the data is not ENA-shaped ' +
|
|
2826
|
+
'(e.g. no binary code columns) and what would be needed — relay this to the user rather than silently failing or ' +
|
|
2827
|
+
'claiming the connector is unavailable. ' +
|
|
2828
|
+
'PRIVACY: prefer data_path over data_csv so raw data stays local. ' +
|
|
2829
|
+
'If the user has not yet indicated whether their data is sensitive, ask before using data_csv. ' +
|
|
2830
|
+
'If they indicate sensitivity, use ena_profile instead (no raw data transmitted).',
|
|
2831
|
+
inputSchema: {
|
|
2832
|
+
type: 'object',
|
|
2833
|
+
properties: DATA_PROPS,
|
|
2834
|
+
},
|
|
2835
|
+
},
|
|
2836
|
+
{
|
|
2837
|
+
name: 'ena_fit',
|
|
2838
|
+
description:
|
|
2839
|
+
'Fit a full ENA model: accumulate → sphere-norm → center → rotate → project → LWS node positions. ' +
|
|
2840
|
+
'Returns a model_id that can be passed to ena_plot for additional views (group mean, unit, subtraction, comparison) without re-fitting. ' +
|
|
2841
|
+
'Repeated calls with identical parameters return the cached model instantly. ' +
|
|
2842
|
+
'Supports rotation="svd", rotation="mean" (requires group_col, group1_value, group2_value), or rotation="generalized" (GMR: dimension 1 built to represent a unit-level variable given by x_var, controlling for covariates; optional second target y_var); omitted, it rotates as the ENA Web Tool does (see rotation). ' +
|
|
2843
|
+
'Includes a stats block (per dimension: Welch t-test, Cohen\'s d, Mann-Whitney U with effect size r, group means/SDs/medians) whenever group_col, group1_value and group2_value are given — which a default rotation by groups fills in. ' +
|
|
2844
|
+
'Defaults and plots follow the ENA Web Tool, so the model and its plot match it there (see ena_export): ' +
|
|
2845
|
+
'the plot is the web tool\'s comparison plot — points scaled to the network, group means with 95% CI boxes, ' +
|
|
2846
|
+
'group colours #cc423a, #218ebf, … in data order, and group 1 − group 2 when two groups are named. ' +
|
|
2847
|
+
'The ENA network plot is written to disk as SVG (svg_plot) and interactive HTML (interactive_plot), and — when the optional PNG rasterizer is installed — also as PNG (png_plot) plus an inline image block. Use these paths to show the plot; do not generate your own visualization. ' +
|
|
2848
|
+
'PRIVACY: use data_path (file stays local, only model results reach Claude) rather than data_csv (raw data passes through conversation). ' +
|
|
2849
|
+
'The WASM model runs entirely on the user\'s machine regardless of which input method is used.',
|
|
2850
|
+
inputSchema: {
|
|
2851
|
+
type: 'object',
|
|
2852
|
+
properties: {
|
|
2853
|
+
...CORE_PROPS,
|
|
2854
|
+
dims: { type: 'number', description: 'Dimensions to return (default 2)' },
|
|
2855
|
+
...FIT_PROPS,
|
|
2856
|
+
...ROTATION_PROPS,
|
|
2857
|
+
group_col: { type: 'string', description: 'Metadata column for group membership (required for rotation="mean"; enables group/subtraction plots). Defaults to the first unit column when there are two or more, as the ENA Web Tool groups units.' },
|
|
2858
|
+
group1_value: { type: 'string', description: 'Group 1 value (required for rotation="mean")' },
|
|
2859
|
+
group2_value: { type: 'string', description: 'Group 2 value (required for rotation="mean")' },
|
|
2860
|
+
},
|
|
2861
|
+
required: ['units', 'conversations', 'codes'],
|
|
2862
|
+
},
|
|
2863
|
+
},
|
|
2864
|
+
{
|
|
2865
|
+
name: 'ena_accumulate',
|
|
2866
|
+
description:
|
|
2867
|
+
'Run only the ENA accumulation step — no rotation or projection. ' +
|
|
2868
|
+
'Returns raw per-unit co-occurrence networks and top connections by weight. ' +
|
|
2869
|
+
'Useful for inspecting co-occurrence patterns before committing to a model.',
|
|
2870
|
+
inputSchema: {
|
|
2871
|
+
type: 'object',
|
|
2872
|
+
properties: CORE_PROPS,
|
|
2873
|
+
required: ['units', 'conversations', 'codes'],
|
|
2874
|
+
},
|
|
2875
|
+
},
|
|
2876
|
+
{
|
|
2877
|
+
name: 'ena_window',
|
|
2878
|
+
description:
|
|
2879
|
+
'Estimate the stanza window size from the data by Cross-Covariance Decay (= rENA ena.ccd): how many lines back ' +
|
|
2880
|
+
'co-occurrence between codes stays above noise. Runs on the coded lines within each conversation — no model is fitted. ' +
|
|
2881
|
+
'Use before ena_fit / tma_fit when the user has no theoretically motivated window size, and report the estimate ' +
|
|
2882
|
+
'and curve so the choice is transparent. ' +
|
|
2883
|
+
'PRIVACY: use data_path rather than data_csv.',
|
|
2884
|
+
inputSchema: {
|
|
2885
|
+
type: 'object',
|
|
2886
|
+
properties: {
|
|
2887
|
+
...DATA_PROPS,
|
|
2888
|
+
conversations: CORE_PROPS.conversations,
|
|
2889
|
+
codes: CORE_PROPS.codes,
|
|
2890
|
+
exclude_conversations: CORE_PROPS.exclude_conversations,
|
|
2891
|
+
order_by: CORE_PROPS.order_by,
|
|
2892
|
+
max_window: { type: 'number', description: 'Largest lag to evaluate (default 20)' },
|
|
2893
|
+
min_overlap: { type: 'number', description: 'Minimum overlapping lines per conversation at a lag (default 10)' },
|
|
2894
|
+
},
|
|
2895
|
+
required: ['conversations', 'codes'],
|
|
2896
|
+
},
|
|
2897
|
+
},
|
|
2898
|
+
{
|
|
2899
|
+
name: 'tma_fit',
|
|
2900
|
+
description:
|
|
2901
|
+
'Fit a full TMA (Transmodal Analysis) or ONA (Ordered Network Analysis) model using context tensor accumulation and directed networks. ' +
|
|
2902
|
+
'Returns a model_id that can be passed to ena_plot. ' +
|
|
2903
|
+
'Supports rotation="svd", rotation="mean" (requires group_col, group1_value, group2_value), or rotation="generalized" (GMR: dimension 1 built to represent a unit-level variable given by x_var, controlling for covariates; optional second target y_var); omitted, it rotates as the ENA Web Tool does (see rotation). ' +
|
|
2904
|
+
'Includes a stats block (per dimension: Welch t-test, Cohen\'s d, Mann-Whitney U with effect size r, group means/SDs/medians) whenever group_col, group1_value and group2_value are given. ' +
|
|
2905
|
+
'Plots follow the ENA Web Tool, as for ena_fit.',
|
|
2906
|
+
inputSchema: {
|
|
2907
|
+
type: 'object',
|
|
2908
|
+
properties: {
|
|
2909
|
+
...CORE_PROPS,
|
|
2910
|
+
stanza_ground: { type: 'array', items: { type: 'string' }, description: 'Column name(s) representing sender/ground roles.' },
|
|
2911
|
+
stanza_response: { type: 'array', items: { type: 'string' }, description: 'Column name(s) representing receiver/response roles.' },
|
|
2912
|
+
stanza_modality: { type: ['string', 'array'], items: { type: 'string' }, description: 'Column name(s) representing modalities/modes.' },
|
|
2913
|
+
window_type: { type: 'string', enum: ['lines', 'time', 'time_duration'], description: 'Measure windows in lines (default) or in time ("time", with time_column and time_unit)' },
|
|
2914
|
+
time_column: { type: 'string', description: 'Timestamp column for timing windows' },
|
|
2915
|
+
time_unit: { type: 'string', enum: ['secs', 'mins', 'hours', 'days', 'weeks'], description: 'Unit of window_size and override windows for time-based windows (default secs). Numeric time columns are used as-is; others are parsed as elapsed time (MM:SS, HH:MM:SS) or a date-time.' },
|
|
2916
|
+
horizons: {
|
|
2917
|
+
type: 'object',
|
|
2918
|
+
description: 'Flexible horizons of observation (= tma contexts() with HOO rules): each unit sees its own horizon instead of the conversations. by is a discriminator column; rules maps each of its values to the columns that define that horizon — a row is in unit U\'s horizon when each of those columns holds a value U has. Rows whose by value has no rule are left out. Example: { "by": "Modality", "rules": { "chat": ["GroupName"], "log": ["UserName"] } } — chat lines are seen by the whole group, log lines only by their author.',
|
|
2919
|
+
properties: {
|
|
2920
|
+
by: { type: 'string' },
|
|
2921
|
+
rules: { type: 'object', additionalProperties: { type: 'array', items: { type: 'string' } } },
|
|
2922
|
+
},
|
|
2923
|
+
required: ['by', 'rules'],
|
|
2924
|
+
},
|
|
2925
|
+
stanza_overrides: {
|
|
2926
|
+
type: 'array',
|
|
2927
|
+
items: {
|
|
2928
|
+
type: 'object',
|
|
2929
|
+
properties: {
|
|
2930
|
+
path: { type: 'array', items: { type: 'string' }, description: 'Combination values path' },
|
|
2931
|
+
weight: { type: 'number', description: 'Weight override' },
|
|
2932
|
+
window: { type: ['number', 'string'], description: 'Window override (a number, or "Inf")' },
|
|
2933
|
+
},
|
|
2934
|
+
required: ['path'],
|
|
2935
|
+
},
|
|
2936
|
+
},
|
|
2937
|
+
ordered: { type: 'boolean', description: 'Directed networks (defaults to true for TMA/ONA)' },
|
|
2938
|
+
dims: { type: 'number', description: 'Dimensions to return (default 2)' },
|
|
2939
|
+
...FIT_PROPS,
|
|
2940
|
+
...ROTATION_PROPS,
|
|
2941
|
+
group_col: { type: 'string', description: 'Metadata column for group membership (required for rotation="mean"; enables group/subtraction plots). Defaults to the first unit column when there are two or more, as the ENA Web Tool groups units.' },
|
|
2942
|
+
group1_value: { type: 'string', description: 'Group 1 value (required for rotation="mean")' },
|
|
2943
|
+
group2_value: { type: 'string', description: 'Group 2 value (required for rotation="mean")' },
|
|
2944
|
+
},
|
|
2945
|
+
required: ['units', 'conversations', 'codes'],
|
|
2946
|
+
},
|
|
2947
|
+
},
|
|
2948
|
+
{
|
|
2949
|
+
name: 'tma_accumulate',
|
|
2950
|
+
description:
|
|
2951
|
+
'Run only the TMA / ONA context tensor accumulation step — no rotation or projection. ' +
|
|
2952
|
+
'Returns raw per-unit directed co-occurrence networks and top connections.',
|
|
2953
|
+
inputSchema: {
|
|
2954
|
+
type: 'object',
|
|
2955
|
+
properties: {
|
|
2956
|
+
...CORE_PROPS,
|
|
2957
|
+
stanza_ground: { type: 'array', items: { type: 'string' }, description: 'Column name(s) representing sender/ground roles.' },
|
|
2958
|
+
stanza_response: { type: 'array', items: { type: 'string' }, description: 'Column name(s) representing receiver/response roles.' },
|
|
2959
|
+
stanza_modality: { type: ['string', 'array'], items: { type: 'string' }, description: 'Column name(s) representing modalities/modes.' },
|
|
2960
|
+
window_type: { type: 'string', enum: ['lines', 'time', 'time_duration'], description: 'Measure windows in lines (default) or in time ("time", with time_column and time_unit)' },
|
|
2961
|
+
time_column: { type: 'string', description: 'Timestamp column for timing windows' },
|
|
2962
|
+
time_unit: { type: 'string', enum: ['secs', 'mins', 'hours', 'days', 'weeks'], description: 'Unit of window_size and override windows for time-based windows (default secs). Numeric time columns are used as-is; others are parsed as elapsed time (MM:SS, HH:MM:SS) or a date-time.' },
|
|
2963
|
+
horizons: {
|
|
2964
|
+
type: 'object',
|
|
2965
|
+
description: 'Flexible horizons of observation (= tma contexts() with HOO rules): each unit sees its own horizon instead of the conversations. by is a discriminator column; rules maps each of its values to the columns that define that horizon — a row is in unit U\'s horizon when each of those columns holds a value U has. Rows whose by value has no rule are left out. Example: { "by": "Modality", "rules": { "chat": ["GroupName"], "log": ["UserName"] } } — chat lines are seen by the whole group, log lines only by their author.',
|
|
2966
|
+
properties: {
|
|
2967
|
+
by: { type: 'string' },
|
|
2968
|
+
rules: { type: 'object', additionalProperties: { type: 'array', items: { type: 'string' } } },
|
|
2969
|
+
},
|
|
2970
|
+
required: ['by', 'rules'],
|
|
2971
|
+
},
|
|
2972
|
+
stanza_overrides: {
|
|
2973
|
+
type: 'array',
|
|
2974
|
+
items: {
|
|
2975
|
+
type: 'object',
|
|
2976
|
+
properties: {
|
|
2977
|
+
path: { type: 'array', items: { type: 'string' }, description: 'Combination values path' },
|
|
2978
|
+
weight: { type: 'number', description: 'Weight override' },
|
|
2979
|
+
window: { type: ['number', 'string'], description: 'Window override (a number, or "Inf")' },
|
|
2980
|
+
},
|
|
2981
|
+
required: ['path'],
|
|
2982
|
+
},
|
|
2983
|
+
},
|
|
2984
|
+
ordered: { type: 'boolean', description: 'Directed networks (defaults to true for TMA/ONA)' },
|
|
2985
|
+
},
|
|
2986
|
+
required: ['units', 'conversations', 'codes'],
|
|
2987
|
+
},
|
|
2988
|
+
},
|
|
2989
|
+
{
|
|
2990
|
+
name: 'ena_compare_groups',
|
|
2991
|
+
description:
|
|
2992
|
+
'Fit an ENA model rotated through two groups and compare them. ' +
|
|
2993
|
+
'The rotation is the ENA Web Tool\'s for grouped units: a GMR whose dimension 1 (GMR1) passes through the two group means, group 1 the reference — the same axes as a means rotation. ' +
|
|
2994
|
+
'Defaults and plots follow the ENA Web Tool, so the model and its plot match it there (see ena_export): ' +
|
|
2995
|
+
'the plot is the web tool\'s comparison plot — points scaled to the network, group means with 95% CI boxes, ' +
|
|
2996
|
+
'group colours #cc423a, #218ebf, … in data order, and group 1 − group 2 when two groups are named. ' +
|
|
2997
|
+
'Returns a stats block testing the groups on each dimension (Welch t-test, Cohen\'s d, Mann-Whitney U with effect size r). ' +
|
|
2998
|
+
'Returns a model_id for use with ena_plot (group, subtraction, comparison, unit views). ' +
|
|
2999
|
+
'Repeated calls with identical parameters return the cached model instantly. ' +
|
|
3000
|
+
'The ENA network plot is written to disk as SVG (svg_plot) and interactive HTML (interactive_plot), and — when the optional PNG rasterizer is installed — also as PNG (png_plot) plus an inline image block. Use these paths to show the plot; do not generate your own visualization. ' +
|
|
3001
|
+
'PRIVACY: use data_path (file stays local, only model results reach Claude) rather than data_csv (raw data passes through conversation). ' +
|
|
3002
|
+
'The WASM model runs entirely on the user\'s machine regardless of which input method is used.',
|
|
3003
|
+
inputSchema: {
|
|
3004
|
+
type: 'object',
|
|
3005
|
+
properties: {
|
|
3006
|
+
...CORE_PROPS,
|
|
3007
|
+
group_col: { type: 'string', description: 'Column identifying group membership' },
|
|
3008
|
+
group1_value: { type: 'string', description: 'Value for group 1' },
|
|
3009
|
+
group2_value: { type: 'string', description: 'Value for group 2' },
|
|
3010
|
+
dims: { type: 'number', description: 'Dimensions to return (default 2)' },
|
|
3011
|
+
...FIT_PROPS,
|
|
3012
|
+
},
|
|
3013
|
+
required: ['units', 'conversations', 'codes', 'group_col', 'group1_value', 'group2_value'],
|
|
3014
|
+
},
|
|
3015
|
+
},
|
|
3016
|
+
{
|
|
3017
|
+
name: 'ena_plot',
|
|
3018
|
+
description:
|
|
3019
|
+
'Generate an ENA network plot from a previously fitted model without re-running the pipeline. ' +
|
|
3020
|
+
'Use the model_id returned by ena_fit or ena_compare_groups. ' +
|
|
3021
|
+
'View types: ' +
|
|
3022
|
+
'"all" — all units and group means, with the mean network of every unit; ' +
|
|
3023
|
+
'"group" — mean network for a single group (requires group_value); ' +
|
|
3024
|
+
'"comparison" — overlay of two group mean networks (requires group1, group2); ' +
|
|
3025
|
+
'"subtraction" — difference network g1 − g2 (the web tool\'s comparison plot), edges in the colour of the group they are stronger in (requires group1, group2, and groups on the source model); ' +
|
|
3026
|
+
'"unit" — individual unit network (requires unit_label). ' +
|
|
3027
|
+
'Every view draws the unit points and group means as the ENA Web Tool\'s comparison plot does. ' +
|
|
3028
|
+
'Display options match the web tool\'s Plot Settings (minimum edge weight, unconnected codes, text size, group colours, CI and outlier boxes); they change only the drawing. ' +
|
|
3029
|
+
'The interactive HTML has its own controls for the same settings, so for fine-tuning point the user to it rather than re-plotting repeatedly. ' +
|
|
3030
|
+
'The ENA network plot is written to disk as SVG (svg_plot) and interactive HTML (interactive_plot), plus PNG (png_plot) and an inline image when the optional PNG rasterizer is installed.',
|
|
3031
|
+
inputSchema: {
|
|
3032
|
+
type: 'object',
|
|
3033
|
+
properties: {
|
|
3034
|
+
model_id: { type: 'string', description: 'model_id from ena_fit or ena_compare_groups' },
|
|
3035
|
+
type: {
|
|
3036
|
+
type: 'string',
|
|
3037
|
+
enum: ['all', 'group', 'comparison', 'subtraction', 'unit'],
|
|
3038
|
+
description: 'Plot view (default: all)',
|
|
3039
|
+
},
|
|
3040
|
+
group_value: { type: 'string', description: 'Group value for type="group"' },
|
|
3041
|
+
group1: { type: 'string', description: 'First group for type="comparison" or "subtraction"' },
|
|
3042
|
+
group2: { type: 'string', description: 'Second group for type="comparison" or "subtraction"' },
|
|
3043
|
+
unit_label: { type: 'string', description: 'Unit label for type="unit"' },
|
|
3044
|
+
magnify: { type: 'number', description: 'The web tool\'s "Scale edge weights" (default 1): multiplies edge widths and node sizes, not colour; the plot is titled "(scaled N.Nx)". A subtraction\'s strongest edge is always drawn 4 px wide, as in the web tool, so magnify only grows its nodes.' },
|
|
3045
|
+
x_dim: { type: 'integer', minimum: 1, description: 'Dimension on the x-axis (default 1). Fits keep up to 6 dimensions, as the ENA Web Tool does, so e.g. x_dim=1, y_dim=3 shows SVD1 against SVD3.' },
|
|
3046
|
+
y_dim: { type: 'integer', minimum: 1, description: 'Dimension on the y-axis (default 2).' },
|
|
3047
|
+
flip_x: { type: 'boolean', description: 'Mirror the x-axis (the web tool\'s Flip X): points, nodes and means change sides; the model is unchanged.' },
|
|
3048
|
+
flip_y: { type: 'boolean', description: 'Mirror the y-axis (the web tool\'s Flip Y).' },
|
|
3049
|
+
min_edge_weight: { type: 'number', minimum: 0, description: 'Hide connections whose |weight| is below this (the web tool\'s Minimum edge weight). Weights are on the plotted network\'s own scale — the result reports max_edge_weight; group means are rescaled to 0.1–1. Use it when a network is too dense to read ("hide the weak connections").' },
|
|
3050
|
+
hide_unconnected_codes: { type: 'boolean', description: 'Hide codes with no drawn connection (the web tool\'s Show unconnected codes, off). Pairs well with min_edge_weight.' },
|
|
3051
|
+
text_size: { type: 'number', minimum: 6, maximum: 48, description: 'Label size in px (default 12), e.g. 16–20 for slides.' },
|
|
3052
|
+
colors: { type: 'object', additionalProperties: { type: 'string' }, description: 'Group colours: { "<group>": "<colour>" }, as #rgb, #rrggbb or a CSS colour name (orange, steelblue, ...). Colours the group\'s points, mean and network.' },
|
|
3053
|
+
confidence_intervals: { type: 'boolean', description: 'Draw the 95% confidence-interval boxes around group means (default true).' },
|
|
3054
|
+
outlier_intervals: { type: 'boolean', description: 'Draw the outlier boxes (Tukey fences, 1.5 × IQR) around group means (default false).' },
|
|
3055
|
+
},
|
|
3056
|
+
required: ['model_id'],
|
|
3057
|
+
},
|
|
3058
|
+
},
|
|
3059
|
+
{
|
|
3060
|
+
name: 'ena_simplify',
|
|
3061
|
+
description:
|
|
3062
|
+
'Simplify a model with PRIA — the ENA Web Tool\'s "Simplify Model" (= R\'s PRIA::pria): find the largest set of codes, up to remove_num and never ' +
|
|
3063
|
+
'leaving fewer than 3, that can be removed while the model stays within threshold of the full one. Runs with the model\'s own settings; if codes ' +
|
|
3064
|
+
'can go, refits without them (window and every other setting unchanged) and returns the new model, with removed_codes and kept_codes. ' +
|
|
3065
|
+
'Needs more than 3 codes.',
|
|
3066
|
+
inputSchema: {
|
|
3067
|
+
type: 'object',
|
|
3068
|
+
properties: {
|
|
3069
|
+
model_id: { type: 'string', description: 'model_id from ena_fit, ena_compare_groups or tma_fit' },
|
|
3070
|
+
threshold: { type: 'number', minimum: 0.8, maximum: 0.99, description: 'How close the simplified model must stay to the full one (default 0.95; the web tool allows 0.80–0.99).' },
|
|
3071
|
+
remove_num: { type: 'integer', minimum: 1, description: 'Most codes to remove (default 3).' },
|
|
3072
|
+
},
|
|
3073
|
+
required: ['model_id'],
|
|
3074
|
+
},
|
|
3075
|
+
},
|
|
3076
|
+
{
|
|
3077
|
+
name: 'ena_lines',
|
|
3078
|
+
description:
|
|
3079
|
+
'Show the lines behind a connection — the ENA Web Tool\'s Data View, for closing the interpretive loop. ' +
|
|
3080
|
+
'Writes a local HTML viewer of every line that contributed to the connection for the chosen units or group, each with the window ' +
|
|
3081
|
+
'of lines it was connected to, grouped by conversation, with the connection\'s codes highlighted (and direction for ordered models). ' +
|
|
3082
|
+
'The lines are exactly the ones the model counted (from its own per-line counts), for ENA, ONA and TMA; a check against the model is reported. ' +
|
|
3083
|
+
'Returns the viewer\'s path, counts and data-row numbers — never the text. The viewer contains rows of the data file: tell the user to open it ' +
|
|
3084
|
+
'in a browser; do not open, read or summarize it yourself. Omit connection to show all of the selection\'s lines.',
|
|
3085
|
+
inputSchema: {
|
|
3086
|
+
type: 'object',
|
|
3087
|
+
properties: {
|
|
3088
|
+
model_id: { type: 'string', description: 'model_id from ena_fit, ena_compare_groups or tma_fit' },
|
|
3089
|
+
connection: { type: 'string', description: 'Connection as in connection_names, e.g. "Data & Design Reasoning" (for ordered models "A & B" is A → B). Omit for all lines.' },
|
|
3090
|
+
units: { type: 'array', items: { type: 'string' }, description: 'Unit labels to show (as in the results, e.g. "FirstGame::steven z").' },
|
|
3091
|
+
group: { type: 'string', description: 'All units of this group (one of the model\'s groups). Use units or group.' },
|
|
3092
|
+
columns: { type: 'array', items: { type: 'string' }, description: 'Extra columns to show, e.g. the text column (the web tool\'s "add metadata column"). By default: unit, conversation and code columns; every column can be switched on in the viewer.' },
|
|
3093
|
+
max_windows: { type: 'integer', minimum: 1, description: 'Most windows to put in the viewer and window_rows (default 500).' },
|
|
3094
|
+
out_path: { type: 'string', description: 'A .html path or a folder. Default: next to the data file, never overwriting.' },
|
|
3095
|
+
},
|
|
3096
|
+
required: ['model_id'],
|
|
3097
|
+
},
|
|
3098
|
+
},
|
|
3099
|
+
{
|
|
3100
|
+
name: 'ena_tables',
|
|
3101
|
+
description:
|
|
3102
|
+
'Write a fitted model\'s data tables as CSV files for R, Excel or a paper — the ENA Web Tool\'s "Download Model" tables: ' +
|
|
3103
|
+
'nodes.csv (code positions per dimension), points.csv (unit points per dimension), line-weights.csv (normalized networks, one "A & B" column per connection) ' +
|
|
3104
|
+
'and connection-counts.csv (raw accumulated counts). The last two lead with ENA_UNIT and the unit columns. ' +
|
|
3105
|
+
'Takes a model_id from ena_fit, ena_compare_groups or tma_fit; writes a new folder (by default next to the data file, never overwriting one) and returns only the paths. ' +
|
|
3106
|
+
'The tables hold unit labels and model values, not rows of the data file.',
|
|
3107
|
+
inputSchema: {
|
|
3108
|
+
type: 'object',
|
|
3109
|
+
properties: {
|
|
3110
|
+
model_id: { type: 'string', description: 'model_id of the model' },
|
|
3111
|
+
out_path: { type: 'string', description: 'Folder to create for the CSVs. Default: "ENA model <id> tables" next to the data file (a number is added rather than overwrite).' },
|
|
3112
|
+
},
|
|
3113
|
+
required: ['model_id'],
|
|
3114
|
+
},
|
|
3115
|
+
},
|
|
3116
|
+
{
|
|
3117
|
+
name: 'ena_export',
|
|
3118
|
+
description:
|
|
3119
|
+
'Export a fitted model (model_id from ena_fit, ena_compare_groups or an ordered tma_fit) as an ENA Web Tool project ZIP: ' +
|
|
3120
|
+
'the data file plus a set holding every setting that produced the model (units, conversations, codes, window, rotation, weight model, ' +
|
|
3121
|
+
'sphere normalization), the group colours, and the plot it opens with. Imported in the web tool (local session → Sets → Import), ' +
|
|
3122
|
+
'it rebuilds the same model and opens on the same comparison plot. ' +
|
|
3123
|
+
'Excluded units (units_used) are stored as excluded in the web tool\'s Units tab. Models the web tool has no setting for (transmodal or time windows, a GMR on x_var, rotations by a column other than the first unit column, ' +
|
|
3124
|
+
'center_align_to_origin=false) are refused with the reason. ' +
|
|
3125
|
+
'PRIVACY: the ZIP contains the data file. It is written locally (by default next to the data file, never overwriting one) and no rows are sent to Claude.',
|
|
3126
|
+
inputSchema: {
|
|
3127
|
+
type: 'object',
|
|
3128
|
+
properties: {
|
|
3129
|
+
model_id: { type: 'string', description: 'model_id of the model to export' },
|
|
3130
|
+
set_name: { type: 'string', description: 'Name of the set in the web tool (default: "<group 1> vs <group 2>", or "ENA model <id>")' },
|
|
3131
|
+
out_path: { type: 'string', description: 'Where to write: a .zip path, or a folder. Default: the data file\'s folder, as "<set name>.zip" (a number is added rather than overwrite).' },
|
|
3132
|
+
},
|
|
3133
|
+
required: ['model_id'],
|
|
3134
|
+
},
|
|
3135
|
+
},
|
|
3136
|
+
];
|
|
3137
|
+
|
|
3138
|
+
// ── Server setup ──────────────────────────────────────────────────────────────
|
|
3139
|
+
|
|
3140
|
+
const PKG = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf8'));
|
|
3141
|
+
const RENA_VERSION = (() => {
|
|
3142
|
+
try { return JSON.parse(readFileSync(_require.resolve('@qe-libs/rena-wasm/package.json'), 'utf8')).version; }
|
|
3143
|
+
catch { return 'unknown'; }
|
|
3144
|
+
})();
|
|
3145
|
+
// Stamped on every result and plot, so results from this server can be told
|
|
3146
|
+
// apart from anything computed in the conversation.
|
|
3147
|
+
const PROVENANCE = `${PKG.name} ${PKG.version} (rena-wasm ${RENA_VERSION})`;
|
|
3148
|
+
|
|
3149
|
+
// Appended to every tool error. When a tool fails, the model must not make
|
|
3150
|
+
// up the analysis some other way and present it as ENA.
|
|
3151
|
+
const NO_SUBSTITUTE =
|
|
3152
|
+
'Do not work around this error by doing the analysis yourself: no accumulation, rotation, projection, ' +
|
|
3153
|
+
'statistics or network plots computed in code (Python, R, JavaScript, a notebook or by hand), and no ' +
|
|
3154
|
+
'estimated, approximate or illustrative results. Fix the call using the message above and retry, or tell ' +
|
|
3155
|
+
'the user what failed and stop. Every ENA result shown to the user must come from these tools.';
|
|
3156
|
+
|
|
3157
|
+
// Tool errors go back to the model, so they must not carry values read from
|
|
3158
|
+
// the data file (rena-wasm quotes the offending value, e.g. a code cell or a
|
|
3159
|
+
// time it could not parse). Quoted strings are withheld unless the model
|
|
3160
|
+
// passed them in itself — column names and other arguments stay readable.
|
|
3161
|
+
const WITHHELD = '[value withheld]';
|
|
3162
|
+
function withholdDataValues(msg, args) {
|
|
3163
|
+
const known = new Set(['Inf']);
|
|
3164
|
+
const collect = (v) => {
|
|
3165
|
+
if (typeof v === 'string' || typeof v === 'number') known.add(String(v));
|
|
3166
|
+
else if (Array.isArray(v)) v.forEach(collect);
|
|
3167
|
+
else if (v && typeof v === 'object') Object.entries(v).forEach(([k, x]) => { known.add(k); collect(x); });
|
|
3168
|
+
};
|
|
3169
|
+
collect(args);
|
|
3170
|
+
// A quote opens after a boundary and closes before one, so apostrophes
|
|
3171
|
+
// inside words ("don't", "units'") are left alone.
|
|
3172
|
+
return String(msg).replace(/(^|[\s(=:,\[])(["'])([^"'\n]{0,200})\2(?=$|[\s).,:;\]])/g,
|
|
3173
|
+
(m, pre, q, inner) => (known.has(inner) ? m : `${pre}${q}${WITHHELD}${q}`));
|
|
3174
|
+
}
|
|
3175
|
+
|
|
3176
|
+
// Sent to the client at connection; MCP clients (Claude, Codex, Gemini CLI,
|
|
3177
|
+
// Cursor, ...) pass it to the model, so the privacy rule reaches every one.
|
|
3178
|
+
const SERVER_INSTRUCTIONS = [
|
|
3179
|
+
'ENA / ONA / TMA analysis that runs locally on the user\'s machine.',
|
|
3180
|
+
'',
|
|
3181
|
+
'Data privacy — follow in every conversation:',
|
|
3182
|
+
'- Pass data files to these tools with data_path. The server reads the file locally; the raw rows never need to enter the conversation.',
|
|
3183
|
+
'- Do not open, read, print, cat, head or summarize the data file yourself with any other tool (file readers, shells, code execution). Learn its structure with ena_profile (column names, counts, binary detection; values only for columns with at most 10 distinct values) or ena_inspect.',
|
|
3184
|
+
'- Do not ask the user to paste their data. If they offer to, ask for the file path instead. data_csv is for small demo or toy data only.',
|
|
3185
|
+
'- Ask whether the data is sensitive before the first analysis; if it is, start with ena_profile.',
|
|
3186
|
+
'- Results include unit labels (the values of the unit columns, often participant names or IDs) and group values. If those are identifying, tell the user before fitting so they can pseudonymize the file first.',
|
|
3187
|
+
'- ena_export writes a ZIP that contains the data file itself, for the ENA Web Tool. Tell the user so before they share it.',
|
|
3188
|
+
'- ena_lines writes an HTML viewer that contains rows of the data file. Give the user its path to open in a browser; never open, read or summarize it yourself.',
|
|
3189
|
+
'',
|
|
3190
|
+
'Results come only from these tools — follow in every conversation:',
|
|
3191
|
+
'- Every ENA, ONA or TMA result shown to the user (accumulated networks, points, rotations, statistics, plots) must come from these tools. Do not compute any part of an ENA analysis yourself — in code, a notebook, R, Python or by hand — and do not draw ENA-style network plots of your own.',
|
|
3192
|
+
'- If a tool fails, read its error, fix the call and retry, or tell the user what failed and stop. Never fill the gap with a substitute that looks like ENA, and never present estimated or illustrative numbers as results.',
|
|
3193
|
+
'- If the user wants something these tools cannot do, say so, and suggest rENA in R or the ENA Web Tool, rather than improvising it.',
|
|
3194
|
+
`- Results carry provenance ("${PKG.name} <version>"); anything without it did not come from this server.`,
|
|
3195
|
+
'',
|
|
3196
|
+
'Before the first analysis in a conversation, call ena_guide (no arguments) for the analysis workflow: triage, window, rotations, masking, ONA/TMA and reading the results. ena_guide with a topic returns one section.',
|
|
3197
|
+
].join('\n');
|
|
3198
|
+
|
|
3199
|
+
// MCP prompts — slash commands / templates in clients that support them.
|
|
3200
|
+
const PROMPTS = [
|
|
3201
|
+
{
|
|
3202
|
+
name: 'analyze_data',
|
|
3203
|
+
description: 'Analyze a coded data file with ENA, following the analysis guide.',
|
|
3204
|
+
arguments: [
|
|
3205
|
+
{ name: 'data_path', description: 'Absolute path to the .csv or .xlsx file', required: true },
|
|
3206
|
+
{ name: 'question', description: 'What you want to learn (optional)', required: false },
|
|
3207
|
+
],
|
|
3208
|
+
text: a => `Analyze the data at ${a.data_path} with Epistemic Network Analysis.` +
|
|
3209
|
+
(a.question ? ` My question: ${a.question}` : '') +
|
|
3210
|
+
' Call ena_guide first and follow it: ask me whether the data is sensitive, learn its structure with ena_profile or ena_inspect, estimate a window, fit, plot, and interpret the result.',
|
|
3211
|
+
},
|
|
3212
|
+
{
|
|
3213
|
+
name: 'compare_groups',
|
|
3214
|
+
description: 'Compare two groups\' networks with ENA: means rotation, difference plot and statistics.',
|
|
3215
|
+
arguments: [
|
|
3216
|
+
{ name: 'data_path', description: 'Absolute path to the .csv or .xlsx file', required: true },
|
|
3217
|
+
{ name: 'group_column', description: 'Column that holds the groups (optional)', required: false },
|
|
3218
|
+
],
|
|
3219
|
+
text: a => `Compare the groups in ${a.data_path}` +
|
|
3220
|
+
(a.group_column ? ` (groups in the ${a.group_column} column)` : '') +
|
|
3221
|
+
' with ENA. Call ena_guide first and follow it: ask me whether the data is sensitive, profile the data, fit with ena_compare_groups, show the difference plot, and report the group statistics and which connections drive the difference.',
|
|
3222
|
+
},
|
|
3223
|
+
{
|
|
3224
|
+
name: 'order_and_sequence',
|
|
3225
|
+
description: 'Model order and sequence with directed ONA / transmodal TMA networks.',
|
|
3226
|
+
arguments: [
|
|
3227
|
+
{ name: 'data_path', description: 'Absolute path to the .csv or .xlsx file', required: true },
|
|
3228
|
+
],
|
|
3229
|
+
text: a => `Analyze order and sequence in ${a.data_path} with directed (ONA) or transmodal (TMA) networks.` +
|
|
3230
|
+
' Call ena_guide with topic "ordered-and-transmodal-networks-ona-tma" first and follow it, after asking me whether the data is sensitive.',
|
|
3231
|
+
},
|
|
3232
|
+
];
|
|
3233
|
+
|
|
3234
|
+
const server = new Server(
|
|
3235
|
+
{ name: 'ena', version: PKG.version },
|
|
3236
|
+
{ capabilities: { tools: {}, prompts: {} }, instructions: SERVER_INSTRUCTIONS }
|
|
3237
|
+
);
|
|
3238
|
+
|
|
3239
|
+
// ── rena-wasm API adapter ─────────────────────────────────────────────────────
|
|
3240
|
+
// rena-wasm ≥ the R-mirroring rewrite returns an R-style ENA set object
|
|
3241
|
+
// (lineWeights / points / rotation.nodes / model.unitLabels) instead of the
|
|
3242
|
+
// original flat object (networks / centroids / positions / unitLabels).
|
|
3243
|
+
// These shims expose the flat field names this server is written against, and
|
|
3244
|
+
// are idempotent no-ops on the older flat API.
|
|
3245
|
+
|
|
3246
|
+
function adaptModel(m) {
|
|
3247
|
+
if (!m) return m;
|
|
3248
|
+
let mClean = m;
|
|
3249
|
+
if (m.lineWeights !== undefined && m.networks === undefined) {
|
|
3250
|
+
mClean = {
|
|
3251
|
+
networks: m.lineWeights, // sphere-normed adjacency (nUnits × nConn)
|
|
3252
|
+
centroids: m.points, // projected unit positions (nUnits × dims)
|
|
3253
|
+
positions: m.rotation?.nodes, // code node positions (nCodes × dims)
|
|
3254
|
+
connectionNames: m.connectionNames,
|
|
3255
|
+
unitLabels: m.model?.unitLabels ?? m.unitLabels,
|
|
3256
|
+
columnNames: m.rotation?.columnNames,
|
|
3257
|
+
eigenvalues: m.rotation?.eigenvalues,
|
|
3258
|
+
variance: m.model?.variance, // share of total variance per dim (rena-wasm >= 0.1.11)
|
|
3259
|
+
rotationMatrix: m.rotationMatrix, // nConn × dims, for rotation_from
|
|
3260
|
+
centerVec: m.rotation?.centerVec, // for rotation_from
|
|
3261
|
+
networkCentroids: m.model?.centroids, // network centroids (nUnits × dims), for goodness of fit
|
|
3262
|
+
connectionCounts: m.connectionCounts, // raw accumulated counts (nUnits × nConn), for ena_tables
|
|
3263
|
+
rowConnectionCounts: m.rowConnectionCounts ?? m.model?.rowConnectionCounts, // per line (nRows × nConn), for ena_lines
|
|
3264
|
+
nUnits: m.nUnits,
|
|
3265
|
+
nConnections: m.nConnections,
|
|
3266
|
+
dims: m.dims,
|
|
3267
|
+
};
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
// Ordered (ONA/TMA) models have n² directed connections. rena-wasm already
|
|
3271
|
+
// names them in network column order ("ground & response", column j*n + i =
|
|
3272
|
+
// codes[i] → codes[j]), so the names are kept as-is — only flag direction.
|
|
3273
|
+
const nCodes = m.rotation?.codes?.length;
|
|
3274
|
+
if (nCodes > 1 && mClean.nConnections === nCodes * nCodes) mClean.ordered = true;
|
|
3275
|
+
|
|
3276
|
+
return mClean;
|
|
3277
|
+
}
|
|
3278
|
+
|
|
3279
|
+
function adaptAccum(a) {
|
|
3280
|
+
if (!a) return a;
|
|
3281
|
+
if (a.networks === undefined && a.connectionCounts !== undefined) {
|
|
3282
|
+
return { ...a, networks: a.connectionCounts };
|
|
3283
|
+
}
|
|
3284
|
+
return a;
|
|
3285
|
+
}
|
|
3286
|
+
|
|
3287
|
+
// A WASM abort (raised by libqe on a degenerate SVD / means rotation) leaves the
|
|
3288
|
+
// module instance unusable — every later call would also abort. Detect it,
|
|
3289
|
+
// discard the cached instance so the next call reloads a fresh one, and surface
|
|
3290
|
+
// an actionable message instead of the raw "Aborted(undefined)".
|
|
3291
|
+
function isWasmAbort(e) {
|
|
3292
|
+
return /abort|RuntimeError|out of bounds|memory access|table index/i.test(e?.message ?? '');
|
|
3293
|
+
}
|
|
3294
|
+
|
|
3295
|
+
const DEGENERATE_FIT_MSG =
|
|
3296
|
+
'ENA rotation could not be computed — the data is too sparse or degenerate. ' +
|
|
3297
|
+
'Common causes: fewer than 2 units have any code co-occurrences within the ' +
|
|
3298
|
+
'stanza window; only one code column; or all units have identical networks. ' +
|
|
3299
|
+
'Use ena_accumulate to inspect the raw co-occurrence networks, and check that ' +
|
|
3300
|
+
'you have 2+ code columns that actually co-occur and 2+ units with non-empty networks.';
|
|
3301
|
+
|
|
3302
|
+
let enaPromise = null;
|
|
3303
|
+
function getENA() {
|
|
3304
|
+
if (!enaPromise) enaPromise = loadENA().then(ena => {
|
|
3305
|
+
const guard = (fn, adapt) => (rows, opts) => {
|
|
3306
|
+
try {
|
|
3307
|
+
return adapt(fn.call(ena, rows, opts));
|
|
3308
|
+
} catch (e) {
|
|
3309
|
+
if (isWasmAbort(e)) {
|
|
3310
|
+
enaPromise = null; // force a fresh WASM instance next call
|
|
3311
|
+
throw new Error(DEGENERATE_FIT_MSG);
|
|
3312
|
+
}
|
|
3313
|
+
throw e;
|
|
3314
|
+
}
|
|
3315
|
+
};
|
|
3316
|
+
return {
|
|
3317
|
+
...ena,
|
|
3318
|
+
fit: guard(ena.fit, adaptModel),
|
|
3319
|
+
accumulate: guard(ena.accumulate, adaptAccum),
|
|
3320
|
+
};
|
|
3321
|
+
});
|
|
3322
|
+
return enaPromise;
|
|
3323
|
+
}
|
|
3324
|
+
|
|
3325
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
|
|
3326
|
+
|
|
3327
|
+
server.setRequestHandler(ListPromptsRequestSchema, async () => ({
|
|
3328
|
+
prompts: PROMPTS.map(({ text, ...p }) => p),
|
|
3329
|
+
}));
|
|
3330
|
+
|
|
3331
|
+
server.setRequestHandler(GetPromptRequestSchema, async (request) => {
|
|
3332
|
+
const { name, arguments: a = {} } = request.params;
|
|
3333
|
+
const p = PROMPTS.find(x => x.name === name);
|
|
3334
|
+
if (!p) throw new Error(`Unknown prompt: ${name}`);
|
|
3335
|
+
for (const arg of p.arguments)
|
|
3336
|
+
if (arg.required && !a[arg.name]) throw new Error(`Missing argument: ${arg.name}`);
|
|
3337
|
+
return {
|
|
3338
|
+
description: p.description,
|
|
3339
|
+
messages: [{ role: 'user', content: { type: 'text', text: p.text(a) } }],
|
|
3340
|
+
};
|
|
3341
|
+
});
|
|
3342
|
+
|
|
3343
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
3344
|
+
const { name, arguments: args } = request.params;
|
|
3345
|
+
try {
|
|
3346
|
+
// rena-wasm windows look backward only (= rENA's window.size.back); a
|
|
3347
|
+
// forward window would otherwise be silently ignored.
|
|
3348
|
+
if (Number(args?.window_forward) > 0)
|
|
3349
|
+
throw new Error('window_forward is not supported by the WASM backend — ' +
|
|
3350
|
+
'stanza windows look backward only. Omit it, or use rENA in R for forward windows.');
|
|
3351
|
+
// The guide is markdown; return it as-is, not JSON-encoded.
|
|
3352
|
+
if (name === 'ena_guide') return { content: [{ type: 'text', text: toolGuide(args) }] };
|
|
3353
|
+
let result;
|
|
3354
|
+
if (name === 'ena_profile') {
|
|
3355
|
+
result = toolProfile(args);
|
|
3356
|
+
} else if (name === 'ena_inspect') {
|
|
3357
|
+
result = toolInspect(args);
|
|
3358
|
+
} else if (name === 'ena_plot') {
|
|
3359
|
+
result = await toolPlot(args);
|
|
3360
|
+
} else {
|
|
3361
|
+
const ena = await getENA();
|
|
3362
|
+
if (name === 'ena_fit') result = await toolFit(ena, args);
|
|
3363
|
+
else if (name === 'tma_fit') result = await toolTmaFit(ena, args);
|
|
3364
|
+
else if (name === 'ena_accumulate') result = await toolAccumulate(ena, args);
|
|
3365
|
+
else if (name === 'tma_accumulate') result = await toolTmaAccumulate(ena, args);
|
|
3366
|
+
else if (name === 'ena_compare_groups') result = await toolCompareGroups(ena, args);
|
|
3367
|
+
else if (name === 'ena_window') result = await toolWindow(ena, args);
|
|
3368
|
+
else if (name === 'ena_tables') result = await toolTables(ena, args);
|
|
3369
|
+
else if (name === 'ena_lines') result = await toolLines(ena, args);
|
|
3370
|
+
else if (name === 'ena_simplify') result = await toolSimplify(ena, args);
|
|
3371
|
+
else if (name === 'ena_export') result = await toolExport(ena, args);
|
|
3372
|
+
else throw new Error(`Unknown tool: ${name}`);
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3375
|
+
// Extract viz fields before stringifying so they never appear in the text block.
|
|
3376
|
+
const { _plot, ...cleanResult } = result;
|
|
3377
|
+
|
|
3378
|
+
cleanResult.provenance = PROVENANCE;
|
|
3379
|
+
const content = [{ type: 'text', text: JSON.stringify(cleanResult, null, 2) }];
|
|
3380
|
+
|
|
3381
|
+
if (_plot) {
|
|
3382
|
+
const slug = vizSlug(cleanResult, args) || name;
|
|
3383
|
+
const artifacts = {}; // fields to merge into the text block
|
|
3384
|
+
let renderError = null;
|
|
3385
|
+
let svg = null;
|
|
3386
|
+
|
|
3387
|
+
try {
|
|
3388
|
+
svg = await renderSVG(_plot);
|
|
3389
|
+
if (svg) {
|
|
3390
|
+
// SVG file — always available (pure JS/WASM), vector, embeddable.
|
|
3391
|
+
try { artifacts.svg_plot = writeVizFile(svg, slug, 'svg'); }
|
|
3392
|
+
catch (e) { process.stderr.write(`[ena-mcp] SVG write failed: ${e.message}\n`); }
|
|
3393
|
+
|
|
3394
|
+
// PNG (inline image block + png_plot file) only when the optional
|
|
3395
|
+
// native rasterizer is present. Absent in cross-platform bundles.
|
|
3396
|
+
const Resvg = await getResvg();
|
|
3397
|
+
if (Resvg) {
|
|
3398
|
+
const png = new Resvg(svg, { fitTo: { mode: 'height', value: SVG_SIZE }, background: '#ffffff' }).render().asPng();
|
|
3399
|
+
content.unshift({
|
|
3400
|
+
type: 'image',
|
|
3401
|
+
data: png.toString('base64'),
|
|
3402
|
+
mimeType: 'image/png',
|
|
3403
|
+
});
|
|
3404
|
+
try { artifacts.png_plot = writeVizFile(png, slug, 'png'); }
|
|
3405
|
+
catch (e) { process.stderr.write(`[ena-mcp] PNG write failed: ${e.message}\n`); }
|
|
3406
|
+
}
|
|
3407
|
+
} else {
|
|
3408
|
+
renderError = 'renderSVG returned null (qe-graph svg_element not found — jsdom/qeviz may have failed silently)';
|
|
3409
|
+
}
|
|
3410
|
+
} catch (e) {
|
|
3411
|
+
renderError = e.message;
|
|
3412
|
+
}
|
|
3413
|
+
if (renderError) {
|
|
3414
|
+
process.stderr.write(`[ena-mcp] render failed: ${renderError}\n`);
|
|
3415
|
+
artifacts._render_error = renderError;
|
|
3416
|
+
artifacts.render_note = 'The model is fine, but the static plot could not be drawn. Give the user interactive_plot ' +
|
|
3417
|
+
'if it is listed, or report the error; do not draw a substitute plot yourself.';
|
|
3418
|
+
}
|
|
3419
|
+
|
|
3420
|
+
// Self-contained interactive HTML for clients that don't render inline images.
|
|
3421
|
+
try {
|
|
3422
|
+
const html = generateHTML(_plot, _plot.title ?? cleanResult.label ?? name);
|
|
3423
|
+
artifacts.interactive_plot = writeVizFile(html, slug, 'html');
|
|
3424
|
+
} catch (e) {
|
|
3425
|
+
process.stderr.write(`[ena-mcp] HTML write failed: ${e.message}\n`);
|
|
3426
|
+
}
|
|
3427
|
+
|
|
3428
|
+
// Merge all artifact paths into the text block in one pass.
|
|
3429
|
+
const textBlock = content.find(c => c.type === 'text');
|
|
3430
|
+
if (textBlock && Object.keys(artifacts).length) {
|
|
3431
|
+
const parsed = JSON.parse(textBlock.text);
|
|
3432
|
+
Object.assign(parsed, artifacts);
|
|
3433
|
+
textBlock.text = JSON.stringify(parsed, null, 2);
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
|
|
3437
|
+
return { content };
|
|
3438
|
+
} catch (err) {
|
|
3439
|
+
return {
|
|
3440
|
+
content: [{ type: 'text', text: `Error: ${withholdDataValues(err.message, args)}\n\n${NO_SUBSTITUTE}` }],
|
|
3441
|
+
isError: true,
|
|
3442
|
+
};
|
|
3443
|
+
}
|
|
3444
|
+
});
|
|
3445
|
+
|
|
3446
|
+
// ── Start ─────────────────────────────────────────────────────────────────────
|
|
3447
|
+
|
|
3448
|
+
const transport = new StdioServerTransport();
|
|
3449
|
+
await server.connect(transport);
|