@qe-libs/rena-wasm 0.1.1 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/package.json +2 -2
- package/src/index.js +260 -47
- package/src/pipeline.js +0 -58
- package/src/tensor.js +90 -8
package/README.md
CHANGED
|
@@ -32,12 +32,14 @@ const model = ena.fit(rows, {
|
|
|
32
32
|
dims: 2,
|
|
33
33
|
});
|
|
34
34
|
|
|
35
|
-
model.centroids
|
|
36
|
-
model.
|
|
37
|
-
model.
|
|
35
|
+
model.model.centroids // Float64Array nUnits × dims
|
|
36
|
+
model.lineWeights // Float64Array nUnits × nConnections (normed)
|
|
37
|
+
model.connectionCounts // Float64Array nUnits × nConnections (raw unit counts)
|
|
38
|
+
model.rowConnectionCounts // Float64Array nRows × nConnections (raw row counts)
|
|
39
|
+
model.rotation.nodes // Float64Array nCodes × dims
|
|
38
40
|
model.connectionNames // ['Data & Technical.Constraints', ...]
|
|
39
|
-
model.unitLabels
|
|
40
|
-
model.columnNames
|
|
41
|
+
model.model.unitLabels // ['UserName1_ConditionA', ...]
|
|
42
|
+
model.rotation.columnNames // ['SVD1', 'SVD2']
|
|
41
43
|
|
|
42
44
|
// Per-unit helpers
|
|
43
45
|
model.centroid('Alice_A') // number[] length = dims
|
|
@@ -70,7 +72,7 @@ const model = ena.fit(rows, {
|
|
|
70
72
|
Returns raw (un-normalised) network vectors without running the full pipeline.
|
|
71
73
|
|
|
72
74
|
```js
|
|
73
|
-
const {
|
|
75
|
+
const { connectionCounts, rowConnectionCounts, unitLabels, connectionNames, nUnits, nConnections } =
|
|
74
76
|
ena.accumulate(rows, { codes, units, conversations, window: 4 });
|
|
75
77
|
```
|
|
76
78
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qe-libs/rena-wasm",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "JavaScript/WebAssembly ENA pipeline — thin orchestration layer over @qe-libs/libqe-wasm",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"resolver": "<rootDir>/jest-resolver.cjs"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@qe-libs/libqe-wasm": "^0.1.
|
|
34
|
+
"@qe-libs/libqe-wasm": "^0.1.4"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"jest": "^29.0.0"
|
package/src/index.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* Top-level fields (= R's set$...):
|
|
14
14
|
* connectionCounts Float64Array (nUnits × nConnections) — raw accumulation
|
|
15
|
+
* rowConnectionCounts Float64Array (nRows × nConnections) — per-row raw accumulation
|
|
15
16
|
* lineWeights Float64Array (nUnits × nConnections) — sphere-normed
|
|
16
17
|
* points Float64Array (nUnits × dims) — projected positions
|
|
17
18
|
* rotationMatrix Float64Array (nConnections × dims) — rotation vectors
|
|
@@ -23,6 +24,7 @@
|
|
|
23
24
|
* dims number
|
|
24
25
|
*
|
|
25
26
|
* model sub-object (= R's set$model$...):
|
|
27
|
+
* model.rowConnectionCounts Float64Array (nRows × nConnections) — per-row raw accumulation
|
|
26
28
|
* model.centroids Float64Array (nUnits × dims) — LWS centroids
|
|
27
29
|
* model.variance number[] variance explained per dim
|
|
28
30
|
* model.unitLabels string[]
|
|
@@ -41,7 +43,7 @@
|
|
|
41
43
|
import loadLibQE from '@qe-libs/libqe-wasm';
|
|
42
44
|
import { parseData } from './data.js';
|
|
43
45
|
import {
|
|
44
|
-
|
|
46
|
+
sphereNorm, center,
|
|
45
47
|
rotateSVD, rotateMeans, rotateGeneralized,
|
|
46
48
|
project, nodePositions, spaceDistCorr,
|
|
47
49
|
} from './pipeline.js';
|
|
@@ -107,6 +109,7 @@ class ENAModel {
|
|
|
107
109
|
constructor(opts) {
|
|
108
110
|
// ── top-level fields (= R's set$...) ────────────────────────────────
|
|
109
111
|
this.connectionCounts = opts.connectionCounts; // raw networks
|
|
112
|
+
this.rowConnectionCounts = opts.rowConnectionCounts;
|
|
110
113
|
this.lineWeights = opts.lineWeights; // sphere-normed networks
|
|
111
114
|
this.points = opts.points; // projected unit positions
|
|
112
115
|
this.rotationMatrix = opts.rotationMatrix; // n_connections × dims
|
|
@@ -118,6 +121,7 @@ class ENAModel {
|
|
|
118
121
|
|
|
119
122
|
// ── model sub-object (= R's set$model$...) ───────────────────────────
|
|
120
123
|
this.model = {
|
|
124
|
+
rowConnectionCounts: opts.rowConnectionCounts,
|
|
121
125
|
centroids: opts.centroids, // LWS positions
|
|
122
126
|
variance: opts.variance, // variance explained
|
|
123
127
|
unitLabels: opts.unitLabels,
|
|
@@ -183,10 +187,33 @@ class ENAModel {
|
|
|
183
187
|
}
|
|
184
188
|
}
|
|
185
189
|
|
|
190
|
+
// ── weight models (= R's `weight.by`) ────────────────────────────────────────
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Map a weight-model name to its element-wise transform, matching the webtool's
|
|
194
|
+
* server.js mapping to R weight.by functions:
|
|
195
|
+
* 'sqrt' → Math.sqrt (R "sqrt")
|
|
196
|
+
* 'log' → Math.log1p (R "log1p", i.e. log(x+1); guards log(0))
|
|
197
|
+
* 'product' → identity (raw non-binary counts; R's "product" string
|
|
198
|
+
* is not a function, so no transform)
|
|
199
|
+
* A falsy value or 'binary' returns null (binary accumulation, no transform).
|
|
200
|
+
* @param {string|false|undefined} name
|
|
201
|
+
* @returns {((x:number)=>number)|null}
|
|
202
|
+
*/
|
|
203
|
+
function weightModelTransform(name) {
|
|
204
|
+
switch (name) {
|
|
205
|
+
case 'sqrt': return Math.sqrt;
|
|
206
|
+
case 'log': return Math.log1p;
|
|
207
|
+
case 'product': return (x) => x;
|
|
208
|
+
default: return null; // binary / off / unknown
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
186
212
|
// ── shared pipeline (post-accumulation) ──────────────────────────────────────
|
|
187
213
|
|
|
188
214
|
function runPipeline(qe, rawNetworks, nUnits, nConnections, codes, unitLabels,
|
|
189
|
-
metaData, rotMethod, groupA, groupB, dims, gParams
|
|
215
|
+
metaData, rotMethod, groupA, groupB, dims, gParams,
|
|
216
|
+
rowConnectionCounts = null) {
|
|
190
217
|
const connectionNames = qe.connection_names(codes);
|
|
191
218
|
|
|
192
219
|
// Sphere norm → lineWeights (= R's set$line.weights)
|
|
@@ -228,10 +255,11 @@ function runPipeline(qe, rawNetworks, nUnits, nConnections, codes, unitLabels,
|
|
|
228
255
|
);
|
|
229
256
|
|
|
230
257
|
// Node positions (LWS) → rotation.nodes + model.centroids
|
|
231
|
-
//
|
|
232
|
-
//
|
|
258
|
+
// Matches rENA's lws.positions.sq, which regresses the projected points onto
|
|
259
|
+
// the SPHERE-normed line weights (enaset$line.weights) — not the centered
|
|
260
|
+
// networks. Verified node-for-node against R rENA on rs.data.new.csv.
|
|
233
261
|
const { nodes, centroids } = nodePositions(
|
|
234
|
-
qe,
|
|
262
|
+
qe, lineWeights, nUnits, nConnections, points, dims
|
|
235
263
|
);
|
|
236
264
|
|
|
237
265
|
// Variance explained (= R's model$variance)
|
|
@@ -243,6 +271,7 @@ function runPipeline(qe, rawNetworks, nUnits, nConnections, codes, unitLabels,
|
|
|
243
271
|
return new ENAModel({
|
|
244
272
|
// top-level
|
|
245
273
|
connectionCounts: rawNetworks,
|
|
274
|
+
rowConnectionCounts,
|
|
246
275
|
lineWeights,
|
|
247
276
|
points,
|
|
248
277
|
rotationMatrix,
|
|
@@ -285,10 +314,11 @@ export default async function loadENA() {
|
|
|
285
314
|
* @param {string[]} opts.codes - Code column names
|
|
286
315
|
* @param {string[]} opts.units - Unit identifier column(s)
|
|
287
316
|
* @param {string[]} opts.conversations - Conversation identifier column(s)
|
|
288
|
-
* @param {number} [opts.window=4] - Backward window (
|
|
289
|
-
* @param {boolean} [opts.binary=true] - Binarise co-occurrences (
|
|
290
|
-
* @param {boolean} [opts.ordered=false] - Directed networks
|
|
291
|
-
* @param {object} [opts.tensor] - Context tensor definition
|
|
317
|
+
* @param {number} [opts.window=4] - Backward window; builds defaultTensor(window) when no tensor is given
|
|
318
|
+
* @param {boolean} [opts.binary=true] - Binarise each line's co-occurrences (unordered; ignored when weightModel is set)
|
|
319
|
+
* @param {boolean} [opts.ordered=false] - Directed networks
|
|
320
|
+
* @param {object} [opts.tensor] - Context tensor definition (overrides window)
|
|
321
|
+
* @param {string} [opts.weightModel] - 'product' | 'sqrt' | 'log' (per-line, before the unit sum)
|
|
292
322
|
* @param {string} [opts.rotation='svd'] - 'svd', 'mean', or 'generalized'
|
|
293
323
|
* @param {number[]} [opts.groupA] - Unit indices for means rotation group A
|
|
294
324
|
* @param {number[]} [opts.groupB] - Unit indices for means rotation group B
|
|
@@ -312,31 +342,36 @@ export default async function loadENA() {
|
|
|
312
342
|
gParams,
|
|
313
343
|
dims = 2,
|
|
314
344
|
codeMask,
|
|
345
|
+
weightModel,
|
|
315
346
|
} = opts;
|
|
316
347
|
|
|
317
348
|
if (!codes?.length) throw new Error('opts.codes is required');
|
|
318
349
|
if (!units?.length) throw new Error('opts.units is required');
|
|
319
350
|
if (!conversations?.length) throw new Error('opts.conversations is required');
|
|
320
351
|
|
|
352
|
+
// Weight model = R's `weight.by`. R applies it per LINE (each row's
|
|
353
|
+
// "product" co-occurrence counts) BEFORE summing per unit, mirroring
|
|
354
|
+
// accumulate.data.R (lapply(.SD, weight.by) over the per-line
|
|
355
|
+
// co-occurrence table, then per-unit aggregation). Because
|
|
356
|
+
// sqrt(Σ) ≠ Σsqrt, the transform is applied to each row's counts
|
|
357
|
+
// inside accumulateTensor, not to the unit-summed network.
|
|
358
|
+
// `product` is the non-binarized row counts themselves (identity);
|
|
359
|
+
// any weight model therefore forces non-binary accumulation.
|
|
360
|
+
const weightFn = weightModelTransform(weightModel);
|
|
361
|
+
const effBinary = weightFn ? false : binary;
|
|
362
|
+
|
|
321
363
|
const { codeMatrix, nRows, nCodes, nUnits, unitLabels,
|
|
322
364
|
unitOf, convoGroups, metaData } =
|
|
323
365
|
parseData(rows, codes, units, conversations);
|
|
324
366
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
} else {
|
|
334
|
-
rawNetworks = accumulate(
|
|
335
|
-
qe, codeMatrix, nRows, nCodes, nUnits,
|
|
336
|
-
unitOf, convoGroups, windowSize, binary
|
|
337
|
-
);
|
|
338
|
-
nConnections = qe.choose_two(nCodes);
|
|
339
|
-
}
|
|
367
|
+
// Every model accumulates through the tensor path; a plain moving
|
|
368
|
+
// window is expressed as defaultTensor(window).
|
|
369
|
+
const { networks: rawNetworks, rowConnectionCounts } = accumulateTensor(
|
|
370
|
+
qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
371
|
+
unitOf, convoGroups, tensorDef ?? defaultTensor(windowSize),
|
|
372
|
+
ordered, effBinary, weightFn
|
|
373
|
+
);
|
|
374
|
+
const nConnections = ordered ? nCodes * nCodes : qe.choose_two(nCodes);
|
|
340
375
|
|
|
341
376
|
// Apply code masking by zeroing out the masked connection columns across all units
|
|
342
377
|
if (codeMask && codeMask.length === codes.length) {
|
|
@@ -368,7 +403,8 @@ export default async function loadENA() {
|
|
|
368
403
|
}
|
|
369
404
|
|
|
370
405
|
return runPipeline(qe, rawNetworks, nUnits, nConnections, codes,
|
|
371
|
-
unitLabels, metaData, rotMethod, groupA, groupB,
|
|
406
|
+
unitLabels, metaData, rotMethod, groupA, groupB,
|
|
407
|
+
dims, gParams, rowConnectionCounts);
|
|
372
408
|
},
|
|
373
409
|
|
|
374
410
|
/**
|
|
@@ -379,6 +415,7 @@ export default async function loadENA() {
|
|
|
379
415
|
* @param {object} opts - codes, units, conversations, window, binary, ordered, tensor
|
|
380
416
|
* @returns {{
|
|
381
417
|
* connectionCounts: Float64Array,
|
|
418
|
+
* rowConnectionCounts: Float64Array | null,
|
|
382
419
|
* unitLabels: string[],
|
|
383
420
|
* connectionNames: string[],
|
|
384
421
|
* metaData: Object[],
|
|
@@ -401,21 +438,12 @@ export default async function loadENA() {
|
|
|
401
438
|
unitOf, convoGroups, metaData } =
|
|
402
439
|
parseData(rows, codes, units, conversations);
|
|
403
440
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
);
|
|
411
|
-
nConnections = ordered ? nCodes * nCodes : qe.choose_two(nCodes);
|
|
412
|
-
} else {
|
|
413
|
-
networks = accumulate(
|
|
414
|
-
qe, codeMatrix, nRows, nCodes, nUnits,
|
|
415
|
-
unitOf, convoGroups, windowSize, binary
|
|
416
|
-
);
|
|
417
|
-
nConnections = qe.choose_two(nCodes);
|
|
418
|
-
}
|
|
441
|
+
const { networks, rowConnectionCounts } = accumulateTensor(
|
|
442
|
+
qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
443
|
+
unitOf, convoGroups, tensorDef ?? defaultTensor(windowSize),
|
|
444
|
+
ordered, binary
|
|
445
|
+
);
|
|
446
|
+
const nConnections = ordered ? nCodes * nCodes : qe.choose_two(nCodes);
|
|
419
447
|
|
|
420
448
|
// Apply code masking by zeroing out the masked connection columns across all units
|
|
421
449
|
if (codeMask && codeMask.length === codes.length) {
|
|
@@ -448,7 +476,7 @@ export default async function loadENA() {
|
|
|
448
476
|
|
|
449
477
|
const connectionNames = qe.connection_names(codes);
|
|
450
478
|
return {
|
|
451
|
-
connectionCounts: networks, unitLabels, connectionNames,
|
|
479
|
+
connectionCounts: networks, rowConnectionCounts, unitLabels, connectionNames,
|
|
452
480
|
metaData, nUnits, nConnections,
|
|
453
481
|
// Retained so tuneWindowSize() can rebuild at other window sizes
|
|
454
482
|
// (= R's ENAAccumulation$`_function.call`).
|
|
@@ -491,21 +519,21 @@ export default async function loadENA() {
|
|
|
491
519
|
'maxSize must be greater than minSize to compare windows.'
|
|
492
520
|
);
|
|
493
521
|
|
|
494
|
-
const { rows, codes, units, conversations, binary } = call;
|
|
522
|
+
const { rows, codes, units, conversations, binary, ordered } = call;
|
|
495
523
|
|
|
496
524
|
// Parse once; only the window size changes between iterations.
|
|
497
525
|
const { codeMatrix, nRows, nCodes, nUnits, unitLabels,
|
|
498
526
|
unitOf, convoGroups, metaData } =
|
|
499
527
|
parseData(rows, codes, units, conversations);
|
|
500
|
-
const nConnections = qe.choose_two(nCodes);
|
|
528
|
+
const nConnections = ordered ? nCodes * nCodes : qe.choose_two(nCodes);
|
|
501
529
|
|
|
502
530
|
// 1. Rebuild + fit at each window, collecting unit points.
|
|
503
531
|
const dims = 2;
|
|
504
532
|
const allPoints = [];
|
|
505
533
|
for (const w of windowRange) {
|
|
506
|
-
const raw =
|
|
507
|
-
qe, codeMatrix, nRows, nCodes, nUnits,
|
|
508
|
-
unitOf, convoGroups, w, binary
|
|
534
|
+
const { networks: raw } = accumulateTensor(
|
|
535
|
+
qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
536
|
+
unitOf, convoGroups, defaultTensor(w), ordered, binary
|
|
509
537
|
);
|
|
510
538
|
const model = runPipeline(
|
|
511
539
|
qe, raw, nUnits, nConnections, codes,
|
|
@@ -531,10 +559,70 @@ export default async function loadENA() {
|
|
|
531
559
|
|
|
532
560
|
// 4. Rebuild the accumulation at the selected window size.
|
|
533
561
|
return api.accumulate(rows, {
|
|
534
|
-
codes, units, conversations, window: bestWindow, binary,
|
|
562
|
+
codes, units, conversations, window: bestWindow, binary, ordered,
|
|
535
563
|
});
|
|
536
564
|
},
|
|
537
565
|
|
|
566
|
+
/**
|
|
567
|
+
* Estimate the moving-window size via Cross-Covariance Decay (CCD)
|
|
568
|
+
* (= R's ena.ccd). Computes the noise-corrected cross-covariance decay
|
|
569
|
+
* curves over lags and returns the half-life lag as the window size.
|
|
570
|
+
*
|
|
571
|
+
* Unlike tuneWindowSize (which rebuilds+fits a full SVD model at each
|
|
572
|
+
* window), CCD runs directly on the raw code matrix per conversation —
|
|
573
|
+
* no accumulation/rotation — via the shared libqe kernel.
|
|
574
|
+
*
|
|
575
|
+
* @param {Object[]} rows
|
|
576
|
+
* @param {object} opts
|
|
577
|
+
* @param {string[]} opts.codes - Code column names
|
|
578
|
+
* @param {string[]} opts.conversations - Conversation identifier column(s)
|
|
579
|
+
* @param {number} [opts.maxWindow=20] - Maximum lag to evaluate
|
|
580
|
+
* @param {number} [opts.minOverlap=10] - Minimum overlapping rows per conversation at a lag
|
|
581
|
+
* @returns {{
|
|
582
|
+
* window_size: number,
|
|
583
|
+
* peak_lag: number,
|
|
584
|
+
* lag: number[],
|
|
585
|
+
* frob: number[],
|
|
586
|
+
* frob_sq_unbiased: number[],
|
|
587
|
+
* frob_unbiased_signed: number[],
|
|
588
|
+
* total_weight: number[],
|
|
589
|
+
* }}
|
|
590
|
+
*/
|
|
591
|
+
ccd(rows, opts = {}) {
|
|
592
|
+
const { codes, conversations, maxWindow = 20, minOverlap = 10 } = opts;
|
|
593
|
+
if (!codes?.length) throw new Error('opts.codes is required');
|
|
594
|
+
if (!conversations?.length) throw new Error('opts.conversations is required');
|
|
595
|
+
|
|
596
|
+
// Units are irrelevant to CCD; reuse conversations as a placeholder
|
|
597
|
+
// so parseData can build codeMatrix + convoGroups.
|
|
598
|
+
const { codeMatrix, nRows, nCodes, convoGroups } =
|
|
599
|
+
parseData(rows, codes, conversations, conversations);
|
|
600
|
+
|
|
601
|
+
// Flatten conversation row groups into (sizes, indices) for the kernel.
|
|
602
|
+
const groupSizes = [];
|
|
603
|
+
const rowIndices = [];
|
|
604
|
+
for (const rowIdxs of convoGroups.values()) {
|
|
605
|
+
groupSizes.push(rowIdxs.length);
|
|
606
|
+
for (const ri of rowIdxs) rowIndices.push(ri);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
return qe.ccd_window(
|
|
610
|
+
codeMatrix, nRows, nCodes, groupSizes, rowIndices, maxWindow, minOverlap
|
|
611
|
+
);
|
|
612
|
+
},
|
|
613
|
+
|
|
614
|
+
/**
|
|
615
|
+
* Convenience wrapper returning only the estimated window size
|
|
616
|
+
* (= R's ena.ccd.window).
|
|
617
|
+
*
|
|
618
|
+
* @param {Object[]} rows
|
|
619
|
+
* @param {object} opts - see ccd()
|
|
620
|
+
* @returns {number} estimated window size
|
|
621
|
+
*/
|
|
622
|
+
ccdWindow(rows, opts = {}) {
|
|
623
|
+
return api.ccd(rows, opts).window_size;
|
|
624
|
+
},
|
|
625
|
+
|
|
538
626
|
/**
|
|
539
627
|
* Per-dimension t-based confidence intervals around the column means.
|
|
540
628
|
* Matches R's conf.ints / libqe::mean_ci.
|
|
@@ -581,10 +669,135 @@ export default async function loadENA() {
|
|
|
581
669
|
return qe.group_stats(g1Points, nG1, dims, g2Points, nG2, dims);
|
|
582
670
|
},
|
|
583
671
|
|
|
672
|
+
/**
|
|
673
|
+
* PRIA — find the largest set of codes (up to removeNum) that can be
|
|
674
|
+
* removed while keeping the model's goodness-of-fit within `threshold`
|
|
675
|
+
* of the full model. Brute-force subset search matching R
|
|
676
|
+
* PRIA::pria(): for k = 1..removeNum, over every k-subset of codes,
|
|
677
|
+
* build the reduced model (fit with a codeMask that drops every
|
|
678
|
+
* connection touching a removed code), gate on
|
|
679
|
+
* min_d(gof_reduced[d] / gof_full[d]) >= threshold where gof is the
|
|
680
|
+
* per-dimension ena_correlation(points, centroids), and keep the subset
|
|
681
|
+
* with the MOST codes removed, then the highest reduced dim-1 variance.
|
|
682
|
+
*
|
|
683
|
+
* @returns {{ removed: string[], removedIndices: number[], k: number,
|
|
684
|
+
* variance: (number|null) }}
|
|
685
|
+
*/
|
|
686
|
+
pria(rows, opts = {}) {
|
|
687
|
+
const {
|
|
688
|
+
codes, units, conversations,
|
|
689
|
+
window: windowSize = 4, binary = true, ordered = false,
|
|
690
|
+
rotation = 'svd', dims = 2, gParams, groupA, groupB,
|
|
691
|
+
removeNum = 3, threshold = 0.95,
|
|
692
|
+
} = opts;
|
|
693
|
+
if (!codes?.length) throw new Error('opts.codes is required');
|
|
694
|
+
if (!units?.length) throw new Error('opts.units is required');
|
|
695
|
+
if (!conversations?.length) throw new Error('opts.conversations is required');
|
|
696
|
+
|
|
697
|
+
const m = codes.length;
|
|
698
|
+
const rn = Math.min(removeNum, m - 3); // never reduce below 3 codes
|
|
699
|
+
const empty = { removed: [], removedIndices: [], k: 0, variance: null };
|
|
700
|
+
if (rn < 1) return empty;
|
|
701
|
+
|
|
702
|
+
const gofDims = Math.min(dims, 2); // R scores on dims 1:2
|
|
703
|
+
const baseOpts = { codes, units, conversations, window: windowSize,
|
|
704
|
+
binary, ordered, rotation, dims, gParams, groupA, groupB };
|
|
705
|
+
|
|
706
|
+
const D = gofDims; // score on the first 2 dims (R's get_pria_scores_2Ds)
|
|
707
|
+
// Extract a D-column submatrix (given row indices) from a flat
|
|
708
|
+
// row-major (nRows × dims) array.
|
|
709
|
+
const sub = (flat, rowIdxs) => {
|
|
710
|
+
const out = new Float64Array(rowIdxs.length * D);
|
|
711
|
+
rowIdxs.forEach((r, t) => { for (let d = 0; d < D; d++) out[t * D + d] = flat[r * dims + d]; });
|
|
712
|
+
return out;
|
|
713
|
+
};
|
|
714
|
+
// ena_correlation(A, B) → [r_dim0, r_dim1] (col 0 of the dims×3 result).
|
|
715
|
+
const corr2 = (Aflat, Bflat, nRows) => {
|
|
716
|
+
const r = qe.ena_correlation(Array.from(Aflat), nRows, D,
|
|
717
|
+
Array.from(Bflat), nRows, D, 0.95);
|
|
718
|
+
const out = [];
|
|
719
|
+
for (let d = 0; d < D; d++) out.push(r.data[d * 3]);
|
|
720
|
+
return out;
|
|
721
|
+
};
|
|
722
|
+
const full = api.fit(rows, baseOpts);
|
|
723
|
+
const nUnits = full.nUnits;
|
|
724
|
+
const unitRows = Array.from({ length: nUnits }, (_, i) => i);
|
|
725
|
+
const gofOf = (mdl) => corr2(sub(mdl.points, unitRows), sub(mdl.model.centroids, unitRows), nUnits);
|
|
726
|
+
const gFull = gofOf(full);
|
|
727
|
+
const fullPts2 = sub(full.points, unitRows);
|
|
728
|
+
|
|
729
|
+
// All k-subsets of [0..m) as index arrays.
|
|
730
|
+
const combos = (n, k) => {
|
|
731
|
+
const res = [], cur = [];
|
|
732
|
+
const rec = (start) => {
|
|
733
|
+
if (cur.length === k) { res.push(cur.slice()); return; }
|
|
734
|
+
for (let i = start; i < n; i++) { cur.push(i); rec(i + 1); cur.pop(); }
|
|
735
|
+
};
|
|
736
|
+
rec(0);
|
|
737
|
+
return res;
|
|
738
|
+
};
|
|
739
|
+
|
|
740
|
+
let bestK = 0, bestVar = -Infinity, bestRemoved = [];
|
|
741
|
+
for (let k = 1; k <= rn; k++) {
|
|
742
|
+
for (const idxs of combos(m, k)) {
|
|
743
|
+
const removedSet = new Set(idxs);
|
|
744
|
+
const mask = [];
|
|
745
|
+
for (let i = 0; i < m; i++) {
|
|
746
|
+
const row = [];
|
|
747
|
+
for (let j = 0; j < m; j++) {
|
|
748
|
+
row.push((removedSet.has(i) || removedSet.has(j)) ? 0 : 1);
|
|
749
|
+
}
|
|
750
|
+
mask.push(row);
|
|
751
|
+
}
|
|
752
|
+
const red = api.fit(rows, { ...baseOpts, codeMask: mask });
|
|
753
|
+
|
|
754
|
+
// Gate 1 — goodness-of-fit ratio vs full, per dim.
|
|
755
|
+
const gRed = gofOf(red);
|
|
756
|
+
let pass = true;
|
|
757
|
+
for (let d = 0; d < D; d++) {
|
|
758
|
+
if (gFull[d] === 0 || gRed[d] / gFull[d] < threshold) { pass = false; break; }
|
|
759
|
+
}
|
|
760
|
+
if (!pass) continue;
|
|
761
|
+
|
|
762
|
+
// Gate 2 — reduced points AND retained-code nodes must each
|
|
763
|
+
// correlate >= threshold with the full model (per dim, with a
|
|
764
|
+
// per-dim sign flip that negates the node corr alongside it).
|
|
765
|
+
const retained = [];
|
|
766
|
+
for (let i = 0; i < m; i++) if (!removedSet.has(i)) retained.push(i);
|
|
767
|
+
let [pc1, pc2] = corr2(fullPts2, sub(red.points, unitRows), nUnits);
|
|
768
|
+
let [nc1, nc2] = corr2(sub(full.rotation.nodes, retained),
|
|
769
|
+
sub(red.rotation.nodes, retained), retained.length);
|
|
770
|
+
if (pc1 < 0) { pc1 = -pc1; nc1 = -nc1; }
|
|
771
|
+
if (pc2 < 0) { pc2 = -pc2; nc2 = -nc2; }
|
|
772
|
+
if (Math.min(pc1, pc2, nc1, nc2) < threshold) continue;
|
|
773
|
+
|
|
774
|
+
// Dim-1 variance proportion over the FULL rotation spectrum
|
|
775
|
+
// (= R's set$model$variance[1] = diag(var(points.rotated))/sum);
|
|
776
|
+
// NOT model.variance, which is the 2-dim projected ratio.
|
|
777
|
+
const ev = red.rotation.eigenvalues;
|
|
778
|
+
let evSum = 0; for (let i = 0; i < ev.length; i++) evSum += ev[i];
|
|
779
|
+
const vr1 = evSum > 0 ? ev[0] / evSum : 0;
|
|
780
|
+
// Prefer more codes removed; tie-break on higher dim-1 variance.
|
|
781
|
+
if (k > bestK || (k === bestK && vr1 > bestVar)) {
|
|
782
|
+
bestK = k; bestVar = vr1; bestRemoved = idxs.slice();
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
return {
|
|
787
|
+
removed: bestRemoved.map(i => codes[i]),
|
|
788
|
+
removedIndices: bestRemoved,
|
|
789
|
+
k: bestK,
|
|
790
|
+
variance: bestVar === -Infinity ? null : bestVar,
|
|
791
|
+
};
|
|
792
|
+
},
|
|
793
|
+
|
|
584
794
|
/**
|
|
585
795
|
* Helpers re-exported for consumers who want to build their own pipeline.
|
|
586
796
|
*/
|
|
587
797
|
defaultTensor,
|
|
798
|
+
|
|
799
|
+
/** The underlying libqe WASM module, for custom low-level pipelines. */
|
|
800
|
+
qe,
|
|
588
801
|
};
|
|
589
802
|
|
|
590
803
|
return api;
|
package/src/pipeline.js
CHANGED
|
@@ -116,64 +116,6 @@ export function spaceDistCorr(A, B, m, d, maxSampleSize = 100000, rand = Math.ra
|
|
|
116
116
|
return pearson(distA, distB);
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
-
// ── accumulation ─────────────────────────────────────────────────────────────
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Accumulate windowed co-occurrences for all units across all conversations.
|
|
123
|
-
*
|
|
124
|
-
* For each conversation group, runs qe.accumulate_stanza() on the conversation's
|
|
125
|
-
* rows to get per-row connection vectors, then folds each row's vector into its
|
|
126
|
-
* owning unit's running sum.
|
|
127
|
-
*
|
|
128
|
-
* @param {object} qe libqe WASM module
|
|
129
|
-
* @param {Float64Array} codeMatrix n_rows × n_codes, row-major
|
|
130
|
-
* @param {number} nRows
|
|
131
|
-
* @param {number} nCodes
|
|
132
|
-
* @param {number} nUnits
|
|
133
|
-
* @param {Int32Array} unitOf unit index per row
|
|
134
|
-
* @param {Map} convoGroups convoIdx → [rowIdx, ...]
|
|
135
|
-
* @param {number} windowSize backward window (rows)
|
|
136
|
-
* @param {boolean} binary binarise co-occurrences
|
|
137
|
-
*
|
|
138
|
-
* @returns {Float64Array} nUnits × nConnections, row-major
|
|
139
|
-
*/
|
|
140
|
-
export function accumulate(qe, codeMatrix, nRows, nCodes, nUnits,
|
|
141
|
-
unitOf, convoGroups, windowSize = 4, binary = true) {
|
|
142
|
-
const nConnections = qe.choose_two(nCodes);
|
|
143
|
-
const networks = new Float64Array(nUnits * nConnections);
|
|
144
|
-
|
|
145
|
-
for (const [, rowIndices] of convoGroups) {
|
|
146
|
-
const nConvo = rowIndices.length;
|
|
147
|
-
|
|
148
|
-
// Extract code rows for this conversation (contiguous sub-matrix)
|
|
149
|
-
const convoCodes = new Float64Array(nConvo * nCodes);
|
|
150
|
-
for (let r = 0; r < nConvo; r++) {
|
|
151
|
-
const src = rowIndices[r];
|
|
152
|
-
convoCodes.set(
|
|
153
|
-
codeMatrix.subarray(src * nCodes, src * nCodes + nCodes),
|
|
154
|
-
r * nCodes
|
|
155
|
-
);
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
// Windowed accumulation — returns per-row connection vectors
|
|
159
|
-
const stanza = qe.accumulate_stanza(
|
|
160
|
-
convoCodes, nConvo, nCodes, windowSize, 0, binary
|
|
161
|
-
);
|
|
162
|
-
// stanza.data: nConvo × nConnections, row-major
|
|
163
|
-
|
|
164
|
-
// Fold each row into its unit's accumulator
|
|
165
|
-
for (let r = 0; r < nConvo; r++) {
|
|
166
|
-
const unit = unitOf[rowIndices[r]];
|
|
167
|
-
const offset = r * nConnections;
|
|
168
|
-
for (let c = 0; c < nConnections; c++) {
|
|
169
|
-
networks[unit * nConnections + c] += stanza.data[offset + c];
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
return networks;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
119
|
// ── normalization (sphere norm) ───────────────────────────────────────────────
|
|
178
120
|
|
|
179
121
|
/**
|
package/src/tensor.js
CHANGED
|
@@ -119,9 +119,61 @@ export function inferFactorLevels(rows, factors) {
|
|
|
119
119
|
return levels;
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Weighted aggregation of a unit's per-response-row directed connection counts.
|
|
124
|
+
*
|
|
125
|
+
* Mirrors the kernel's aggregate_row_connections but applies a weight-model
|
|
126
|
+
* transform (R's weight.by) per LINE before the per-unit sum, so weight models
|
|
127
|
+
* behave identically on the tensor path and the windowed path. Non-binary by
|
|
128
|
+
* construction (the weight replaces the binarize step).
|
|
129
|
+
*
|
|
130
|
+
* unordered: fold each directed row (m + mᵀ, upper triangle, column-major —
|
|
131
|
+
* the same order as choose_two/connection_names), weight each
|
|
132
|
+
* folded cell, then sum across rows.
|
|
133
|
+
* ordered: weight each directed cell, then sum across rows.
|
|
134
|
+
*
|
|
135
|
+
* @param {Float64Array|number[]} data Row-major (nRccRows × nCodes²) counts.
|
|
136
|
+
* @param {number} nRccRows Number of response rows for this unit.
|
|
137
|
+
* @param {number} nCodes
|
|
138
|
+
* @param {boolean} ordered
|
|
139
|
+
* @param {(x:number)=>number} weightFn
|
|
140
|
+
* @returns {Float64Array} length nCodes² (ordered) or choose_two(nCodes).
|
|
141
|
+
*/
|
|
142
|
+
function aggregateRowConnectionsWeighted(data, nRccRows, nCodes, ordered, weightFn) {
|
|
143
|
+
const nSq = nCodes * nCodes;
|
|
144
|
+
if (ordered) {
|
|
145
|
+
const out = new Float64Array(nSq);
|
|
146
|
+
for (let r = 0; r < nRccRows; r++) {
|
|
147
|
+
const base = r * nSq;
|
|
148
|
+
for (let c = 0; c < nSq; c++) out[c] += weightFn(data[base + c]);
|
|
149
|
+
}
|
|
150
|
+
return out;
|
|
151
|
+
}
|
|
152
|
+
const nTri = (nCodes * (nCodes - 1)) / 2;
|
|
153
|
+
const out = new Float64Array(nTri);
|
|
154
|
+
for (let r = 0; r < nRccRows; r++) {
|
|
155
|
+
const base = r * nSq;
|
|
156
|
+
let k = 0;
|
|
157
|
+
for (let col = 1; col < nCodes; col++) {
|
|
158
|
+
for (let row = 0; row < col; row++) {
|
|
159
|
+
// fold: m(row,col) + m(col,row), column-major reshape of the row
|
|
160
|
+
const folded = data[base + col * nCodes + row] +
|
|
161
|
+
data[base + row * nCodes + col];
|
|
162
|
+
out[k] += weightFn(folded);
|
|
163
|
+
k++;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return out;
|
|
168
|
+
}
|
|
169
|
+
|
|
122
170
|
/**
|
|
123
171
|
* Accumulate tensor networks for all units across all conversations.
|
|
124
172
|
*
|
|
173
|
+
* Every accumulation — including plain windowed ENA, via defaultTensor(window)
|
|
174
|
+
* — runs through here, so the per-line finalisation (fold / binarize / weight)
|
|
175
|
+
* lives in exactly one place.
|
|
176
|
+
*
|
|
125
177
|
* @param {object} qe libqe WASM module
|
|
126
178
|
* @param {Object[]} rows Full dataset
|
|
127
179
|
* @param {Float64Array} codeMatrix n_rows × n_codes, row-major
|
|
@@ -132,11 +184,18 @@ export function inferFactorLevels(rows, factors) {
|
|
|
132
184
|
* @param {Map} convoGroups convoIdx → [rowIdx, ...]
|
|
133
185
|
* @param {object} tensorDef Tensor definition (see module docstring)
|
|
134
186
|
* @param {boolean} ordered true → directed (n²); false → undirected
|
|
187
|
+
* @param {boolean} [binary=true] binarize each folded row (unordered only)
|
|
188
|
+
* @param {((x:number)=>number)|null} [weightFn=null] per-line weight transform
|
|
135
189
|
*
|
|
136
|
-
* @returns {
|
|
190
|
+
* @returns {{ networks: Float64Array, rowConnectionCounts: Float64Array }}
|
|
191
|
+
* networks nUnits × nConnections, row-major
|
|
192
|
+
* rowConnectionCounts nRows × nConnections, row-major — each row's finalised
|
|
193
|
+
* (folded / binarized / weighted) connection vector;
|
|
194
|
+
* rows sum by unit to `networks`
|
|
137
195
|
*/
|
|
138
196
|
export function accumulateTensor(qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
139
|
-
unitOf, convoGroups, tensorDef, ordered = false
|
|
197
|
+
unitOf, convoGroups, tensorDef, ordered = false,
|
|
198
|
+
binary = true, weightFn = null) {
|
|
140
199
|
const {
|
|
141
200
|
dims,
|
|
142
201
|
dimsSender = [],
|
|
@@ -154,7 +213,8 @@ export function accumulateTensor(qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
|
154
213
|
? nCodes * nCodes
|
|
155
214
|
: qe.choose_two(nCodes);
|
|
156
215
|
|
|
157
|
-
const networks
|
|
216
|
+
const networks = new Float64Array(nUnits * nConnections);
|
|
217
|
+
const rowConnectionCounts = new Float64Array(nRows * nConnections);
|
|
158
218
|
|
|
159
219
|
const dimsArr = new Int32Array(dims);
|
|
160
220
|
const senderArr = new Int32Array(dimsSender);
|
|
@@ -191,6 +251,12 @@ export function accumulateTensor(qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
|
191
251
|
for (const [unit, localRows] of unitConvoRows) {
|
|
192
252
|
const unitRowsArr = new Int32Array(localRows);
|
|
193
253
|
|
|
254
|
+
// Always accumulate the DIRECTED per-response-row counts (ordered
|
|
255
|
+
// kernel), exactly as tma does — it calls apply_tensor with the
|
|
256
|
+
// default ordered=TRUE and defers the ordered-vs-unordered decision
|
|
257
|
+
// to aggregation. Passing the unordered flag here would make the
|
|
258
|
+
// kernel emit an already-symmetric matrix that the fold below would
|
|
259
|
+
// then double-count.
|
|
194
260
|
const result = qe.accumulate_tensor_unit(
|
|
195
261
|
tensorData, dimsArr,
|
|
196
262
|
senderArr, receiverArr, modeArr,
|
|
@@ -198,17 +264,33 @@ export function accumulateTensor(qe, rows, codeMatrix, nRows, nCodes, nUnits,
|
|
|
198
264
|
unitRowsArr,
|
|
199
265
|
convoCodes, nConvo, nCodes,
|
|
200
266
|
times,
|
|
201
|
-
|
|
267
|
+
true
|
|
202
268
|
);
|
|
203
269
|
|
|
204
|
-
//
|
|
205
|
-
|
|
206
|
-
|
|
270
|
+
// Finalise each raw per-response-row connection vector exactly as
|
|
271
|
+
// tma does in R: fold to the upper triangle and binarize per row
|
|
272
|
+
// (unordered), or keep the directed row (ordered). A weight model
|
|
273
|
+
// (R's weight.by) replaces the binarize step with its transform,
|
|
274
|
+
// applied per line before the per-unit sum. Unweighted rows go
|
|
275
|
+
// through the libqe kernel so fold/binarize semantics are shared
|
|
276
|
+
// across every binding. Row i of rcc is local row localRows[i].
|
|
277
|
+
const rcc = result.row_connection_counts;
|
|
278
|
+
for (let i = 0; i < rcc.rows; i++) {
|
|
279
|
+
const one = rcc.data.slice(i * rcc.cols, (i + 1) * rcc.cols);
|
|
280
|
+
const rowVec = weightFn
|
|
281
|
+
? aggregateRowConnectionsWeighted(one, 1, nCodes, ordered, weightFn)
|
|
282
|
+
: qe.aggregate_row_connections(one, 1, rcc.cols, nCodes, ordered, binary);
|
|
283
|
+
const rowOff = rowIndices[localRows[i]] * nConnections;
|
|
284
|
+
const unitOff = unit * nConnections;
|
|
285
|
+
for (let c = 0; c < nConnections; c++) {
|
|
286
|
+
rowConnectionCounts[rowOff + c] = rowVec[c];
|
|
287
|
+
networks[unitOff + c] += rowVec[c];
|
|
288
|
+
}
|
|
207
289
|
}
|
|
208
290
|
}
|
|
209
291
|
}
|
|
210
292
|
|
|
211
|
-
return networks;
|
|
293
|
+
return { networks, rowConnectionCounts };
|
|
212
294
|
}
|
|
213
295
|
|
|
214
296
|
/**
|