@datagrok/proteomics 1.0.1 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/CHANGELOG.md +52 -3
  2. package/CLAUDE.md +178 -0
  3. package/README.md +195 -3
  4. package/detectors.js +152 -9
  5. package/dist/package-test.js +1 -1744
  6. package/dist/package-test.js.map +1 -1
  7. package/dist/package.js +1 -146
  8. package/dist/package.js.map +1 -1
  9. package/docs/personas-and-capabilities.md +165 -0
  10. package/files/demo/README.md +264 -0
  11. package/files/demo/cptac-spike-in.txt +1571 -0
  12. package/files/demo/enrichment-demo.csv +120 -0
  13. package/files/demo/fragpipe-smoke-test.tsv +11 -0
  14. package/files/demo/proteinGroups.txt +28 -0
  15. package/files/demo/spectronaut-hye-candidates.tsv +94 -0
  16. package/files/demo/spectronaut-hye-demo.tsv +8761 -0
  17. package/files/demo/spectronaut-hye-mix.tsv +8761 -0
  18. package/files/demo/spectronaut-hye-precursor-golden.json +938 -0
  19. package/files/demo/spectronaut-hye-precursor-golden.tsv +235 -0
  20. package/files/demo/spectronaut-hye-precursor.tsv +493 -0
  21. package/images/enrichment-crosslink.png +0 -0
  22. package/images/enrichment-term-selected.png +0 -0
  23. package/images/hero.png +0 -0
  24. package/images/pipeline.svg +80 -0
  25. package/package.json +88 -63
  26. package/scripts/deqms_de.R +60 -0
  27. package/scripts/limma_de.R +42 -0
  28. package/scripts/vsn_normalize.R +19 -0
  29. package/src/analysis/differential-expression.ts +450 -0
  30. package/src/analysis/enrichment-export.ts +101 -0
  31. package/src/analysis/enrichment.ts +602 -0
  32. package/src/analysis/experiment-setup.ts +199 -0
  33. package/src/analysis/imputation.ts +407 -0
  34. package/src/analysis/log2-scale.ts +139 -0
  35. package/src/analysis/normalization.ts +255 -0
  36. package/src/analysis/pca.ts +254 -0
  37. package/src/analysis/spc-storage.ts +515 -0
  38. package/src/analysis/spc.ts +544 -0
  39. package/src/analysis/subcellular-location.ts +431 -0
  40. package/src/demo/enrichment-demo.ts +94 -0
  41. package/src/demo/proteomics-demo.ts +123 -0
  42. package/src/global.d.ts +15 -0
  43. package/src/menu.ts +133 -0
  44. package/src/package-api.ts +136 -14
  45. package/src/package-test.ts +45 -20
  46. package/src/package.g.ts +161 -0
  47. package/src/package.ts +1029 -17
  48. package/src/panels/protein-focus.ts +63 -0
  49. package/src/panels/published-analysis-panel.ts +151 -0
  50. package/src/panels/uniprot-panel.ts +349 -0
  51. package/src/parsers/fragpipe-parser.ts +200 -0
  52. package/src/parsers/generic-parser.ts +197 -0
  53. package/src/parsers/maxquant-parser.ts +162 -0
  54. package/src/parsers/shared-utils.ts +163 -0
  55. package/src/parsers/spectronaut-candidates-parser.ts +307 -0
  56. package/src/parsers/spectronaut-parser.ts +604 -0
  57. package/src/publishing/assert-published-shape.ts +260 -0
  58. package/src/publishing/post-open-recovery.ts +104 -0
  59. package/src/publishing/publish-project.ts +515 -0
  60. package/src/publishing/publish-settings.ts +59 -0
  61. package/src/publishing/publish-state.ts +316 -0
  62. package/src/publishing/share-dialog.ts +171 -0
  63. package/src/publishing/trim-dataframe.ts +247 -0
  64. package/src/tests/analysis.ts +658 -0
  65. package/src/tests/enrichment-export.ts +61 -0
  66. package/src/tests/enrichment-visualization.ts +340 -0
  67. package/src/tests/enrichment.ts +224 -0
  68. package/src/tests/fragpipe-e2e.ts +74 -0
  69. package/src/tests/fragpipe-parser.ts +147 -0
  70. package/src/tests/gene-label-resolver.ts +387 -0
  71. package/src/tests/generic-parser.ts +152 -0
  72. package/src/tests/group-mean-correlation.ts +139 -0
  73. package/src/tests/log2-scale.ts +93 -0
  74. package/src/tests/organisms.ts +56 -0
  75. package/src/tests/parsers.ts +182 -0
  76. package/src/tests/publish-roundtrip.ts +584 -0
  77. package/src/tests/publish-spike.ts +377 -0
  78. package/src/tests/qc-dashboard.ts +210 -0
  79. package/src/tests/smart-pathway-filter.ts +193 -0
  80. package/src/tests/spc-formula-lines-spike.ts +129 -0
  81. package/src/tests/spc.ts +640 -0
  82. package/src/tests/spectronaut-candidates-e2e.ts +140 -0
  83. package/src/tests/spectronaut-candidates-parser.ts +398 -0
  84. package/src/tests/spectronaut-parser.ts +668 -0
  85. package/src/tests/subcellular-location.ts +361 -0
  86. package/src/tests/uniprot-panel.ts +202 -0
  87. package/src/tests/volcano.ts +603 -0
  88. package/src/utils/column-detection.ts +28 -0
  89. package/src/utils/gene-label-resolver.ts +443 -0
  90. package/src/utils/organisms.ts +82 -0
  91. package/src/utils/proteomics-types.ts +30 -0
  92. package/src/viewers/enrichment-viewers.ts +274 -0
  93. package/src/viewers/group-mean-correlation.ts +218 -0
  94. package/src/viewers/heatmap.ts +168 -0
  95. package/src/viewers/pca-plot.ts +169 -0
  96. package/src/viewers/qc-computations.ts +266 -0
  97. package/src/viewers/qc-dashboard.ts +176 -0
  98. package/src/viewers/spc-dashboard.ts +755 -0
  99. package/src/viewers/volcano.ts +690 -0
  100. package/test-console-output-1.log +2073 -0
  101. package/test-record-1.mp4 +0 -0
  102. package/tools/derive-precursor-golden-sidecar.mjs +81 -0
  103. package/tools/generate-enrichment-fixture.sh +160 -0
  104. package/tools/generate-spectronaut-candidates-fixture.mjs +212 -0
  105. package/tools/generate-spectronaut-precursor-fixture.mjs +128 -0
  106. package/tools/spectronaut-aggregate.sh +46 -0
  107. package/tools/spectronaut-aggregate.sql +80 -0
  108. package/tsconfig.json +18 -71
  109. package/webpack.config.js +86 -45
  110. package/.eslintignore +0 -1
  111. package/.eslintrc.json +0 -56
  112. package/LICENSE +0 -674
  113. package/package.png +0 -0
  114. package/scripts/number_antibody.py +0 -190
  115. package/scripts/number_antibody_abnumber.py +0 -177
  116. package/scripts/number_antibody_anarci.py +0 -200
@@ -0,0 +1,602 @@
1
+ import * as grok from 'datagrok-api/grok';
2
+ import * as ui from 'datagrok-api/ui';
3
+ import * as DG from 'datagrok-api/dg';
4
+ import {parseAccession} from '../panels/uniprot-panel';
5
+ import {findProteomicsColumns} from '../utils/column-detection';
6
+ import {SEMTYPE, DEFAULT_FC_THRESHOLD, DEFAULT_P_THRESHOLD} from '../utils/proteomics-types';
7
+ import {openEnrichmentVisualization} from '../viewers/enrichment-viewers';
8
+ import {requireDifferentialExpression} from './differential-expression';
9
+ import {getOrganism, setOrganism} from './experiment-setup';
10
+ import {ORGANISM_LIST, detectOrganismCode, organismDisplayForCode} from '../utils/organisms';
11
+
12
+ // ORGANISM_LIST now lives in utils/organisms (single source); re-exported here so
13
+ // existing importers (and the enrichment tests) keep resolving it from this module.
14
+ export {ORGANISM_LIST};
15
+
16
+ // --- Constants ---
17
+
18
+ const GPROFILER_BASE = 'https://biit.cs.ut.ee/gprofiler';
19
+
20
+ // --- Types ---
21
+
22
+ export interface ConvertResult {
23
+ incoming: string;
24
+ converted: string;
25
+ name: string;
26
+ description: string;
27
+ namespaces: string;
28
+ n_incoming: number;
29
+ n_converted: number;
30
+ }
31
+
32
+ export interface GostResult {
33
+ native: string;
34
+ name: string;
35
+ source: string;
36
+ p_value: number;
37
+ significant: boolean;
38
+ term_size: number;
39
+ query_size: number;
40
+ intersection_size: number;
41
+ effective_domain_size: number;
42
+ precision: number;
43
+ recall: number;
44
+ intersections: string[][];
45
+ }
46
+
47
+ // --- API Client ---
48
+
49
+ async function fetchWithTimeout(
50
+ url: string, options: RequestInit, timeoutMs: number = 30000,
51
+ ): Promise<Response> {
52
+ // External URLs go through grok.dapi.fetchProxy to avoid browser CORS.
53
+ // Datagrok's proxy does not currently honor AbortSignal, so we race the
54
+ // request against a timeout promise to enforce the bound.
55
+ const proxied = grok.dapi.fetchProxy(url, options);
56
+ const timeout = new Promise<never>((_, reject) =>
57
+ setTimeout(() => reject(new Error('g:Profiler API request timed out. The service may be temporarily unavailable.')), timeoutMs),
58
+ );
59
+ const resp = await Promise.race([proxied, timeout]);
60
+ if (!resp.ok)
61
+ throw new Error(`g:Profiler API returned status ${resp.status}`);
62
+ return resp;
63
+ }
64
+
65
+ export async function gConvert(
66
+ accessions: string[],
67
+ organism: string = 'hsapiens',
68
+ ): Promise<ConvertResult[]> {
69
+ const resp = await fetchWithTimeout(`${GPROFILER_BASE}/api/convert/convert/`, {
70
+ method: 'POST',
71
+ headers: {'Content-Type': 'application/json'},
72
+ body: JSON.stringify({
73
+ organism: organism,
74
+ query: accessions,
75
+ target: 'ENSG',
76
+ numeric_ns: '',
77
+ output: 'json',
78
+ }),
79
+ });
80
+ const data = await resp.json();
81
+ return data.result ?? [];
82
+ }
83
+
84
+ export async function gGOSt(
85
+ queryGenes: string[],
86
+ backgroundGenes: string[],
87
+ organism: string,
88
+ sources: string[],
89
+ threshold: number = DEFAULT_P_THRESHOLD,
90
+ ): Promise<GostResult[]> {
91
+ const resp = await fetchWithTimeout(`${GPROFILER_BASE}/api/gost/profile/`, {
92
+ method: 'POST',
93
+ headers: {'Content-Type': 'application/json'},
94
+ body: JSON.stringify({
95
+ organism: organism,
96
+ query: queryGenes,
97
+ sources: sources,
98
+ user_threshold: threshold,
99
+ significance_threshold_method: 'fdr',
100
+ domain_scope: 'custom',
101
+ background: backgroundGenes,
102
+ all_results: true,
103
+ ordered: false,
104
+ no_evidences: false,
105
+ combined: false,
106
+ measure_underrepresentation: false,
107
+ no_iea: false,
108
+ numeric_ns: '',
109
+ output: 'json',
110
+ }),
111
+ });
112
+ const data = await resp.json();
113
+ // g:GOSt nests results: data.result is an array of per-query result objects,
114
+ // each containing a `.result` array of term results.
115
+ if (!data.result)
116
+ return [];
117
+ // Handle array of query results (standard structure)
118
+ if (Array.isArray(data.result) && data.result.length > 0 && data.result[0]?.result)
119
+ return data.result[0].result as GostResult[];
120
+ // Fallback: if data.result is directly the results array
121
+ if (Array.isArray(data.result))
122
+ return data.result as GostResult[];
123
+ return [];
124
+ }
125
+
126
+ // --- Smart Pathway Filter (R5/D-13/D-14) ---
127
+
128
+ // LOCKED CLIENT CONTRACT — port verbatim from
129
+ // ~/Downloads/ck/CKomics_tool2.py:4685-4736 (apply_smart_pathway_filtering).
130
+ // Do not re-derive heuristics; any change requires a CONTEXT update.
131
+
132
+ export const GENERIC_PARENT_TERMS = [
133
+ 'localization', 'cellular component organization', 'transport',
134
+ 'cellular process', 'biological process', 'metabolic process',
135
+ ] as const;
136
+
137
+ export const SPECIFIC_CHILD_TERMS = ['actin', 'vesicle', 'endocytosis', 'cytoskeleton'] as const;
138
+
139
+ export interface SmartFilterStats {
140
+ total: number;
141
+ kept: number;
142
+ droppedParents: number;
143
+ cappedAtN: number;
144
+ }
145
+
146
+ /** Verbatim port of CK-omics apply_smart_pathway_filtering (lines 4685-4736).
147
+ *
148
+ * Algorithm:
149
+ * 1. Partition into GO:BP rows and `other_data` (everything else).
150
+ * 2. Sort GO:BP by p_value ASCENDING.
151
+ * 3. Walk GO:BP in sorted order: drop a row when its name contains any
152
+ * GENERIC_PARENT term AND the already-kept list already contains a
153
+ * SPECIFIC_CHILD-term row. Otherwise keep it. Stop at `maxPerSource`.
154
+ * 4. Non-GO:BP rows are sorted by p_value ASC and `.head(maxPerSource)` —
155
+ * a COMBINED cap across all non-GO:BP sources, not per-source
156
+ * (Assumption A4; CK-omics line 4731 explicitly uses combined .head). */
157
+ export function applySmartPathwayFilter(
158
+ results: GostResult[], maxPerSource = 15,
159
+ ): {kept: GostResult[]; stats: SmartFilterStats} {
160
+ if (results.length === 0) {
161
+ return {kept: [], stats: {total: 0, kept: 0, droppedParents: 0, cappedAtN: maxPerSource}};
162
+ }
163
+
164
+ const goBp = results.filter((r) => r.source === 'GO:BP');
165
+ const other = results.filter((r) => r.source !== 'GO:BP');
166
+
167
+ let droppedParents = 0;
168
+ const filteredGoBp: GostResult[] = [];
169
+ if (goBp.length > 0) {
170
+ const sorted = [...goBp].sort((a, b) => a.p_value - b.p_value);
171
+ for (const r of sorted) {
172
+ const name = (r.name ?? '').toLowerCase();
173
+ const isGeneric = GENERIC_PARENT_TERMS.some((g) => name.includes(g));
174
+ if (isGeneric) {
175
+ const hasSpecificChild = filteredGoBp.some((existing) =>
176
+ SPECIFIC_CHILD_TERMS.some((s) => (existing.name ?? '').toLowerCase().includes(s)));
177
+ if (hasSpecificChild) { droppedParents++; continue; }
178
+ }
179
+ filteredGoBp.push(r);
180
+ if (filteredGoBp.length >= maxPerSource) break;
181
+ }
182
+ }
183
+
184
+ const otherSorted = [...other].sort((a, b) => a.p_value - b.p_value);
185
+ const otherCapped = otherSorted.slice(0, maxPerSource);
186
+
187
+ const kept = [...filteredGoBp, ...otherCapped].sort((a, b) => a.p_value - b.p_value);
188
+ return {
189
+ kept,
190
+ stats: {
191
+ total: results.length,
192
+ kept: kept.length,
193
+ droppedParents,
194
+ cappedAtN: maxPerSource,
195
+ },
196
+ };
197
+ }
198
+
199
+ // --- Result Builder ---
200
+
201
+ export function buildEnrichmentDf(
202
+ results: GostResult[],
203
+ queryGenes: string[],
204
+ pThreshold: number = DEFAULT_P_THRESHOLD,
205
+ direction?: 'Up' | 'Down',
206
+ ): DG.DataFrame {
207
+ const n = results.length;
208
+ const df = DG.DataFrame.create(n);
209
+ df.name = 'Enrichment Results';
210
+
211
+ // Pre-derive the comma-joined member-gene string per row so the column init
212
+ // doesn't repeat the intersections-array walk per cell.
213
+ const memberGeneStrs = results.map((r) => {
214
+ if (!r.intersections) return '';
215
+ const members: string[] = [];
216
+ const lim = Math.min(queryGenes.length, r.intersections.length);
217
+ for (let j = 0; j < lim; j++) {
218
+ if (r.intersections[j] && r.intersections[j].length > 0)
219
+ members.push(queryGenes[j]);
220
+ }
221
+ return members.join(', ');
222
+ });
223
+
224
+ const sourceCol = df.columns.addNewString('Source');
225
+ const termIdCol = df.columns.addNewString('Term ID');
226
+ const termNameCol = df.columns.addNewString('Term Name');
227
+ // g:GOSt with significance_threshold_method='fdr' returns the FDR-corrected
228
+ // p-value (and no separate raw p-value) — we expose it under the FDR column
229
+ // to keep semantics honest. Downstream consumers should read FDR.
230
+ const fdrCol = df.columns.addNewFloat('FDR');
231
+ const geneCountCol = df.columns.addNewInt('Gene Count');
232
+ const geneRatioCol = df.columns.addNewFloat('Gene Ratio');
233
+ const intersectionCol = df.columns.addNewString('Intersection');
234
+
235
+ sourceCol.init((i) => results[i].source);
236
+ termIdCol.init((i) => results[i].native);
237
+ termNameCol.init((i) => results[i].name);
238
+ fdrCol.init((i) => results[i].p_value);
239
+ geneCountCol.init((i) => results[i].intersection_size);
240
+ geneRatioCol.init((i) => results[i].precision);
241
+ intersectionCol.init((i) => memberGeneStrs[i]);
242
+
243
+ const fdrRaw = fdrCol.getRawData() as Float32Array | Float64Array;
244
+ const sigCol = df.columns.addNewBool('Significant');
245
+ sigCol.init((i) => fdrRaw[i] !== DG.FLOAT_NULL && fdrRaw[i] <= pThreshold);
246
+
247
+ // Optional directional label (R2 up/down split). Categorical, bulk-init
248
+ // (memory feedback_dg_column_bulk_init — never per-row set).
249
+ if (direction) {
250
+ const directionCol = df.columns.addNewString('Direction');
251
+ directionCol.init(() => direction);
252
+ }
253
+
254
+ // Color coding on FDR column
255
+ fdrCol.setTag('color-coding-type', 'Linear');
256
+ fdrCol.setTag('color-coding-linear', '{"0":"#2ecc71","0.05":"#f39c12","1":"#e74c3c"}');
257
+
258
+ df.setTag('proteomics.enrichment', 'true');
259
+ return df;
260
+ }
261
+
262
+ /**
263
+ * Splits the detected genes into up/down significant query sets and the shared
264
+ * all-detected background. Ported from CK-omics run_gprofiler_analysis: strict
265
+ * `fc > 0` / `fc < 0`; background is every detected gene (used by BOTH
266
+ * directional g:Profiler calls). Null fc/p rows count toward background only.
267
+ */
268
+ export function splitGenesByDirection(
269
+ geneForRow: Map<number, string>,
270
+ fcRaw: Float32Array | Float64Array,
271
+ pRaw: Float32Array | Float64Array,
272
+ fcThreshold: number,
273
+ pThreshold: number,
274
+ ): {upGenes: string[]; downGenes: string[]; background: string[]} {
275
+ const up: Set<string> = new Set();
276
+ const down: Set<string> = new Set();
277
+ const background: Set<string> = new Set();
278
+ for (const [row, gene] of geneForRow) {
279
+ background.add(gene);
280
+ const fc = fcRaw[row];
281
+ const adjP = pRaw[row];
282
+ if (fc !== DG.FLOAT_NULL && adjP !== DG.FLOAT_NULL &&
283
+ adjP <= pThreshold && Math.abs(fc) >= fcThreshold) {
284
+ if (fc > 0) up.add(gene);
285
+ else if (fc < 0) down.add(gene);
286
+ }
287
+ }
288
+ return {upGenes: [...up], downGenes: [...down], background: [...background]};
289
+ }
290
+
291
+ // --- Significant Protein Counter ---
292
+
293
+ export function countSignificantProteins(
294
+ df: DG.DataFrame,
295
+ fcThreshold: number,
296
+ pThreshold: number,
297
+ log2fcCol: DG.Column,
298
+ pValCol: DG.Column,
299
+ ): {significant: number; total: number} {
300
+ // Bulk-read via getRawData so we don't pay a 3× JS/native bridge hop per row.
301
+ const n = df.rowCount;
302
+ const fcRaw = log2fcCol.getRawData() as Float32Array | Float64Array;
303
+ const pRaw = pValCol.getRawData() as Float32Array | Float64Array;
304
+ let significant = 0;
305
+ for (let i = 0; i < n; i++) {
306
+ const fc = fcRaw[i];
307
+ const p = pRaw[i];
308
+ if (fc === DG.FLOAT_NULL || p === DG.FLOAT_NULL) continue;
309
+ if (Math.abs(fc) >= fcThreshold && p <= pThreshold)
310
+ significant++;
311
+ }
312
+ return {significant, total: n};
313
+ }
314
+
315
+ // --- Pipeline Orchestrator ---
316
+
317
+ export async function runEnrichmentPipeline(
318
+ df: DG.DataFrame,
319
+ fcThreshold: number,
320
+ pThreshold: number,
321
+ organismCode: string,
322
+ sources: string[],
323
+ smartFilterEnabled: boolean = true,
324
+ maxTermsPerSource: number = 15,
325
+ ): Promise<{enrichmentDf: DG.DataFrame; mapped: number; total: number; unmapped: number}> {
326
+ const cols = findProteomicsColumns(df);
327
+ if (!cols.proteinId)
328
+ throw new Error('No protein ID column found');
329
+ if (!cols.log2fc || !cols.pValue)
330
+ throw new Error('No log2FC or adjusted p-value columns found. Run Differential Expression first.');
331
+
332
+ // Step 1: Get gene symbols (existing or mapped)
333
+ let geneForRow: Map<number, string>;
334
+ let mapped = 0;
335
+ let total = 0;
336
+ let unmapped = 0;
337
+
338
+ if (cols.geneName) {
339
+ // Use existing gene name column. Truthiness check covers null/empty;
340
+ // skipping the redundant isNone() halves the bridge hops.
341
+ geneForRow = new Map();
342
+ const geneNameCol = cols.geneName;
343
+ for (let i = 0; i < df.rowCount; i++) {
344
+ const val = geneNameCol.get(i) as string | null;
345
+ if (val) geneForRow.set(i, val);
346
+ }
347
+ mapped = geneForRow.size;
348
+ total = df.rowCount;
349
+ unmapped = total - mapped;
350
+ } else {
351
+ // Extract clean accessions and map via g:Convert
352
+ const accToRows = new Map<string, number[]>();
353
+ for (let i = 0; i < df.rowCount; i++) {
354
+ const raw = cols.proteinId!.get(i) as string;
355
+ const acc = parseAccession(raw);
356
+ if (acc) {
357
+ if (!accToRows.has(acc))
358
+ accToRows.set(acc, []);
359
+ accToRows.get(acc)!.push(i);
360
+ }
361
+ }
362
+
363
+ const uniqueAccessions = [...accToRows.keys()];
364
+ const convertResults = await gConvert(uniqueAccessions, organismCode);
365
+
366
+ // Build accession -> gene symbol map (first match per accession)
367
+ const accToGene = new Map<string, string>();
368
+ for (const r of convertResults) {
369
+ if (r.incoming && r.name && r.name !== 'N/A' && !accToGene.has(r.incoming))
370
+ accToGene.set(r.incoming, r.name);
371
+ }
372
+
373
+ // Map rows to gene symbols
374
+ geneForRow = new Map();
375
+ for (const [acc, rows] of accToRows) {
376
+ const gene = accToGene.get(acc);
377
+ if (gene) {
378
+ for (const row of rows)
379
+ geneForRow.set(row, gene);
380
+ }
381
+ }
382
+
383
+ // Add gene symbol column to main table. Materialize the row->gene array
384
+ // first, then bulk-init the column to skip per-row set() bridge hops.
385
+ const geneArr: string[] = new Array(df.rowCount).fill('');
386
+ for (const [row, gene] of geneForRow)
387
+ geneArr[row] = gene;
388
+
389
+ const geneCol = df.columns.addNewString('Gene Symbol (mapped)');
390
+ geneCol.semType = SEMTYPE.GENE_SYMBOL;
391
+ geneCol.init((i) => geneArr[i]);
392
+
393
+ mapped = accToGene.size;
394
+ total = uniqueAccessions.length;
395
+ unmapped = total - mapped;
396
+ }
397
+
398
+ // Step 2: Split significant genes by fc sign; background = all detected,
399
+ // shared by both directional queries (CK-omics run_gprofiler_analysis).
400
+ // Bulk-read the numeric columns once via getRawData (no per-row bridge hops).
401
+ const fcRaw = cols.log2fc.getRawData() as Float32Array | Float64Array;
402
+ const pRaw = cols.pValue.getRawData() as Float32Array | Float64Array;
403
+ const {upGenes, downGenes, background: bgArray} =
404
+ splitGenesByDirection(geneForRow, fcRaw, pRaw, fcThreshold, pThreshold);
405
+
406
+ if (upGenes.length === 0 && downGenes.length === 0)
407
+ throw new Error('No significant proteins found with the given thresholds');
408
+
409
+ // Step 3: One g:GOSt call per non-empty direction, identical background.
410
+ // A direction with zero genes is skipped (no empty-query request).
411
+ // Pitfall 8: when smart filter is enabled, run it BEFORE buildEnrichmentDf so
412
+ // the per-row Intersection strings are built from the kept-set indices and
413
+ // never get reindexed after the fact.
414
+ const directionDfs: DG.DataFrame[] = [];
415
+ let upStats: SmartFilterStats | null = null;
416
+ let downStats: SmartFilterStats | null = null;
417
+ if (upGenes.length > 0) {
418
+ let upResults = await gGOSt(upGenes, bgArray, organismCode, sources, pThreshold);
419
+ if (smartFilterEnabled) {
420
+ const filtered = applySmartPathwayFilter(upResults, maxTermsPerSource);
421
+ upResults = filtered.kept;
422
+ upStats = filtered.stats;
423
+ }
424
+ directionDfs.push(buildEnrichmentDf(upResults, upGenes, pThreshold, 'Up'));
425
+ }
426
+ if (downGenes.length > 0) {
427
+ let downResults = await gGOSt(downGenes, bgArray, organismCode, sources, pThreshold);
428
+ if (smartFilterEnabled) {
429
+ const filtered = applySmartPathwayFilter(downResults, maxTermsPerSource);
430
+ downResults = filtered.kept;
431
+ downStats = filtered.stats;
432
+ }
433
+ directionDfs.push(buildEnrichmentDf(downResults, downGenes, pThreshold, 'Down'));
434
+ }
435
+
436
+ // Step 4: Merge directional results into one DataFrame. Phase-9 column shape
437
+ // is intact (Direction is just an extra categorical). append() returns a NEW
438
+ // frame and may drop frame/column metadata, so re-apply the enrichment tag,
439
+ // name, and FDR color-coding the viewers/cross-link key on.
440
+ let enrichmentDf = directionDfs[0];
441
+ for (let i = 1; i < directionDfs.length; i++)
442
+ enrichmentDf = enrichmentDf.append(directionDfs[i]);
443
+ // Name the result after its source table (e.g. "MyStudy — Enrichment") so it's
444
+ // distinguishable when several analyses are open — mirrors the "<source> — QC"
445
+ // convention. Falls back to the generic name if the source is unnamed.
446
+ enrichmentDf.name = df.name ? `${df.name} — Enrichment` : 'Enrichment Results';
447
+ enrichmentDf.setTag('proteomics.enrichment', 'true');
448
+ const fdrCol = enrichmentDf.col('FDR');
449
+ if (fdrCol) {
450
+ fdrCol.setTag('color-coding-type', 'Linear');
451
+ fdrCol.setTag('color-coding-linear', '{"0":"#2ecc71","0.05":"#f39c12","1":"#e74c3c"}');
452
+ }
453
+
454
+ // D-14 — set the smart-filter banner tags only when the filter actually
455
+ // dropped rows (kept < total). Banner is suppressed when the filter was off
456
+ // OR when kept == total. Tag names mirror UI-SPEC §"Enrichment table:
457
+ // smart pathway filter banner".
458
+ if (smartFilterEnabled) {
459
+ const totalInput = (upStats?.total ?? 0) + (downStats?.total ?? 0);
460
+ const totalKept = (upStats?.kept ?? 0) + (downStats?.kept ?? 0);
461
+ const totalDropped = (upStats?.droppedParents ?? 0) + (downStats?.droppedParents ?? 0);
462
+ if (totalKept < totalInput) {
463
+ enrichmentDf.setTag('proteomics.enrichment_smart_filtered', 'true');
464
+ enrichmentDf.setTag('proteomics.enrichment_smart_filtered_kept', String(totalKept));
465
+ enrichmentDf.setTag('proteomics.enrichment_smart_filtered_total', String(totalInput));
466
+ enrichmentDf.setTag('proteomics.enrichment_smart_filtered_dropped_parents', String(totalDropped));
467
+ enrichmentDf.setTag('proteomics.enrichment_smart_filtered_cap', String(maxTermsPerSource));
468
+ }
469
+ }
470
+
471
+ return {enrichmentDf, mapped, total, unmapped};
472
+ }
473
+
474
+ // --- Dialog ---
475
+
476
+ export function showEnrichmentDialog(df: DG.DataFrame): void {
477
+ if (!requireDifferentialExpression(df, 'Run Differential Expression first')) return;
478
+
479
+ const cols = findProteomicsColumns(df);
480
+ if (!cols.log2fc || !cols.pValue) {
481
+ grok.shell.warning('No log2FC or p-value columns found');
482
+ return;
483
+ }
484
+
485
+ const fcInput = ui.input.float('|log2FC| threshold', {value: DEFAULT_FC_THRESHOLD});
486
+ fcInput.setTooltip('Minimum absolute fold change for significance');
487
+
488
+ const pInput = ui.input.float('Adj. p-value threshold', {value: DEFAULT_P_THRESHOLD});
489
+ pInput.setTooltip('Maximum adjusted p-value for significance');
490
+
491
+ // Seed from the chosen/persisted organism, else auto-detect from the data's
492
+ // organism column (covers the Candidates path, which skips Annotate), else human.
493
+ // Sticky + shared with the subcellular-location fetch via the proteomics.organism tag.
494
+ const seededDisplay = organismDisplayForCode(getOrganism(df) ?? detectOrganismCode(df));
495
+ const organismInput = ui.input.choice('Organism', {
496
+ value: (seededDisplay ?? 'Homo sapiens (Human)') as string,
497
+ items: ORGANISM_LIST.map((o) => o.display) as unknown as string[],
498
+ nullable: false,
499
+ });
500
+
501
+ // Enrichment source checkboxes
502
+ const goBpInput = ui.input.bool('GO: Biological Process', {value: true});
503
+ const goMfInput = ui.input.bool('GO: Molecular Function', {value: true});
504
+ const goCcInput = ui.input.bool('GO: Cellular Component', {value: true});
505
+ const keggInput = ui.input.bool('KEGG Pathways', {value: true});
506
+ const reactomeInput = ui.input.bool('Reactome Pathways', {value: true});
507
+ const wpInput = ui.input.bool('WikiPathways', {value: true});
508
+
509
+ // D-14 — smart pathway filter checkbox, default checked.
510
+ const smartFilterInput = ui.input.bool('Apply smart pathway filter', {value: true});
511
+ smartFilterInput.setTooltip(
512
+ 'Drops generic GO parent terms when more specific child terms are present; ' +
513
+ 'caps each source at top-N by FDR. Recommended for cleaner results; ' +
514
+ 'uncheck for raw g:Profiler output.');
515
+
516
+ // M3 — the per-source top-N cap the smart filter applies (was hardcoded 15).
517
+ // Only meaningful when the smart filter is on; greyed out otherwise.
518
+ const maxTermsInput = ui.input.int('Max terms per source', {value: 15, min: 1});
519
+ maxTermsInput.setTooltip(
520
+ 'Upper bound on terms kept per enrichment source (top-N by FDR). ' +
521
+ 'Applies only when the smart pathway filter is on.');
522
+ const syncMaxTermsEnabled = () => maxTermsInput.enabled = smartFilterInput.value ?? true;
523
+ smartFilterInput.onChanged.subscribe(syncMaxTermsEnabled);
524
+ syncMaxTermsEnabled();
525
+
526
+ // Live count of significant proteins
527
+ const countDiv = ui.divText('');
528
+ const updateCount = () => {
529
+ const fc = fcInput.value ?? DEFAULT_FC_THRESHOLD;
530
+ const p = pInput.value ?? DEFAULT_P_THRESHOLD;
531
+ const result = countSignificantProteins(df, fc, p, cols.log2fc!, cols.pValue!);
532
+ const totalFmt = result.total.toLocaleString();
533
+ countDiv.textContent =
534
+ `${result.significant} of ${totalFmt} proteins are significant with these thresholds`;
535
+ };
536
+ updateCount();
537
+ fcInput.onChanged.subscribe(updateCount);
538
+ pInput.onChanged.subscribe(updateCount);
539
+
540
+ ui.dialog('Enrichment Analysis')
541
+ .add(countDiv)
542
+ .add(fcInput)
543
+ .add(pInput)
544
+ .add(organismInput)
545
+ .add(goBpInput)
546
+ .add(goMfInput)
547
+ .add(goCcInput)
548
+ .add(keggInput)
549
+ .add(reactomeInput)
550
+ .add(wpInput)
551
+ .add(smartFilterInput)
552
+ .add(maxTermsInput)
553
+ .onOK(async () => {
554
+ const pi = DG.TaskBarProgressIndicator.create('Running enrichment analysis...');
555
+ try {
556
+ const fc = fcInput.value ?? DEFAULT_FC_THRESHOLD;
557
+ const p = pInput.value ?? DEFAULT_P_THRESHOLD;
558
+
559
+ // Resolve organism code from display name
560
+ const selectedDisplay = organismInput.value;
561
+ const organism = ORGANISM_LIST.find((o) => o.display === selectedDisplay);
562
+ const organismCode = organism?.code ?? 'hsapiens';
563
+ // Persist so the subcellular-location fetch (and next enrichment run)
564
+ // narrow to the same species.
565
+ setOrganism(df, organismCode);
566
+
567
+ // Build sources array from checkbox states
568
+ const selectedSources: string[] = [];
569
+ if (goBpInput.value) selectedSources.push('GO:BP');
570
+ if (goMfInput.value) selectedSources.push('GO:MF');
571
+ if (goCcInput.value) selectedSources.push('GO:CC');
572
+ if (keggInput.value) selectedSources.push('KEGG');
573
+ if (reactomeInput.value) selectedSources.push('REAC');
574
+ if (wpInput.value) selectedSources.push('WP');
575
+
576
+ if (selectedSources.length === 0) {
577
+ grok.shell.warning('Please select at least one enrichment source');
578
+ return;
579
+ }
580
+
581
+ const result = await runEnrichmentPipeline(
582
+ df, fc, p, organismCode, selectedSources, smartFilterInput.value ?? true,
583
+ maxTermsInput.value ?? 15);
584
+
585
+ // Show mapping stats
586
+ const pct = result.total > 0 ? (result.mapped / result.total * 100).toFixed(1) : '0.0';
587
+ grok.shell.info(
588
+ `${result.mapped}/${result.total} proteins mapped (${pct}%). ${result.unmapped} unmapped.`,
589
+ );
590
+
591
+ // openEnrichmentVisualization creates (or reuses) the enrichment results
592
+ // view itself — adding one here too spawned a duplicate "Enrichment
593
+ // Results (2)" tab every run.
594
+ openEnrichmentVisualization(result.enrichmentDf, df);
595
+ } catch (e: any) {
596
+ grok.shell.error(`Enrichment analysis failed: ${e.message}`);
597
+ } finally {
598
+ pi.close();
599
+ }
600
+ })
601
+ .show();
602
+ }