@datagrok/proteomics 1.0.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -3
- package/CLAUDE.md +178 -0
- package/README.md +195 -3
- package/detectors.js +152 -9
- package/dist/package-test.js +1 -1744
- package/dist/package-test.js.map +1 -1
- package/dist/package.js +1 -146
- package/dist/package.js.map +1 -1
- package/docs/personas-and-capabilities.md +165 -0
- package/files/demo/README.md +264 -0
- package/files/demo/cptac-spike-in.txt +1571 -0
- package/files/demo/enrichment-demo.csv +120 -0
- package/files/demo/fragpipe-smoke-test.tsv +11 -0
- package/files/demo/proteinGroups.txt +28 -0
- package/files/demo/spectronaut-hye-candidates.tsv +94 -0
- package/files/demo/spectronaut-hye-demo.tsv +8761 -0
- package/files/demo/spectronaut-hye-mix.tsv +8761 -0
- package/files/demo/spectronaut-hye-precursor-golden.json +938 -0
- package/files/demo/spectronaut-hye-precursor-golden.tsv +235 -0
- package/files/demo/spectronaut-hye-precursor.tsv +493 -0
- package/images/enrichment-crosslink.png +0 -0
- package/images/enrichment-term-selected.png +0 -0
- package/images/hero.png +0 -0
- package/images/pipeline.svg +80 -0
- package/package.json +88 -63
- package/scripts/deqms_de.R +60 -0
- package/scripts/limma_de.R +42 -0
- package/scripts/vsn_normalize.R +19 -0
- package/src/analysis/differential-expression.ts +450 -0
- package/src/analysis/enrichment-export.ts +101 -0
- package/src/analysis/enrichment.ts +602 -0
- package/src/analysis/experiment-setup.ts +199 -0
- package/src/analysis/imputation.ts +407 -0
- package/src/analysis/log2-scale.ts +139 -0
- package/src/analysis/normalization.ts +255 -0
- package/src/analysis/pca.ts +254 -0
- package/src/analysis/spc-storage.ts +515 -0
- package/src/analysis/spc.ts +544 -0
- package/src/analysis/subcellular-location.ts +431 -0
- package/src/demo/enrichment-demo.ts +94 -0
- package/src/demo/proteomics-demo.ts +123 -0
- package/src/global.d.ts +15 -0
- package/src/menu.ts +133 -0
- package/src/package-api.ts +136 -14
- package/src/package-test.ts +45 -20
- package/src/package.g.ts +161 -0
- package/src/package.ts +1029 -17
- package/src/panels/protein-focus.ts +63 -0
- package/src/panels/published-analysis-panel.ts +151 -0
- package/src/panels/uniprot-panel.ts +349 -0
- package/src/parsers/fragpipe-parser.ts +200 -0
- package/src/parsers/generic-parser.ts +197 -0
- package/src/parsers/maxquant-parser.ts +162 -0
- package/src/parsers/shared-utils.ts +163 -0
- package/src/parsers/spectronaut-candidates-parser.ts +307 -0
- package/src/parsers/spectronaut-parser.ts +604 -0
- package/src/publishing/assert-published-shape.ts +260 -0
- package/src/publishing/post-open-recovery.ts +104 -0
- package/src/publishing/publish-project.ts +515 -0
- package/src/publishing/publish-settings.ts +59 -0
- package/src/publishing/publish-state.ts +316 -0
- package/src/publishing/share-dialog.ts +171 -0
- package/src/publishing/trim-dataframe.ts +247 -0
- package/src/tests/analysis.ts +658 -0
- package/src/tests/enrichment-export.ts +61 -0
- package/src/tests/enrichment-visualization.ts +340 -0
- package/src/tests/enrichment.ts +224 -0
- package/src/tests/fragpipe-e2e.ts +74 -0
- package/src/tests/fragpipe-parser.ts +147 -0
- package/src/tests/gene-label-resolver.ts +387 -0
- package/src/tests/generic-parser.ts +152 -0
- package/src/tests/group-mean-correlation.ts +139 -0
- package/src/tests/log2-scale.ts +93 -0
- package/src/tests/organisms.ts +56 -0
- package/src/tests/parsers.ts +182 -0
- package/src/tests/publish-roundtrip.ts +584 -0
- package/src/tests/publish-spike.ts +377 -0
- package/src/tests/qc-dashboard.ts +210 -0
- package/src/tests/smart-pathway-filter.ts +193 -0
- package/src/tests/spc-formula-lines-spike.ts +129 -0
- package/src/tests/spc.ts +640 -0
- package/src/tests/spectronaut-candidates-e2e.ts +140 -0
- package/src/tests/spectronaut-candidates-parser.ts +398 -0
- package/src/tests/spectronaut-parser.ts +668 -0
- package/src/tests/subcellular-location.ts +361 -0
- package/src/tests/uniprot-panel.ts +202 -0
- package/src/tests/volcano.ts +603 -0
- package/src/utils/column-detection.ts +28 -0
- package/src/utils/gene-label-resolver.ts +443 -0
- package/src/utils/organisms.ts +82 -0
- package/src/utils/proteomics-types.ts +30 -0
- package/src/viewers/enrichment-viewers.ts +274 -0
- package/src/viewers/group-mean-correlation.ts +218 -0
- package/src/viewers/heatmap.ts +168 -0
- package/src/viewers/pca-plot.ts +169 -0
- package/src/viewers/qc-computations.ts +266 -0
- package/src/viewers/qc-dashboard.ts +176 -0
- package/src/viewers/spc-dashboard.ts +755 -0
- package/src/viewers/volcano.ts +690 -0
- package/test-console-output-1.log +2073 -0
- package/test-record-1.mp4 +0 -0
- package/tools/derive-precursor-golden-sidecar.mjs +81 -0
- package/tools/generate-enrichment-fixture.sh +160 -0
- package/tools/generate-spectronaut-candidates-fixture.mjs +212 -0
- package/tools/generate-spectronaut-precursor-fixture.mjs +128 -0
- package/tools/spectronaut-aggregate.sh +46 -0
- package/tools/spectronaut-aggregate.sql +80 -0
- package/tsconfig.json +18 -71
- package/webpack.config.js +86 -45
- package/.eslintignore +0 -1
- package/.eslintrc.json +0 -56
- package/LICENSE +0 -674
- package/package.png +0 -0
- package/scripts/number_antibody.py +0 -190
- package/scripts/number_antibody_abnumber.py +0 -177
- package/scripts/number_antibody_anarci.py +0 -200
|
@@ -0,0 +1,602 @@
|
|
|
1
|
+
import * as grok from 'datagrok-api/grok';
|
|
2
|
+
import * as ui from 'datagrok-api/ui';
|
|
3
|
+
import * as DG from 'datagrok-api/dg';
|
|
4
|
+
import {parseAccession} from '../panels/uniprot-panel';
|
|
5
|
+
import {findProteomicsColumns} from '../utils/column-detection';
|
|
6
|
+
import {SEMTYPE, DEFAULT_FC_THRESHOLD, DEFAULT_P_THRESHOLD} from '../utils/proteomics-types';
|
|
7
|
+
import {openEnrichmentVisualization} from '../viewers/enrichment-viewers';
|
|
8
|
+
import {requireDifferentialExpression} from './differential-expression';
|
|
9
|
+
import {getOrganism, setOrganism} from './experiment-setup';
|
|
10
|
+
import {ORGANISM_LIST, detectOrganismCode, organismDisplayForCode} from '../utils/organisms';
|
|
11
|
+
|
|
12
|
+
// ORGANISM_LIST now lives in utils/organisms (single source); re-exported here so
|
|
13
|
+
// existing importers (and the enrichment tests) keep resolving it from this module.
|
|
14
|
+
export {ORGANISM_LIST};
|
|
15
|
+
|
|
16
|
+
// --- Constants ---
|
|
17
|
+
|
|
18
|
+
const GPROFILER_BASE = 'https://biit.cs.ut.ee/gprofiler';
|
|
19
|
+
|
|
20
|
+
// --- Types ---
|
|
21
|
+
|
|
22
|
+
export interface ConvertResult {
|
|
23
|
+
incoming: string;
|
|
24
|
+
converted: string;
|
|
25
|
+
name: string;
|
|
26
|
+
description: string;
|
|
27
|
+
namespaces: string;
|
|
28
|
+
n_incoming: number;
|
|
29
|
+
n_converted: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface GostResult {
|
|
33
|
+
native: string;
|
|
34
|
+
name: string;
|
|
35
|
+
source: string;
|
|
36
|
+
p_value: number;
|
|
37
|
+
significant: boolean;
|
|
38
|
+
term_size: number;
|
|
39
|
+
query_size: number;
|
|
40
|
+
intersection_size: number;
|
|
41
|
+
effective_domain_size: number;
|
|
42
|
+
precision: number;
|
|
43
|
+
recall: number;
|
|
44
|
+
intersections: string[][];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// --- API Client ---
|
|
48
|
+
|
|
49
|
+
async function fetchWithTimeout(
|
|
50
|
+
url: string, options: RequestInit, timeoutMs: number = 30000,
|
|
51
|
+
): Promise<Response> {
|
|
52
|
+
// External URLs go through grok.dapi.fetchProxy to avoid browser CORS.
|
|
53
|
+
// Datagrok's proxy does not currently honor AbortSignal, so we race the
|
|
54
|
+
// request against a timeout promise to enforce the bound.
|
|
55
|
+
const proxied = grok.dapi.fetchProxy(url, options);
|
|
56
|
+
const timeout = new Promise<never>((_, reject) =>
|
|
57
|
+
setTimeout(() => reject(new Error('g:Profiler API request timed out. The service may be temporarily unavailable.')), timeoutMs),
|
|
58
|
+
);
|
|
59
|
+
const resp = await Promise.race([proxied, timeout]);
|
|
60
|
+
if (!resp.ok)
|
|
61
|
+
throw new Error(`g:Profiler API returned status ${resp.status}`);
|
|
62
|
+
return resp;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function gConvert(
|
|
66
|
+
accessions: string[],
|
|
67
|
+
organism: string = 'hsapiens',
|
|
68
|
+
): Promise<ConvertResult[]> {
|
|
69
|
+
const resp = await fetchWithTimeout(`${GPROFILER_BASE}/api/convert/convert/`, {
|
|
70
|
+
method: 'POST',
|
|
71
|
+
headers: {'Content-Type': 'application/json'},
|
|
72
|
+
body: JSON.stringify({
|
|
73
|
+
organism: organism,
|
|
74
|
+
query: accessions,
|
|
75
|
+
target: 'ENSG',
|
|
76
|
+
numeric_ns: '',
|
|
77
|
+
output: 'json',
|
|
78
|
+
}),
|
|
79
|
+
});
|
|
80
|
+
const data = await resp.json();
|
|
81
|
+
return data.result ?? [];
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export async function gGOSt(
|
|
85
|
+
queryGenes: string[],
|
|
86
|
+
backgroundGenes: string[],
|
|
87
|
+
organism: string,
|
|
88
|
+
sources: string[],
|
|
89
|
+
threshold: number = DEFAULT_P_THRESHOLD,
|
|
90
|
+
): Promise<GostResult[]> {
|
|
91
|
+
const resp = await fetchWithTimeout(`${GPROFILER_BASE}/api/gost/profile/`, {
|
|
92
|
+
method: 'POST',
|
|
93
|
+
headers: {'Content-Type': 'application/json'},
|
|
94
|
+
body: JSON.stringify({
|
|
95
|
+
organism: organism,
|
|
96
|
+
query: queryGenes,
|
|
97
|
+
sources: sources,
|
|
98
|
+
user_threshold: threshold,
|
|
99
|
+
significance_threshold_method: 'fdr',
|
|
100
|
+
domain_scope: 'custom',
|
|
101
|
+
background: backgroundGenes,
|
|
102
|
+
all_results: true,
|
|
103
|
+
ordered: false,
|
|
104
|
+
no_evidences: false,
|
|
105
|
+
combined: false,
|
|
106
|
+
measure_underrepresentation: false,
|
|
107
|
+
no_iea: false,
|
|
108
|
+
numeric_ns: '',
|
|
109
|
+
output: 'json',
|
|
110
|
+
}),
|
|
111
|
+
});
|
|
112
|
+
const data = await resp.json();
|
|
113
|
+
// g:GOSt nests results: data.result is an array of per-query result objects,
|
|
114
|
+
// each containing a `.result` array of term results.
|
|
115
|
+
if (!data.result)
|
|
116
|
+
return [];
|
|
117
|
+
// Handle array of query results (standard structure)
|
|
118
|
+
if (Array.isArray(data.result) && data.result.length > 0 && data.result[0]?.result)
|
|
119
|
+
return data.result[0].result as GostResult[];
|
|
120
|
+
// Fallback: if data.result is directly the results array
|
|
121
|
+
if (Array.isArray(data.result))
|
|
122
|
+
return data.result as GostResult[];
|
|
123
|
+
return [];
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// --- Smart Pathway Filter (R5/D-13/D-14) ---
|
|
127
|
+
|
|
128
|
+
// LOCKED CLIENT CONTRACT — port verbatim from
|
|
129
|
+
// ~/Downloads/ck/CKomics_tool2.py:4685-4736 (apply_smart_pathway_filtering).
|
|
130
|
+
// Do not re-derive heuristics; any change requires a CONTEXT update.
|
|
131
|
+
|
|
132
|
+
export const GENERIC_PARENT_TERMS = [
|
|
133
|
+
'localization', 'cellular component organization', 'transport',
|
|
134
|
+
'cellular process', 'biological process', 'metabolic process',
|
|
135
|
+
] as const;
|
|
136
|
+
|
|
137
|
+
export const SPECIFIC_CHILD_TERMS = ['actin', 'vesicle', 'endocytosis', 'cytoskeleton'] as const;
|
|
138
|
+
|
|
139
|
+
export interface SmartFilterStats {
|
|
140
|
+
total: number;
|
|
141
|
+
kept: number;
|
|
142
|
+
droppedParents: number;
|
|
143
|
+
cappedAtN: number;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Verbatim port of CK-omics apply_smart_pathway_filtering (lines 4685-4736).
|
|
147
|
+
*
|
|
148
|
+
* Algorithm:
|
|
149
|
+
* 1. Partition into GO:BP rows and `other_data` (everything else).
|
|
150
|
+
* 2. Sort GO:BP by p_value ASCENDING.
|
|
151
|
+
* 3. Walk GO:BP in sorted order: drop a row when its name contains any
|
|
152
|
+
* GENERIC_PARENT term AND the already-kept list already contains a
|
|
153
|
+
* SPECIFIC_CHILD-term row. Otherwise keep it. Stop at `maxPerSource`.
|
|
154
|
+
* 4. Non-GO:BP rows are sorted by p_value ASC and `.head(maxPerSource)` —
|
|
155
|
+
* a COMBINED cap across all non-GO:BP sources, not per-source
|
|
156
|
+
* (Assumption A4; CK-omics line 4731 explicitly uses combined .head). */
|
|
157
|
+
export function applySmartPathwayFilter(
|
|
158
|
+
results: GostResult[], maxPerSource = 15,
|
|
159
|
+
): {kept: GostResult[]; stats: SmartFilterStats} {
|
|
160
|
+
if (results.length === 0) {
|
|
161
|
+
return {kept: [], stats: {total: 0, kept: 0, droppedParents: 0, cappedAtN: maxPerSource}};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const goBp = results.filter((r) => r.source === 'GO:BP');
|
|
165
|
+
const other = results.filter((r) => r.source !== 'GO:BP');
|
|
166
|
+
|
|
167
|
+
let droppedParents = 0;
|
|
168
|
+
const filteredGoBp: GostResult[] = [];
|
|
169
|
+
if (goBp.length > 0) {
|
|
170
|
+
const sorted = [...goBp].sort((a, b) => a.p_value - b.p_value);
|
|
171
|
+
for (const r of sorted) {
|
|
172
|
+
const name = (r.name ?? '').toLowerCase();
|
|
173
|
+
const isGeneric = GENERIC_PARENT_TERMS.some((g) => name.includes(g));
|
|
174
|
+
if (isGeneric) {
|
|
175
|
+
const hasSpecificChild = filteredGoBp.some((existing) =>
|
|
176
|
+
SPECIFIC_CHILD_TERMS.some((s) => (existing.name ?? '').toLowerCase().includes(s)));
|
|
177
|
+
if (hasSpecificChild) { droppedParents++; continue; }
|
|
178
|
+
}
|
|
179
|
+
filteredGoBp.push(r);
|
|
180
|
+
if (filteredGoBp.length >= maxPerSource) break;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const otherSorted = [...other].sort((a, b) => a.p_value - b.p_value);
|
|
185
|
+
const otherCapped = otherSorted.slice(0, maxPerSource);
|
|
186
|
+
|
|
187
|
+
const kept = [...filteredGoBp, ...otherCapped].sort((a, b) => a.p_value - b.p_value);
|
|
188
|
+
return {
|
|
189
|
+
kept,
|
|
190
|
+
stats: {
|
|
191
|
+
total: results.length,
|
|
192
|
+
kept: kept.length,
|
|
193
|
+
droppedParents,
|
|
194
|
+
cappedAtN: maxPerSource,
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// --- Result Builder ---
|
|
200
|
+
|
|
201
|
+
export function buildEnrichmentDf(
|
|
202
|
+
results: GostResult[],
|
|
203
|
+
queryGenes: string[],
|
|
204
|
+
pThreshold: number = DEFAULT_P_THRESHOLD,
|
|
205
|
+
direction?: 'Up' | 'Down',
|
|
206
|
+
): DG.DataFrame {
|
|
207
|
+
const n = results.length;
|
|
208
|
+
const df = DG.DataFrame.create(n);
|
|
209
|
+
df.name = 'Enrichment Results';
|
|
210
|
+
|
|
211
|
+
// Pre-derive the comma-joined member-gene string per row so the column init
|
|
212
|
+
// doesn't repeat the intersections-array walk per cell.
|
|
213
|
+
const memberGeneStrs = results.map((r) => {
|
|
214
|
+
if (!r.intersections) return '';
|
|
215
|
+
const members: string[] = [];
|
|
216
|
+
const lim = Math.min(queryGenes.length, r.intersections.length);
|
|
217
|
+
for (let j = 0; j < lim; j++) {
|
|
218
|
+
if (r.intersections[j] && r.intersections[j].length > 0)
|
|
219
|
+
members.push(queryGenes[j]);
|
|
220
|
+
}
|
|
221
|
+
return members.join(', ');
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
const sourceCol = df.columns.addNewString('Source');
|
|
225
|
+
const termIdCol = df.columns.addNewString('Term ID');
|
|
226
|
+
const termNameCol = df.columns.addNewString('Term Name');
|
|
227
|
+
// g:GOSt with significance_threshold_method='fdr' returns the FDR-corrected
|
|
228
|
+
// p-value (and no separate raw p-value) — we expose it under the FDR column
|
|
229
|
+
// to keep semantics honest. Downstream consumers should read FDR.
|
|
230
|
+
const fdrCol = df.columns.addNewFloat('FDR');
|
|
231
|
+
const geneCountCol = df.columns.addNewInt('Gene Count');
|
|
232
|
+
const geneRatioCol = df.columns.addNewFloat('Gene Ratio');
|
|
233
|
+
const intersectionCol = df.columns.addNewString('Intersection');
|
|
234
|
+
|
|
235
|
+
sourceCol.init((i) => results[i].source);
|
|
236
|
+
termIdCol.init((i) => results[i].native);
|
|
237
|
+
termNameCol.init((i) => results[i].name);
|
|
238
|
+
fdrCol.init((i) => results[i].p_value);
|
|
239
|
+
geneCountCol.init((i) => results[i].intersection_size);
|
|
240
|
+
geneRatioCol.init((i) => results[i].precision);
|
|
241
|
+
intersectionCol.init((i) => memberGeneStrs[i]);
|
|
242
|
+
|
|
243
|
+
const fdrRaw = fdrCol.getRawData() as Float32Array | Float64Array;
|
|
244
|
+
const sigCol = df.columns.addNewBool('Significant');
|
|
245
|
+
sigCol.init((i) => fdrRaw[i] !== DG.FLOAT_NULL && fdrRaw[i] <= pThreshold);
|
|
246
|
+
|
|
247
|
+
// Optional directional label (R2 up/down split). Categorical, bulk-init
|
|
248
|
+
// (memory feedback_dg_column_bulk_init — never per-row set).
|
|
249
|
+
if (direction) {
|
|
250
|
+
const directionCol = df.columns.addNewString('Direction');
|
|
251
|
+
directionCol.init(() => direction);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Color coding on FDR column
|
|
255
|
+
fdrCol.setTag('color-coding-type', 'Linear');
|
|
256
|
+
fdrCol.setTag('color-coding-linear', '{"0":"#2ecc71","0.05":"#f39c12","1":"#e74c3c"}');
|
|
257
|
+
|
|
258
|
+
df.setTag('proteomics.enrichment', 'true');
|
|
259
|
+
return df;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Splits the detected genes into up/down significant query sets and the shared
|
|
264
|
+
* all-detected background. Ported from CK-omics run_gprofiler_analysis: strict
|
|
265
|
+
* `fc > 0` / `fc < 0`; background is every detected gene (used by BOTH
|
|
266
|
+
* directional g:Profiler calls). Null fc/p rows count toward background only.
|
|
267
|
+
*/
|
|
268
|
+
export function splitGenesByDirection(
|
|
269
|
+
geneForRow: Map<number, string>,
|
|
270
|
+
fcRaw: Float32Array | Float64Array,
|
|
271
|
+
pRaw: Float32Array | Float64Array,
|
|
272
|
+
fcThreshold: number,
|
|
273
|
+
pThreshold: number,
|
|
274
|
+
): {upGenes: string[]; downGenes: string[]; background: string[]} {
|
|
275
|
+
const up: Set<string> = new Set();
|
|
276
|
+
const down: Set<string> = new Set();
|
|
277
|
+
const background: Set<string> = new Set();
|
|
278
|
+
for (const [row, gene] of geneForRow) {
|
|
279
|
+
background.add(gene);
|
|
280
|
+
const fc = fcRaw[row];
|
|
281
|
+
const adjP = pRaw[row];
|
|
282
|
+
if (fc !== DG.FLOAT_NULL && adjP !== DG.FLOAT_NULL &&
|
|
283
|
+
adjP <= pThreshold && Math.abs(fc) >= fcThreshold) {
|
|
284
|
+
if (fc > 0) up.add(gene);
|
|
285
|
+
else if (fc < 0) down.add(gene);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
return {upGenes: [...up], downGenes: [...down], background: [...background]};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// --- Significant Protein Counter ---
|
|
292
|
+
|
|
293
|
+
export function countSignificantProteins(
|
|
294
|
+
df: DG.DataFrame,
|
|
295
|
+
fcThreshold: number,
|
|
296
|
+
pThreshold: number,
|
|
297
|
+
log2fcCol: DG.Column,
|
|
298
|
+
pValCol: DG.Column,
|
|
299
|
+
): {significant: number; total: number} {
|
|
300
|
+
// Bulk-read via getRawData so we don't pay a 3× JS/native bridge hop per row.
|
|
301
|
+
const n = df.rowCount;
|
|
302
|
+
const fcRaw = log2fcCol.getRawData() as Float32Array | Float64Array;
|
|
303
|
+
const pRaw = pValCol.getRawData() as Float32Array | Float64Array;
|
|
304
|
+
let significant = 0;
|
|
305
|
+
for (let i = 0; i < n; i++) {
|
|
306
|
+
const fc = fcRaw[i];
|
|
307
|
+
const p = pRaw[i];
|
|
308
|
+
if (fc === DG.FLOAT_NULL || p === DG.FLOAT_NULL) continue;
|
|
309
|
+
if (Math.abs(fc) >= fcThreshold && p <= pThreshold)
|
|
310
|
+
significant++;
|
|
311
|
+
}
|
|
312
|
+
return {significant, total: n};
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// --- Pipeline Orchestrator ---
|
|
316
|
+
|
|
317
|
+
export async function runEnrichmentPipeline(
|
|
318
|
+
df: DG.DataFrame,
|
|
319
|
+
fcThreshold: number,
|
|
320
|
+
pThreshold: number,
|
|
321
|
+
organismCode: string,
|
|
322
|
+
sources: string[],
|
|
323
|
+
smartFilterEnabled: boolean = true,
|
|
324
|
+
maxTermsPerSource: number = 15,
|
|
325
|
+
): Promise<{enrichmentDf: DG.DataFrame; mapped: number; total: number; unmapped: number}> {
|
|
326
|
+
const cols = findProteomicsColumns(df);
|
|
327
|
+
if (!cols.proteinId)
|
|
328
|
+
throw new Error('No protein ID column found');
|
|
329
|
+
if (!cols.log2fc || !cols.pValue)
|
|
330
|
+
throw new Error('No log2FC or adjusted p-value columns found. Run Differential Expression first.');
|
|
331
|
+
|
|
332
|
+
// Step 1: Get gene symbols (existing or mapped)
|
|
333
|
+
let geneForRow: Map<number, string>;
|
|
334
|
+
let mapped = 0;
|
|
335
|
+
let total = 0;
|
|
336
|
+
let unmapped = 0;
|
|
337
|
+
|
|
338
|
+
if (cols.geneName) {
|
|
339
|
+
// Use existing gene name column. Truthiness check covers null/empty;
|
|
340
|
+
// skipping the redundant isNone() halves the bridge hops.
|
|
341
|
+
geneForRow = new Map();
|
|
342
|
+
const geneNameCol = cols.geneName;
|
|
343
|
+
for (let i = 0; i < df.rowCount; i++) {
|
|
344
|
+
const val = geneNameCol.get(i) as string | null;
|
|
345
|
+
if (val) geneForRow.set(i, val);
|
|
346
|
+
}
|
|
347
|
+
mapped = geneForRow.size;
|
|
348
|
+
total = df.rowCount;
|
|
349
|
+
unmapped = total - mapped;
|
|
350
|
+
} else {
|
|
351
|
+
// Extract clean accessions and map via g:Convert
|
|
352
|
+
const accToRows = new Map<string, number[]>();
|
|
353
|
+
for (let i = 0; i < df.rowCount; i++) {
|
|
354
|
+
const raw = cols.proteinId!.get(i) as string;
|
|
355
|
+
const acc = parseAccession(raw);
|
|
356
|
+
if (acc) {
|
|
357
|
+
if (!accToRows.has(acc))
|
|
358
|
+
accToRows.set(acc, []);
|
|
359
|
+
accToRows.get(acc)!.push(i);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const uniqueAccessions = [...accToRows.keys()];
|
|
364
|
+
const convertResults = await gConvert(uniqueAccessions, organismCode);
|
|
365
|
+
|
|
366
|
+
// Build accession -> gene symbol map (first match per accession)
|
|
367
|
+
const accToGene = new Map<string, string>();
|
|
368
|
+
for (const r of convertResults) {
|
|
369
|
+
if (r.incoming && r.name && r.name !== 'N/A' && !accToGene.has(r.incoming))
|
|
370
|
+
accToGene.set(r.incoming, r.name);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Map rows to gene symbols
|
|
374
|
+
geneForRow = new Map();
|
|
375
|
+
for (const [acc, rows] of accToRows) {
|
|
376
|
+
const gene = accToGene.get(acc);
|
|
377
|
+
if (gene) {
|
|
378
|
+
for (const row of rows)
|
|
379
|
+
geneForRow.set(row, gene);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// Add gene symbol column to main table. Materialize the row->gene array
|
|
384
|
+
// first, then bulk-init the column to skip per-row set() bridge hops.
|
|
385
|
+
const geneArr: string[] = new Array(df.rowCount).fill('');
|
|
386
|
+
for (const [row, gene] of geneForRow)
|
|
387
|
+
geneArr[row] = gene;
|
|
388
|
+
|
|
389
|
+
const geneCol = df.columns.addNewString('Gene Symbol (mapped)');
|
|
390
|
+
geneCol.semType = SEMTYPE.GENE_SYMBOL;
|
|
391
|
+
geneCol.init((i) => geneArr[i]);
|
|
392
|
+
|
|
393
|
+
mapped = accToGene.size;
|
|
394
|
+
total = uniqueAccessions.length;
|
|
395
|
+
unmapped = total - mapped;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Step 2: Split significant genes by fc sign; background = all detected,
|
|
399
|
+
// shared by both directional queries (CK-omics run_gprofiler_analysis).
|
|
400
|
+
// Bulk-read the numeric columns once via getRawData (no per-row bridge hops).
|
|
401
|
+
const fcRaw = cols.log2fc.getRawData() as Float32Array | Float64Array;
|
|
402
|
+
const pRaw = cols.pValue.getRawData() as Float32Array | Float64Array;
|
|
403
|
+
const {upGenes, downGenes, background: bgArray} =
|
|
404
|
+
splitGenesByDirection(geneForRow, fcRaw, pRaw, fcThreshold, pThreshold);
|
|
405
|
+
|
|
406
|
+
if (upGenes.length === 0 && downGenes.length === 0)
|
|
407
|
+
throw new Error('No significant proteins found with the given thresholds');
|
|
408
|
+
|
|
409
|
+
// Step 3: One g:GOSt call per non-empty direction, identical background.
|
|
410
|
+
// A direction with zero genes is skipped (no empty-query request).
|
|
411
|
+
// Pitfall 8: when smart filter is enabled, run it BEFORE buildEnrichmentDf so
|
|
412
|
+
// the per-row Intersection strings are built from the kept-set indices and
|
|
413
|
+
// never get reindexed after the fact.
|
|
414
|
+
const directionDfs: DG.DataFrame[] = [];
|
|
415
|
+
let upStats: SmartFilterStats | null = null;
|
|
416
|
+
let downStats: SmartFilterStats | null = null;
|
|
417
|
+
if (upGenes.length > 0) {
|
|
418
|
+
let upResults = await gGOSt(upGenes, bgArray, organismCode, sources, pThreshold);
|
|
419
|
+
if (smartFilterEnabled) {
|
|
420
|
+
const filtered = applySmartPathwayFilter(upResults, maxTermsPerSource);
|
|
421
|
+
upResults = filtered.kept;
|
|
422
|
+
upStats = filtered.stats;
|
|
423
|
+
}
|
|
424
|
+
directionDfs.push(buildEnrichmentDf(upResults, upGenes, pThreshold, 'Up'));
|
|
425
|
+
}
|
|
426
|
+
if (downGenes.length > 0) {
|
|
427
|
+
let downResults = await gGOSt(downGenes, bgArray, organismCode, sources, pThreshold);
|
|
428
|
+
if (smartFilterEnabled) {
|
|
429
|
+
const filtered = applySmartPathwayFilter(downResults, maxTermsPerSource);
|
|
430
|
+
downResults = filtered.kept;
|
|
431
|
+
downStats = filtered.stats;
|
|
432
|
+
}
|
|
433
|
+
directionDfs.push(buildEnrichmentDf(downResults, downGenes, pThreshold, 'Down'));
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// Step 4: Merge directional results into one DataFrame. Phase-9 column shape
|
|
437
|
+
// is intact (Direction is just an extra categorical). append() returns a NEW
|
|
438
|
+
// frame and may drop frame/column metadata, so re-apply the enrichment tag,
|
|
439
|
+
// name, and FDR color-coding the viewers/cross-link key on.
|
|
440
|
+
let enrichmentDf = directionDfs[0];
|
|
441
|
+
for (let i = 1; i < directionDfs.length; i++)
|
|
442
|
+
enrichmentDf = enrichmentDf.append(directionDfs[i]);
|
|
443
|
+
// Name the result after its source table (e.g. "MyStudy — Enrichment") so it's
|
|
444
|
+
// distinguishable when several analyses are open — mirrors the "<source> — QC"
|
|
445
|
+
// convention. Falls back to the generic name if the source is unnamed.
|
|
446
|
+
enrichmentDf.name = df.name ? `${df.name} — Enrichment` : 'Enrichment Results';
|
|
447
|
+
enrichmentDf.setTag('proteomics.enrichment', 'true');
|
|
448
|
+
const fdrCol = enrichmentDf.col('FDR');
|
|
449
|
+
if (fdrCol) {
|
|
450
|
+
fdrCol.setTag('color-coding-type', 'Linear');
|
|
451
|
+
fdrCol.setTag('color-coding-linear', '{"0":"#2ecc71","0.05":"#f39c12","1":"#e74c3c"}');
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// D-14 — set the smart-filter banner tags only when the filter actually
|
|
455
|
+
// dropped rows (kept < total). Banner is suppressed when the filter was off
|
|
456
|
+
// OR when kept == total. Tag names mirror UI-SPEC §"Enrichment table:
|
|
457
|
+
// smart pathway filter banner".
|
|
458
|
+
if (smartFilterEnabled) {
|
|
459
|
+
const totalInput = (upStats?.total ?? 0) + (downStats?.total ?? 0);
|
|
460
|
+
const totalKept = (upStats?.kept ?? 0) + (downStats?.kept ?? 0);
|
|
461
|
+
const totalDropped = (upStats?.droppedParents ?? 0) + (downStats?.droppedParents ?? 0);
|
|
462
|
+
if (totalKept < totalInput) {
|
|
463
|
+
enrichmentDf.setTag('proteomics.enrichment_smart_filtered', 'true');
|
|
464
|
+
enrichmentDf.setTag('proteomics.enrichment_smart_filtered_kept', String(totalKept));
|
|
465
|
+
enrichmentDf.setTag('proteomics.enrichment_smart_filtered_total', String(totalInput));
|
|
466
|
+
enrichmentDf.setTag('proteomics.enrichment_smart_filtered_dropped_parents', String(totalDropped));
|
|
467
|
+
enrichmentDf.setTag('proteomics.enrichment_smart_filtered_cap', String(maxTermsPerSource));
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
return {enrichmentDf, mapped, total, unmapped};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// --- Dialog ---
|
|
475
|
+
|
|
476
|
+
export function showEnrichmentDialog(df: DG.DataFrame): void {
|
|
477
|
+
if (!requireDifferentialExpression(df, 'Run Differential Expression first')) return;
|
|
478
|
+
|
|
479
|
+
const cols = findProteomicsColumns(df);
|
|
480
|
+
if (!cols.log2fc || !cols.pValue) {
|
|
481
|
+
grok.shell.warning('No log2FC or p-value columns found');
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
const fcInput = ui.input.float('|log2FC| threshold', {value: DEFAULT_FC_THRESHOLD});
|
|
486
|
+
fcInput.setTooltip('Minimum absolute fold change for significance');
|
|
487
|
+
|
|
488
|
+
const pInput = ui.input.float('Adj. p-value threshold', {value: DEFAULT_P_THRESHOLD});
|
|
489
|
+
pInput.setTooltip('Maximum adjusted p-value for significance');
|
|
490
|
+
|
|
491
|
+
// Seed from the chosen/persisted organism, else auto-detect from the data's
|
|
492
|
+
// organism column (covers the Candidates path, which skips Annotate), else human.
|
|
493
|
+
// Sticky + shared with the subcellular-location fetch via the proteomics.organism tag.
|
|
494
|
+
const seededDisplay = organismDisplayForCode(getOrganism(df) ?? detectOrganismCode(df));
|
|
495
|
+
const organismInput = ui.input.choice('Organism', {
|
|
496
|
+
value: (seededDisplay ?? 'Homo sapiens (Human)') as string,
|
|
497
|
+
items: ORGANISM_LIST.map((o) => o.display) as unknown as string[],
|
|
498
|
+
nullable: false,
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
// Enrichment source checkboxes
|
|
502
|
+
const goBpInput = ui.input.bool('GO: Biological Process', {value: true});
|
|
503
|
+
const goMfInput = ui.input.bool('GO: Molecular Function', {value: true});
|
|
504
|
+
const goCcInput = ui.input.bool('GO: Cellular Component', {value: true});
|
|
505
|
+
const keggInput = ui.input.bool('KEGG Pathways', {value: true});
|
|
506
|
+
const reactomeInput = ui.input.bool('Reactome Pathways', {value: true});
|
|
507
|
+
const wpInput = ui.input.bool('WikiPathways', {value: true});
|
|
508
|
+
|
|
509
|
+
// D-14 — smart pathway filter checkbox, default checked.
|
|
510
|
+
const smartFilterInput = ui.input.bool('Apply smart pathway filter', {value: true});
|
|
511
|
+
smartFilterInput.setTooltip(
|
|
512
|
+
'Drops generic GO parent terms when more specific child terms are present; ' +
|
|
513
|
+
'caps each source at top-N by FDR. Recommended for cleaner results; ' +
|
|
514
|
+
'uncheck for raw g:Profiler output.');
|
|
515
|
+
|
|
516
|
+
// M3 — the per-source top-N cap the smart filter applies (was hardcoded 15).
|
|
517
|
+
// Only meaningful when the smart filter is on; greyed out otherwise.
|
|
518
|
+
const maxTermsInput = ui.input.int('Max terms per source', {value: 15, min: 1});
|
|
519
|
+
maxTermsInput.setTooltip(
|
|
520
|
+
'Upper bound on terms kept per enrichment source (top-N by FDR). ' +
|
|
521
|
+
'Applies only when the smart pathway filter is on.');
|
|
522
|
+
const syncMaxTermsEnabled = () => maxTermsInput.enabled = smartFilterInput.value ?? true;
|
|
523
|
+
smartFilterInput.onChanged.subscribe(syncMaxTermsEnabled);
|
|
524
|
+
syncMaxTermsEnabled();
|
|
525
|
+
|
|
526
|
+
// Live count of significant proteins
|
|
527
|
+
const countDiv = ui.divText('');
|
|
528
|
+
const updateCount = () => {
|
|
529
|
+
const fc = fcInput.value ?? DEFAULT_FC_THRESHOLD;
|
|
530
|
+
const p = pInput.value ?? DEFAULT_P_THRESHOLD;
|
|
531
|
+
const result = countSignificantProteins(df, fc, p, cols.log2fc!, cols.pValue!);
|
|
532
|
+
const totalFmt = result.total.toLocaleString();
|
|
533
|
+
countDiv.textContent =
|
|
534
|
+
`${result.significant} of ${totalFmt} proteins are significant with these thresholds`;
|
|
535
|
+
};
|
|
536
|
+
updateCount();
|
|
537
|
+
fcInput.onChanged.subscribe(updateCount);
|
|
538
|
+
pInput.onChanged.subscribe(updateCount);
|
|
539
|
+
|
|
540
|
+
ui.dialog('Enrichment Analysis')
|
|
541
|
+
.add(countDiv)
|
|
542
|
+
.add(fcInput)
|
|
543
|
+
.add(pInput)
|
|
544
|
+
.add(organismInput)
|
|
545
|
+
.add(goBpInput)
|
|
546
|
+
.add(goMfInput)
|
|
547
|
+
.add(goCcInput)
|
|
548
|
+
.add(keggInput)
|
|
549
|
+
.add(reactomeInput)
|
|
550
|
+
.add(wpInput)
|
|
551
|
+
.add(smartFilterInput)
|
|
552
|
+
.add(maxTermsInput)
|
|
553
|
+
.onOK(async () => {
|
|
554
|
+
const pi = DG.TaskBarProgressIndicator.create('Running enrichment analysis...');
|
|
555
|
+
try {
|
|
556
|
+
const fc = fcInput.value ?? DEFAULT_FC_THRESHOLD;
|
|
557
|
+
const p = pInput.value ?? DEFAULT_P_THRESHOLD;
|
|
558
|
+
|
|
559
|
+
// Resolve organism code from display name
|
|
560
|
+
const selectedDisplay = organismInput.value;
|
|
561
|
+
const organism = ORGANISM_LIST.find((o) => o.display === selectedDisplay);
|
|
562
|
+
const organismCode = organism?.code ?? 'hsapiens';
|
|
563
|
+
// Persist so the subcellular-location fetch (and next enrichment run)
|
|
564
|
+
// narrow to the same species.
|
|
565
|
+
setOrganism(df, organismCode);
|
|
566
|
+
|
|
567
|
+
// Build sources array from checkbox states
|
|
568
|
+
const selectedSources: string[] = [];
|
|
569
|
+
if (goBpInput.value) selectedSources.push('GO:BP');
|
|
570
|
+
if (goMfInput.value) selectedSources.push('GO:MF');
|
|
571
|
+
if (goCcInput.value) selectedSources.push('GO:CC');
|
|
572
|
+
if (keggInput.value) selectedSources.push('KEGG');
|
|
573
|
+
if (reactomeInput.value) selectedSources.push('REAC');
|
|
574
|
+
if (wpInput.value) selectedSources.push('WP');
|
|
575
|
+
|
|
576
|
+
if (selectedSources.length === 0) {
|
|
577
|
+
grok.shell.warning('Please select at least one enrichment source');
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
const result = await runEnrichmentPipeline(
|
|
582
|
+
df, fc, p, organismCode, selectedSources, smartFilterInput.value ?? true,
|
|
583
|
+
maxTermsInput.value ?? 15);
|
|
584
|
+
|
|
585
|
+
// Show mapping stats
|
|
586
|
+
const pct = result.total > 0 ? (result.mapped / result.total * 100).toFixed(1) : '0.0';
|
|
587
|
+
grok.shell.info(
|
|
588
|
+
`${result.mapped}/${result.total} proteins mapped (${pct}%). ${result.unmapped} unmapped.`,
|
|
589
|
+
);
|
|
590
|
+
|
|
591
|
+
// openEnrichmentVisualization creates (or reuses) the enrichment results
|
|
592
|
+
// view itself — adding one here too spawned a duplicate "Enrichment
|
|
593
|
+
// Results (2)" tab every run.
|
|
594
|
+
openEnrichmentVisualization(result.enrichmentDf, df);
|
|
595
|
+
} catch (e: any) {
|
|
596
|
+
grok.shell.error(`Enrichment analysis failed: ${e.message}`);
|
|
597
|
+
} finally {
|
|
598
|
+
pi.close();
|
|
599
|
+
}
|
|
600
|
+
})
|
|
601
|
+
.show();
|
|
602
|
+
}
|