@datagrok/proteomics 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -3
- package/CLAUDE.md +178 -0
- package/README.md +195 -3
- package/detectors.js +152 -9
- package/dist/package-test.js +1 -1744
- package/dist/package-test.js.map +1 -1
- package/dist/package.js +1 -146
- package/dist/package.js.map +1 -1
- package/docs/personas-and-capabilities.md +165 -0
- package/files/demo/README.md +264 -0
- package/files/demo/cptac-spike-in.txt +1571 -0
- package/files/demo/enrichment-demo.csv +120 -0
- package/files/demo/fragpipe-smoke-test.tsv +11 -0
- package/files/demo/proteinGroups.txt +28 -0
- package/files/demo/spectronaut-hye-candidates.tsv +94 -0
- package/files/demo/spectronaut-hye-demo.tsv +8761 -0
- package/files/demo/spectronaut-hye-mix.tsv +8761 -0
- package/files/demo/spectronaut-hye-precursor-golden.json +938 -0
- package/files/demo/spectronaut-hye-precursor-golden.tsv +235 -0
- package/files/demo/spectronaut-hye-precursor.tsv +493 -0
- package/images/enrichment-crosslink.png +0 -0
- package/images/enrichment-term-selected.png +0 -0
- package/images/hero.png +0 -0
- package/images/pipeline.svg +80 -0
- package/package.json +87 -63
- package/scripts/deqms_de.R +60 -0
- package/scripts/limma_de.R +42 -0
- package/scripts/vsn_normalize.R +19 -0
- package/src/analysis/differential-expression.ts +450 -0
- package/src/analysis/enrichment-export.ts +101 -0
- package/src/analysis/enrichment.ts +602 -0
- package/src/analysis/experiment-setup.ts +199 -0
- package/src/analysis/imputation.ts +407 -0
- package/src/analysis/log2-scale.ts +139 -0
- package/src/analysis/normalization.ts +255 -0
- package/src/analysis/pca.ts +254 -0
- package/src/analysis/spc-storage.ts +515 -0
- package/src/analysis/spc.ts +544 -0
- package/src/analysis/subcellular-location.ts +431 -0
- package/src/demo/enrichment-demo.ts +94 -0
- package/src/demo/proteomics-demo.ts +123 -0
- package/src/global.d.ts +15 -0
- package/src/menu.ts +133 -0
- package/src/package-api.ts +136 -14
- package/src/package-test.ts +45 -20
- package/src/package.g.ts +161 -0
- package/src/package.ts +1029 -17
- package/src/panels/protein-focus.ts +63 -0
- package/src/panels/published-analysis-panel.ts +151 -0
- package/src/panels/uniprot-panel.ts +349 -0
- package/src/parsers/fragpipe-parser.ts +200 -0
- package/src/parsers/generic-parser.ts +197 -0
- package/src/parsers/maxquant-parser.ts +162 -0
- package/src/parsers/shared-utils.ts +163 -0
- package/src/parsers/spectronaut-candidates-parser.ts +307 -0
- package/src/parsers/spectronaut-parser.ts +604 -0
- package/src/publishing/assert-published-shape.ts +260 -0
- package/src/publishing/post-open-recovery.ts +104 -0
- package/src/publishing/publish-project.ts +515 -0
- package/src/publishing/publish-settings.ts +59 -0
- package/src/publishing/publish-state.ts +316 -0
- package/src/publishing/share-dialog.ts +171 -0
- package/src/publishing/trim-dataframe.ts +247 -0
- package/src/tests/analysis.ts +658 -0
- package/src/tests/enrichment-export.ts +61 -0
- package/src/tests/enrichment-visualization.ts +340 -0
- package/src/tests/enrichment.ts +224 -0
- package/src/tests/fragpipe-e2e.ts +74 -0
- package/src/tests/fragpipe-parser.ts +147 -0
- package/src/tests/gene-label-resolver.ts +387 -0
- package/src/tests/generic-parser.ts +152 -0
- package/src/tests/group-mean-correlation.ts +139 -0
- package/src/tests/log2-scale.ts +93 -0
- package/src/tests/organisms.ts +56 -0
- package/src/tests/parsers.ts +182 -0
- package/src/tests/publish-roundtrip.ts +584 -0
- package/src/tests/publish-spike.ts +377 -0
- package/src/tests/qc-dashboard.ts +210 -0
- package/src/tests/smart-pathway-filter.ts +193 -0
- package/src/tests/spc-formula-lines-spike.ts +129 -0
- package/src/tests/spc.ts +640 -0
- package/src/tests/spectronaut-candidates-e2e.ts +140 -0
- package/src/tests/spectronaut-candidates-parser.ts +398 -0
- package/src/tests/spectronaut-parser.ts +668 -0
- package/src/tests/subcellular-location.ts +361 -0
- package/src/tests/uniprot-panel.ts +202 -0
- package/src/tests/volcano.ts +603 -0
- package/src/utils/column-detection.ts +28 -0
- package/src/utils/gene-label-resolver.ts +443 -0
- package/src/utils/organisms.ts +82 -0
- package/src/utils/proteomics-types.ts +30 -0
- package/src/viewers/enrichment-viewers.ts +274 -0
- package/src/viewers/group-mean-correlation.ts +218 -0
- package/src/viewers/heatmap.ts +168 -0
- package/src/viewers/pca-plot.ts +169 -0
- package/src/viewers/qc-computations.ts +266 -0
- package/src/viewers/qc-dashboard.ts +176 -0
- package/src/viewers/spc-dashboard.ts +755 -0
- package/src/viewers/volcano.ts +690 -0
- package/test-console-output-1.log +2055 -0
- package/test-record-1.mp4 +0 -0
- package/tools/derive-precursor-golden-sidecar.mjs +81 -0
- package/tools/generate-enrichment-fixture.sh +160 -0
- package/tools/generate-spectronaut-candidates-fixture.mjs +212 -0
- package/tools/generate-spectronaut-precursor-fixture.mjs +128 -0
- package/tools/spectronaut-aggregate.sh +46 -0
- package/tools/spectronaut-aggregate.sql +80 -0
- package/tsconfig.json +18 -71
- package/webpack.config.js +86 -45
- package/.eslintignore +0 -1
- package/.eslintrc.json +0 -56
- package/LICENSE +0 -674
- package/package.png +0 -0
- package/scripts/number_antibody.py +0 -190
- package/scripts/number_antibody_abnumber.py +0 -177
- package/scripts/number_antibody_anarci.py +0 -200
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
import * as DG from 'datagrok-api/dg';
|
|
2
|
+
import * as grok from 'datagrok-api/grok';
|
|
3
|
+
import {findColumn} from './column-detection';
|
|
4
|
+
import {SEMTYPE} from './proteomics-types';
|
|
5
|
+
import {runWithConcurrency} from '../analysis/subcellular-location';
|
|
6
|
+
|
|
7
|
+
// ---------------------------------------------------------------------------
|
|
8
|
+
// LOCKED CLIENT CONTRACT — verbatim port of CKomics_tool2.py
|
|
9
|
+
// • improve_gene_labels_with_ensrnog_marking (lines 895-1059)
|
|
10
|
+
// • extract_readable_description (lines 1062-1104)
|
|
11
|
+
// • get_ensembl_annotations request shape (lines 756-855)
|
|
12
|
+
// D-08: inline `*` (grouped) / `†` (predicted/reclassified) markers; raw ID
|
|
13
|
+
// kept in `Source ID` column.
|
|
14
|
+
// D-09: Ensembl REST POST /lookup/id batched ≤1000 IDs per request, cached
|
|
15
|
+
// cross-session via grok.dapi.userDataStorage with `__schema_v` invalidation.
|
|
16
|
+
// D-10: duplicate gene names disambiguated with `(Source ID)` suffix; one
|
|
17
|
+
// grok.shell.warning emitted per import with affected-row count.
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
/** userDataStorage key for the cross-session id → EnsemblEntry cache.
|
|
21
|
+
* Exported so callers (and tests) can peek the cache directly. */
|
|
22
|
+
export const STORE_GENE_LABELS = 'proteomics-gene-labels';
|
|
23
|
+
|
|
24
|
+
/** Bump only if the locked CK-omics resolver contract changes. Mismatch in
|
|
25
|
+
* userDataStorage → stale cache discarded. */
|
|
26
|
+
export const SCHEMA_V_GENE_LABELS = '14-r1-1';
|
|
27
|
+
const SCHEMA_KEY = '__schema_v';
|
|
28
|
+
|
|
29
|
+
/** Ensembl rest.ensembl.org /lookup/id hard cap (Pitfall 3). */
|
|
30
|
+
const ENS_CHUNK = 1000;
|
|
31
|
+
|
|
32
|
+
/** Matches the Phase 13 UniProt cap. Ensembl rate budget is 55k req/hour
|
|
33
|
+
* (≈15 req/s); FETCH_CONCURRENCY = 6 keeps us well under that bound. */
|
|
34
|
+
const FETCH_CONCURRENCY = 6;
|
|
35
|
+
|
|
36
|
+
/** Predicted-pattern prefix list (CK-omics line 898-909, in this exact order). */
|
|
37
|
+
export const PREDICTED_PREFIXES = [
|
|
38
|
+
'ENSRNOG', 'ENSMUSG', 'LOC', 'ENSG', 'ENSDARG',
|
|
39
|
+
'ENSRNO', 'MGP_', 'RGD', 'AABR',
|
|
40
|
+
] as const;
|
|
41
|
+
|
|
42
|
+
/** Ensembl-eligible test per CK-omics line 933: only IDs starting with 'ENS'
|
|
43
|
+
* OR 'MGP_' are sent to /lookup/id. LOC / RGD / AABR are NCBI / RGD /
|
|
44
|
+
* Affymetrix respectively — they keep the raw ID + `†` marker only. */
|
|
45
|
+
export function isEnsemblEligible(id: string): boolean {
|
|
46
|
+
return id.startsWith('ENS') || id.startsWith('MGP_');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** True iff `id` starts with any locked predicted-pattern prefix. */
|
|
50
|
+
export function isPredicted(id: string): boolean {
|
|
51
|
+
for (const p of PREDICTED_PREFIXES)
|
|
52
|
+
if (id.startsWith(p)) return true;
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export type SpeciesCode =
|
|
57
|
+
'homo_sapiens' | 'mus_musculus' | 'rattus_norvegicus' | 'danio_rerio';
|
|
58
|
+
|
|
59
|
+
/** Prefix → Ensembl species code. ENSRNO falls back to rattus_norvegicus.
|
|
60
|
+
* Non-Ensembl-eligible prefixes (LOC / RGD / AABR) return null so callers
|
|
61
|
+
* skip them at the POST stage. */
|
|
62
|
+
export function detectSpecies(id: string): SpeciesCode | null {
|
|
63
|
+
if (id.startsWith('ENSG')) return 'homo_sapiens';
|
|
64
|
+
if (id.startsWith('ENSMUSG')) return 'mus_musculus';
|
|
65
|
+
if (id.startsWith('MGP_')) return 'mus_musculus';
|
|
66
|
+
if (id.startsWith('ENSRNOG')) return 'rattus_norvegicus';
|
|
67
|
+
if (id.startsWith('ENSRNO')) return 'rattus_norvegicus';
|
|
68
|
+
if (id.startsWith('ENSDARG')) return 'danio_rerio';
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface EnsemblEntry {
|
|
73
|
+
display_name?: string;
|
|
74
|
+
external_name?: string;
|
|
75
|
+
description?: string;
|
|
76
|
+
species?: string;
|
|
77
|
+
biotype?: string;
|
|
78
|
+
object_type?: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Verbatim port of CKomics extract_readable_description (lines 1062-1104).
|
|
82
|
+
* Returns null when the cleaned description is empty, still says
|
|
83
|
+
* "uncharacterized", or is too short to be useful. */
|
|
84
|
+
export function extractReadableDescription(description: string | null | undefined): string | null {
|
|
85
|
+
if (description == null || description === '') return null;
|
|
86
|
+
let s = String(description);
|
|
87
|
+
|
|
88
|
+
// Strip [organism] suffix.
|
|
89
|
+
s = s.split('[')[0].trim();
|
|
90
|
+
|
|
91
|
+
// Strip the various "Predicted to ..." preambles, case-insensitive, in
|
|
92
|
+
// order — `enable` / `be involved in` / `be located in` / `be part of` /
|
|
93
|
+
// `be` first because the bare `Predicted to ` would otherwise consume them.
|
|
94
|
+
s = s.replace(/^Predicted to enable\s+/i, '');
|
|
95
|
+
s = s.replace(/^Predicted to be involved in\s+/i, '');
|
|
96
|
+
s = s.replace(/^Predicted to be located in\s+/i, '');
|
|
97
|
+
s = s.replace(/^Predicted to be part of\s+/i, '');
|
|
98
|
+
s = s.replace(/^Predicted to be\s+/i, '');
|
|
99
|
+
s = s.replace(/^Predicted to\s+/i, '');
|
|
100
|
+
|
|
101
|
+
// First sentence only.
|
|
102
|
+
const sentences = s.split('.');
|
|
103
|
+
if (sentences.length > 0) s = sentences[0].trim();
|
|
104
|
+
|
|
105
|
+
// Strip species suffixes.
|
|
106
|
+
s = s.replace(/_RAT$/i, '');
|
|
107
|
+
s = s.replace(/_MOUSE$/i, '');
|
|
108
|
+
s = s.replace(/_HUMAN$/i, '');
|
|
109
|
+
|
|
110
|
+
// Capitalize first letter if lowercase.
|
|
111
|
+
if (s.length > 0 && s[0] === s[0].toLowerCase() && s[0] !== s[0].toUpperCase())
|
|
112
|
+
s = s[0].toUpperCase() + s.slice(1);
|
|
113
|
+
|
|
114
|
+
// Truncate runaway descriptions (CK-omics line 1097-1098).
|
|
115
|
+
if (s.length > 60) s = s.slice(0, 57) + '...';
|
|
116
|
+
|
|
117
|
+
if (s === '' || s.length < 3) return null;
|
|
118
|
+
if (s.toLowerCase().includes('uncharacterized')) return null;
|
|
119
|
+
|
|
120
|
+
return s;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** CK-omics line 1011 — `improved_name + '*'? + '†'?`. The dagger is appended
|
|
124
|
+
* UNCONDITIONALLY when the resolver acted on a predicted row (CK-omics
|
|
125
|
+
* `final_label = improved_name + ('*' if has_asterisk else '') + '†'`). */
|
|
126
|
+
export function applyMarkerRules(improvedName: string,
|
|
127
|
+
hadAsterisk: boolean, wasReclassified: boolean): string {
|
|
128
|
+
return improvedName + (hadAsterisk ? '*' : '') + (wasReclassified ? '†' : '');
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function chunk<T>(arr: T[], size: number): T[][] {
|
|
132
|
+
const out: T[][] = [];
|
|
133
|
+
for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
|
|
134
|
+
return out;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function loadCache(): Promise<Record<string, EnsemblEntry | string>> {
|
|
138
|
+
try {
|
|
139
|
+
const raw = (await grok.dapi.userDataStorage.get(STORE_GENE_LABELS)) ?? {};
|
|
140
|
+
if ((raw as any)[SCHEMA_KEY] !== SCHEMA_V_GENE_LABELS)
|
|
141
|
+
return {[SCHEMA_KEY]: SCHEMA_V_GENE_LABELS};
|
|
142
|
+
return raw as Record<string, EnsemblEntry | string>;
|
|
143
|
+
} catch {
|
|
144
|
+
return {[SCHEMA_KEY]: SCHEMA_V_GENE_LABELS};
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function flushCache(cache: Record<string, EnsemblEntry | string>,
|
|
149
|
+
fetched: Record<string, EnsemblEntry>): Promise<void> {
|
|
150
|
+
try {
|
|
151
|
+
await grok.dapi.userDataStorage.put(STORE_GENE_LABELS,
|
|
152
|
+
{...cache, ...fetched, [SCHEMA_KEY]: SCHEMA_V_GENE_LABELS});
|
|
153
|
+
} catch (e: any) {
|
|
154
|
+
console.warn(`Gene-label cache write failed: ${e?.message ?? e}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** One POST to https://rest.ensembl.org/lookup/id with body `{ids: [...]}`.
|
|
159
|
+
* Honours a single 429 retry-after backoff per chunk. Warn-and-continue on
|
|
160
|
+
* any other error; the resolver must not block import on Ensembl flakes. */
|
|
161
|
+
export async function lookupEnsemblBatch(ids: string[]): Promise<Map<string, EnsemblEntry>> {
|
|
162
|
+
const out = new Map<string, EnsemblEntry>();
|
|
163
|
+
if (ids.length === 0) return out;
|
|
164
|
+
|
|
165
|
+
const post = async (): Promise<Response> => grok.dapi.fetchProxy(
|
|
166
|
+
'https://rest.ensembl.org/lookup/id',
|
|
167
|
+
{
|
|
168
|
+
method: 'POST',
|
|
169
|
+
headers: {'Content-Type': 'application/json', 'Accept': 'application/json'},
|
|
170
|
+
body: JSON.stringify({ids}),
|
|
171
|
+
},
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
let resp: Response;
|
|
175
|
+
try {
|
|
176
|
+
resp = await post();
|
|
177
|
+
} catch (e: any) {
|
|
178
|
+
console.warn(`Ensembl lookup/id batch failed (network): ${e?.message ?? e}`);
|
|
179
|
+
return out;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (resp.status === 429) {
|
|
183
|
+
// Single retry per CK-omics line 822-825 contract; Retry-After header in
|
|
184
|
+
// seconds (fractional allowed). Cap the back-off at 30s to keep import
|
|
185
|
+
// responsive — the resolver degrades gracefully on continued failure.
|
|
186
|
+
const retryAfter = parseFloat(resp.headers.get('Retry-After') ?? '1');
|
|
187
|
+
const waitMs = Math.min(30_000, Math.max(0, retryAfter * 1000));
|
|
188
|
+
await new Promise((res) => setTimeout(res, waitMs));
|
|
189
|
+
try {
|
|
190
|
+
resp = await post();
|
|
191
|
+
} catch (e: any) {
|
|
192
|
+
console.warn(`Ensembl lookup/id retry failed: ${e?.message ?? e}`);
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (!resp.ok) {
|
|
198
|
+
console.warn(`Ensembl lookup/id returned status ${resp.status}; continuing`);
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
let body: any;
|
|
203
|
+
try {
|
|
204
|
+
body = await resp.json();
|
|
205
|
+
} catch (e: any) {
|
|
206
|
+
console.warn(`Ensembl lookup/id JSON parse failed: ${e?.message ?? e}`);
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (body && typeof body === 'object') {
|
|
211
|
+
for (const [id, entry] of Object.entries(body)) {
|
|
212
|
+
if (entry && typeof entry === 'object') {
|
|
213
|
+
const e = entry as Record<string, unknown>;
|
|
214
|
+
// Type-guard every field read (threat T-14-01-T1).
|
|
215
|
+
const safe: EnsemblEntry = {};
|
|
216
|
+
if (typeof e.display_name === 'string') safe.display_name = e.display_name;
|
|
217
|
+
if (typeof e.external_name === 'string') safe.external_name = e.external_name;
|
|
218
|
+
if (typeof e.description === 'string') safe.description = e.description;
|
|
219
|
+
if (typeof e.species === 'string') safe.species = e.species;
|
|
220
|
+
if (typeof e.biotype === 'string') safe.biotype = e.biotype;
|
|
221
|
+
if (typeof e.object_type === 'string') safe.object_type = e.object_type;
|
|
222
|
+
out.set(id, safe);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return out;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Three-level fallback per CK-omics lines 800-804:
|
|
230
|
+
* external_name → display_name → description.split('[')[0].strip() → raw ID.
|
|
231
|
+
* Acceptance gate per CK-omics lines 977-988: reject the candidate if it equals
|
|
232
|
+
* the raw ID OR starts with a predicted prefix. */
|
|
233
|
+
function pickBestName(rawId: string, entry: EnsemblEntry | undefined): string | null {
|
|
234
|
+
if (!entry) return null;
|
|
235
|
+
const candidates: Array<string | undefined> = [
|
|
236
|
+
entry.external_name,
|
|
237
|
+
entry.display_name,
|
|
238
|
+
entry.description ? entry.description.split('[')[0].trim() : undefined,
|
|
239
|
+
];
|
|
240
|
+
for (const c of candidates) {
|
|
241
|
+
if (c && c !== rawId && !isPredicted(c)) return c;
|
|
242
|
+
}
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export type GeneLabelProgress = (done: number, total: number,
|
|
247
|
+
phase: 'lookup' | 'apply') => void;
|
|
248
|
+
|
|
249
|
+
/** Public entry — runs after every parser produces its DataFrame. Always
|
|
250
|
+
* creates `Display Name` and `Source ID` columns even when no predicted IDs
|
|
251
|
+
* are present (Pitfall 9: downstream Volcano label bindings depend on the
|
|
252
|
+
* `Display Name` invariant). The resolver degrades gracefully on Ensembl
|
|
253
|
+
* outages — analyst sees raw IDs + a single warning toast.
|
|
254
|
+
*/
|
|
255
|
+
export async function resolveGeneLabels(df: DG.DataFrame,
|
|
256
|
+
progress?: GeneLabelProgress): Promise<void> {
|
|
257
|
+
const geneCol = findColumn(df, SEMTYPE.GENE_SYMBOL, ['gene name', 'gene names', 'gene symbol']);
|
|
258
|
+
const n = df.rowCount;
|
|
259
|
+
const displayArr: string[] = new Array(n).fill('');
|
|
260
|
+
const sourceArr: string[] = new Array(n).fill('');
|
|
261
|
+
|
|
262
|
+
// Pre-fill from Gene name (when present); the resolver overrides specific
|
|
263
|
+
// rows below. Rows without a Gene name source still get '' for both, so the
|
|
264
|
+
// columns are always rectangular.
|
|
265
|
+
if (geneCol != null) {
|
|
266
|
+
for (let i = 0; i < n; i++) {
|
|
267
|
+
const raw = geneCol.get(i);
|
|
268
|
+
const s = (raw == null) ? '' : String(raw);
|
|
269
|
+
displayArr[i] = s;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Identify rows whose Gene name (stripped of pre-existing * and †) matches a
|
|
274
|
+
// predicted-pattern prefix. CK-omics line 916-925: also walks FirstGene /
|
|
275
|
+
// FirstUniProt columns. Datagrok's parsers normalize to a single Gene name
|
|
276
|
+
// column, so checking that column is sufficient — but if the proteomics
|
|
277
|
+
// primary protein id column happens to also be a predicted Ensembl ID, we
|
|
278
|
+
// honour the same logic on it.
|
|
279
|
+
const idCol = findColumn(df, SEMTYPE.PROTEIN_ID,
|
|
280
|
+
['protein ids', 'primary protein id', 'protein id', 'majority protein ids', 'accession', 'uniprot']);
|
|
281
|
+
|
|
282
|
+
/** Per-row work item carrying the raw ID we want to resolve. */
|
|
283
|
+
type Predicted = {row: number; clean: string; hadAsterisk: boolean; source: 'gene' | 'protein'};
|
|
284
|
+
const predicted: Predicted[] = [];
|
|
285
|
+
|
|
286
|
+
const recordIfPredicted = (row: number, raw: string, source: 'gene' | 'protein'): boolean => {
|
|
287
|
+
if (!raw) return false;
|
|
288
|
+
const hadAsterisk = raw.includes('*');
|
|
289
|
+
const clean = raw.replace(/[*†]/g, '');
|
|
290
|
+
if (!isPredicted(clean)) return false;
|
|
291
|
+
predicted.push({row, clean, hadAsterisk, source});
|
|
292
|
+
return true;
|
|
293
|
+
};
|
|
294
|
+
|
|
295
|
+
for (let i = 0; i < n; i++) {
|
|
296
|
+
const g = geneCol != null ? geneCol.get(i) : null;
|
|
297
|
+
const gs = g == null ? '' : String(g);
|
|
298
|
+
if (recordIfPredicted(i, gs, 'gene')) continue;
|
|
299
|
+
// Fall through to protein id only when gene name isn't a predicted ID.
|
|
300
|
+
if (idCol != null) {
|
|
301
|
+
const p = idCol.get(i);
|
|
302
|
+
const ps = p == null ? '' : String(p).split(';')[0];
|
|
303
|
+
recordIfPredicted(i, ps, 'protein');
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// Always ensure the two new columns exist (Pitfall 9). Remove first if a
|
|
308
|
+
// re-run is in progress so we re-init cleanly.
|
|
309
|
+
for (const name of ['Display Name', 'Source ID']) {
|
|
310
|
+
const existing = df.col(name);
|
|
311
|
+
if (existing != null) df.columns.remove(name);
|
|
312
|
+
}
|
|
313
|
+
// Fast path: no predicted IDs → write columns and exit, but always create
|
|
314
|
+
// them. Display Name = raw gene name (or ''), Source ID = ''.
|
|
315
|
+
if (predicted.length === 0) {
|
|
316
|
+
const dCol = df.columns.addNewString('Display Name');
|
|
317
|
+
dCol.init((i) => displayArr[i]);
|
|
318
|
+
dCol.semType = SEMTYPE.DISPLAY_NAME;
|
|
319
|
+
const sCol = df.columns.addNewString('Source ID');
|
|
320
|
+
sCol.init((i) => sourceArr[i]);
|
|
321
|
+
sCol.semType = SEMTYPE.SOURCE_ID;
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// Collect unique predicted IDs and the Ensembl-eligible subset.
|
|
326
|
+
const uniqueIds = [...new Set(predicted.map((p) => p.clean))];
|
|
327
|
+
const ensIds = uniqueIds.filter((id) => isEnsemblEligible(id));
|
|
328
|
+
|
|
329
|
+
const cache = await loadCache();
|
|
330
|
+
const fetched: Record<string, EnsemblEntry> = {};
|
|
331
|
+
const lookupResult = new Map<string, EnsemblEntry>();
|
|
332
|
+
|
|
333
|
+
// Seed from cache.
|
|
334
|
+
for (const id of ensIds) {
|
|
335
|
+
const c = cache[id];
|
|
336
|
+
if (c && typeof c === 'object') lookupResult.set(id, c as EnsemblEntry);
|
|
337
|
+
}
|
|
338
|
+
const misses = ensIds.filter((id) => !lookupResult.has(id));
|
|
339
|
+
|
|
340
|
+
let unresolvedAny = false;
|
|
341
|
+
if (misses.length > 0) {
|
|
342
|
+
const chunks = chunk(misses, ENS_CHUNK);
|
|
343
|
+
const total = chunks.length;
|
|
344
|
+
let done = 0;
|
|
345
|
+
try {
|
|
346
|
+
await runWithConcurrency(chunks, FETCH_CONCURRENCY, async (group) => {
|
|
347
|
+
const got = await lookupEnsemblBatch(group);
|
|
348
|
+
for (const [id, entry] of got) {
|
|
349
|
+
lookupResult.set(id, entry);
|
|
350
|
+
fetched[id] = entry;
|
|
351
|
+
}
|
|
352
|
+
// Anything in `group` that didn't come back is unresolved by Ensembl —
|
|
353
|
+
// we still mark the row reclassified, but Display Name falls back to
|
|
354
|
+
// the raw ID. Track this so the warning toast can name it.
|
|
355
|
+
for (const id of group) if (!got.has(id)) unresolvedAny = true;
|
|
356
|
+
done++;
|
|
357
|
+
progress?.(done, total, 'lookup');
|
|
358
|
+
});
|
|
359
|
+
} finally {
|
|
360
|
+
await flushCache(cache, fetched);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// Apply the marker / fallback rules per predicted row.
|
|
365
|
+
// CK-omics fallback chain when Ensembl yielded nothing: try the
|
|
366
|
+
// ProteinDescriptions column (CK-omics line 997-1005). Datagrok parsers
|
|
367
|
+
// surface this as "Protein Descriptions" (Spectronaut) or "Protein names"
|
|
368
|
+
// (MaxQuant); we accept either by name.
|
|
369
|
+
const descCol = df.col('Protein Descriptions') ?? df.col('Protein names')
|
|
370
|
+
?? df.col('Description') ?? df.col('protein_descriptions');
|
|
371
|
+
|
|
372
|
+
let reclassified = 0;
|
|
373
|
+
for (const p of predicted) {
|
|
374
|
+
const entry = lookupResult.get(p.clean);
|
|
375
|
+
let improved: string | null = pickBestName(p.clean, entry);
|
|
376
|
+
|
|
377
|
+
if (improved == null && entry != null) {
|
|
378
|
+
const cleaned = extractReadableDescription(entry.description ?? null);
|
|
379
|
+
if (cleaned != null && cleaned !== p.clean && !isPredicted(cleaned))
|
|
380
|
+
improved = cleaned;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// ProteinDescriptions fallback.
|
|
384
|
+
if (improved == null && descCol != null) {
|
|
385
|
+
const dRaw = descCol.get(p.row);
|
|
386
|
+
const cleaned = extractReadableDescription(dRaw == null ? null : String(dRaw));
|
|
387
|
+
if (cleaned != null && cleaned !== p.clean && !isPredicted(cleaned) && cleaned.length > 3)
|
|
388
|
+
improved = cleaned;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// If still nothing, keep raw ID — marker still appended (CK-omics line 1011).
|
|
392
|
+
const finalName = applyMarkerRules(improved ?? p.clean, p.hadAsterisk, true);
|
|
393
|
+
displayArr[p.row] = finalName;
|
|
394
|
+
sourceArr[p.row] = p.clean;
|
|
395
|
+
reclassified++;
|
|
396
|
+
}
|
|
397
|
+
progress?.(predicted.length, predicted.length, 'apply');
|
|
398
|
+
|
|
399
|
+
// D-10 duplicate disambiguation. Group rows by Display Name; if a name maps
|
|
400
|
+
// to >1 distinct Source ID, append `(Source ID)` to each of those rows.
|
|
401
|
+
const byName = new Map<string, Set<string>>();
|
|
402
|
+
const rowsByName = new Map<string, number[]>();
|
|
403
|
+
for (let i = 0; i < n; i++) {
|
|
404
|
+
const name = displayArr[i];
|
|
405
|
+
const src = sourceArr[i];
|
|
406
|
+
if (!name || !src) continue;
|
|
407
|
+
if (!byName.has(name)) byName.set(name, new Set());
|
|
408
|
+
byName.get(name)!.add(src);
|
|
409
|
+
if (!rowsByName.has(name)) rowsByName.set(name, []);
|
|
410
|
+
rowsByName.get(name)!.push(i);
|
|
411
|
+
}
|
|
412
|
+
let duplicateRows = 0;
|
|
413
|
+
for (const [name, srcs] of byName) {
|
|
414
|
+
if (srcs.size <= 1) continue;
|
|
415
|
+
for (const r of rowsByName.get(name) ?? []) {
|
|
416
|
+
const sid = sourceArr[r];
|
|
417
|
+
if (!sid) continue;
|
|
418
|
+
displayArr[r] = `${name} (${sid})`;
|
|
419
|
+
duplicateRows++;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
if (duplicateRows > 0) {
|
|
423
|
+
grok.shell.warning(`${duplicateRows} duplicate gene names disambiguated with source IDs.`);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
if (unresolvedAny) {
|
|
427
|
+
const unresolvedCount = predicted.filter((p) =>
|
|
428
|
+
isEnsemblEligible(p.clean) && !lookupResult.has(p.clean)).length;
|
|
429
|
+
grok.shell.warning(
|
|
430
|
+
`Ensembl gene-label resolution unavailable. Showing raw IDs (${unresolvedCount} unresolved). Re-import to retry.`,
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const dCol = df.columns.addNewString('Display Name');
|
|
435
|
+
dCol.init((i) => displayArr[i]);
|
|
436
|
+
dCol.semType = SEMTYPE.DISPLAY_NAME;
|
|
437
|
+
const sCol = df.columns.addNewString('Source ID');
|
|
438
|
+
sCol.init((i) => sourceArr[i]);
|
|
439
|
+
sCol.semType = SEMTYPE.SOURCE_ID;
|
|
440
|
+
|
|
441
|
+
// Silence unused-warning for `reclassified` — kept for future telemetry.
|
|
442
|
+
void reclassified;
|
|
443
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import * as DG from 'datagrok-api/dg';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Single source of truth for the organisms the package supports for g:Profiler
|
|
5
|
+
* enrichment and UniProt subcellular-location narrowing. `code` is the g:Profiler
|
|
6
|
+
* organism id; `display` is the user-facing dropdown label. Moved here (a leaf
|
|
7
|
+
* module) from `analysis/enrichment.ts` so parsers, the annotate dialog, and
|
|
8
|
+
* enrichment can all share it without a circular import.
|
|
9
|
+
*/
|
|
10
|
+
export const ORGANISM_LIST = [
|
|
11
|
+
{display: 'Homo sapiens (Human)', code: 'hsapiens'},
|
|
12
|
+
{display: 'Mus musculus (Mouse)', code: 'mmusculus'},
|
|
13
|
+
{display: 'Rattus norvegicus (Rat)', code: 'rnorvegicus'},
|
|
14
|
+
{display: 'Saccharomyces cerevisiae (Yeast)', code: 'scerevisiae'},
|
|
15
|
+
{display: 'Escherichia coli (K12)', code: 'ecoli'},
|
|
16
|
+
{display: 'Danio rerio (Zebrafish)', code: 'drerio'},
|
|
17
|
+
{display: 'Drosophila melanogaster (Fruit fly)', code: 'dmelanogaster'},
|
|
18
|
+
{display: 'Arabidopsis thaliana', code: 'athaliana'},
|
|
19
|
+
{display: 'Caenorhabditis elegans', code: 'celegans'},
|
|
20
|
+
] as const;
|
|
21
|
+
|
|
22
|
+
/** Scientific-name prefix (lowercased, parenthetical stripped) → g:Profiler code,
|
|
23
|
+
* derived once from ORGANISM_LIST. e.g. 'homo sapiens' → 'hsapiens'. */
|
|
24
|
+
const SCIENTIFIC_TO_CODE: ReadonlyArray<{scientific: string; code: string}> =
|
|
25
|
+
ORGANISM_LIST.map((o) => ({
|
|
26
|
+
scientific: o.display.split('(')[0].trim().toLowerCase(),
|
|
27
|
+
code: o.code,
|
|
28
|
+
}));
|
|
29
|
+
|
|
30
|
+
/** Display label → code and code → display, for dialog seeding. */
|
|
31
|
+
export function organismDisplayForCode(code: string | undefined): string | undefined {
|
|
32
|
+
return ORGANISM_LIST.find((o) => o.code === code)?.display;
|
|
33
|
+
}
|
|
34
|
+
export function organismCodeForDisplay(display: string | undefined): string | undefined {
|
|
35
|
+
return ORGANISM_LIST.find((o) => o.display === display)?.code;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Maps a raw UniProt/Spectronaut organism-name string to a g:Profiler code, or
|
|
40
|
+
* undefined if it matches none of the supported organisms. Matches on the
|
|
41
|
+
* scientific-name prefix so strain-qualified values resolve too — e.g.
|
|
42
|
+
* "Escherichia coli (strain K12)" → 'ecoli', "Rattus norvegicus" → 'rnorvegicus'.
|
|
43
|
+
*/
|
|
44
|
+
export function resolveOrganismCode(name: string | null | undefined): string | undefined {
|
|
45
|
+
if (!name) return undefined;
|
|
46
|
+
const lower = name.toLowerCase();
|
|
47
|
+
for (const {scientific, code} of SCIENTIFIC_TO_CODE)
|
|
48
|
+
if (lower.includes(scientific)) return code;
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Column-name hints for the per-protein organism column that some vendors emit
|
|
53
|
+
* (Spectronaut's `PG.Organisms`; generic "Organism"/"Species"). */
|
|
54
|
+
const ORGANISM_COLUMN_HINTS = ['pg.organisms', 'organism', 'organisms', 'species'];
|
|
55
|
+
|
|
56
|
+
function findOrganismColumn(df: DG.DataFrame): DG.Column | null {
|
|
57
|
+
for (const col of df.columns.toList()) {
|
|
58
|
+
const n = col.name.toLowerCase();
|
|
59
|
+
if (ORGANISM_COLUMN_HINTS.some((h) => n === h || n.includes(h))) return col;
|
|
60
|
+
}
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Infers the experiment's organism from an organism column in the data, when the
|
|
66
|
+
* data is unambiguously a single supported species. Returns the g:Profiler code,
|
|
67
|
+
* or undefined when there is no organism column, nothing resolves, or MORE THAN
|
|
68
|
+
* ONE supported organism is present (e.g. a HYE multi-species mix) — in which
|
|
69
|
+
* case the caller should leave the choice to the user rather than guess.
|
|
70
|
+
*/
|
|
71
|
+
export function detectOrganismCode(df: DG.DataFrame): string | undefined {
|
|
72
|
+
const col = findOrganismColumn(df);
|
|
73
|
+
if (!col) return undefined;
|
|
74
|
+
const codes = new Set<string>();
|
|
75
|
+
for (let i = 0; i < col.length; i++) {
|
|
76
|
+
if (col.isNone(i)) continue;
|
|
77
|
+
const code = resolveOrganismCode(col.get(i) as string);
|
|
78
|
+
if (code) codes.add(code);
|
|
79
|
+
if (codes.size > 1) return undefined; // ambiguous — don't guess
|
|
80
|
+
}
|
|
81
|
+
return codes.size === 1 ? [...codes][0] : undefined;
|
|
82
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export const SEMTYPE = {
|
|
2
|
+
PROTEIN_ID: 'Proteomics-ProteinId',
|
|
3
|
+
GENE_SYMBOL: 'Proteomics-GeneSymbol',
|
|
4
|
+
LOG2FC: 'Proteomics-Log2FC',
|
|
5
|
+
P_VALUE: 'Proteomics-PValue',
|
|
6
|
+
INTENSITY: 'Proteomics-Intensity',
|
|
7
|
+
SUBCELLULAR_LOCATION: 'Proteomics-SubcellularLocation',
|
|
8
|
+
DISPLAY_NAME: 'Proteomics-DisplayName',
|
|
9
|
+
SOURCE_ID: 'Proteomics-SourceId',
|
|
10
|
+
NUMERATOR_MEAN: 'Proteomics-NumeratorMean',
|
|
11
|
+
DENOMINATOR_MEAN: 'Proteomics-DenominatorMean',
|
|
12
|
+
} as const;
|
|
13
|
+
|
|
14
|
+
/** Default significance cutoffs — the SINGLE source for the |log2FC| ≥ 1 and
|
|
15
|
+
* adj.p ≤ 0.05 thresholds used as dialog defaults, function-parameter defaults,
|
|
16
|
+
* and `?? ` fallbacks across DE, enrichment, the volcano, and the demos. Never
|
|
17
|
+
* re-inline the literals; import these so the pipeline stays internally
|
|
18
|
+
* consistent when a default is retuned. */
|
|
19
|
+
export const DEFAULT_FC_THRESHOLD = 1.0;
|
|
20
|
+
export const DEFAULT_P_THRESHOLD = 0.05;
|
|
21
|
+
|
|
22
|
+
/** Locked direction palette (D-04): numerator/group1 = magenta, denominator/
|
|
23
|
+
* group2 = cyan, not-significant = gray, as ARGB ints. Single source shared by
|
|
24
|
+
* the volcano direction column and the UniProt panel's per-group bar chart —
|
|
25
|
+
* lives here (a leaf module) so both can import without a circular dependency. */
|
|
26
|
+
export const DIRECTION_COLORS_BASE = {
|
|
27
|
+
enrichedG1: 0xFFFF00FF, // magenta (ARGB)
|
|
28
|
+
enrichedG2: 0xFF00FFFF, // cyan (ARGB)
|
|
29
|
+
notSig: 0xFFAAAAAA, // gray (ARGB)
|
|
30
|
+
} as const;
|