@datagrok/proteomics 1.0.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -3
- package/CLAUDE.md +178 -0
- package/README.md +195 -3
- package/detectors.js +152 -9
- package/dist/package-test.js +1 -1744
- package/dist/package-test.js.map +1 -1
- package/dist/package.js +1 -146
- package/dist/package.js.map +1 -1
- package/docs/personas-and-capabilities.md +165 -0
- package/files/demo/README.md +264 -0
- package/files/demo/cptac-spike-in.txt +1571 -0
- package/files/demo/enrichment-demo.csv +120 -0
- package/files/demo/fragpipe-smoke-test.tsv +11 -0
- package/files/demo/proteinGroups.txt +28 -0
- package/files/demo/spectronaut-hye-candidates.tsv +94 -0
- package/files/demo/spectronaut-hye-demo.tsv +8761 -0
- package/files/demo/spectronaut-hye-mix.tsv +8761 -0
- package/files/demo/spectronaut-hye-precursor-golden.json +938 -0
- package/files/demo/spectronaut-hye-precursor-golden.tsv +235 -0
- package/files/demo/spectronaut-hye-precursor.tsv +493 -0
- package/images/enrichment-crosslink.png +0 -0
- package/images/enrichment-term-selected.png +0 -0
- package/images/hero.png +0 -0
- package/images/pipeline.svg +80 -0
- package/package.json +88 -63
- package/scripts/deqms_de.R +60 -0
- package/scripts/limma_de.R +42 -0
- package/scripts/vsn_normalize.R +19 -0
- package/src/analysis/differential-expression.ts +450 -0
- package/src/analysis/enrichment-export.ts +101 -0
- package/src/analysis/enrichment.ts +602 -0
- package/src/analysis/experiment-setup.ts +199 -0
- package/src/analysis/imputation.ts +407 -0
- package/src/analysis/log2-scale.ts +139 -0
- package/src/analysis/normalization.ts +255 -0
- package/src/analysis/pca.ts +254 -0
- package/src/analysis/spc-storage.ts +515 -0
- package/src/analysis/spc.ts +544 -0
- package/src/analysis/subcellular-location.ts +431 -0
- package/src/demo/enrichment-demo.ts +94 -0
- package/src/demo/proteomics-demo.ts +123 -0
- package/src/global.d.ts +15 -0
- package/src/menu.ts +133 -0
- package/src/package-api.ts +136 -14
- package/src/package-test.ts +45 -20
- package/src/package.g.ts +161 -0
- package/src/package.ts +1029 -17
- package/src/panels/protein-focus.ts +63 -0
- package/src/panels/published-analysis-panel.ts +151 -0
- package/src/panels/uniprot-panel.ts +349 -0
- package/src/parsers/fragpipe-parser.ts +200 -0
- package/src/parsers/generic-parser.ts +197 -0
- package/src/parsers/maxquant-parser.ts +162 -0
- package/src/parsers/shared-utils.ts +163 -0
- package/src/parsers/spectronaut-candidates-parser.ts +307 -0
- package/src/parsers/spectronaut-parser.ts +604 -0
- package/src/publishing/assert-published-shape.ts +260 -0
- package/src/publishing/post-open-recovery.ts +104 -0
- package/src/publishing/publish-project.ts +515 -0
- package/src/publishing/publish-settings.ts +59 -0
- package/src/publishing/publish-state.ts +316 -0
- package/src/publishing/share-dialog.ts +171 -0
- package/src/publishing/trim-dataframe.ts +247 -0
- package/src/tests/analysis.ts +658 -0
- package/src/tests/enrichment-export.ts +61 -0
- package/src/tests/enrichment-visualization.ts +340 -0
- package/src/tests/enrichment.ts +224 -0
- package/src/tests/fragpipe-e2e.ts +74 -0
- package/src/tests/fragpipe-parser.ts +147 -0
- package/src/tests/gene-label-resolver.ts +387 -0
- package/src/tests/generic-parser.ts +152 -0
- package/src/tests/group-mean-correlation.ts +139 -0
- package/src/tests/log2-scale.ts +93 -0
- package/src/tests/organisms.ts +56 -0
- package/src/tests/parsers.ts +182 -0
- package/src/tests/publish-roundtrip.ts +584 -0
- package/src/tests/publish-spike.ts +377 -0
- package/src/tests/qc-dashboard.ts +210 -0
- package/src/tests/smart-pathway-filter.ts +193 -0
- package/src/tests/spc-formula-lines-spike.ts +129 -0
- package/src/tests/spc.ts +640 -0
- package/src/tests/spectronaut-candidates-e2e.ts +140 -0
- package/src/tests/spectronaut-candidates-parser.ts +398 -0
- package/src/tests/spectronaut-parser.ts +668 -0
- package/src/tests/subcellular-location.ts +361 -0
- package/src/tests/uniprot-panel.ts +202 -0
- package/src/tests/volcano.ts +603 -0
- package/src/utils/column-detection.ts +28 -0
- package/src/utils/gene-label-resolver.ts +443 -0
- package/src/utils/organisms.ts +82 -0
- package/src/utils/proteomics-types.ts +30 -0
- package/src/viewers/enrichment-viewers.ts +274 -0
- package/src/viewers/group-mean-correlation.ts +218 -0
- package/src/viewers/heatmap.ts +168 -0
- package/src/viewers/pca-plot.ts +169 -0
- package/src/viewers/qc-computations.ts +266 -0
- package/src/viewers/qc-dashboard.ts +176 -0
- package/src/viewers/spc-dashboard.ts +755 -0
- package/src/viewers/volcano.ts +690 -0
- package/test-console-output-1.log +2073 -0
- package/test-record-1.mp4 +0 -0
- package/tools/derive-precursor-golden-sidecar.mjs +81 -0
- package/tools/generate-enrichment-fixture.sh +160 -0
- package/tools/generate-spectronaut-candidates-fixture.mjs +212 -0
- package/tools/generate-spectronaut-precursor-fixture.mjs +128 -0
- package/tools/spectronaut-aggregate.sh +46 -0
- package/tools/spectronaut-aggregate.sql +80 -0
- package/tsconfig.json +18 -71
- package/webpack.config.js +86 -45
- package/.eslintignore +0 -1
- package/.eslintrc.json +0 -56
- package/LICENSE +0 -674
- package/package.png +0 -0
- package/scripts/number_antibody.py +0 -190
- package/scripts/number_antibody_abnumber.py +0 -177
- package/scripts/number_antibody_anarci.py +0 -200
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import * as grok from 'datagrok-api/grok';
|
|
2
|
+
import * as ui from 'datagrok-api/ui';
|
|
3
|
+
import * as DG from 'datagrok-api/dg';
|
|
4
|
+
|
|
5
|
+
import {SEMTYPE} from '../utils/proteomics-types';
|
|
6
|
+
import {log2TransformColumns, copyAsLog2Columns, detectLog2Status} from '../parsers/shared-utils';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Post-import correction for the log2 scale of intensity data.
|
|
10
|
+
*
|
|
11
|
+
* At import each parser decides whether the intensities are raw (needs a log2
|
|
12
|
+
* transform) or already log2-transformed (copy as-is). That decision is a
|
|
13
|
+
* magnitude heuristic (`detectLog2Status`) and has misfired on small-magnitude
|
|
14
|
+
* linear intensities — values that happen to sit in the [0, 30] "looks like
|
|
15
|
+
* log2" band get copied instead of transformed, producing nonsense ±thousands
|
|
16
|
+
* fold changes downstream. This module lets the analyst override that call
|
|
17
|
+
* without re-importing.
|
|
18
|
+
*
|
|
19
|
+
* The rebuild works off the ORIGINAL (non-`log2(...)`) intensity columns, which
|
|
20
|
+
* every parser keeps pristine — normalization/imputation only ever mutate the
|
|
21
|
+
* `log2(...)` copies in place. So re-deriving the `log2(...)` columns is always
|
|
22
|
+
* valid and, by design, discards any normalization/imputation/DE that ran on the
|
|
23
|
+
* old values; those steps must be re-run, which the dialog warns about.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** Original (pre-`log2(...)`) intensity column names — the pristine source the
|
|
27
|
+
* rebuild derives from. Excludes the `log2(...)` copies themselves. */
|
|
28
|
+
export function getIntensityOriginals(df: DG.DataFrame): string[] {
|
|
29
|
+
return df.columns.toList()
|
|
30
|
+
.filter((c) => c.semType === SEMTYPE.INTENSITY && !c.name.startsWith('log2('))
|
|
31
|
+
.map((c) => c.name);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Infers whether the current `log2(...)` columns were produced by a transform
|
|
35
|
+
* (`Math.log2(orig)`) or a straight copy (`orig`, i.e. treated as already log2).
|
|
36
|
+
* Scans until the first decisive non-null positive row — the two branches never
|
|
37
|
+
* coincide for a positive value (`log2(x) ≠ x` for all x > 0). Returns 'unknown'
|
|
38
|
+
* when no `log2(...)` column or no decisive row exists. */
|
|
39
|
+
export function detectCurrentLog2Applied(
|
|
40
|
+
df: DG.DataFrame, originalNames: string[],
|
|
41
|
+
): 'transform' | 'copy' | 'unknown' {
|
|
42
|
+
for (const name of originalNames) {
|
|
43
|
+
const orig = df.col(name);
|
|
44
|
+
const l2 = df.col(`log2(${name})`);
|
|
45
|
+
if (!orig || !l2) continue;
|
|
46
|
+
for (let i = 0; i < df.rowCount; i++) {
|
|
47
|
+
if (orig.isNone(i) || l2.isNone(i)) continue;
|
|
48
|
+
const o = Number(orig.get(i));
|
|
49
|
+
const v = Number(l2.get(i));
|
|
50
|
+
if (!(o > 0) || !Number.isFinite(v)) continue;
|
|
51
|
+
if (Math.abs(v - Math.log2(o)) < 1e-4) return 'transform';
|
|
52
|
+
if (Math.abs(v - o) < 1e-4) return 'copy';
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return 'unknown';
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Rebuilds the `log2(...)` columns from the originals under the chosen scale:
|
|
59
|
+
* `alreadyLog2` copies as-is, otherwise applies the log2 transform. Removes the
|
|
60
|
+
* existing `log2(...)` columns first (so a re-run doesn't duplicate) and clears
|
|
61
|
+
* stale downstream pipeline tags — the base data changed, so any prior
|
|
62
|
+
* normalize/impute/DE is invalid and must be re-run. No-op (returns false) when
|
|
63
|
+
* the requested scale already matches what's applied. */
|
|
64
|
+
export function applyLog2Scale(df: DG.DataFrame, alreadyLog2: boolean): boolean {
|
|
65
|
+
const originals = getIntensityOriginals(df);
|
|
66
|
+
if (originals.length === 0) return false;
|
|
67
|
+
|
|
68
|
+
const current = detectCurrentLog2Applied(df, originals);
|
|
69
|
+
const target = alreadyLog2 ? 'copy' : 'transform';
|
|
70
|
+
if (current === target) return false;
|
|
71
|
+
|
|
72
|
+
for (const name of originals) {
|
|
73
|
+
const l2 = `log2(${name})`;
|
|
74
|
+
if (df.col(l2)) df.columns.remove(l2);
|
|
75
|
+
}
|
|
76
|
+
if (alreadyLog2)
|
|
77
|
+
copyAsLog2Columns(df, originals);
|
|
78
|
+
else
|
|
79
|
+
log2TransformColumns(df, originals);
|
|
80
|
+
|
|
81
|
+
// Base intensities were re-derived — anything computed from the old log2
|
|
82
|
+
// columns is stale. Drop the completion tags so viewers gate correctly and
|
|
83
|
+
// the analyst re-runs from Normalize.
|
|
84
|
+
for (const tag of ['proteomics.normalized', 'proteomics.imputed',
|
|
85
|
+
'proteomics.de_complete', 'proteomics.de_method'])
|
|
86
|
+
df.setTag(tag, '');
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Dialog to review and override the log2 scale of the imported intensities.
|
|
91
|
+
* Seeds the checkbox from the scale actually applied at import (inferred), not a
|
|
92
|
+
* re-detection, so it reflects reality; shows the magnitude heuristic's read as
|
|
93
|
+
* an advisory hint. */
|
|
94
|
+
export function showLog2ScaleDialog(df: DG.DataFrame): void {
|
|
95
|
+
const originals = getIntensityOriginals(df);
|
|
96
|
+
if (originals.length === 0) {
|
|
97
|
+
grok.shell.warning('No intensity columns found — nothing to rescale.');
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const current = detectCurrentLog2Applied(df, originals);
|
|
102
|
+
const detection = detectLog2Status(df, originals);
|
|
103
|
+
|
|
104
|
+
const alreadyLog2 = ui.input.bool('Data is already log2-transformed', {
|
|
105
|
+
value: current === 'copy',
|
|
106
|
+
tooltipText: 'On: intensities are used as-is. Off: a log2 transform is applied. ' +
|
|
107
|
+
'Flip this if fold changes look wildly off — the import heuristic can misread ' +
|
|
108
|
+
'small-magnitude raw intensities as already-log2.',
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
const hint = ui.divText(`Auto-detection: ${detection.message}.` +
|
|
112
|
+
(current !== 'unknown' ? ` Currently applied: ${current === 'copy' ?
|
|
113
|
+
'used as-is (no transform)' : 'log2-transformed'}.` : ''));
|
|
114
|
+
hint.style.cssText = 'font-style:italic;color:#888;margin:4px 0;font-size:12px;max-width:420px;';
|
|
115
|
+
|
|
116
|
+
const pipelineRan = ['proteomics.normalized', 'proteomics.imputed', 'proteomics.de_complete']
|
|
117
|
+
.some((t) => df.getTag(t) === 'true');
|
|
118
|
+
const warn = ui.divText(
|
|
119
|
+
'Normalize / Impute / Differential Expression have already run. Changing the ' +
|
|
120
|
+
'scale resets those steps — re-run them from Analyze.');
|
|
121
|
+
warn.style.cssText = 'color:#b26a00;margin:4px 0;font-size:12px;max-width:420px;';
|
|
122
|
+
warn.style.display = pipelineRan ? '' : 'none';
|
|
123
|
+
|
|
124
|
+
ui.dialog('Set Log2 Scale')
|
|
125
|
+
.add(alreadyLog2)
|
|
126
|
+
.add(hint)
|
|
127
|
+
.add(warn)
|
|
128
|
+
.onOK(() => {
|
|
129
|
+
const changed = applyLog2Scale(df, alreadyLog2.value);
|
|
130
|
+
if (!changed) {
|
|
131
|
+
grok.shell.info('Scale unchanged.');
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
grok.shell.info(alreadyLog2.value ?
|
|
135
|
+
'Intensities are now used as-is (no log2 transform). Re-run analysis steps.' :
|
|
136
|
+
'Intensities are now log2-transformed. Re-run analysis steps.');
|
|
137
|
+
})
|
|
138
|
+
.show();
|
|
139
|
+
}
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import * as grok from 'datagrok-api/grok';
|
|
2
|
+
import * as ui from 'datagrok-api/ui';
|
|
3
|
+
import * as DG from 'datagrok-api/dg';
|
|
4
|
+
|
|
5
|
+
import {SEMTYPE} from '../utils/proteomics-types';
|
|
6
|
+
import {unpivotIntensities} from '../viewers/qc-computations';
|
|
7
|
+
|
|
8
|
+
/** Median-centers each specified column in-place.
|
|
9
|
+
* Subtracts the column median from every non-null value.
|
|
10
|
+
* Uses getRawData() for bulk array access instead of per-element get/set. */
|
|
11
|
+
export function medianNormalize(df: DG.DataFrame, colNames: string[]): void {
|
|
12
|
+
for (const name of colNames) {
|
|
13
|
+
const col = df.col(name);
|
|
14
|
+
if (!col) continue;
|
|
15
|
+
if (col.type !== DG.COLUMN_TYPE.FLOAT) {
|
|
16
|
+
console.warn(`medianNormalize: skipping non-float column "${name}" (type=${col.type})`);
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const median = col.stats.med;
|
|
21
|
+
if (isNaN(median)) continue;
|
|
22
|
+
|
|
23
|
+
const raw = col.getRawData() as Float32Array | Float64Array;
|
|
24
|
+
for (let i = 0; i < raw.length; i++) {
|
|
25
|
+
if (!col.isNone(i))
|
|
26
|
+
raw[i] -= median;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
df.fireValuesChanged();
|
|
30
|
+
df.setTag('proteomics.normalized', 'true');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Quantile-normalizes the specified columns in-place.
|
|
34
|
+
* Aligns all column distributions to the same shape by replacing values
|
|
35
|
+
* with rank-based averages. Missing values are preserved.
|
|
36
|
+
* Uses getRawData() for bulk array access in tight loops. */
|
|
37
|
+
export function quantileNormalize(df: DG.DataFrame, colNames: string[]): void {
|
|
38
|
+
if (colNames.length < 2) return;
|
|
39
|
+
|
|
40
|
+
const cols = colNames.map((n) => df.col(n))
|
|
41
|
+
.filter((c) => c != null && c.type === DG.COLUMN_TYPE.FLOAT) as DG.Column[];
|
|
42
|
+
if (cols.length < 2) return;
|
|
43
|
+
|
|
44
|
+
const nRows = df.rowCount;
|
|
45
|
+
|
|
46
|
+
// For each column, collect indices of non-missing values and sort by value
|
|
47
|
+
const colData: {indices: number[]; raw: Float32Array | Float64Array}[] = [];
|
|
48
|
+
for (const col of cols) {
|
|
49
|
+
const raw = col.getRawData() as Float32Array | Float64Array;
|
|
50
|
+
const indices: number[] = [];
|
|
51
|
+
for (let i = 0; i < nRows; i++) {
|
|
52
|
+
if (!col.isNone(i))
|
|
53
|
+
indices.push(i);
|
|
54
|
+
}
|
|
55
|
+
// Sort indices by value (ascending)
|
|
56
|
+
indices.sort((a, b) => raw[a] - raw[b]);
|
|
57
|
+
colData.push({indices, raw});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Find the maximum number of non-null values across columns
|
|
61
|
+
const maxValid = Math.max(...colData.map((d) => d.indices.length));
|
|
62
|
+
if (maxValid === 0) return;
|
|
63
|
+
|
|
64
|
+
// Compute rank means using scaled rank alignment
|
|
65
|
+
// For each rank position r (0..maxValid-1), compute the mean of values
|
|
66
|
+
// at the corresponding fractional position in each column
|
|
67
|
+
const rankMeans = new Float64Array(maxValid);
|
|
68
|
+
for (let r = 0; r < maxValid; r++) {
|
|
69
|
+
let sum = 0;
|
|
70
|
+
let count = 0;
|
|
71
|
+
for (const d of colData) {
|
|
72
|
+
if (d.indices.length === 0) continue;
|
|
73
|
+
// Map rank r in [0..maxValid-1] to position in this column's sorted values
|
|
74
|
+
const pos = (r / (maxValid - 1)) * (d.indices.length - 1);
|
|
75
|
+
// Interpolate between floor and ceil positions
|
|
76
|
+
const lo = Math.floor(pos);
|
|
77
|
+
const hi = Math.min(Math.ceil(pos), d.indices.length - 1);
|
|
78
|
+
const frac = pos - lo;
|
|
79
|
+
const val = (1 - frac) * d.raw[d.indices[lo]] + frac * d.raw[d.indices[hi]];
|
|
80
|
+
sum += val;
|
|
81
|
+
count++;
|
|
82
|
+
}
|
|
83
|
+
rankMeans[r] = sum / count;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Replace each column's sorted values with the corresponding rank mean
|
|
87
|
+
for (const d of colData) {
|
|
88
|
+
const n = d.indices.length;
|
|
89
|
+
if (n === 0) continue;
|
|
90
|
+
for (let r = 0; r < n; r++) {
|
|
91
|
+
// Map this column's rank to the global rank mean position
|
|
92
|
+
const globalPos = maxValid === 1 ? 0 : (r / (n - 1)) * (maxValid - 1);
|
|
93
|
+
const lo = Math.floor(globalPos);
|
|
94
|
+
const hi = Math.min(Math.ceil(globalPos), maxValid - 1);
|
|
95
|
+
const frac = globalPos - lo;
|
|
96
|
+
const val = (1 - frac) * rankMeans[lo] + frac * rankMeans[hi];
|
|
97
|
+
d.raw[d.indices[r]] = val;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
df.fireValuesChanged();
|
|
102
|
+
df.setTag('proteomics.normalized', 'true');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Performs Variance-Stabilizing Normalization via server-side R script.
|
|
106
|
+
* Operates on raw (non-log2) intensity columns and writes results to log2 columns.
|
|
107
|
+
* Falls back to quantile normalization if R environment is unavailable. */
|
|
108
|
+
export async function vsnNormalize(df: DG.DataFrame, colNames: string[]): Promise<void> {
|
|
109
|
+
try {
|
|
110
|
+
// Find raw intensity column names by stripping log2() prefix
|
|
111
|
+
const rawColNames: string[] = [];
|
|
112
|
+
const log2ColNames: string[] = [];
|
|
113
|
+
for (const name of colNames) {
|
|
114
|
+
if (name.startsWith('log2(') && name.endsWith(')')) {
|
|
115
|
+
const rawName = name.slice(5, -1);
|
|
116
|
+
if (df.col(rawName)) {
|
|
117
|
+
rawColNames.push(rawName);
|
|
118
|
+
log2ColNames.push(name);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (rawColNames.length === 0) {
|
|
124
|
+
// No raw columns found, fall back to quantile
|
|
125
|
+
if (colNames.length < 2) {
|
|
126
|
+
grok.shell.warning('Cannot normalize — VSN needs raw intensity columns and quantile fallback needs at least 2 columns');
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
grok.shell.warning('No raw intensity columns found for VSN — using quantile normalization');
|
|
130
|
+
quantileNormalize(df, colNames);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Build clean DataFrame with simple column names (s1, s2, ...)
|
|
135
|
+
const exprDf = DG.DataFrame.create(df.rowCount);
|
|
136
|
+
for (let i = 0; i < rawColNames.length; i++) {
|
|
137
|
+
const src = df.col(rawColNames[i])!;
|
|
138
|
+
const srcRaw = src.getRawData() as Float32Array | Float64Array;
|
|
139
|
+
const dst = exprDf.columns.addNewFloat(`s${i + 1}`);
|
|
140
|
+
dst.init((r) => srcRaw[r]);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const result: DG.DataFrame = await grok.functions.call('Proteomics:VsnNormalize', {exprDf});
|
|
144
|
+
|
|
145
|
+
// Copy VSN output (glog2 scale) into log2 intensity columns
|
|
146
|
+
for (let i = 0; i < log2ColNames.length; i++) {
|
|
147
|
+
const dst = df.col(log2ColNames[i])!;
|
|
148
|
+
const src = result.col(`s${i + 1}`)!;
|
|
149
|
+
const dstRaw = dst.getRawData() as Float32Array | Float64Array;
|
|
150
|
+
const srcRaw = src.getRawData() as Float32Array | Float64Array;
|
|
151
|
+
for (let r = 0; r < df.rowCount; r++) {
|
|
152
|
+
if (srcRaw[r] !== DG.FLOAT_NULL)
|
|
153
|
+
dstRaw[r] = srcRaw[r];
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
df.fireValuesChanged();
|
|
158
|
+
df.setTag('proteomics.normalized', 'true');
|
|
159
|
+
} catch (e: any) {
|
|
160
|
+
console.warn('VSN normalization failed, using quantile fallback:', e);
|
|
161
|
+
if (colNames.length < 2) {
|
|
162
|
+
grok.shell.warning('VSN failed and quantile fallback needs at least 2 columns — data left unnormalized');
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
grok.shell.warning('R environment unavailable — using quantile normalization');
|
|
166
|
+
quantileNormalize(df, colNames);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Shows a dialog for normalizing intensity columns.
|
|
171
|
+
* Supports Median Centering, Quantile, and VSN methods with reactive box plot preview.
|
|
172
|
+
* Displays an inline warning when Spectronaut pre-normalized data is detected. */
|
|
173
|
+
export function showNormalizationDialog(df: DG.DataFrame): void {
|
|
174
|
+
if (df.getTag('proteomics.normalized') === 'true') {
|
|
175
|
+
grok.shell.warning('Data already normalized');
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const log2ColNames = df.columns.toList()
|
|
180
|
+
.filter((c) => c.semType === SEMTYPE.INTENSITY && c.name.startsWith('log2('))
|
|
181
|
+
.map((c) => c.name);
|
|
182
|
+
|
|
183
|
+
const log2Cols = log2ColNames.map((n) => df.col(n)!);
|
|
184
|
+
|
|
185
|
+
// Pre-normalized warning banner (NORM-04)
|
|
186
|
+
const warningDiv = ui.div([], {style: {
|
|
187
|
+
background: '#FFF3CD', border: '1px solid #FFEEBA', color: '#856404',
|
|
188
|
+
padding: '8px', margin: '4px 0', borderRadius: '4px',
|
|
189
|
+
display: df.getTag('proteomics.preNormalized') === 'true' ? '' : 'none',
|
|
190
|
+
}});
|
|
191
|
+
warningDiv.textContent =
|
|
192
|
+
'This data may be pre-normalized (Spectronaut). Additional normalization may distort results.';
|
|
193
|
+
|
|
194
|
+
// Method selector (NORM-03)
|
|
195
|
+
const methodInput = ui.input.choice('Method', {
|
|
196
|
+
value: 'Median Centering',
|
|
197
|
+
items: ['Median Centering', 'Quantile', 'VSN'],
|
|
198
|
+
nullable: false,
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
// Column picker
|
|
202
|
+
const colsInput = ui.input.columns('Intensity columns', {
|
|
203
|
+
table: df, available: log2ColNames, value: log2Cols,
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
// Box plot preview container
|
|
207
|
+
const plotContainer = ui.div([], {style: {
|
|
208
|
+
width: '100%', height: '250px', border: '1px solid #ddd', marginTop: '8px',
|
|
209
|
+
}});
|
|
210
|
+
|
|
211
|
+
function updatePreview(): void {
|
|
212
|
+
plotContainer.innerHTML = '';
|
|
213
|
+
const selected = colsInput.value.map((c: DG.Column) => c.name);
|
|
214
|
+
if (selected.length === 0) return;
|
|
215
|
+
|
|
216
|
+
// Clone DataFrame and apply selected normalization for preview
|
|
217
|
+
const clone = df.clone(null, selected);
|
|
218
|
+
const method = methodInput.value;
|
|
219
|
+
if (method === 'Median Centering')
|
|
220
|
+
medianNormalize(clone, selected);
|
|
221
|
+
else if (method === 'Quantile')
|
|
222
|
+
quantileNormalize(clone, selected);
|
|
223
|
+
// VSN requires async R call -- show un-normalized distributions as preview
|
|
224
|
+
|
|
225
|
+
const longDf = unpivotIntensities(clone, selected);
|
|
226
|
+
const boxPlot = DG.Viewer.boxPlot(longDf, {
|
|
227
|
+
valueColumnName: 'Intensity', categoryColumnName: 'Sample',
|
|
228
|
+
} as any);
|
|
229
|
+
boxPlot.root.style.width = '100%';
|
|
230
|
+
boxPlot.root.style.height = '220px';
|
|
231
|
+
plotContainer.appendChild(boxPlot.root);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
methodInput.onChanged.subscribe(() => updatePreview());
|
|
235
|
+
colsInput.onChanged.subscribe(() => updatePreview());
|
|
236
|
+
updatePreview();
|
|
237
|
+
|
|
238
|
+
ui.dialog('Normalize')
|
|
239
|
+
.add(warningDiv)
|
|
240
|
+
.add(methodInput)
|
|
241
|
+
.add(colsInput)
|
|
242
|
+
.add(plotContainer)
|
|
243
|
+
.onOK(async () => {
|
|
244
|
+
const selected = colsInput.value.map((c: DG.Column) => c.name);
|
|
245
|
+
const method = methodInput.value as string;
|
|
246
|
+
if (method === 'Median Centering')
|
|
247
|
+
medianNormalize(df, selected);
|
|
248
|
+
else if (method === 'Quantile')
|
|
249
|
+
quantileNormalize(df, selected);
|
|
250
|
+
else if (method === 'VSN')
|
|
251
|
+
await vsnNormalize(df, selected);
|
|
252
|
+
grok.shell.info(`Normalized ${selected.length} columns (${method})`);
|
|
253
|
+
})
|
|
254
|
+
.show();
|
|
255
|
+
}
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import * as DG from 'datagrok-api/dg';
|
|
2
|
+
import {GroupAssignment} from './experiment-setup';
|
|
3
|
+
|
|
4
|
+
/** Transpose a matrix (rows x cols) -> (cols x rows). */
|
|
5
|
+
function transpose(matrix: number[][], rows: number, cols: number): number[][] {
|
|
6
|
+
const result: number[][] = [];
|
|
7
|
+
for (let j = 0; j < cols; j++) {
|
|
8
|
+
const row: number[] = new Array(rows);
|
|
9
|
+
for (let i = 0; i < rows; i++)
|
|
10
|
+
row[i] = matrix[i][j];
|
|
11
|
+
result.push(row);
|
|
12
|
+
}
|
|
13
|
+
return result;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Multiply matrices A (m x n) and B (n x p) -> C (m x p). */
|
|
17
|
+
function matmul(A: number[][], B: number[][], m: number, n: number, p: number): number[][] {
|
|
18
|
+
const C: number[][] = [];
|
|
19
|
+
for (let i = 0; i < m; i++) {
|
|
20
|
+
const row = new Array(p).fill(0);
|
|
21
|
+
for (let k = 0; k < n; k++) {
|
|
22
|
+
const aik = A[i][k];
|
|
23
|
+
for (let j = 0; j < p; j++)
|
|
24
|
+
row[j] += aik * B[k][j];
|
|
25
|
+
}
|
|
26
|
+
C.push(row);
|
|
27
|
+
}
|
|
28
|
+
return C;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Jacobi eigendecomposition for a symmetric matrix.
|
|
32
|
+
* Returns eigenvalues and eigenvectors sorted descending by eigenvalue. */
|
|
33
|
+
function jacobi(matrix: number[][], n: number): {eigenvalues: number[]; eigenvectors: number[][]} {
|
|
34
|
+
// Deep copy the matrix
|
|
35
|
+
const A: number[][] = matrix.map((row) => [...row]);
|
|
36
|
+
|
|
37
|
+
// Initialize eigenvectors as identity matrix
|
|
38
|
+
const V: number[][] = [];
|
|
39
|
+
for (let i = 0; i < n; i++) {
|
|
40
|
+
const row = new Array(n).fill(0);
|
|
41
|
+
row[i] = 1;
|
|
42
|
+
V.push(row);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const maxIter = 100;
|
|
46
|
+
const tolerance = 1e-10;
|
|
47
|
+
let converged = false;
|
|
48
|
+
|
|
49
|
+
for (let iter = 0; iter < maxIter; iter++) {
|
|
50
|
+
// Find largest off-diagonal element
|
|
51
|
+
let maxVal = 0;
|
|
52
|
+
let p = 0;
|
|
53
|
+
let q = 1;
|
|
54
|
+
for (let i = 0; i < n; i++) {
|
|
55
|
+
for (let j = i + 1; j < n; j++) {
|
|
56
|
+
if (Math.abs(A[i][j]) > maxVal) {
|
|
57
|
+
maxVal = Math.abs(A[i][j]);
|
|
58
|
+
p = i;
|
|
59
|
+
q = j;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (maxVal < tolerance) {
|
|
65
|
+
converged = true;
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Compute rotation angle
|
|
70
|
+
const theta = (A[q][q] - A[p][p]) / (2 * A[p][q]);
|
|
71
|
+
const t = Math.sign(theta) / (Math.abs(theta) + Math.sqrt(theta * theta + 1));
|
|
72
|
+
const c = 1 / Math.sqrt(t * t + 1);
|
|
73
|
+
const s = t * c;
|
|
74
|
+
|
|
75
|
+
// Apply rotation to A
|
|
76
|
+
const app = A[p][p];
|
|
77
|
+
const aqq = A[q][q];
|
|
78
|
+
const apq = A[p][q];
|
|
79
|
+
|
|
80
|
+
A[p][p] = c * c * app - 2 * s * c * apq + s * s * aqq;
|
|
81
|
+
A[q][q] = s * s * app + 2 * s * c * apq + c * c * aqq;
|
|
82
|
+
A[p][q] = 0;
|
|
83
|
+
A[q][p] = 0;
|
|
84
|
+
|
|
85
|
+
for (let i = 0; i < n; i++) {
|
|
86
|
+
if (i !== p && i !== q) {
|
|
87
|
+
const aip = A[i][p];
|
|
88
|
+
const aiq = A[i][q];
|
|
89
|
+
A[i][p] = c * aip - s * aiq;
|
|
90
|
+
A[p][i] = A[i][p];
|
|
91
|
+
A[i][q] = s * aip + c * aiq;
|
|
92
|
+
A[q][i] = A[i][q];
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Update eigenvectors
|
|
97
|
+
for (let i = 0; i < n; i++) {
|
|
98
|
+
const vip = V[i][p];
|
|
99
|
+
const viq = V[i][q];
|
|
100
|
+
V[i][p] = c * vip - s * viq;
|
|
101
|
+
V[i][q] = s * vip + c * viq;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (!converged)
|
|
106
|
+
console.warn(`PCA Jacobi did not converge in ${maxIter} iterations (n=${n}); results may be approximate`);
|
|
107
|
+
|
|
108
|
+
// Extract eigenvalues and sort descending
|
|
109
|
+
const eigenvalues = new Array(n);
|
|
110
|
+
for (let i = 0; i < n; i++)
|
|
111
|
+
eigenvalues[i] = A[i][i];
|
|
112
|
+
|
|
113
|
+
const indices = Array.from({length: n}, (_, i) => i);
|
|
114
|
+
indices.sort((a, b) => eigenvalues[b] - eigenvalues[a]);
|
|
115
|
+
|
|
116
|
+
const sortedEigenvalues = indices.map((i) => eigenvalues[i]);
|
|
117
|
+
const sortedEigenvectors: number[][] = [];
|
|
118
|
+
for (let i = 0; i < n; i++) {
|
|
119
|
+
const row = new Array(n);
|
|
120
|
+
for (let j = 0; j < n; j++)
|
|
121
|
+
row[j] = V[i][indices[j]];
|
|
122
|
+
sortedEigenvectors.push(row);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return {eigenvalues: sortedEigenvalues, eigenvectors: sortedEigenvectors};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Compute sample-level PCA on intensity columns.
|
|
129
|
+
* Returns a NEW sample-level DataFrame (rows = samples, not proteins)
|
|
130
|
+
* and the variance explained percentages. */
|
|
131
|
+
export function computePCA(
|
|
132
|
+
df: DG.DataFrame,
|
|
133
|
+
intensityCols: string[],
|
|
134
|
+
groupAssignment: GroupAssignment,
|
|
135
|
+
nComponents: number = 2,
|
|
136
|
+
): {pcaDf: DG.DataFrame; varianceExplained: number[]} {
|
|
137
|
+
const nSamples = intensityCols.length;
|
|
138
|
+
const nProteins = df.rowCount;
|
|
139
|
+
|
|
140
|
+
// Step 1: Build transposed matrix (nSamples x nProteins)
|
|
141
|
+
// Each row = one sample, each column = one protein's intensity across that sample
|
|
142
|
+
const matrix: number[][] = [];
|
|
143
|
+
|
|
144
|
+
// Compute protein means for imputation (mean across samples for each protein)
|
|
145
|
+
const proteinMeans: number[] = new Array(nProteins).fill(0);
|
|
146
|
+
const proteinCounts: number[] = new Array(nProteins).fill(0);
|
|
147
|
+
|
|
148
|
+
for (const colName of intensityCols) {
|
|
149
|
+
const col = df.col(colName);
|
|
150
|
+
if (!col) continue;
|
|
151
|
+
for (let i = 0; i < nProteins; i++) {
|
|
152
|
+
if (!col.isNone(i)) {
|
|
153
|
+
proteinMeans[i] += col.get(i) as number;
|
|
154
|
+
proteinCounts[i]++;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
for (let i = 0; i < nProteins; i++)
|
|
159
|
+
proteinMeans[i] = proteinCounts[i] > 0 ? proteinMeans[i] / proteinCounts[i] : 0;
|
|
160
|
+
|
|
161
|
+
// Build sample rows with imputation
|
|
162
|
+
for (const colName of intensityCols) {
|
|
163
|
+
const col = df.col(colName);
|
|
164
|
+
const row: number[] = new Array(nProteins);
|
|
165
|
+
for (let i = 0; i < nProteins; i++) {
|
|
166
|
+
if (!col || col.isNone(i))
|
|
167
|
+
row[i] = proteinMeans[i];
|
|
168
|
+
else
|
|
169
|
+
row[i] = col.get(i) as number;
|
|
170
|
+
}
|
|
171
|
+
matrix.push(row);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Step 2: Center each column (protein) by subtracting its mean across samples
|
|
175
|
+
const colMeans: number[] = new Array(nProteins).fill(0);
|
|
176
|
+
for (let j = 0; j < nProteins; j++) {
|
|
177
|
+
for (let i = 0; i < nSamples; i++)
|
|
178
|
+
colMeans[j] += matrix[i][j];
|
|
179
|
+
colMeans[j] /= nSamples;
|
|
180
|
+
}
|
|
181
|
+
for (let i = 0; i < nSamples; i++) {
|
|
182
|
+
for (let j = 0; j < nProteins; j++)
|
|
183
|
+
matrix[i][j] -= colMeans[j];
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Step 3: Compute covariance matrix S (nSamples x nSamples)
|
|
187
|
+
// S = X * X^T / (nProteins - 1) -- small matrix since nSamples << nProteins
|
|
188
|
+
const Xt = transpose(matrix, nSamples, nProteins);
|
|
189
|
+
const S = matmul(matrix, Xt, nSamples, nProteins, nSamples);
|
|
190
|
+
const divisor = nProteins > 1 ? nProteins - 1 : 1;
|
|
191
|
+
for (let i = 0; i < nSamples; i++) {
|
|
192
|
+
for (let j = 0; j < nSamples; j++)
|
|
193
|
+
S[i][j] /= divisor;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Step 4: Eigendecompose S
|
|
197
|
+
const {eigenvalues, eigenvectors} = jacobi(S, nSamples);
|
|
198
|
+
|
|
199
|
+
// Step 5: PC scores are the eigenvectors scaled by sqrt(eigenvalue)
|
|
200
|
+
const scores: number[][] = [];
|
|
201
|
+
for (let i = 0; i < nSamples; i++) {
|
|
202
|
+
const row: number[] = [];
|
|
203
|
+
for (let k = 0; k < Math.min(nComponents, nSamples); k++) {
|
|
204
|
+
const scale = eigenvalues[k] > 0 ? Math.sqrt(eigenvalues[k]) : 0;
|
|
205
|
+
row.push(eigenvectors[i][k] * scale);
|
|
206
|
+
}
|
|
207
|
+
scores.push(row);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Step 6: Compute variance explained. Clamp eigenvalues to >= 0 for both
|
|
211
|
+
// totalVariance and per-PC pct so numerical noise can't produce negative percentages.
|
|
212
|
+
const totalVariance = eigenvalues.reduce((sum, v) => sum + Math.max(v, 0), 0);
|
|
213
|
+
const varianceExplained: number[] = [];
|
|
214
|
+
for (let k = 0; k < Math.min(nComponents, nSamples); k++) {
|
|
215
|
+
const pct = totalVariance > 0 ? (Math.max(eigenvalues[k], 0) / totalVariance) * 100 : 0;
|
|
216
|
+
varianceExplained.push(Math.round(pct * 10) / 10);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// Step 7: Create sample-level DataFrame. PCA on <2 samples is meaningless;
|
|
220
|
+
// produce stable column labels rather than "PC1 (NaN%)".
|
|
221
|
+
const pc1Pct = varianceExplained.length > 0 && isFinite(varianceExplained[0]) ? varianceExplained[0] : 0;
|
|
222
|
+
const pc1Name = nSamples >= 2 ? `PC1 (${pc1Pct}%)` : 'PC1';
|
|
223
|
+
const pc2Pct = varianceExplained.length >= 2 && isFinite(varianceExplained[1]) ? varianceExplained[1] : 0;
|
|
224
|
+
const pc2Name = nComponents >= 2 && varianceExplained.length >= 2 && nSamples >= 2
|
|
225
|
+
? `PC2 (${pc2Pct}%)`
|
|
226
|
+
: 'PC2';
|
|
227
|
+
|
|
228
|
+
const sampleNames = DG.Column.fromStrings('SampleName', intensityCols);
|
|
229
|
+
const pc1Col = DG.Column.fromFloat32Array(pc1Name,
|
|
230
|
+
new Float32Array(scores.map((s) => s[0])));
|
|
231
|
+
const columns: DG.Column[] = [sampleNames, pc1Col];
|
|
232
|
+
|
|
233
|
+
if (nComponents >= 2) {
|
|
234
|
+
const pc2Col = DG.Column.fromFloat32Array(pc2Name,
|
|
235
|
+
new Float32Array(scores.map((s) => s.length > 1 ? s[1] : 0)));
|
|
236
|
+
columns.push(pc2Col);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Map each sample to its group
|
|
240
|
+
const groupNames: string[] = intensityCols.map((colName) => {
|
|
241
|
+
if (groupAssignment.group1.columns.includes(colName))
|
|
242
|
+
return groupAssignment.group1.name;
|
|
243
|
+
if (groupAssignment.group2.columns.includes(colName))
|
|
244
|
+
return groupAssignment.group2.name;
|
|
245
|
+
return 'Unknown';
|
|
246
|
+
});
|
|
247
|
+
const groupCol = DG.Column.fromStrings('Group', groupNames);
|
|
248
|
+
columns.push(groupCol);
|
|
249
|
+
|
|
250
|
+
const pcaDf = DG.DataFrame.fromColumns(columns);
|
|
251
|
+
pcaDf.name = 'PCA';
|
|
252
|
+
|
|
253
|
+
return {pcaDf, varianceExplained};
|
|
254
|
+
}
|