@datagrok/proteomics 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/CHANGELOG.md +52 -3
  2. package/CLAUDE.md +178 -0
  3. package/README.md +195 -3
  4. package/detectors.js +152 -9
  5. package/dist/package-test.js +1 -1744
  6. package/dist/package-test.js.map +1 -1
  7. package/dist/package.js +1 -146
  8. package/dist/package.js.map +1 -1
  9. package/docs/personas-and-capabilities.md +165 -0
  10. package/files/demo/README.md +264 -0
  11. package/files/demo/cptac-spike-in.txt +1571 -0
  12. package/files/demo/enrichment-demo.csv +120 -0
  13. package/files/demo/fragpipe-smoke-test.tsv +11 -0
  14. package/files/demo/proteinGroups.txt +28 -0
  15. package/files/demo/spectronaut-hye-candidates.tsv +94 -0
  16. package/files/demo/spectronaut-hye-demo.tsv +8761 -0
  17. package/files/demo/spectronaut-hye-mix.tsv +8761 -0
  18. package/files/demo/spectronaut-hye-precursor-golden.json +938 -0
  19. package/files/demo/spectronaut-hye-precursor-golden.tsv +235 -0
  20. package/files/demo/spectronaut-hye-precursor.tsv +493 -0
  21. package/images/enrichment-crosslink.png +0 -0
  22. package/images/enrichment-term-selected.png +0 -0
  23. package/images/hero.png +0 -0
  24. package/images/pipeline.svg +80 -0
  25. package/package.json +87 -63
  26. package/scripts/deqms_de.R +60 -0
  27. package/scripts/limma_de.R +42 -0
  28. package/scripts/vsn_normalize.R +19 -0
  29. package/src/analysis/differential-expression.ts +450 -0
  30. package/src/analysis/enrichment-export.ts +101 -0
  31. package/src/analysis/enrichment.ts +602 -0
  32. package/src/analysis/experiment-setup.ts +199 -0
  33. package/src/analysis/imputation.ts +407 -0
  34. package/src/analysis/log2-scale.ts +139 -0
  35. package/src/analysis/normalization.ts +255 -0
  36. package/src/analysis/pca.ts +254 -0
  37. package/src/analysis/spc-storage.ts +515 -0
  38. package/src/analysis/spc.ts +544 -0
  39. package/src/analysis/subcellular-location.ts +431 -0
  40. package/src/demo/enrichment-demo.ts +94 -0
  41. package/src/demo/proteomics-demo.ts +123 -0
  42. package/src/global.d.ts +15 -0
  43. package/src/menu.ts +133 -0
  44. package/src/package-api.ts +136 -14
  45. package/src/package-test.ts +45 -20
  46. package/src/package.g.ts +161 -0
  47. package/src/package.ts +1029 -17
  48. package/src/panels/protein-focus.ts +63 -0
  49. package/src/panels/published-analysis-panel.ts +151 -0
  50. package/src/panels/uniprot-panel.ts +349 -0
  51. package/src/parsers/fragpipe-parser.ts +200 -0
  52. package/src/parsers/generic-parser.ts +197 -0
  53. package/src/parsers/maxquant-parser.ts +162 -0
  54. package/src/parsers/shared-utils.ts +163 -0
  55. package/src/parsers/spectronaut-candidates-parser.ts +307 -0
  56. package/src/parsers/spectronaut-parser.ts +604 -0
  57. package/src/publishing/assert-published-shape.ts +260 -0
  58. package/src/publishing/post-open-recovery.ts +104 -0
  59. package/src/publishing/publish-project.ts +515 -0
  60. package/src/publishing/publish-settings.ts +59 -0
  61. package/src/publishing/publish-state.ts +316 -0
  62. package/src/publishing/share-dialog.ts +171 -0
  63. package/src/publishing/trim-dataframe.ts +247 -0
  64. package/src/tests/analysis.ts +658 -0
  65. package/src/tests/enrichment-export.ts +61 -0
  66. package/src/tests/enrichment-visualization.ts +340 -0
  67. package/src/tests/enrichment.ts +224 -0
  68. package/src/tests/fragpipe-e2e.ts +74 -0
  69. package/src/tests/fragpipe-parser.ts +147 -0
  70. package/src/tests/gene-label-resolver.ts +387 -0
  71. package/src/tests/generic-parser.ts +152 -0
  72. package/src/tests/group-mean-correlation.ts +139 -0
  73. package/src/tests/log2-scale.ts +93 -0
  74. package/src/tests/organisms.ts +56 -0
  75. package/src/tests/parsers.ts +182 -0
  76. package/src/tests/publish-roundtrip.ts +584 -0
  77. package/src/tests/publish-spike.ts +377 -0
  78. package/src/tests/qc-dashboard.ts +210 -0
  79. package/src/tests/smart-pathway-filter.ts +193 -0
  80. package/src/tests/spc-formula-lines-spike.ts +129 -0
  81. package/src/tests/spc.ts +640 -0
  82. package/src/tests/spectronaut-candidates-e2e.ts +140 -0
  83. package/src/tests/spectronaut-candidates-parser.ts +398 -0
  84. package/src/tests/spectronaut-parser.ts +668 -0
  85. package/src/tests/subcellular-location.ts +361 -0
  86. package/src/tests/uniprot-panel.ts +202 -0
  87. package/src/tests/volcano.ts +603 -0
  88. package/src/utils/column-detection.ts +28 -0
  89. package/src/utils/gene-label-resolver.ts +443 -0
  90. package/src/utils/organisms.ts +82 -0
  91. package/src/utils/proteomics-types.ts +30 -0
  92. package/src/viewers/enrichment-viewers.ts +274 -0
  93. package/src/viewers/group-mean-correlation.ts +218 -0
  94. package/src/viewers/heatmap.ts +168 -0
  95. package/src/viewers/pca-plot.ts +169 -0
  96. package/src/viewers/qc-computations.ts +266 -0
  97. package/src/viewers/qc-dashboard.ts +176 -0
  98. package/src/viewers/spc-dashboard.ts +755 -0
  99. package/src/viewers/volcano.ts +690 -0
  100. package/test-console-output-1.log +2055 -0
  101. package/test-record-1.mp4 +0 -0
  102. package/tools/derive-precursor-golden-sidecar.mjs +81 -0
  103. package/tools/generate-enrichment-fixture.sh +160 -0
  104. package/tools/generate-spectronaut-candidates-fixture.mjs +212 -0
  105. package/tools/generate-spectronaut-precursor-fixture.mjs +128 -0
  106. package/tools/spectronaut-aggregate.sh +46 -0
  107. package/tools/spectronaut-aggregate.sql +80 -0
  108. package/tsconfig.json +18 -71
  109. package/webpack.config.js +86 -45
  110. package/.eslintignore +0 -1
  111. package/.eslintrc.json +0 -56
  112. package/LICENSE +0 -674
  113. package/package.png +0 -0
  114. package/scripts/number_antibody.py +0 -190
  115. package/scripts/number_antibody_abnumber.py +0 -177
  116. package/scripts/number_antibody_anarci.py +0 -200
@@ -0,0 +1,139 @@
1
+ import * as grok from 'datagrok-api/grok';
2
+ import * as ui from 'datagrok-api/ui';
3
+ import * as DG from 'datagrok-api/dg';
4
+
5
+ import {SEMTYPE} from '../utils/proteomics-types';
6
+ import {log2TransformColumns, copyAsLog2Columns, detectLog2Status} from '../parsers/shared-utils';
7
+
8
+ /**
9
+ * Post-import correction for the log2 scale of intensity data.
10
+ *
11
+ * At import each parser decides whether the intensities are raw (needs a log2
12
+ * transform) or already log2-transformed (copy as-is). That decision is a
13
+ * magnitude heuristic (`detectLog2Status`) and has misfired on small-magnitude
14
+ * linear intensities — values that happen to sit in the [0, 30] "looks like
15
+ * log2" band get copied instead of transformed, producing nonsense ±thousands
16
+ * fold changes downstream. This module lets the analyst override that call
17
+ * without re-importing.
18
+ *
19
+ * The rebuild works off the ORIGINAL (non-`log2(...)`) intensity columns, which
20
+ * every parser keeps pristine — normalization/imputation only ever mutate the
21
+ * `log2(...)` copies in place. So re-deriving the `log2(...)` columns is always
22
+ * valid and, by design, discards any normalization/imputation/DE that ran on the
23
+ * old values; those steps must be re-run, which the dialog warns about.
24
+ */
25
+
26
+ /** Original (pre-`log2(...)`) intensity column names — the pristine source the
27
+ * rebuild derives from. Excludes the `log2(...)` copies themselves. */
28
+ export function getIntensityOriginals(df: DG.DataFrame): string[] {
29
+ return df.columns.toList()
30
+ .filter((c) => c.semType === SEMTYPE.INTENSITY && !c.name.startsWith('log2('))
31
+ .map((c) => c.name);
32
+ }
33
+
34
+ /** Infers whether the current `log2(...)` columns were produced by a transform
35
+ * (`Math.log2(orig)`) or a straight copy (`orig`, i.e. treated as already log2).
36
+ * Scans until the first decisive non-null positive row — the two branches never
37
+ * coincide for a positive value (`log2(x) ≠ x` for all x > 0). Returns 'unknown'
38
+ * when no `log2(...)` column or no decisive row exists. */
39
+ export function detectCurrentLog2Applied(
40
+ df: DG.DataFrame, originalNames: string[],
41
+ ): 'transform' | 'copy' | 'unknown' {
42
+ for (const name of originalNames) {
43
+ const orig = df.col(name);
44
+ const l2 = df.col(`log2(${name})`);
45
+ if (!orig || !l2) continue;
46
+ for (let i = 0; i < df.rowCount; i++) {
47
+ if (orig.isNone(i) || l2.isNone(i)) continue;
48
+ const o = Number(orig.get(i));
49
+ const v = Number(l2.get(i));
50
+ if (!(o > 0) || !Number.isFinite(v)) continue;
51
+ if (Math.abs(v - Math.log2(o)) < 1e-4) return 'transform';
52
+ if (Math.abs(v - o) < 1e-4) return 'copy';
53
+ }
54
+ }
55
+ return 'unknown';
56
+ }
57
+
58
+ /** Rebuilds the `log2(...)` columns from the originals under the chosen scale:
59
+ * `alreadyLog2` copies as-is, otherwise applies the log2 transform. Removes the
60
+ * existing `log2(...)` columns first (so a re-run doesn't duplicate) and clears
61
+ * stale downstream pipeline tags — the base data changed, so any prior
62
+ * normalize/impute/DE is invalid and must be re-run. No-op (returns false) when
63
+ * the requested scale already matches what's applied. */
64
+ export function applyLog2Scale(df: DG.DataFrame, alreadyLog2: boolean): boolean {
65
+ const originals = getIntensityOriginals(df);
66
+ if (originals.length === 0) return false;
67
+
68
+ const current = detectCurrentLog2Applied(df, originals);
69
+ const target = alreadyLog2 ? 'copy' : 'transform';
70
+ if (current === target) return false;
71
+
72
+ for (const name of originals) {
73
+ const l2 = `log2(${name})`;
74
+ if (df.col(l2)) df.columns.remove(l2);
75
+ }
76
+ if (alreadyLog2)
77
+ copyAsLog2Columns(df, originals);
78
+ else
79
+ log2TransformColumns(df, originals);
80
+
81
+ // Base intensities were re-derived — anything computed from the old log2
82
+ // columns is stale. Drop the completion tags so viewers gate correctly and
83
+ // the analyst re-runs from Normalize.
84
+ for (const tag of ['proteomics.normalized', 'proteomics.imputed',
85
+ 'proteomics.de_complete', 'proteomics.de_method'])
86
+ df.setTag(tag, '');
87
+ return true;
88
+ }
89
+
90
+ /** Dialog to review and override the log2 scale of the imported intensities.
91
+ * Seeds the checkbox from the scale actually applied at import (inferred), not a
92
+ * re-detection, so it reflects reality; shows the magnitude heuristic's read as
93
+ * an advisory hint. */
94
+ export function showLog2ScaleDialog(df: DG.DataFrame): void {
95
+ const originals = getIntensityOriginals(df);
96
+ if (originals.length === 0) {
97
+ grok.shell.warning('No intensity columns found — nothing to rescale.');
98
+ return;
99
+ }
100
+
101
+ const current = detectCurrentLog2Applied(df, originals);
102
+ const detection = detectLog2Status(df, originals);
103
+
104
+ const alreadyLog2 = ui.input.bool('Data is already log2-transformed', {
105
+ value: current === 'copy',
106
+ tooltipText: 'On: intensities are used as-is. Off: a log2 transform is applied. ' +
107
+ 'Flip this if fold changes look wildly off — the import heuristic can misread ' +
108
+ 'small-magnitude raw intensities as already-log2.',
109
+ });
110
+
111
+ const hint = ui.divText(`Auto-detection: ${detection.message}.` +
112
+ (current !== 'unknown' ? ` Currently applied: ${current === 'copy' ?
113
+ 'used as-is (no transform)' : 'log2-transformed'}.` : ''));
114
+ hint.style.cssText = 'font-style:italic;color:#888;margin:4px 0;font-size:12px;max-width:420px;';
115
+
116
+ const pipelineRan = ['proteomics.normalized', 'proteomics.imputed', 'proteomics.de_complete']
117
+ .some((t) => df.getTag(t) === 'true');
118
+ const warn = ui.divText(
119
+ 'Normalize / Impute / Differential Expression have already run. Changing the ' +
120
+ 'scale resets those steps — re-run them from Analyze.');
121
+ warn.style.cssText = 'color:#b26a00;margin:4px 0;font-size:12px;max-width:420px;';
122
+ warn.style.display = pipelineRan ? '' : 'none';
123
+
124
+ ui.dialog('Set Log2 Scale')
125
+ .add(alreadyLog2)
126
+ .add(hint)
127
+ .add(warn)
128
+ .onOK(() => {
129
+ const changed = applyLog2Scale(df, alreadyLog2.value);
130
+ if (!changed) {
131
+ grok.shell.info('Scale unchanged.');
132
+ return;
133
+ }
134
+ grok.shell.info(alreadyLog2.value ?
135
+ 'Intensities are now used as-is (no log2 transform). Re-run analysis steps.' :
136
+ 'Intensities are now log2-transformed. Re-run analysis steps.');
137
+ })
138
+ .show();
139
+ }
@@ -0,0 +1,255 @@
1
+ import * as grok from 'datagrok-api/grok';
2
+ import * as ui from 'datagrok-api/ui';
3
+ import * as DG from 'datagrok-api/dg';
4
+
5
+ import {SEMTYPE} from '../utils/proteomics-types';
6
+ import {unpivotIntensities} from '../viewers/qc-computations';
7
+
8
+ /** Median-centers each specified column in-place.
9
+ * Subtracts the column median from every non-null value.
10
+ * Uses getRawData() for bulk array access instead of per-element get/set. */
11
+ export function medianNormalize(df: DG.DataFrame, colNames: string[]): void {
12
+ for (const name of colNames) {
13
+ const col = df.col(name);
14
+ if (!col) continue;
15
+ if (col.type !== DG.COLUMN_TYPE.FLOAT) {
16
+ console.warn(`medianNormalize: skipping non-float column "${name}" (type=${col.type})`);
17
+ continue;
18
+ }
19
+
20
+ const median = col.stats.med;
21
+ if (isNaN(median)) continue;
22
+
23
+ const raw = col.getRawData() as Float32Array | Float64Array;
24
+ for (let i = 0; i < raw.length; i++) {
25
+ if (!col.isNone(i))
26
+ raw[i] -= median;
27
+ }
28
+ }
29
+ df.fireValuesChanged();
30
+ df.setTag('proteomics.normalized', 'true');
31
+ }
32
+
33
+ /** Quantile-normalizes the specified columns in-place.
34
+ * Aligns all column distributions to the same shape by replacing values
35
+ * with rank-based averages. Missing values are preserved.
36
+ * Uses getRawData() for bulk array access in tight loops. */
37
+ export function quantileNormalize(df: DG.DataFrame, colNames: string[]): void {
38
+ if (colNames.length < 2) return;
39
+
40
+ const cols = colNames.map((n) => df.col(n))
41
+ .filter((c) => c != null && c.type === DG.COLUMN_TYPE.FLOAT) as DG.Column[];
42
+ if (cols.length < 2) return;
43
+
44
+ const nRows = df.rowCount;
45
+
46
+ // For each column, collect indices of non-missing values and sort by value
47
+ const colData: {indices: number[]; raw: Float32Array | Float64Array}[] = [];
48
+ for (const col of cols) {
49
+ const raw = col.getRawData() as Float32Array | Float64Array;
50
+ const indices: number[] = [];
51
+ for (let i = 0; i < nRows; i++) {
52
+ if (!col.isNone(i))
53
+ indices.push(i);
54
+ }
55
+ // Sort indices by value (ascending)
56
+ indices.sort((a, b) => raw[a] - raw[b]);
57
+ colData.push({indices, raw});
58
+ }
59
+
60
+ // Find the maximum number of non-null values across columns
61
+ const maxValid = Math.max(...colData.map((d) => d.indices.length));
62
+ if (maxValid === 0) return;
63
+
64
+ // Compute rank means using scaled rank alignment
65
+ // For each rank position r (0..maxValid-1), compute the mean of values
66
+ // at the corresponding fractional position in each column
67
+ const rankMeans = new Float64Array(maxValid);
68
+ for (let r = 0; r < maxValid; r++) {
69
+ let sum = 0;
70
+ let count = 0;
71
+ for (const d of colData) {
72
+ if (d.indices.length === 0) continue;
73
+ // Map rank r in [0..maxValid-1] to position in this column's sorted values
74
+ const pos = (r / (maxValid - 1)) * (d.indices.length - 1);
75
+ // Interpolate between floor and ceil positions
76
+ const lo = Math.floor(pos);
77
+ const hi = Math.min(Math.ceil(pos), d.indices.length - 1);
78
+ const frac = pos - lo;
79
+ const val = (1 - frac) * d.raw[d.indices[lo]] + frac * d.raw[d.indices[hi]];
80
+ sum += val;
81
+ count++;
82
+ }
83
+ rankMeans[r] = sum / count;
84
+ }
85
+
86
+ // Replace each column's sorted values with the corresponding rank mean
87
+ for (const d of colData) {
88
+ const n = d.indices.length;
89
+ if (n === 0) continue;
90
+ for (let r = 0; r < n; r++) {
91
+ // Map this column's rank to the global rank mean position
92
+ const globalPos = maxValid === 1 ? 0 : (r / (n - 1)) * (maxValid - 1);
93
+ const lo = Math.floor(globalPos);
94
+ const hi = Math.min(Math.ceil(globalPos), maxValid - 1);
95
+ const frac = globalPos - lo;
96
+ const val = (1 - frac) * rankMeans[lo] + frac * rankMeans[hi];
97
+ d.raw[d.indices[r]] = val;
98
+ }
99
+ }
100
+
101
+ df.fireValuesChanged();
102
+ df.setTag('proteomics.normalized', 'true');
103
+ }
104
+
105
+ /** Performs Variance-Stabilizing Normalization via server-side R script.
106
+ * Operates on raw (non-log2) intensity columns and writes results to log2 columns.
107
+ * Falls back to quantile normalization if R environment is unavailable. */
108
+ export async function vsnNormalize(df: DG.DataFrame, colNames: string[]): Promise<void> {
109
+ try {
110
+ // Find raw intensity column names by stripping log2() prefix
111
+ const rawColNames: string[] = [];
112
+ const log2ColNames: string[] = [];
113
+ for (const name of colNames) {
114
+ if (name.startsWith('log2(') && name.endsWith(')')) {
115
+ const rawName = name.slice(5, -1);
116
+ if (df.col(rawName)) {
117
+ rawColNames.push(rawName);
118
+ log2ColNames.push(name);
119
+ }
120
+ }
121
+ }
122
+
123
+ if (rawColNames.length === 0) {
124
+ // No raw columns found, fall back to quantile
125
+ if (colNames.length < 2) {
126
+ grok.shell.warning('Cannot normalize — VSN needs raw intensity columns and quantile fallback needs at least 2 columns');
127
+ return;
128
+ }
129
+ grok.shell.warning('No raw intensity columns found for VSN — using quantile normalization');
130
+ quantileNormalize(df, colNames);
131
+ return;
132
+ }
133
+
134
+ // Build clean DataFrame with simple column names (s1, s2, ...)
135
+ const exprDf = DG.DataFrame.create(df.rowCount);
136
+ for (let i = 0; i < rawColNames.length; i++) {
137
+ const src = df.col(rawColNames[i])!;
138
+ const srcRaw = src.getRawData() as Float32Array | Float64Array;
139
+ const dst = exprDf.columns.addNewFloat(`s${i + 1}`);
140
+ dst.init((r) => srcRaw[r]);
141
+ }
142
+
143
+ const result: DG.DataFrame = await grok.functions.call('Proteomics:VsnNormalize', {exprDf});
144
+
145
+ // Copy VSN output (glog2 scale) into log2 intensity columns
146
+ for (let i = 0; i < log2ColNames.length; i++) {
147
+ const dst = df.col(log2ColNames[i])!;
148
+ const src = result.col(`s${i + 1}`)!;
149
+ const dstRaw = dst.getRawData() as Float32Array | Float64Array;
150
+ const srcRaw = src.getRawData() as Float32Array | Float64Array;
151
+ for (let r = 0; r < df.rowCount; r++) {
152
+ if (srcRaw[r] !== DG.FLOAT_NULL)
153
+ dstRaw[r] = srcRaw[r];
154
+ }
155
+ }
156
+
157
+ df.fireValuesChanged();
158
+ df.setTag('proteomics.normalized', 'true');
159
+ } catch (e: any) {
160
+ console.warn('VSN normalization failed, using quantile fallback:', e);
161
+ if (colNames.length < 2) {
162
+ grok.shell.warning('VSN failed and quantile fallback needs at least 2 columns — data left unnormalized');
163
+ return;
164
+ }
165
+ grok.shell.warning('R environment unavailable — using quantile normalization');
166
+ quantileNormalize(df, colNames);
167
+ }
168
+ }
169
+
170
+ /** Shows a dialog for normalizing intensity columns.
171
+ * Supports Median Centering, Quantile, and VSN methods with reactive box plot preview.
172
+ * Displays an inline warning when Spectronaut pre-normalized data is detected. */
173
+ export function showNormalizationDialog(df: DG.DataFrame): void {
174
+ if (df.getTag('proteomics.normalized') === 'true') {
175
+ grok.shell.warning('Data already normalized');
176
+ return;
177
+ }
178
+
179
+ const log2ColNames = df.columns.toList()
180
+ .filter((c) => c.semType === SEMTYPE.INTENSITY && c.name.startsWith('log2('))
181
+ .map((c) => c.name);
182
+
183
+ const log2Cols = log2ColNames.map((n) => df.col(n)!);
184
+
185
+ // Pre-normalized warning banner (NORM-04)
186
+ const warningDiv = ui.div([], {style: {
187
+ background: '#FFF3CD', border: '1px solid #FFEEBA', color: '#856404',
188
+ padding: '8px', margin: '4px 0', borderRadius: '4px',
189
+ display: df.getTag('proteomics.preNormalized') === 'true' ? '' : 'none',
190
+ }});
191
+ warningDiv.textContent =
192
+ 'This data may be pre-normalized (Spectronaut). Additional normalization may distort results.';
193
+
194
+ // Method selector (NORM-03)
195
+ const methodInput = ui.input.choice('Method', {
196
+ value: 'Median Centering',
197
+ items: ['Median Centering', 'Quantile', 'VSN'],
198
+ nullable: false,
199
+ });
200
+
201
+ // Column picker
202
+ const colsInput = ui.input.columns('Intensity columns', {
203
+ table: df, available: log2ColNames, value: log2Cols,
204
+ });
205
+
206
+ // Box plot preview container
207
+ const plotContainer = ui.div([], {style: {
208
+ width: '100%', height: '250px', border: '1px solid #ddd', marginTop: '8px',
209
+ }});
210
+
211
+ function updatePreview(): void {
212
+ plotContainer.innerHTML = '';
213
+ const selected = colsInput.value.map((c: DG.Column) => c.name);
214
+ if (selected.length === 0) return;
215
+
216
+ // Clone DataFrame and apply selected normalization for preview
217
+ const clone = df.clone(null, selected);
218
+ const method = methodInput.value;
219
+ if (method === 'Median Centering')
220
+ medianNormalize(clone, selected);
221
+ else if (method === 'Quantile')
222
+ quantileNormalize(clone, selected);
223
+ // VSN requires async R call -- show un-normalized distributions as preview
224
+
225
+ const longDf = unpivotIntensities(clone, selected);
226
+ const boxPlot = DG.Viewer.boxPlot(longDf, {
227
+ valueColumnName: 'Intensity', categoryColumnName: 'Sample',
228
+ } as any);
229
+ boxPlot.root.style.width = '100%';
230
+ boxPlot.root.style.height = '220px';
231
+ plotContainer.appendChild(boxPlot.root);
232
+ }
233
+
234
+ methodInput.onChanged.subscribe(() => updatePreview());
235
+ colsInput.onChanged.subscribe(() => updatePreview());
236
+ updatePreview();
237
+
238
+ ui.dialog('Normalize')
239
+ .add(warningDiv)
240
+ .add(methodInput)
241
+ .add(colsInput)
242
+ .add(plotContainer)
243
+ .onOK(async () => {
244
+ const selected = colsInput.value.map((c: DG.Column) => c.name);
245
+ const method = methodInput.value as string;
246
+ if (method === 'Median Centering')
247
+ medianNormalize(df, selected);
248
+ else if (method === 'Quantile')
249
+ quantileNormalize(df, selected);
250
+ else if (method === 'VSN')
251
+ await vsnNormalize(df, selected);
252
+ grok.shell.info(`Normalized ${selected.length} columns (${method})`);
253
+ })
254
+ .show();
255
+ }
@@ -0,0 +1,254 @@
1
+ import * as DG from 'datagrok-api/dg';
2
+ import {GroupAssignment} from './experiment-setup';
3
+
4
+ /** Transpose a matrix (rows x cols) -> (cols x rows). */
5
+ function transpose(matrix: number[][], rows: number, cols: number): number[][] {
6
+ const result: number[][] = [];
7
+ for (let j = 0; j < cols; j++) {
8
+ const row: number[] = new Array(rows);
9
+ for (let i = 0; i < rows; i++)
10
+ row[i] = matrix[i][j];
11
+ result.push(row);
12
+ }
13
+ return result;
14
+ }
15
+
16
+ /** Multiply matrices A (m x n) and B (n x p) -> C (m x p). */
17
+ function matmul(A: number[][], B: number[][], m: number, n: number, p: number): number[][] {
18
+ const C: number[][] = [];
19
+ for (let i = 0; i < m; i++) {
20
+ const row = new Array(p).fill(0);
21
+ for (let k = 0; k < n; k++) {
22
+ const aik = A[i][k];
23
+ for (let j = 0; j < p; j++)
24
+ row[j] += aik * B[k][j];
25
+ }
26
+ C.push(row);
27
+ }
28
+ return C;
29
+ }
30
+
31
+ /** Jacobi eigendecomposition for a symmetric matrix.
32
+ * Returns eigenvalues and eigenvectors sorted descending by eigenvalue. */
33
+ function jacobi(matrix: number[][], n: number): {eigenvalues: number[]; eigenvectors: number[][]} {
34
+ // Deep copy the matrix
35
+ const A: number[][] = matrix.map((row) => [...row]);
36
+
37
+ // Initialize eigenvectors as identity matrix
38
+ const V: number[][] = [];
39
+ for (let i = 0; i < n; i++) {
40
+ const row = new Array(n).fill(0);
41
+ row[i] = 1;
42
+ V.push(row);
43
+ }
44
+
45
+ const maxIter = 100;
46
+ const tolerance = 1e-10;
47
+ let converged = false;
48
+
49
+ for (let iter = 0; iter < maxIter; iter++) {
50
+ // Find largest off-diagonal element
51
+ let maxVal = 0;
52
+ let p = 0;
53
+ let q = 1;
54
+ for (let i = 0; i < n; i++) {
55
+ for (let j = i + 1; j < n; j++) {
56
+ if (Math.abs(A[i][j]) > maxVal) {
57
+ maxVal = Math.abs(A[i][j]);
58
+ p = i;
59
+ q = j;
60
+ }
61
+ }
62
+ }
63
+
64
+ if (maxVal < tolerance) {
65
+ converged = true;
66
+ break;
67
+ }
68
+
69
+ // Compute rotation angle
70
+ const theta = (A[q][q] - A[p][p]) / (2 * A[p][q]);
71
+ const t = Math.sign(theta) / (Math.abs(theta) + Math.sqrt(theta * theta + 1));
72
+ const c = 1 / Math.sqrt(t * t + 1);
73
+ const s = t * c;
74
+
75
+ // Apply rotation to A
76
+ const app = A[p][p];
77
+ const aqq = A[q][q];
78
+ const apq = A[p][q];
79
+
80
+ A[p][p] = c * c * app - 2 * s * c * apq + s * s * aqq;
81
+ A[q][q] = s * s * app + 2 * s * c * apq + c * c * aqq;
82
+ A[p][q] = 0;
83
+ A[q][p] = 0;
84
+
85
+ for (let i = 0; i < n; i++) {
86
+ if (i !== p && i !== q) {
87
+ const aip = A[i][p];
88
+ const aiq = A[i][q];
89
+ A[i][p] = c * aip - s * aiq;
90
+ A[p][i] = A[i][p];
91
+ A[i][q] = s * aip + c * aiq;
92
+ A[q][i] = A[i][q];
93
+ }
94
+ }
95
+
96
+ // Update eigenvectors
97
+ for (let i = 0; i < n; i++) {
98
+ const vip = V[i][p];
99
+ const viq = V[i][q];
100
+ V[i][p] = c * vip - s * viq;
101
+ V[i][q] = s * vip + c * viq;
102
+ }
103
+ }
104
+
105
+ if (!converged)
106
+ console.warn(`PCA Jacobi did not converge in ${maxIter} iterations (n=${n}); results may be approximate`);
107
+
108
+ // Extract eigenvalues and sort descending
109
+ const eigenvalues = new Array(n);
110
+ for (let i = 0; i < n; i++)
111
+ eigenvalues[i] = A[i][i];
112
+
113
+ const indices = Array.from({length: n}, (_, i) => i);
114
+ indices.sort((a, b) => eigenvalues[b] - eigenvalues[a]);
115
+
116
+ const sortedEigenvalues = indices.map((i) => eigenvalues[i]);
117
+ const sortedEigenvectors: number[][] = [];
118
+ for (let i = 0; i < n; i++) {
119
+ const row = new Array(n);
120
+ for (let j = 0; j < n; j++)
121
+ row[j] = V[i][indices[j]];
122
+ sortedEigenvectors.push(row);
123
+ }
124
+
125
+ return {eigenvalues: sortedEigenvalues, eigenvectors: sortedEigenvectors};
126
+ }
127
+
128
+ /** Compute sample-level PCA on intensity columns.
129
+ * Returns a NEW sample-level DataFrame (rows = samples, not proteins)
130
+ * and the variance explained percentages. */
131
+ export function computePCA(
132
+ df: DG.DataFrame,
133
+ intensityCols: string[],
134
+ groupAssignment: GroupAssignment,
135
+ nComponents: number = 2,
136
+ ): {pcaDf: DG.DataFrame; varianceExplained: number[]} {
137
+ const nSamples = intensityCols.length;
138
+ const nProteins = df.rowCount;
139
+
140
+ // Step 1: Build transposed matrix (nSamples x nProteins)
141
+ // Each row = one sample, each column = one protein's intensity across that sample
142
+ const matrix: number[][] = [];
143
+
144
+ // Compute protein means for imputation (mean across samples for each protein)
145
+ const proteinMeans: number[] = new Array(nProteins).fill(0);
146
+ const proteinCounts: number[] = new Array(nProteins).fill(0);
147
+
148
+ for (const colName of intensityCols) {
149
+ const col = df.col(colName);
150
+ if (!col) continue;
151
+ for (let i = 0; i < nProteins; i++) {
152
+ if (!col.isNone(i)) {
153
+ proteinMeans[i] += col.get(i) as number;
154
+ proteinCounts[i]++;
155
+ }
156
+ }
157
+ }
158
+ for (let i = 0; i < nProteins; i++)
159
+ proteinMeans[i] = proteinCounts[i] > 0 ? proteinMeans[i] / proteinCounts[i] : 0;
160
+
161
+ // Build sample rows with imputation
162
+ for (const colName of intensityCols) {
163
+ const col = df.col(colName);
164
+ const row: number[] = new Array(nProteins);
165
+ for (let i = 0; i < nProteins; i++) {
166
+ if (!col || col.isNone(i))
167
+ row[i] = proteinMeans[i];
168
+ else
169
+ row[i] = col.get(i) as number;
170
+ }
171
+ matrix.push(row);
172
+ }
173
+
174
+ // Step 2: Center each column (protein) by subtracting its mean across samples
175
+ const colMeans: number[] = new Array(nProteins).fill(0);
176
+ for (let j = 0; j < nProteins; j++) {
177
+ for (let i = 0; i < nSamples; i++)
178
+ colMeans[j] += matrix[i][j];
179
+ colMeans[j] /= nSamples;
180
+ }
181
+ for (let i = 0; i < nSamples; i++) {
182
+ for (let j = 0; j < nProteins; j++)
183
+ matrix[i][j] -= colMeans[j];
184
+ }
185
+
186
+ // Step 3: Compute covariance matrix S (nSamples x nSamples)
187
+ // S = X * X^T / (nProteins - 1) -- small matrix since nSamples << nProteins
188
+ const Xt = transpose(matrix, nSamples, nProteins);
189
+ const S = matmul(matrix, Xt, nSamples, nProteins, nSamples);
190
+ const divisor = nProteins > 1 ? nProteins - 1 : 1;
191
+ for (let i = 0; i < nSamples; i++) {
192
+ for (let j = 0; j < nSamples; j++)
193
+ S[i][j] /= divisor;
194
+ }
195
+
196
+ // Step 4: Eigendecompose S
197
+ const {eigenvalues, eigenvectors} = jacobi(S, nSamples);
198
+
199
+ // Step 5: PC scores are the eigenvectors scaled by sqrt(eigenvalue)
200
+ const scores: number[][] = [];
201
+ for (let i = 0; i < nSamples; i++) {
202
+ const row: number[] = [];
203
+ for (let k = 0; k < Math.min(nComponents, nSamples); k++) {
204
+ const scale = eigenvalues[k] > 0 ? Math.sqrt(eigenvalues[k]) : 0;
205
+ row.push(eigenvectors[i][k] * scale);
206
+ }
207
+ scores.push(row);
208
+ }
209
+
210
+ // Step 6: Compute variance explained. Clamp eigenvalues to >= 0 for both
211
+ // totalVariance and per-PC pct so numerical noise can't produce negative percentages.
212
+ const totalVariance = eigenvalues.reduce((sum, v) => sum + Math.max(v, 0), 0);
213
+ const varianceExplained: number[] = [];
214
+ for (let k = 0; k < Math.min(nComponents, nSamples); k++) {
215
+ const pct = totalVariance > 0 ? (Math.max(eigenvalues[k], 0) / totalVariance) * 100 : 0;
216
+ varianceExplained.push(Math.round(pct * 10) / 10);
217
+ }
218
+
219
+ // Step 7: Create sample-level DataFrame. PCA on <2 samples is meaningless;
220
+ // produce stable column labels rather than "PC1 (NaN%)".
221
+ const pc1Pct = varianceExplained.length > 0 && isFinite(varianceExplained[0]) ? varianceExplained[0] : 0;
222
+ const pc1Name = nSamples >= 2 ? `PC1 (${pc1Pct}%)` : 'PC1';
223
+ const pc2Pct = varianceExplained.length >= 2 && isFinite(varianceExplained[1]) ? varianceExplained[1] : 0;
224
+ const pc2Name = nComponents >= 2 && varianceExplained.length >= 2 && nSamples >= 2
225
+ ? `PC2 (${pc2Pct}%)`
226
+ : 'PC2';
227
+
228
+ const sampleNames = DG.Column.fromStrings('SampleName', intensityCols);
229
+ const pc1Col = DG.Column.fromFloat32Array(pc1Name,
230
+ new Float32Array(scores.map((s) => s[0])));
231
+ const columns: DG.Column[] = [sampleNames, pc1Col];
232
+
233
+ if (nComponents >= 2) {
234
+ const pc2Col = DG.Column.fromFloat32Array(pc2Name,
235
+ new Float32Array(scores.map((s) => s.length > 1 ? s[1] : 0)));
236
+ columns.push(pc2Col);
237
+ }
238
+
239
+ // Map each sample to its group
240
+ const groupNames: string[] = intensityCols.map((colName) => {
241
+ if (groupAssignment.group1.columns.includes(colName))
242
+ return groupAssignment.group1.name;
243
+ if (groupAssignment.group2.columns.includes(colName))
244
+ return groupAssignment.group2.name;
245
+ return 'Unknown';
246
+ });
247
+ const groupCol = DG.Column.fromStrings('Group', groupNames);
248
+ columns.push(groupCol);
249
+
250
+ const pcaDf = DG.DataFrame.fromColumns(columns);
251
+ pcaDf.name = 'PCA';
252
+
253
+ return {pcaDf, varianceExplained};
254
+ }