@datagrok/proteomics 1.0.1 → 1.2.0

package/src/viewers/enrichment-viewers.ts

@@ -0,0 +1,274 @@
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+import * as rxjs from 'rxjs';
+import {findColumn} from '../utils/column-detection';
+import {SEMTYPE} from '../utils/proteomics-types';
+
+/** Module-level subscriptions for cleanup on re-open. */
+let activeSubscriptions: rxjs.Subscription[] = [];
+
+/** Default number of terms each dot/bar chart shows (per direction). */
+export const CHART_TOP_N = 15;
+
+/** Value/color column the dot + bar charts read. Kept as a plain (non-`~`)
+ * name because it is a viewer VALUE binding — a missing value column hard-fails
+ * a bar chart on reopen (the original "Column negLog10FDR does not exist" bug). */
+export const NEG_LOG10_FDR_COL = 'negLog10FDR';
+/** Hidden marker column (auto-hidden via `~` prefix, ignored by the publish
+ * shape contract) flagging the top-N terms per direction. Used only inside the
+ * per-viewer formula filter, where a parse failure degrades to "show all"
+ * rather than crashing — so the `~` name is safe here but not for the value. */
+export const CHART_TOP_MARKER_COL = '~enrichChartTop';
+
+/**
+ * Adds the two derived columns the enrichment dot/bar charts bind to, directly
+ * onto `enrichDf` (idempotent — ensureFreshFloat pattern):
+ *   - `negLog10FDR` — -log10(FDR); dot color + bar value. FDR=0 (underflow /
+ *     extreme enrichment) maps to the float-underflow ceiling so it stays
+ *     visible without colliding with real values like FDR=1e-10.
+ *   - `~enrichChartTop` — true for the top-N terms by FDR (ascending) within
+ *     each Direction, or overall when there is no Direction column.
+ *
+ * Both charts bind to `enrichDf` itself rather than to cloned subsets: docked
+ * viewers rebind to their host view's primary frame on project reopen, so the
+ * columns they read must live on that frame (else the bar chart fatal-errors).
+ * Keeping everything on one frame also stops the subset tables from leaking
+ * into a published project's table tree.
+ */
+export function prepareEnrichmentChartColumns(enrichDf: DG.DataFrame, topN: number = CHART_TOP_N): void {
+  const fdrCol = enrichDf.col('FDR');
+  if (!fdrCol) return;
+
+  const UNDERFLOW_NEGLOG10 = -Math.log10(Number.MIN_VALUE);
+  const fdrRaw = fdrCol.getRawData() as Float32Array | Float64Array;
+  if (enrichDf.columns.contains(NEG_LOG10_FDR_COL)) enrichDf.columns.remove(NEG_LOG10_FDR_COL);
+  const negLogCol = enrichDf.columns.addNewFloat(NEG_LOG10_FDR_COL);
+  negLogCol.init((i) => fdrRaw[i] > 0 ? -Math.log10(fdrRaw[i]) : UNDERFLOW_NEGLOG10);
+  // Keep it out of the reviewer's grid (best-effort; `~` isn't usable on a
+  // viewer value binding). If the tag is stripped on round-trip the only cost
+  // is one extra, meaningful numeric column in the grid.
+  negLogCol.setTag('.hidden', 'true');
+
+  // Rank terms by FDR (ascending) within each direction and mark the top N.
+  const dirCol = enrichDf.col('Direction');
+  const groups = new Map<string, {idx: number; fdr: number}[]>();
+  for (let i = 0; i < enrichDf.rowCount; i++) {
+    if (fdrCol.isNone(i)) continue;
+    const key = dirCol ? String(dirCol.get(i)) : '';
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key)!.push({idx: i, fdr: fdrCol.get(i) as number});
+  }
+  const topMask = new Array<boolean>(enrichDf.rowCount).fill(false);
+  for (const arr of groups.values()) {
+    arr.sort((a, b) => a.fdr - b.fdr);
+    for (let k = 0; k < Math.min(topN, arr.length); k++)
+      topMask[arr[k].idx] = true;
+  }
+  if (enrichDf.columns.contains(CHART_TOP_MARKER_COL)) enrichDf.columns.remove(CHART_TOP_MARKER_COL);
+  enrichDf.columns.addNewBool(CHART_TOP_MARKER_COL).init((i) => topMask[i]);
+}
+
+/** Per-viewer formula filter: the top-N marked terms, optionally narrowed to one
+ * regulation direction. Degrades to "show all rows" if the platform drops the
+ * formula on reopen — never a missing-column crash. */
+function chartFilter(direction?: 'Up' | 'Down'): string {
+  const clauses = [`\${${CHART_TOP_MARKER_COL}}`];
+  if (direction) clauses.push(`\${Direction} == "${direction}"`);
+  return clauses.join(' and ');
+}
+
+/**
+ * Creates an enrichment dot plot bound to the shared enrichment frame.
+ * X = Gene Ratio, Y = Term Name, size = Gene Count, color = -log10(FDR).
+ * Pass `direction` to split into Up/Down via a per-viewer formula filter.
+ */
+export function createEnrichmentDotPlot(enrichDf: DG.DataFrame, direction?: 'Up' | 'Down'): DG.ScatterPlotViewer {
+  const sp = DG.Viewer.scatterPlot(enrichDf, {
+    x: 'Gene Ratio',
+    y: 'Term Name',
+    sizeColumnName: 'Gene Count',
+    colorColumnName: NEG_LOG10_FDR_COL,
+    markerMinSize: 5,
+    markerMaxSize: 25,
+    filter: chartFilter(direction),
+    title: direction ? `Enrichment — ${direction}` : 'Enrichment Dot Plot',
+  } as any);
+  return sp;
+}
+
+/**
+ * Creates an enrichment bar chart (top terms ranked by -log10(FDR)) bound to the
+ * shared enrichment frame. Pass `direction` to split into Up/Down.
+ */
+export function createEnrichmentBarChart(enrichDf: DG.DataFrame, direction?: 'Up' | 'Down'): DG.Viewer {
+  const bar = DG.Viewer.barChart(enrichDf, {
+    splitColumnName: 'Term Name',
+    valueColumnName: NEG_LOG10_FDR_COL,
+    valueAggrType: 'avg',
+    barSortType: 'by value',
+    barSortOrder: 'desc',
+    orientation: 'Horizontal',
+    filter: chartFilter(direction),
+    title: direction ? `Top Enriched — ${direction}` : 'Top Enriched Terms',
+  } as any);
+  return bar;
+}
+
+/**
+ * Wires enrichment row changes to protein DataFrame selection.
+ * When a row is selected in the enrichment table, matching protein rows
+ * (by gene symbol from the Intersection column) are highlighted.
+ */
+export function wireEnrichmentToVolcano(
+  enrichDf: DG.DataFrame,
+  proteinDf: DG.DataFrame,
+): rxjs.Subscription {
+  const geneCol = findColumn(proteinDf, SEMTYPE.GENE_SYMBOL, ['gene name', 'gene symbol']);
+  if (!geneCol) return rxjs.Subscription.EMPTY;
+
+  // Build gene-to-row index
+  const geneToRows = new Map<string, number[]>();
+  for (let i = 0; i < proteinDf.rowCount; i++) {
+    if (!geneCol.isNone(i)) {
+      const gene = (geneCol.get(i) as string).trim();
+      if (!geneToRows.has(gene)) geneToRows.set(gene, []);
+      geneToRows.get(gene)!.push(i);
+    }
+  }
+
+  return enrichDf.onCurrentRowChanged.subscribe(() => {
+    const rowIdx = enrichDf.currentRowIdx;
+    if (rowIdx < 0) return;
+
+    const intersectionCol = enrichDf.col('Intersection');
+    if (!intersectionCol) return;
+
+    const memberGenesStr = intersectionCol.get(rowIdx) as string;
+    if (!memberGenesStr) return;
+
+    const memberGenes = memberGenesStr.split(',').map((g) => g.trim()).filter(Boolean);
+
+    // Clear previous selection
+    proteinDf.selection.setAll(false, false);
+
+    // Set selection for matching genes
+    for (const gene of memberGenes) {
+      const rows = geneToRows.get(gene);
+      if (rows) {
+        for (const row of rows)
+          proteinDf.selection.set(row, true, false);
+      }
+    }
+    proteinDf.selection.fireChanged();
+  });
+}
+
+/** True when `enrichDf` carries a Direction column with at least one row in
+ * `direction`. Drives the Up/Down 2×2 vs single-chart layout choice. */
+function hasDirection(enrichDf: DG.DataFrame, direction: 'Up' | 'Down'): boolean {
+  const dirCol = enrichDf.col('Direction');
+  if (!dirCol) return false;
+  for (let i = 0; i < enrichDf.rowCount; i++) {
+    if (dirCol.get(i) === direction) return true;
+  }
+  return false;
+}
+
+/**
+ * Opens a full enrichment visualization dashboard with dot plot, bar chart,
+ * and cross-DataFrame selection wiring to the protein table. When the result
+ * carries an R2 Direction column with both Up and Down rows, the Up and Down
+ * dot+bar charts are docked side-by-side; otherwise the original single
+ * dot+bar layout is used. The Phase-9 volcano cross-link is unchanged.
+ */
+export function openEnrichmentVisualization(
+  enrichDf: DG.DataFrame,
+  proteinDf: DG.DataFrame,
+): DG.TableView {
+  // Clean up previous subscriptions
+  for (const sub of activeSubscriptions)
+    sub.unsubscribe();
+  activeSubscriptions = [];
+
+  // Find or reuse the enrichment results table view. Match by stable .dart
+  // identity, NOT `=== enrichDf`: grok.shell.tableViews returns a fresh toJs
+  // wrapper per access, so strict-equality misses an already-open view and
+  // spawns a duplicate "Enrichment Results (2)" tab (precedent: the protein-TV
+  // lookup that used to live here; 13-07 SUMMARY).
+  let existing: DG.TableView | undefined;
+  for (const v of grok.shell.tableViews) {
+    if ((v.dataFrame as any)?.dart === (enrichDf as any).dart) { existing = v; break; }
+  }
+  const tv = existing ?? grok.shell.addTableView(enrichDf);
+
+  // Dock the dot/bar charts onto the ENRICHMENT results view — not the protein
+  // view where the volcano lives — so the volcano tab stays uncluttered
+  // (light-touch declutter). Surface this view FIRST: docking onto a
+  // backgrounded host leaves zero-dimension dock splitters and crashes the Dart
+  // scatter plot's smart-labels with
+  //   "Failed to execute 'drawImage'... canvas with width or height of 0".
+  grok.shell.v = tv;
+  const chartHost = tv;
+
+  // D-14 — surface the smart-filter transform stats above the grid when it
+  // actually dropped rows. Tag is set only by runEnrichmentPipeline when
+  // kept < total; the banner copy is locked in 14-UI-SPEC.md §"Copywriting".
+  if (enrichDf.getTag('proteomics.enrichment_smart_filtered') === 'true') {
+    const kept = enrichDf.getTag('proteomics.enrichment_smart_filtered_kept') ?? '?';
+    const total = enrichDf.getTag('proteomics.enrichment_smart_filtered_total') ?? '?';
+    const parents = enrichDf.getTag('proteomics.enrichment_smart_filtered_dropped_parents') ?? '?';
+    const cap = enrichDf.getTag('proteomics.enrichment_smart_filtered_cap') ?? '15';
+    const banner = ui.divText(
+      `Smart pathway filter active: showing ${kept} of ${total} terms ` +
+      `(dropped ${parents} generic parents, capped at ${cap} per source). ` +
+      `Re-run with the filter off to see all.`);
+    banner.dataset.smartFilterBanner = 'true';
+    tv.dockManager.dock(banner, DG.DOCK_TYPE.TOP, null, 'Smart Filter Info', 0.1);
+  }
+
+  // Dock the dot/bar charts (extracted so the publish path reuses the exact
+  // 2×2) and cross-link the frame. Every chart binds to `enrichDf` itself, so
+  // the single full-frame wiring covers all of them (and keeps the Intersection
+  // column the Phase-9 wiring keys on).
+  dockEnrichmentCharts(chartHost, enrichDf);
+  activeSubscriptions.push(wireEnrichmentToVolcano(enrichDf, proteinDf));
+
+  // The dot/bar charts, the merged enrichment grid, and the smart-filter banner
+  // all share this enrichment tab; the volcano stays on its own (protein) tab.
+  // The cross-link keeps the two tabs in sync on selection.
+  return tv;
+}
+
+/** Docks the directional (Up|Down 2×2) or single Dot+Bar enrichment charts onto
+ * `host`. Every chart binds to `enrichDf` itself — the derived `negLog10FDR` /
+ * `~enrichChartTop` columns are added to that frame and the Up/Down split is a
+ * per-viewer formula filter. No cloned subsets, so nothing leaks into a
+ * published project's table tree and the charts survive a project round-trip
+ * (docked viewers rebind to their host frame on reopen). Pure layout — no
+ * subscriptions, no focus changes — shared by the live and publish paths. */
+export function dockEnrichmentCharts(host: DG.TableView, enrichDf: DG.DataFrame): void {
+  prepareEnrichmentChartColumns(enrichDf);
+
+  if (hasDirection(enrichDf, 'Up') && hasDirection(enrichDf, 'Down')) {
+    // Balanced 2×2: [Up Dot | Down Dot] over [Up Bar | Down Bar]. Split the
+    // chart region into the two columns FIRST, THEN drop each bar under its dot
+    // — docking the bars before the split carved up only the top-left cell and
+    // produced a lopsided layout.
+    const upDotNode = host.dockManager.dock(
+      createEnrichmentDotPlot(enrichDf, 'Up'), DG.DOCK_TYPE.RIGHT, null, 'Up — Dot Plot', 0.6);
+    const downDotNode = host.dockManager.dock(
+      createEnrichmentDotPlot(enrichDf, 'Down'), DG.DOCK_TYPE.RIGHT, upDotNode, 'Down — Dot Plot', 0.5);
+    host.dockManager.dock(
+      createEnrichmentBarChart(enrichDf, 'Up'), DG.DOCK_TYPE.DOWN, upDotNode, 'Up — Bar Chart', 0.5);
+    host.dockManager.dock(
+      createEnrichmentBarChart(enrichDf, 'Down'), DG.DOCK_TYPE.DOWN, downDotNode, 'Down — Bar Chart', 0.5);
+    return;
+  }
+
+  // Single-direction or no Direction column → one Dot + Bar.
+  const dotNode = host.dockManager.dock(
+    createEnrichmentDotPlot(enrichDf), DG.DOCK_TYPE.RIGHT, null, 'Dot Plot', 0.5);
+  host.dockManager.dock(
+    createEnrichmentBarChart(enrichDf), DG.DOCK_TYPE.DOWN, dotNode, 'Bar Chart', 0.5);
+}

package/src/viewers/group-mean-correlation.ts

@@ -0,0 +1,218 @@
+import * as DG from 'datagrok-api/dg';
+import {findColumn} from '../utils/column-detection';
+import {SEMTYPE} from '../utils/proteomics-types';
+import {getGroups, GroupAssignment} from '../analysis/experiment-setup';
+
+/**
+ * R4 / D-12 — Group-Mean Correlation viewer.
+ *
+ * Adds two derived columns to the host DataFrame (Numerator Mean, Denominator
+ * Mean — means of the group1/group2 intensity columns per row) and renders
+ * them as a scatter colored by the existing `direction` column (magenta/cyan/
+ * gray per D-04). The viewer title carries inline Pearson r and Spearman ρ
+ * over the current filter; a y=x diagonal reference line is drawn in #888888.
+ *
+ * The derived columns are re-run-safe via `ensureFreshFloat` — a second
+ * invocation replaces the columns instead of duplicating them.
+ *
+ * Statistics: `@datagrok-libraries/statistics` exports only Kendall's tau,
+ * so Pearson and Spearman are implemented inline (Pearson over raw values,
+ * Spearman = Pearson over fractional ranks).
+ */
+
+const NUMERATOR_MEAN_COL = 'Numerator Mean';
+const DENOMINATOR_MEAN_COL = 'Denominator Mean';
+const DIRECTION_COL = 'direction';
+const DIAGONAL_FORMULA = `\${${NUMERATOR_MEAN_COL}} = \${${DENOMINATOR_MEAN_COL}}`;
+const DIAGONAL_COLOR = '#888888';
+
+/** Bulk-init pattern — removes an existing column first to avoid duplicates on
+ * re-run. Mirrors src/viewers/qc-computations.ts:28-32. */
+function ensureFreshFloat(df: DG.DataFrame, name: string): DG.Column {
+  if (df.columns.contains(name))
+    df.columns.remove(name);
+  return df.columns.addNewFloat(name);
+}
+
+/** Mean of non-null values across `cols` at `rowIdx`. Returns NaN if every
+ * value is null. Mirrors qc-computations.ts:15-25. */
+function groupMean(cols: DG.Column[], rowIdx: number): number {
+  let sum = 0;
+  let n = 0;
+  for (const c of cols) {
+    if (c.isNone(rowIdx)) continue;
+    sum += c.get(rowIdx) as number;
+    n++;
+  }
+  return n > 0 ? sum / n : NaN;
+}
+
+/**
+ * Derives Numerator Mean (group1) and Denominator Mean (group2) columns on
+ * `df`. The join key is the protein row index — never a derived key like
+ * gene name (project memory project_proteomics_deqms_result_misalignment).
+ *
+ * The columns carry the dedicated SEMTYPE.NUMERATOR_MEAN / SEMTYPE.DENOMINATOR_MEAN
+ * tags so downstream tooling can find them via `findColumn`.
+ */
+export function computeGroupMeans(df: DG.DataFrame, groups: GroupAssignment): void {
+  const g1Cols = groups.group1.columns
+    .map((n) => df.col(n))
+    .filter((c): c is DG.Column => c != null);
+  const g2Cols = groups.group2.columns
+    .map((n) => df.col(n))
+    .filter((c): c is DG.Column => c != null);
+
+  const nRows = df.rowCount;
+  const numArr = new Float32Array(nRows);
+  const denArr = new Float32Array(nRows);
+  for (let i = 0; i < nRows; i++) {
+    const m1 = groupMean(g1Cols, i);
+    const m2 = groupMean(g2Cols, i);
+    numArr[i] = isNaN(m1) ? DG.FLOAT_NULL : m1;
+    denArr[i] = isNaN(m2) ? DG.FLOAT_NULL : m2;
+  }
+
+  const numCol = ensureFreshFloat(df, NUMERATOR_MEAN_COL);
+  numCol.init((i) => numArr[i]);
+  numCol.semType = SEMTYPE.NUMERATOR_MEAN;
+
+  const denCol = ensureFreshFloat(df, DENOMINATOR_MEAN_COL);
+  denCol.init((i) => denArr[i]);
+  denCol.semType = SEMTYPE.DENOMINATOR_MEAN;
+}
+
+/** Pearson correlation. Skips paired entries containing NaN or FLOAT_NULL. */
+export function pearson(xs: number[], ys: number[]): number {
+  let n = 0;
+  let sumX = 0;
+  let sumY = 0;
+  for (let i = 0; i < xs.length; i++) {
+    const x = xs[i];
+    const y = ys[i];
+    if (!Number.isFinite(x) || !Number.isFinite(y) || x === DG.FLOAT_NULL || y === DG.FLOAT_NULL) continue;
+    sumX += x;
+    sumY += y;
+    n++;
+  }
+  if (n === 0) return NaN;
+  const meanX = sumX / n;
+  const meanY = sumY / n;
+  let num = 0;
+  let denX = 0;
+  let denY = 0;
+  for (let i = 0; i < xs.length; i++) {
+    const x = xs[i];
+    const y = ys[i];
+    if (!Number.isFinite(x) || !Number.isFinite(y) || x === DG.FLOAT_NULL || y === DG.FLOAT_NULL) continue;
+    const dx = x - meanX;
+    const dy = y - meanY;
+    num += dx * dy;
+    denX += dx * dx;
+    denY += dy * dy;
+  }
+  const den = Math.sqrt(denX * denY);
+  return den === 0 ? NaN : num / den;
+}
+
+/** Fractional ranks (1-based). Ties get the average of the positions they span. */
+function rank(arr: number[]): number[] {
+  const indexed = arr.map((v, i) => ({v, i}));
+  indexed.sort((a, b) => a.v - b.v);
+  const ranks = new Array<number>(arr.length);
+  let i = 0;
+  while (i < indexed.length) {
+    let j = i;
+    while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
+    const avg = (i + j) / 2 + 1; // 1-based average of positions [i..j]
+    for (let k = i; k <= j; k++) ranks[indexed[k].i] = avg;
+    i = j + 1;
+  }
+  return ranks;
+}
+
+/** Spearman = Pearson over fractional ranks. */
+export function spearman(xs: number[], ys: number[]): number {
+  return pearson(rank(xs), rank(ys));
+}
+
+/** Computes Pearson + Spearman over the filtered subset of two columns. */
+function computePearsonSpearman(
+  df: DG.DataFrame, xName: string, yName: string,
+): {r: number; rho: number} {
+  const xCol = df.col(xName);
+  const yCol = df.col(yName);
+  if (!xCol || !yCol) return {r: NaN, rho: NaN};
+  const xRaw = xCol.getRawData() as Float32Array | Float64Array;
+  const yRaw = yCol.getRawData() as Float32Array | Float64Array;
+
+  const xs: number[] = [];
+  const ys: number[] = [];
+  for (let i = 0; i < df.rowCount; i++) {
+    if (!df.filter.get(i)) continue;
+    const x = xRaw[i];
+    const y = yRaw[i];
+    if (x === DG.FLOAT_NULL || y === DG.FLOAT_NULL) continue;
+    if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
+    xs.push(x);
+    ys.push(y);
+  }
+  return {r: pearson(xs, ys), rho: spearman(xs, ys)};
+}
+
+/**
+ * Main factory — derives Numerator/Denominator Mean, plots them as a scatter
+ * colored by direction, draws the y=x diagonal, and writes an inline
+ * Pearson/Spearman annotation into the title.
+ */
+export function createGroupMeanCorrelation(
+  df: DG.DataFrame,
+  options?: {title?: string},
+): DG.ScatterPlotViewer {
+  const groups = getGroups(df);
+  if (!groups)
+    throw new Error('Annotate experimental groups first (Proteomics | Annotate Experiment)');
+
+  computeGroupMeans(df, groups);
+
+  const sp = df.plot.scatter({
+    x: NUMERATOR_MEAN_COL,
+    y: DENOMINATOR_MEAN_COL,
+    color: DIRECTION_COL,
+  });
+
+  // Display Name primary, Gene name fallback (Plan 02 pattern).
+  const labelCol = findColumn(df, SEMTYPE.DISPLAY_NAME, ['display name']) ??
+    findColumn(df, SEMTYPE.GENE_SYMBOL, ['gene name', 'gene symbol']);
+  if (labelCol) {
+    sp.props.labelColumnNames = [labelCol.name];
+    sp.props.showLabelsFor = 'MouseOverRow';
+  }
+
+  // y = x diagonal reference — replace any prior diagonal so re-run doesn't stack.
+  df.meta.formulaLines.items = df.meta.formulaLines.items.filter((line) => {
+    const f = (line.formula ?? '') as string;
+    return !(typeof f === 'string' &&
+      f.includes(NUMERATOR_MEAN_COL) && f.includes(DENOMINATOR_MEAN_COL));
+  });
+  df.meta.formulaLines.addLine({
+    formula: DIAGONAL_FORMULA,
+    color: DIAGONAL_COLOR,
+    width: 1,
+    visible: true,
+  });
+  sp.props.showViewerFormulaLines = true;
+
+  // Inline correlation annotation over the current filter.
+  const {r, rho} = computePearsonSpearman(df, NUMERATOR_MEAN_COL, DENOMINATOR_MEAN_COL);
+  const baseTitle = options?.title ?? 'Group-Mean Correlation';
+  const rStr = Number.isFinite(r) ? r.toFixed(2) : 'NA';
+  const rhoStr = Number.isFinite(rho) ? rho.toFixed(2) : 'NA';
+  sp.setOptions({
+    title: `${baseTitle} — r=${rStr} (Pearson), ρ=${rhoStr} (Spearman)`,
+    xColumnLabel: `${groups.group1.name} mean (log2 intensity)`,
+    yColumnLabel: `${groups.group2.name} mean (log2 intensity)`,
+  } as any);
+
+  return sp;
+}

package/src/viewers/heatmap.ts

@@ -0,0 +1,168 @@
+import * as DG from 'datagrok-api/dg';
+import {getGroups} from '../analysis/experiment-setup';
+import {findColumn} from '../utils/column-detection';
+import {SEMTYPE} from '../utils/proteomics-types';
+
+/** Z-score normalize a value given mean and standard deviation. */
+function zScore(value: number, mean: number, std: number): number {
+  return std > 0 ? (value - mean) / std : 0;
+}
+
+/**
+ * Creates an expression heatmap (Grid in heatmap mode) on a cloned DataFrame
+ * containing only the top-N most significant proteins. The clone isolates
+ * the heatmap's filter, z-score columns, and sort order from the original
+ * DataFrame so that other viewers (volcano plot) are not affected.
+ *
+ * Rows are hierarchically clustered via the Dendrogram package service if available,
+ * otherwise sorted by adj.p-value ascending (significance-based fallback).
+ *
+ * Z-score normalization uses statistics computed across ALL rows of the original
+ * DataFrame for statistical correctness, but z-score columns are only added to the
+ * cloned DataFrame.
+ */
+export async function createExpressionHeatmap(
+  df: DG.DataFrame,
+  options?: {topN?: number; labelCol?: string; title?: string},
+): Promise<DG.Grid> {
+  const topN = options?.topN ?? 50;
+  const labelColName = options?.labelCol ?? 'Gene Name';
+
+  // 1. Check prerequisites
+  if (df.getTag('proteomics.de_complete') !== 'true')
+    throw new Error('Differential expression must be run first');
+
+  const groups = getGroups(df);
+  if (!groups)
+    throw new Error('Experimental groups must be annotated first');
+
+  // 2. Select top N proteins by adj.p-value (ascending)
+  const adjPCol = df.col('adj.p-value');
+  if (!adjPCol)
+    throw new Error('adj.p-value column not found');
+
+  // Collect row indices with non-null adj.p-value, sorted ascending
+  const indexedPValues: {idx: number; pVal: number}[] = [];
+  for (let i = 0; i < df.rowCount; i++) {
+    if (!adjPCol.isNone(i))
+      indexedPValues.push({idx: i, pVal: adjPCol.get(i) as number});
+  }
+  indexedPValues.sort((a, b) => a.pVal - b.pVal);
+
+  const topNIndices = new Set<number>();
+  for (let i = 0; i < Math.min(topN, indexedPValues.length); i++)
+    topNIndices.add(indexedPValues[i].idx);
+
+  // 3. Clone the DataFrame with only top-N rows (filter isolation)
+  const filter = DG.BitSet.create(df.rowCount);
+  for (const idx of topNIndices)
+    filter.set(idx, true);
+  const heatmapDf = df.clone(filter);
+
+  // 4. Compute z-score columns on the cloned DataFrame
+  // Statistics (mean, std) are computed across ALL rows of the original df for correctness,
+  // then z-scores are computed on the cloned rows only.
+  const allIntensityCols = [...groups.group1.columns, ...groups.group2.columns];
+  const zScoreColNames: string[] = [];
+
+  for (const colName of allIntensityCols) {
+    const zColName = `_zscore_${colName}`;
+    zScoreColNames.push(zColName);
+
+    const srcColOrig = df.col(colName);
+    if (!srcColOrig) continue;
+
+    // Compute mean and std across all non-null values in the ORIGINAL column.
+    // Use direct typed-array access to skip the per-row JS->native interop.
+    const origRaw = srcColOrig.getRawData() as Float32Array | Float64Array;
+    let sum = 0;
+    let sumSq = 0;
+    let count = 0;
+    for (let i = 0; i < df.rowCount; i++) {
+      const v = origRaw[i];
+      if (v !== DG.FLOAT_NULL) {
+        sum += v;
+        sumSq += v * v;
+        count++;
+      }
+    }
+
+    const colMean = count > 0 ? sum / count : 0;
+    const variance = count > 1 ? (sumSq - sum * sum / count) / (count - 1) : 0;
+    const colStd = Math.sqrt(variance);
+
+    // Apply z-scores to the cloned DataFrame
+    const srcColClone = heatmapDf.col(colName);
+    if (!srcColClone) continue;
+
+    const srcRaw = srcColClone.getRawData() as Float32Array | Float64Array;
+    const zCol = heatmapDf.columns.addNewFloat(zColName);
+    zCol.init((i) => srcRaw[i] === DG.FLOAT_NULL ? DG.FLOAT_NULL : zScore(srcRaw[i], colMean, colStd));
+  }
+
+  // 5. Create Grid on the cloned DataFrame
+  const grid = heatmapDf.plot.grid();
+
+  // 6. Column visibility -- hide everything, then show label + z-score columns
+  for (let i = 0; i < grid.columns.length; i++) {
+    const gc = grid.columns.byIndex(i);
+    if (gc) gc.visible = false;
+  }
+
+  // Show label column (gene symbol preferred, protein ID as fallback)
+  const labelCol = findColumn(heatmapDf, SEMTYPE.GENE_SYMBOL, [labelColName.toLowerCase(), 'gene symbol'])
+    ?? findColumn(heatmapDf, SEMTYPE.PROTEIN_ID, ['protein id', 'majority protein id', 'accession']);
+  if (labelCol) {
+    const labelGc = grid.columns.byName(labelCol.name);
+    if (labelGc) labelGc.visible = true;
+  }
+
+  // Show z-score columns in group order (control left, treatment right)
+  const group1ZCols = groups.group1.columns.map((c) => `_zscore_${c}`);
+  const group2ZCols = groups.group2.columns.map((c) => `_zscore_${c}`);
+
+  for (const zColName of [...group1ZCols, ...group2ZCols]) {
+    const gc = grid.columns.byName(zColName);
+    if (gc) gc.visible = true;
+  }
+
+  // 7. Attempt hierarchical clustering via Dendrogram package; fall back to
+  // significance-based row ordering if the package is unavailable or its
+  // function signature has drifted.
+  let clustered = false;
+  const pi = DG.TaskBarProgressIndicator.create('Calculating distance matrix...');
+  try {
+    const dFunc = DG.Func.find({package: 'Dendrogram', name: 'hierarchicalClustering'})[0];
+    if (!dFunc)
+      throw new Error('Dendrogram package not installed (hierarchicalClustering not found)');
+    if (dFunc.inputs.length !== 4)
+      throw new Error(`Dendrogram.hierarchicalClustering signature changed (expected 4 inputs, got ${dFunc.inputs.length})`);
+    await dFunc.apply({
+      df: grid, colNameList: zScoreColNames,
+      distance: 'euclidean', linkage: 'complete',
+    });
+    clustered = true;
+  } catch (e: any) {
+    console.warn(`Dendrogram clustering unavailable, falling back to significance-based row ordering: ${e?.message ?? e}`);
+  } finally {
+    pi.close();
+  }
+
+  if (!clustered) {
+    // Sort by adjusted p-value ascending (most significant first). R scripts
+    // sometimes return the column as `adj.p.value` (dots) instead of `adj.p-value`.
+    const sortCol = heatmapDf.col('adj.p-value') ? 'adj.p-value' :
+      (heatmapDf.col('adj.p.value') ? 'adj.p.value' : null);
+    if (sortCol)
+      grid.sort([sortCol]);
+  }
+
+  // 8. Configure Grid for heatmap display
+  grid.props.isHeatmap = true;
+  grid.props.isGrid = false;
+
+  if (options?.title)
+    grid.setOptions({title: options.title});
+
+  return grid;
+}