@datagrok/proteomics 1.0.0 → 1.2.0

package/src/parsers/fragpipe-parser.ts

@@ -0,0 +1,200 @@
+import * as DG from 'datagrok-api/dg';
+import {SEMTYPE} from '../utils/proteomics-types';
+import {log2TransformColumns, addPrimaryColumnIfNeeded} from './shared-utils';
+import {resolveGeneLabels} from '../utils/gene-label-resolver';
+
+/** Per-sample intensity column suffixes in FragPipe's combined_protein.tsv,
+ * in order of quantitative preference. MaxLFQ is the FragPipe-recommended
+ * LFQ quant; bare " Intensity" is the summed peptide intensity. Razor is
+ * the razor-protein attribution and not normally used as the primary quant. */
+const INTENSITY_SUFFIXES = [' MaxLFQ Intensity', ' Intensity', ' Razor Intensity'] as const;
+
+/** Column name variants for protein-level identifiers in FragPipe output.
+ * `Protein ID` is the UniProt accession; `Protein` is the full FASTA header
+ * (e.g., `sp|P12345|GENE_HUMAN`) which we prefer only if `Protein ID` is missing. */
+const PROTEIN_ID_COLUMNS = ['Protein ID', 'Protein'];
+
+/** Column name variants for gene symbols. */
+const GENE_NAME_COLUMNS = ['Gene', 'Gene Names', 'Gene Symbol'];
+
+/** Decoy / contaminant prefixes to filter out at the row level.
+ * `contam_` is FragPipe/Philosopher's convention; `rev_` is Philosopher's decoy
+ * prefix (usually filtered upstream, but defensive); CON__/REV__ cover the
+ * case where the FASTA was MaxQuant-formatted. */
+const CONTAMINANT_PREFIXES = ['contam_', 'rev_', 'CON__', 'REV__'];
+
+/** Finds the first existing column from a list of name candidates. */
+function findCol(df: DG.DataFrame, names: readonly string[]): DG.Column | null {
+  for (const name of names) {
+    const col = df.col(name);
+    if (col)
+      return col;
+  }
+  return null;
+}
+
+/** Checks whether a header is a per-sample intensity column.
+ * Returns the matched suffix (lowercased) or null. */
+function matchIntensitySuffix(header: string): string | null {
+  const lower = header.toLowerCase();
+  for (const suffix of INTENSITY_SUFFIXES) {
+    if (lower.endsWith(suffix.toLowerCase()) && lower.length > suffix.length)
+      return suffix.toLowerCase();
+  }
+  return null;
+}
+
+/** Marks rows for exclusion when their primary protein identifier starts with
+ * any contaminant/decoy prefix. Checks both `Protein ID` and `Protein` because
+ * Philosopher contam_/rev_ prefixes appear on whichever column is populated. */
+function filterContaminantRows(df: DG.DataFrame): void {
+  const candidates = ['Protein ID', 'Protein'];
+  const cols: DG.Column[] = [];
+  for (const name of candidates) {
+    const c = df.col(name);
+    if (c) cols.push(c);
+  }
+  if (cols.length === 0) return;
+
+  for (let i = 0; i < df.rowCount; i++) {
+    for (const col of cols) {
+      const val = col.get(i);
+      if (typeof val !== 'string') continue;
+      let matched = false;
+      for (const prefix of CONTAMINANT_PREFIXES) {
+        if (val.startsWith(prefix)) {
+          matched = true;
+          break;
+        }
+      }
+      if (matched) {
+        df.filter.set(i, false, false);
+        break;
+      }
+    }
+  }
+}
+
+/** Scans the header row to find intensity columns and forces them to `double`
+ * during CSV import. Without this, fromCsv can mistype large-integer intensities
+ * (or all-zero / all-blank samples) as boolean or int. Mirrors the same guard
+ * used by the MaxQuant parser. */
+function buildIntensityColumnOptions(text: string): DG.CsvImportColumnOptions[] {
+  const newlineIdx = text.indexOf('\n');
+  const firstLine = (newlineIdx >= 0 ? text.substring(0, newlineIdx) : text).replace(/\r$/, '');
+  const headers = firstLine.split('\t');
+  const opts: DG.CsvImportColumnOptions[] = [];
+  for (const header of headers) {
+    if (matchIntensitySuffix(header))
+      opts.push({name: header, type: 'double'});
+  }
+  return opts;
+}
+
+/** Returns the set of intensity columns to log2-transform. If MaxLFQ Intensity
+ * columns exist, prefer them and drop the redundant bare " Intensity" / " Razor
+ * Intensity" columns for the same sample. If MaxLFQ is absent (e.g. spectral-
+ * count-only run, or LFQ not enabled), fall back to " Intensity". Razor is only
+ * used as a last resort. Returned columns still get SEMTYPE.INTENSITY below. */
+function pickQuantColumns(df: DG.DataFrame): string[] {
+  type Bucket = {maxLfq?: string; intensity?: string; razor?: string};
+  const samples = new Map<string, Bucket>();
+
+  for (const name of df.columns.names()) {
+    const col = df.col(name);
+    if (!col) continue;
+    if (col.type !== DG.COLUMN_TYPE.FLOAT && col.type !== DG.COLUMN_TYPE.INT &&
+        col.type !== DG.COLUMN_TYPE.BIG_INT)
+      continue;
+    const lower = name.toLowerCase();
+    let sample: string | null = null;
+    let kind: keyof Bucket | null = null;
+    if (lower.endsWith(' maxlfq intensity')) {
+      sample = name.substring(0, name.length - ' MaxLFQ Intensity'.length);
+      kind = 'maxLfq';
+    } else if (lower.endsWith(' razor intensity')) {
+      sample = name.substring(0, name.length - ' Razor Intensity'.length);
+      kind = 'razor';
+    } else if (lower.endsWith(' intensity')) {
+      sample = name.substring(0, name.length - ' Intensity'.length);
+      kind = 'intensity';
+    }
+    if (!sample || !kind) continue;
+    if (!samples.has(sample)) samples.set(sample, {});
+    samples.get(sample)![kind] = name;
+  }
+
+  const chosen: string[] = [];
+  for (const bucket of samples.values()) {
+    if (bucket.maxLfq) chosen.push(bucket.maxLfq);
+    else if (bucket.intensity) chosen.push(bucket.intensity);
+    else if (bucket.razor) chosen.push(bucket.razor);
+  }
+  return chosen;
+}
+
+/** Assigns semantic types to ID, gene, and the chosen per-sample intensity columns.
+ * Only `chosenQuant` columns receive SEMTYPE.INTENSITY — the redundant Razor /
+ * bare-Intensity columns we dropped from log2 transform must not surface as
+ * intensities in downstream pickers. */
+function assignSemanticTypes(df: DG.DataFrame, chosenQuant: string[]): void {
+  const proteinCol = findCol(df, PROTEIN_ID_COLUMNS);
+  if (proteinCol)
+    proteinCol.semType = SEMTYPE.PROTEIN_ID;
+
+  const geneCol = findCol(df, GENE_NAME_COLUMNS);
+  if (geneCol)
+    geneCol.semType = SEMTYPE.GENE_SYMBOL;
+
+  for (const name of chosenQuant) {
+    const col = df.col(name);
+    if (col) col.semType = SEMTYPE.INTENSITY;
+  }
+}
+
+/** Finds the source column for primary extraction from FragPipe-specific name
+ * variants and delegates to shared addPrimaryColumnIfNeeded. */
+function addPrimaryColumnFromVariants(df: DG.DataFrame, sourceNames: readonly string[],
+  newName: string, semType: string): void {
+  const srcCol = findCol(df, sourceNames);
+  if (!srcCol) return;
+  addPrimaryColumnIfNeeded(df, srcCol.name, newName, semType);
+}
+
+/** Parses a FragPipe `combined_protein.tsv` into a filtered, typed DataFrame.
+ *
+ * Applies:
+ * - Contaminant/decoy row filtering (`contam_`, `rev_`, plus MaxQuant-style
+ *   CON__/REV__ as a fallback)
+ * - Semantic type assignment (PROTEIN_ID, GENE_SYMBOL, INTENSITY)
+ * - Primary protein/gene split for semicolon-delimited identifier lists
+ * - Log2 transform of the preferred quantitative intensity columns
+ *   (MaxLFQ Intensity > Intensity > Razor Intensity, picked per sample)
+ *
+ * The returned DataFrame is ready for the standard pipeline: Annotate
+ * Experiment -> Normalize -> Impute -> Differential Expression -> Viewers. */
+export async function parseFragPipeText(text: string): Promise<DG.DataFrame> {
+  const columnImportOptions = buildIntensityColumnOptions(text);
+  const raw = DG.DataFrame.fromCsv(text, {delimiter: '\t', columnImportOptions});
+
+  raw.filter.init((_i) => true);
+  filterContaminantRows(raw);
+  raw.filter.fireChanged();
+
+  const df = raw.clone(raw.filter);
+
+  // Choose one quant column per sample for log2 transform.
+  const chosen = pickQuantColumns(df);
+  assignSemanticTypes(df, chosen);
+
+  addPrimaryColumnFromVariants(df, PROTEIN_ID_COLUMNS, 'Primary Protein ID', SEMTYPE.PROTEIN_ID);
+  addPrimaryColumnFromVariants(df, GENE_NAME_COLUMNS, 'Primary Gene Name', SEMTYPE.GENE_SYMBOL);
+
+  log2TransformColumns(df, chosen);
+
+  df.setTag('proteomics.source', 'fragpipe');
+
+  await resolveGeneLabels(df);
+
+  return df;
+}

package/src/parsers/generic-parser.ts

@@ -0,0 +1,197 @@
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+
+import {SEMTYPE} from '../utils/proteomics-types';
+import {focusProtein} from '../panels/protein-focus';
+import {
+  log2TransformColumns,
+  copyAsLog2Columns,
+  addPrimaryColumnIfNeeded,
+  detectLog2Status,
+  detectDelimiter,
+  autoSuggestProteinIdColumn,
+  autoSuggestIntensityColumns,
+  autoSuggestGeneNameColumn,
+} from './shared-utils';
+import {resolveGeneLabels} from '../utils/gene-label-resolver';
+
+/** Opens a file picker and shows the column mapping dialog for generic matrix import. */
+export function showGenericImportDialog(): void {
+  DG.Utils.openFile({
+    accept: '.csv,.tsv,.txt',
+    open: (file: File) => {
+      const reader = new FileReader();
+      reader.onload = () => {
+        try {
+          const text = reader.result as string;
+          const delimiter = detectDelimiter(text);
+          const df = DG.DataFrame.fromCsv(text, {delimiter});
+          showMappingDialog(df, file.name);
+        } catch (e: any) {
+          grok.shell.error('Failed to parse file: ' + e.message);
+        }
+      };
+      reader.readAsText(file);
+    },
+  });
+}
+
+/** Shows the column mapping dialog for a parsed DataFrame.
+ * @param df - Parsed DataFrame from the imported file
+ * @param fileName - Original file name for naming the output DataFrame */
+function showMappingDialog(df: DG.DataFrame, fileName: string): void {
+  // Auto-suggest columns
+  const suggestedProtein = autoSuggestProteinIdColumn(df);
+  const suggestedGene = autoSuggestGeneNameColumn(df);
+  const suggestedIntensityNames = autoSuggestIntensityColumns(df);
+
+  // Protein ID input -- string columns only
+  const proteinIdInput = ui.input.column('Protein ID', {
+    table: df,
+    filter: (c: DG.Column) => c.type === DG.COLUMN_TYPE.STRING,
+    value: suggestedProtein ?? undefined,
+  });
+
+  // Gene Name input -- optional, string columns only
+  const geneNameInput = ui.input.column('Gene Name (optional)', {
+    table: df,
+    filter: (c: DG.Column) => c.type === DG.COLUMN_TYPE.STRING,
+    value: suggestedGene ?? undefined,
+    nullable: true,
+  });
+
+  // Intensity columns multi-select — show all numeric columns, pre-select keyword matches
+  const numericColNames = df.columns.toList()
+    .filter((c) => c.type === DG.COLUMN_TYPE.FLOAT || c.type === DG.COLUMN_TYPE.INT || c.type === DG.COLUMN_TYPE.BIG_INT)
+    .map((c) => c.name);
+  const intensityColsInput = ui.input.columns('Intensity Columns', {
+    table: df,
+    available: numericColNames,
+    value: suggestedIntensityNames.map((n) => df.col(n)!).filter((c) => c != null),
+  });
+
+  // Log2 detection on initial intensity columns
+  const initialDetection = detectLog2Status(df, suggestedIntensityNames);
+
+  // Log2 toggle -- ON means "needs transform" (raw intensities)
+  const log2Toggle = ui.input.bool('Log2 Transform', {value: !initialDetection.isLog2});
+
+  // Hint label for log2 detection message
+  const hintDiv = ui.divText(initialDetection.message);
+  hintDiv.style.cssText = 'font-style:italic;color:#888;margin-bottom:8px;font-size:12px;';
+
+  // Preview container
+  const previewContainer = document.createElement('div');
+  previewContainer.style.cssText = 'border:1px solid #ddd;margin-top:8px;width:100%;';
+
+  /** Updates the preview grid with currently selected columns. */
+  function updatePreview(): void {
+    const selectedNames: string[] = [];
+    const proteinCol = proteinIdInput.value;
+    if (proteinCol) selectedNames.push(proteinCol.name);
+    const geneCol = geneNameInput.value;
+    if (geneCol) selectedNames.push(geneCol.name);
+    const intensityCols: DG.Column[] = intensityColsInput.value ?? [];
+    const intensityNames = intensityCols.map((c) => c.name);
+    selectedNames.push(...intensityNames);
+
+    if (selectedNames.length === 0) {
+      previewContainer.innerHTML = '<div style="padding:8px;color:#888">Select columns to preview</div>';
+      return;
+    }
+
+    try {
+      const mask = DG.BitSet.create(df.rowCount, (i) => i < 5);
+      const previewDf = df.clone(mask, selectedNames);
+      const grid = DG.Viewer.grid(previewDf);
+      previewContainer.innerHTML = '';
+      const countLabel = ui.divText(`Preview: ${selectedNames.length} columns, 5 rows`);
+      countLabel.style.cssText = 'font-size:11px;color:#888;padding:2px 4px;';
+      grid.root.style.cssText = 'width:100%;height:180px;';
+      previewContainer.appendChild(countLabel);
+      previewContainer.appendChild(grid.root);
+    } catch {
+      previewContainer.innerHTML = '<div style="padding:8px;color:#888">Unable to generate preview</div>';
+    }
+  }
+
+  /** Updates log2 detection hint based on current intensity columns. */
+  function updateLog2Detection(): void {
+    const intensityCols: DG.Column[] = intensityColsInput.value ?? [];
+    const names = intensityCols.map((c) => c.name);
+    if (names.length === 0) {
+      hintDiv.textContent = 'Select intensity columns';
+      return;
+    }
+    const detection = detectLog2Status(df, names);
+    hintDiv.textContent = detection.message;
+    log2Toggle.value = !detection.isLog2;
+  }
+
+  // Wire up reactive updates
+  proteinIdInput.onChanged.subscribe(() => updatePreview());
+  geneNameInput.onChanged.subscribe(() => updatePreview());
+  intensityColsInput.onChanged.subscribe(() => {
+    updateLog2Detection();
+    updatePreview();
+  });
+
+  // Initial preview
+  updatePreview();
+
+  // Build dialog
+  ui.dialog('Import Generic Matrix')
+    .add(proteinIdInput)
+    .add(geneNameInput)
+    .add(intensityColsInput)
+    .add(log2Toggle)
+    .add(ui.div([hintDiv]))
+    .add(previewContainer)
+    .onOK(async () => {
+      const proteinCol = proteinIdInput.value;
+      if (!proteinCol) {
+        grok.shell.warning('Please select a Protein ID column');
+        return;
+      }
+
+      const intensityCols: DG.Column[] = intensityColsInput.value ?? [];
+      const intensityNames = intensityCols.map((c) => c.name);
+      if (intensityNames.length === 0) {
+        grok.shell.warning('Please select at least one intensity column');
+        return;
+      }
+
+      // Assign semantic types
+      proteinCol.semType = SEMTYPE.PROTEIN_ID;
+
+      const geneCol = geneNameInput.value;
+      if (geneCol)
+        geneCol.semType = SEMTYPE.GENE_SYMBOL;
+
+      // Log2 transform or copy
+      if (log2Toggle.value)
+        log2TransformColumns(df, intensityNames);
+      else
+        copyAsLog2Columns(df, intensityNames);
+
+      // Primary columns (splits semicolon-delimited values)
+      addPrimaryColumnIfNeeded(df, proteinCol.name, 'Primary Protein ID', SEMTYPE.PROTEIN_ID);
+      if (geneCol)
+        addPrimaryColumnIfNeeded(df, geneCol.name, 'Primary Gene Name', SEMTYPE.GENE_SYMBOL);
+
+      // Source tag
+      df.setTag('proteomics.source', 'generic');
+
+      await resolveGeneLabels(df);
+
+      // Name from file
+      df.name = fileName.replace(/\.[^.]+$/, '');
+
+      // Open in table view
+      grok.shell.addTableView(df);
+      grok.shell.info(`Imported ${df.rowCount} proteins from ${fileName}`);
+      focusProtein(df);
+    })
+    .show();
+}

package/src/parsers/maxquant-parser.ts

@@ -0,0 +1,162 @@
+import * as DG from 'datagrok-api/dg';
+import {SEMTYPE} from '../utils/proteomics-types';
+import {log2TransformColumns, addPrimaryColumnIfNeeded} from './shared-utils';
+import {resolveGeneLabels} from '../utils/gene-label-resolver';
+
+/** Intensity column prefixes to detect, checked in this order.
+ * "intensity" must be last since it is a substring of "lfq intensity". */
+const INTENSITY_PREFIXES = ['lfq intensity', 'ibaq', 'reporter intensity', 'intensity'];
+
+/** Column name variants for filter columns across MaxQuant versions. */
+const FILTER_COLUMNS = {
+  contaminant: ['Potential contaminant', 'Potential.contaminant'],
+  reverse: ['Reverse'],
+  onlyBySite: ['Only identified by site', 'Only.identified.by.site'],
+} as const;
+
+/** Column name variants for protein IDs. */
+const PROTEIN_ID_COLUMNS = ['Protein IDs', 'Majority protein IDs'];
+
+/** Column name variants for gene names. */
+const GENE_NAME_COLUMNS = ['Gene names', 'Gene name'];
+
+/** Finds the first existing column from a list of name candidates. */
+function findCol(df: DG.DataFrame, names: readonly string[]): DG.Column | null {
+  for (const name of names) {
+    const col = df.col(name);
+    if (col)
+      return col;
+  }
+  return null;
+}
+
+/** Marks rows for exclusion where the given column has value "+". */
+function filterByMarker(df: DG.DataFrame, colNames: readonly string[]): void {
+  const col = findCol(df, colNames);
+  if (!col) return;
+  for (let i = 0; i < df.rowCount; i++) {
+    if (col.get(i) === '+')
+      df.filter.set(i, false, false);
+  }
+}
+
+/** Marks rows for exclusion when any protein-id column starts with CON__/REV__.
+ * Scans every column in PROTEIN_ID_COLUMNS rather than the first match — MaxQuant
+ * sometimes drops the prefix from `Protein IDs` but retains it in `Majority protein IDs`. */
+function filterByIdPrefix(df: DG.DataFrame): void {
+  const cols: DG.Column[] = [];
+  for (const name of PROTEIN_ID_COLUMNS) {
+    const c = df.col(name);
+    if (c) cols.push(c);
+  }
+  if (cols.length === 0) return;
+  for (let i = 0; i < df.rowCount; i++) {
+    for (const col of cols) {
+      const val = col.get(i);
+      if (typeof val === 'string' && (val.startsWith('CON__') || val.startsWith('REV__'))) {
+        df.filter.set(i, false, false);
+        break;
+      }
+    }
+  }
+}
+
+/** Finds the source column for primary extraction from MaxQuant-specific name variants,
+ * then delegates to shared addPrimaryColumnIfNeeded which handles semicolon detection. */
+function addPrimaryColumnFromVariants(df: DG.DataFrame, sourceNames: readonly string[],
+  newName: string, semType: string): void {
+  const srcCol = findCol(df, sourceNames);
+  if (!srcCol) return;
+  addPrimaryColumnIfNeeded(df, srcCol.name, newName, semType);
+}
+
+/** Scans header row to find intensity column names and build columnImportOptions
+ * that force them to double. Prevents fromCsv from mistyping large-integer
+ * intensity columns as boolean. */
+function buildIntensityColumnOptions(text: string): DG.CsvImportColumnOptions[] {
+  const newlineIdx = text.indexOf('\n');
+  const firstLine = (newlineIdx >= 0 ? text.substring(0, newlineIdx) : text).replace(/\r$/, '');
+  const headers = firstLine.split('\t');
+  const opts: DG.CsvImportColumnOptions[] = [];
+  for (const header of headers) {
+    const lower = header.toLowerCase();
+    for (const prefix of INTENSITY_PREFIXES) {
+      if (lower.startsWith(prefix)) {
+        opts.push({name: header, type: 'double'});
+        break;
+      }
+    }
+  }
+  return opts;
+}
+
+/** Detects intensity columns by MaxQuant-specific prefix matching and
+ * delegates to shared log2TransformColumns for the actual transformation. */
+function processIntensityColumns(df: DG.DataFrame): void {
+  const intensityNames: string[] = [];
+  for (const name of df.columns.names()) {
+    const col = df.col(name);
+    if (!col) continue;
+    if (col.type !== DG.COLUMN_TYPE.FLOAT && col.type !== DG.COLUMN_TYPE.INT)
+      continue;
+    const lowerName = name.toLowerCase();
+    for (const prefix of INTENSITY_PREFIXES) {
+      if (lowerName.startsWith(prefix)) {
+        intensityNames.push(name);
+        break;
+      }
+    }
+  }
+  log2TransformColumns(df, intensityNames);
+}
+
+/** Assigns semantic types to protein ID and gene name columns. */
+function assignSemanticTypes(df: DG.DataFrame): void {
+  const proteinCol = findCol(df, PROTEIN_ID_COLUMNS);
+  if (proteinCol)
+    proteinCol.semType = SEMTYPE.PROTEIN_ID;
+
+  const geneCol = findCol(df, GENE_NAME_COLUMNS);
+  if (geneCol)
+    geneCol.semType = SEMTYPE.GENE_SYMBOL;
+}
+
+/** Parses MaxQuant proteinGroups.txt TSV text into a filtered, typed DataFrame.
+ *
+ * Applies standard proteomics filtering:
+ * - Removes contaminant rows ('+' marker or CON__ prefix)
+ * - Removes reverse hits ('+' marker or REV__ prefix)
+ * - Removes only-identified-by-site rows
+ *
+ * Adds log2-transformed intensity columns and semantic type annotations. */
+export async function parseMaxQuantText(text: string): Promise<DG.DataFrame> {
+  const columnImportOptions = buildIntensityColumnOptions(text);
+  const raw = DG.DataFrame.fromCsv(text, {delimiter: '\t', columnImportOptions});
+
+  // Apply row filters
+  raw.filter.init((_i) => true);
+  filterByMarker(raw, FILTER_COLUMNS.contaminant);
+  filterByMarker(raw, FILTER_COLUMNS.reverse);
+  filterByMarker(raw, FILTER_COLUMNS.onlyBySite);
+  filterByIdPrefix(raw);
+  raw.filter.fireChanged();
+
+  // Clone filtered rows
+  const df = raw.clone(raw.filter);
+
+  // Assign semantic types to ID and gene columns
+  assignSemanticTypes(df);
+
+  // Parse semicolon-delimited fields into primary columns (only if semicolons present)
+  addPrimaryColumnFromVariants(df, PROTEIN_ID_COLUMNS, 'Primary Protein ID', SEMTYPE.PROTEIN_ID);
+  addPrimaryColumnFromVariants(df, GENE_NAME_COLUMNS, 'Primary Gene Name', SEMTYPE.GENE_SYMBOL);
+
+  // Detect and log2-transform intensity columns
+  processIntensityColumns(df);
+
+  df.setTag('proteomics.source', 'maxquant');
+
+  await resolveGeneLabels(df);
+
+  return df;
+}