@jbrowse/plugin-spreadsheet-view 2.6.1

package/src/SpreadsheetView/importAdapters/BedImport.ts

@@ -0,0 +1,155 @@
+import { ParseOptions, parseTsvBuffer } from './ImportUtils'
+
+const browserBytes = 'browser '.split('').map(c => c.charCodeAt(0))
+const trackBytes = 'track '.split('').map(c => c.charCodeAt(0))
+const commentBytes = '#'.split('').map(c => c.charCodeAt(0))
+
+function bytesAreFoundAt(position: number, buffer: Buffer, bytes: number[]) {
+  let i = 0
+  for (; i < bytes.length; i += 1) {
+    if (buffer[position + i] !== bytes[i]) {
+      return false
+    }
+  }
+  return true
+}
+export function removeBedHeaders(buffer: Buffer) {
+  // slice off the first lines of the buffer if it starts with one or more
+  // header lines
+  let i = 0
+  for (; i < buffer.length; i += 1) {
+    if (
+      bytesAreFoundAt(i, buffer, browserBytes) ||
+      bytesAreFoundAt(i, buffer, trackBytes) ||
+      bytesAreFoundAt(i, buffer, commentBytes)
+    ) {
+      // consume up to the next newline
+      do {
+        i += 1
+      } while (buffer[i] !== 10)
+    } else {
+      // end of headers, return
+      break
+    }
+  }
+  if (i) {
+    return buffer.slice(i)
+  }
+  return buffer
+}
+
+export async function parseBedBuffer(buffer: Buffer, options: ParseOptions) {
+  const b = removeBedHeaders(buffer)
+  const data = await parseTsvBuffer(b)
+  const bedColumns = [
+    { name: 'chrom', dataType: { type: 'LocRef' } },
+    { name: 'chromStart', dataType: { type: 'LocStart' } },
+    { name: 'chromEnd', dataType: { type: 'LocEnd' } },
+    { name: 'name', dataType: { type: 'Text' } },
+    { name: 'score', dataType: { type: 'Number' } },
+    { name: 'strand', dataType: { type: 'Text' } },
+  ]
+  data.columns.forEach((col, colNumber) => {
+    const bedColumn = bedColumns[colNumber]
+    if (bedColumn) {
+      col.name = bedColumn.name
+      col.dataType = bedColumn.dataType
+    }
+  })
+  data.hasColumnNames = true
+  data.assemblyName = options.selectedAssemblyName
+
+  data.columnDisplayOrder.push(data.columnDisplayOrder.length)
+  data.columns.unshift({
+    name: 'Location',
+    dataType: { type: 'LocString' },
+    isDerived: true,
+    derivationFunctionText: `jexl:{text:row.cells[0].text+':'+row.cells[1].text+'..'+row.cells[2].text,\n
+    extendedData: {refName: row.cells.ref.text, start: parseInt(row.cells.start.text,10), end: parseInt(row.cells.end.text,10)}}`,
+  })
+  return data
+}
+
+export async function parseBedPEBuffer(buffer: Buffer, options: ParseOptions) {
+  const b = removeBedHeaders(buffer)
+  const data = await parseTsvBuffer(b)
+  interface BedColumn {
+    name: string
+    dataType: {
+      type: string
+    }
+    featureField: string[]
+  }
+  const bedColumns: BedColumn[] = [
+    { name: 'chrom1', dataType: { type: 'Text' }, featureField: ['refName'] },
+    { name: 'start1', dataType: { type: 'Number' }, featureField: ['start'] },
+    { name: 'end1', dataType: { type: 'Number' }, featureField: ['end'] },
+    {
+      name: 'chrom2',
+      dataType: { type: 'Text' },
+      featureField: ['mate', 'refName'],
+    },
+    {
+      name: 'start2',
+      dataType: { type: 'Number' },
+      featureField: ['mate', 'start'],
+    },
+    {
+      name: 'end2',
+      dataType: { type: 'Number' },
+      featureField: ['mate', 'end'],
+    },
+    { name: 'name', dataType: { type: 'Text' }, featureField: ['name'] },
+    { name: 'score', dataType: { type: 'Number' }, featureField: ['score'] },
+    { name: 'strand1', dataType: { type: 'Text' }, featureField: ['strand'] },
+    {
+      name: 'strand2',
+      dataType: { type: 'Text' },
+      featureField: ['mate', 'strand'],
+    },
+  ]
+  data.columns.forEach((col, colNumber) => {
+    const bedColumn = bedColumns[colNumber]
+    if (bedColumn) {
+      col.name = bedColumn.name
+      col.dataType = bedColumn.dataType
+    }
+  })
+  data.hasColumnNames = true
+
+  // decorate each row with a feature object in its extendedData
+  data.rowSet.rows.forEach((row, rowNumber) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const featureData: Record<string, any> = {}
+    row.cells.forEach(({ text }, columnNumber) => {
+      const bedColumn = bedColumns[columnNumber]
+      const val =
+        bedColumn && bedColumn.dataType.type === 'Number' && text
+          ? parseFloat(text)
+          : text
+      if (bedColumn) {
+        // a predefined column
+        if (bedColumn.featureField.length === 2) {
+          if (!featureData[bedColumn.featureField[0]]) {
+            featureData[bedColumn.featureField[0]] = {}
+          }
+          featureData[bedColumn.featureField[0]][bedColumn.featureField[1]] =
+            val
+        } else {
+          featureData[bedColumn.featureField[0]] = val
+        }
+      } else {
+        // some other column
+        featureData[`column${columnNumber + 1}`] = val
+      }
+    })
+    featureData.uniqueId = `bedpe-${rowNumber}`
+    row.extendedData = {
+      feature: featureData,
+    }
+  })
+
+  data.assemblyName = options.selectedAssemblyName
+
+  return data
+}

package/src/SpreadsheetView/importAdapters/ImportUtils.test.ts

@@ -0,0 +1,25 @@
+import { promises as fsPromises } from 'fs'
+import path from 'path'
+
+// locals
+import { parseCsvBuffer } from './ImportUtils'
+import SpreadsheetModel from '../models/Spreadsheet'
+
+test('csv to spreadsheet snapshot', async () => {
+  const filepath = path.join(
+    __dirname,
+    '..',
+    'test_data',
+    'breast_cancer.subset.csv',
+  )
+  const buf = await fsPromises.readFile(filepath)
+  const spreadsheetSnap = await parseCsvBuffer(buf, {
+    hasColumnNameLine: true,
+    columnNameLineNumber: 1,
+    isValidRefName: () => true,
+  })
+  expect(spreadsheetSnap).toMatchSnapshot()
+  // @ts-expect-error
+  const spreadsheet = SpreadsheetModel.create(spreadsheetSnap)
+  expect(spreadsheet.rowSet.rows.length).toBe(49)
+})

package/src/SpreadsheetView/importAdapters/ImportUtils.ts

@@ -0,0 +1,152 @@
+import { parseLocString } from '@jbrowse/core/util'
+
+export function bufferToString(buffer: Buffer) {
+  return new TextDecoder('utf8', { fatal: true }).decode(buffer)
+}
+
+async function parseWith(buffer: Buffer, options = {}) {
+  const csv = await import('csvtojson').then(module => module.default)
+  return csv({ noheader: true, output: 'csv', ...options }).fromString(
+    bufferToString(buffer),
+  )
+}
+
+export interface Row {
+  id: string
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  extendedData?: any
+  cells: {
+    text: string
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    extendedData?: any
+  }[]
+}
+
+export interface RowSet {
+  isLoaded: boolean
+  rows: Row[]
+}
+
+export interface ParseOptions {
+  hasColumnNameLine?: boolean
+  columnNameLineNumber?: number
+  selectedAssemblyName?: string
+  isValidRefName?: (refName: string, assemblyName?: string) => boolean
+}
+
+export interface Column {
+  name: string
+  dataType: { type: string }
+  isDerived?: boolean
+  derivationFunctionText?: string
+}
+
+function guessColumnType(
+  rowSet: RowSet,
+  columnNumber: number,
+  isValidRefName: (refName: string, assemblyName?: string) => boolean,
+) {
+  const text = rowSet.rows[0].cells[columnNumber].text || ''
+
+  let guessedType = 'Text'
+
+  let parsedLoc
+  try {
+    parsedLoc = parseLocString(text, isValidRefName)
+  } catch (error) {
+    //
+  }
+  if (parsedLoc && parsedLoc.refName && typeof parsedLoc.start === 'number') {
+    guessedType = 'LocString'
+  } else if (/^\d+(\.\d+)?$/.test(text)) {
+    guessedType = 'Number'
+  }
+
+  // MAYBE TODO: iterate over the rest of the rows to confirm
+  // the type for all the rows
+
+  return guessedType
+}
+
+function dataToSpreadsheetSnapshot(
+  rows: string[][],
+  options: ParseOptions = {},
+) {
+  const {
+    hasColumnNameLine = false,
+    columnNameLineNumber = 1,
+    isValidRefName = () => false,
+    selectedAssemblyName,
+  } = options
+  // rows is an array of row objects and columnNames
+  // is an array of column names (in import order)
+  let maxCols = 0
+  const rowSet: RowSet = {
+    isLoaded: true,
+    rows: rows.map((row, rowNumber) => {
+      const id = rowNumber + (hasColumnNameLine ? 0 : 1)
+      if (row.length > maxCols) {
+        maxCols = row.length
+      }
+      return {
+        id: String(id),
+        cells: row.map((text, columnNumber) => {
+          return { columnNumber, text }
+        }),
+      }
+    }),
+  }
+
+  // process the column names row if present
+  const columnNames: Record<string, string> = {}
+  if (hasColumnNameLine && columnNameLineNumber !== undefined) {
+    const [colNamesRow] = rowSet.rows.splice(columnNameLineNumber - 1, 1)
+
+    if (colNamesRow) {
+      colNamesRow.cells.forEach((cell, columnNumber) => {
+        columnNames[columnNumber] = cell.text || ''
+      })
+    }
+  }
+
+  // make our column definitions
+  const columns: Column[] = []
+  const columnDisplayOrder = []
+  for (let columnNumber = 0; columnNumber < maxCols; columnNumber += 1) {
+    columnDisplayOrder.push(columnNumber)
+    const guessedType = guessColumnType(rowSet, columnNumber, isValidRefName)
+
+    // store extendeddata for LocString column
+    if (guessedType === 'LocString') {
+      for (const row of rowSet.rows) {
+        const cell = row.cells[columnNumber]
+        cell.extendedData = parseLocString(cell.text, isValidRefName)
+      }
+    }
+
+    columns[columnNumber] = {
+      name: columnNames[columnNumber],
+      dataType: {
+        type: guessedType,
+      },
+    }
+  }
+
+  return {
+    rowSet,
+    columnDisplayOrder,
+    hasColumnNames: !!hasColumnNameLine,
+    columns,
+    assemblyName: selectedAssemblyName,
+  }
+}
+
+export async function parseCsvBuffer(buffer: Buffer, options?: ParseOptions) {
+  const rows = await parseWith(buffer)
+  return dataToSpreadsheetSnapshot(rows, options)
+}
+
+export async function parseTsvBuffer(buffer: Buffer, options?: ParseOptions) {
+  const rows = await parseWith(buffer, { delimiter: '\t' })
+  return dataToSpreadsheetSnapshot(rows, options)
+}

package/src/SpreadsheetView/importAdapters/STARFusionImport.test.ts

@@ -0,0 +1,27 @@
+import { promises as fsPromises } from 'fs'
+import path from 'path'
+
+// locals
+import { parseSTARFusionBuffer } from './STARFusionImport'
+import SpreadsheetModel from '../models/Spreadsheet'
+
+test('starfusion import', async () => {
+  const filepath = path.join(
+    __dirname,
+    '..',
+    'test_data',
+    'starfusion_example.fusion_predictions.tsv',
+  )
+  const buf = await fsPromises.readFile(filepath)
+  const spreadsheetSnap = await parseSTARFusionBuffer(buf, {
+    selectedAssemblyName: 'fogbat',
+    isValidRefName() {
+      return true
+    },
+  })
+  expect(spreadsheetSnap).toMatchSnapshot()
+
+  // @ts-expect-error
+  const spreadsheet = SpreadsheetModel.create(spreadsheetSnap)
+  expect(spreadsheet.rowSet.rows.length).toBe(24)
+})

package/src/SpreadsheetView/importAdapters/STARFusionImport.ts

@@ -0,0 +1,74 @@
+import { parseTsvBuffer, ParseOptions } from './ImportUtils'
+
+function parseSTARFusionBreakpointString(str: string) {
+  const fields = str.split(':')
+  const refName = fields[0]
+  const pos = Number.parseInt(fields[1], 10)
+  const strand = fields[2] === '-' ? -1 : 1
+  return { refName, pos, strand }
+}
+
+const numericColumns: Record<string, boolean> = {
+  SpanningFragCount: true,
+  FFPM: true,
+  LeftBreakEntropy: true,
+  RightBreakEntropy: true,
+  JunctionReadCount: true,
+}
+
+export async function parseSTARFusionBuffer(
+  buffer: Buffer,
+  options: ParseOptions,
+) {
+  const data = await parseTsvBuffer(buffer, {
+    hasColumnNameLine: true,
+    columnNameLineNumber: 1,
+    selectedAssemblyName: options.selectedAssemblyName,
+    isValidRefName: () => false,
+  })
+
+  // remove the # in #FusionName
+  data.columns[0].name = data.columns[0].name.replace('#', '')
+  // set some columns to be numeric
+  data.columns.forEach(col => {
+    if (numericColumns[col.name]) {
+      col.dataType = { type: 'Number' }
+    }
+  })
+
+  // decorate each row with a feature object in its extendedData
+  data.rowSet.rows.forEach((row, rowNumber) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const featureData: Record<string, any> = {}
+    row.cells.forEach(({ text }, columnNumber) => {
+      const column = data.columns[columnNumber]
+      if (column.name === 'LeftBreakpoint' && text) {
+        const { refName, pos, strand } = parseSTARFusionBreakpointString(text)
+        featureData.refName = refName
+        featureData.start = pos
+        featureData.end = pos
+        featureData.strand = strand
+      } else if (column.name === 'RightBreakpoint' && text) {
+        const { refName, pos, strand } = parseSTARFusionBreakpointString(text)
+        featureData.mate = {
+          refName,
+          start: pos,
+          end: pos,
+          strand,
+        }
+      } else if (text && numericColumns[column.name]) {
+        // some other column, numeric
+        featureData[column.name] = parseFloat(text)
+      } else {
+        // some other column, text
+        featureData[column.name] = text
+      }
+    })
+    featureData.uniqueId = `sf-${rowNumber + 1}`
+    row.extendedData = {
+      feature: featureData,
+    }
+  })
+
+  return data
+}

package/src/SpreadsheetView/importAdapters/VcfImport.test.ts

@@ -0,0 +1,42 @@
+import { promises as fsPromises } from 'fs'
+import path from 'path'
+import { parseVcfBuffer, splitVcfFileHeaderAndBody } from './VcfImport'
+import SpreadsheetModel from '../models/Spreadsheet'
+
+describe('vcf file splitter', () => {
+  const cases: [string, {}][] = [
+    [
+      '##fileformat=VCFv4.3\nfogbat\n',
+      { header: '##fileformat=VCFv4.3\n', body: 'fogbat\n' },
+    ],
+    [
+      '##fileformat=VCFv4.3\n##zonker\n##deek\n##donk\nfogbat\n',
+      {
+        header: '##fileformat=VCFv4.3\n##zonker\n##deek\n##donk\n',
+        body: 'fogbat\n',
+      },
+    ],
+  ]
+
+  cases.forEach(([input, output], caseNumber) => {
+    test(`case ${caseNumber}`, () => {
+      expect(splitVcfFileHeaderAndBody(input)).toEqual(output)
+    })
+  })
+})
+
+test('vcf file import', async () => {
+  const filepath = path.join(
+    __dirname,
+    '..',
+    'test_data',
+    '1801160099-N32519_26611_S51_56704.hard-filtered.vcf',
+  )
+  const buf = await fsPromises.readFile(filepath)
+  const spreadsheetSnap = parseVcfBuffer(buf)
+  expect(spreadsheetSnap).toMatchSnapshot()
+
+  // @ts-expect-error
+  const spreadsheet = SpreadsheetModel.create(spreadsheetSnap)
+  expect(spreadsheet.rowSet.rows.length).toBe(101)
+})

package/src/SpreadsheetView/importAdapters/VcfImport.ts

@@ -0,0 +1,118 @@
+import VCF from '@gmod/vcf'
+import { VcfFeature } from '@jbrowse/plugin-variants'
+import {
+  bufferToString,
+  Row,
+  RowSet,
+  Column,
+  ParseOptions,
+} from './ImportUtils'
+
+const vcfCoreColumns: { name: string; type: string }[] = [
+  { name: 'CHROM', type: 'Text' }, // 0
+  { name: 'POS', type: 'Number' }, // 1
+  { name: 'ID', type: 'Text' }, // 2
+  { name: 'REF', type: 'Text' }, // 3
+  { name: 'ALT', type: 'Text' }, // 4
+  { name: 'QUAL', type: 'Number' }, // 5
+  { name: 'FILTER', type: 'Text' }, // 6
+  { name: 'INFO', type: 'Text' }, // 7
+  { name: 'FORMAT', type: 'Text' }, // 8
+]
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function vcfRecordToRow(vcfParser: any, line: string, lineNumber: number): Row {
+  const vcfVariant = vcfParser.parseLine(line)
+  const vcfFeature = new VcfFeature({
+    variant: vcfVariant,
+    parser: vcfParser,
+    id: `vcf-${lineNumber}`,
+  })
+
+  const data = line.split('\t').map(d => (d === '.' ? '' : d))
+  // no format column, add blank
+  if (data.length === 8) {
+    data.push('')
+  }
+  const row: Row = {
+    id: String(lineNumber + 1),
+    extendedData: { vcfFeature: vcfFeature.toJSON() },
+    cells: data.map((text, columnNumber) => {
+      return {
+        columnNumber,
+        text,
+      }
+    }),
+  }
+  return row
+}
+
+export function parseVcfBuffer(buffer: Buffer, options: ParseOptions = {}) {
+  const { selectedAssemblyName } = options
+  let { header, body } = splitVcfFileHeaderAndBody(bufferToString(buffer))
+  const rows: Row[] = []
+  const vcfParser = new VCF({ header })
+  header = '' // garbage collect
+  body.split(/\n|\r\n|\r/).forEach((line: string, lineNumber) => {
+    if (/\S/.test(line)) {
+      rows.push(vcfRecordToRow(vcfParser, line, lineNumber))
+    }
+  })
+  body = '' // garbage collect
+
+  const rowSet: RowSet = {
+    isLoaded: true,
+    rows,
+  }
+
+  const columnDisplayOrder: number[] = []
+  const columns: Column[] = []
+  for (let i = 0; i < vcfCoreColumns.length; i += 1) {
+    columnDisplayOrder.push(i)
+    columns[i] = {
+      name: vcfCoreColumns[i].name,
+      dataType: { type: vcfCoreColumns[i].type },
+    }
+  }
+  for (let i = 0; i < vcfParser.samples.length; i += 1) {
+    const oi = vcfCoreColumns.length + i
+    columnDisplayOrder.push(oi)
+    columns[oi] = { name: vcfParser.samples[i], dataType: { type: 'Text' } }
+  }
+
+  columnDisplayOrder.push(columnDisplayOrder.length)
+  columns.unshift({
+    name: 'Location',
+    dataType: { type: 'LocString' },
+    isDerived: true,
+    derivationFunctionText: `jexl:{text:row.extendedData.vcfFeature.refName+':'\n
+    +row.extendedData.vcfFeature.start+'..'+row.extendedData.vcfFeature.end, extendedData:\n
+    {refName:row.extendedData.vcfFeature.refName,start:row.extendedData.vcfFeature.start,end:row.extendedData.vcfFeature.end}}`,
+  })
+
+  return {
+    rowSet,
+    columnDisplayOrder,
+    hasColumnNames: true,
+    columns,
+    assemblyName: selectedAssemblyName,
+  }
+}
+
+export function splitVcfFileHeaderAndBody(wholeFile: string) {
+  // split into header and the rest of the file
+  let headerEndIndex = 0
+  let prevChar
+  for (; headerEndIndex < wholeFile.length; headerEndIndex += 1) {
+    const c = wholeFile[headerEndIndex]
+    if (prevChar === '\n' && c !== '#') {
+      break
+    }
+    prevChar = c
+  }
+
+  return {
+    header: wholeFile.slice(0, Math.max(0, headerEndIndex)),
+    body: wholeFile.slice(headerEndIndex),
+  }
+}