@jbrowse/text-indexing 2.6.1

package/src/types/common.ts

@@ -0,0 +1,170 @@
+import fs from 'fs'
+import fetch from 'node-fetch'
+import path from 'path'
+import { LocalPathLocation, UriLocation, Track } from '../util'
+
+// Method for handing off the parsing of a gff3 file URL.
+// Calls the proper parser depending on if it is gzipped or not.
+// Returns a @gmod/gff stream.
+export async function createRemoteStream(urlIn: string) {
+  const response = await fetch(urlIn)
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch ${urlIn} status ${response.status} ${response.statusText}`,
+    )
+  }
+  return response
+}
+
+// Checks if the passed in string is a valid URL.
+// Returns a boolean.
+export function isURL(FileName: string) {
+  let url
+
+  try {
+    url = new URL(FileName)
+  } catch (_) {
+    return false
+  }
+
+  return url.protocol === 'http:' || url.protocol === 'https:'
+}
+
+export async function getLocalOrRemoteStream(uri: string, out: string) {
+  let stream
+  let totalBytes = 0
+  if (isURL(uri)) {
+    const result = await createRemoteStream(uri)
+    totalBytes = +(result.headers?.get('Content-Length') || 0)
+    stream = result.body
+  } else {
+    const filename = path.isAbsolute(uri) ? uri : path.join(out, uri)
+    totalBytes = fs.statSync(filename).size
+    stream = fs.createReadStream(filename)
+  }
+  return { totalBytes, stream }
+}
+
+export function makeLocation(location: string, protocol: string) {
+  if (protocol === 'uri') {
+    return { uri: location, locationType: 'UriLocation' } as UriLocation
+  }
+  if (protocol === 'localPath') {
+    return {
+      localPath: path.resolve(location),
+      locationType: 'LocalPathLocation',
+    } as LocalPathLocation
+  }
+  throw new Error(`invalid protocol ${protocol}`)
+}
+
+export function guessAdapterFromFileName(filePath: string): Track {
+  // const uri = isURL(filePath) ? filePath : path.resolve(filePath)
+  const protocol = isURL(filePath) ? 'uri' : 'localPath'
+  const name = path.basename(filePath)
+  if (/\.vcf\.b?gz$/i.test(filePath)) {
+    return {
+      trackId: name,
+      name: name,
+      assemblyNames: [],
+      adapter: {
+        type: 'VcfTabixAdapter',
+        vcfGzLocation: makeLocation(filePath, protocol),
+      },
+    }
+  } else if (/\.gff3?\.b?gz$/i.test(filePath)) {
+    return {
+      trackId: name,
+      name,
+      assemblyNames: [],
+      adapter: {
+        type: 'Gff3TabixAdapter',
+        gffGzLocation: makeLocation(filePath, protocol),
+      },
+    }
+  } else if (/\.gtf?$/i.test(filePath)) {
+    return {
+      trackId: name,
+      name,
+      assemblyNames: [],
+      adapter: {
+        type: 'GtfAdapter',
+        gtfLocation: makeLocation(filePath, protocol),
+      },
+    }
+  } else if (/\.vcf$/i.test(filePath)) {
+    return {
+      trackId: name,
+      name,
+      assemblyNames: [],
+      adapter: {
+        type: 'VcfAdapter',
+        vcfLocation: makeLocation(filePath, protocol),
+      },
+    }
+  } else if (/\.gff3?$/i.test(filePath)) {
+    return {
+      trackId: name,
+      name,
+      assemblyNames: [],
+      adapter: {
+        type: 'Gff3Adapter',
+        gffLocation: makeLocation(filePath, protocol),
+      },
+    }
+  } else {
+    throw new Error(`Unsupported file type ${filePath}`)
+  }
+}
+
+/**
+ * Generates metadata of index given a filename (trackId or assembly)
+ * @param name -  assembly name or trackId
+ * @param attributes -  attributes indexed
+ * @param include -  feature types included from index
+ * @param exclude -  feature types excluded from index
+ * @param configs - list of track
+ */
+export async function generateMeta({
+  configs,
+  attributes,
+  outDir,
+  name,
+  exclude,
+  assemblyNames,
+}: {
+  configs: Track[]
+  attributes: string[]
+  outDir: string
+  name: string
+  exclude: string[]
+  assemblyNames: string[]
+}) {
+  const tracks = configs.map(config => {
+    const { trackId, textSearching, adapter } = config
+
+    const includeExclude =
+      textSearching?.indexingFeatureTypesToExclude || exclude
+
+    const metaAttrs = textSearching?.indexingAttributes || attributes
+
+    return {
+      trackId: trackId,
+      attributesIndexed: metaAttrs,
+      excludedTypes: includeExclude,
+      adapterConf: adapter,
+    }
+  })
+  fs.writeFileSync(
+    path.join(outDir, 'trix', `${name}_meta.json`),
+    JSON.stringify(
+      {
+        dateCreated: new Date().toISOString(),
+        tracks,
+        assemblyNames,
+      },
+      null,
+      2,
+    ),
+  )
+}

package/src/types/gff3Adapter.ts

@@ -0,0 +1,75 @@
+import { createGunzip } from 'zlib'
+import readline from 'readline'
+import { Track } from '../util'
+import { getLocalOrRemoteStream } from './common'
+import { checkAbortSignal } from '@jbrowse/core/util'
+
+export async function* indexGff3(
+  config: Track,
+  attributes: string[],
+  inLocation: string,
+  outLocation: string,
+  typesToExclude: string[],
+  quiet: boolean,
+  statusCallback: (message: string) => void,
+  signal?: AbortSignal,
+) {
+  const { trackId } = config
+  let receivedBytes = 0
+  const { totalBytes, stream } = await getLocalOrRemoteStream(
+    inLocation,
+    outLocation,
+  )
+  stream.on('data', chunk => {
+    receivedBytes += chunk.length
+    // send an update?
+    const progress = Math.round((receivedBytes / totalBytes) * 100)
+    statusCallback(`${progress}`)
+  })
+  const rl = readline.createInterface({
+    input: inLocation.match(/.b?gz$/) ? stream.pipe(createGunzip()) : stream,
+  })
+
+  for await (const line of rl) {
+    if (line.startsWith('#')) {
+      continue
+    } else if (line.startsWith('>')) {
+      break
+    }
+
+    const [seq_id, , type, start, end, , , , col9] = line.split('\t')
+    const locStr = `${seq_id}:${start}..${end}`
+
+    if (!typesToExclude.includes(type)) {
+      // turns gff3 attrs into a map, and converts the arrays into space
+      // separated strings
+      const col9attrs = Object.fromEntries(
+        col9
+          .split(';')
+          .map(f => f.trim())
+          .filter(f => !!f)
+          .map(f => f.split('='))
+          .map(([key, val]) => [
+            key.trim(),
+            decodeURIComponent(val).trim().split(',').join(' '),
+          ]),
+      )
+      const attrs = attributes
+        .map(attr => col9attrs[attr])
+        .filter((f): f is string => !!f)
+
+      if (attrs.length) {
+        const record = JSON.stringify([
+          encodeURIComponent(locStr),
+          encodeURIComponent(trackId),
+          ...attrs.map(a => encodeURIComponent(a)),
+        ]).replaceAll(',', '|')
+
+        // Check abort signal
+        checkAbortSignal(signal)
+        yield `${record} ${[...new Set(attrs)].join(' ')}\n`
+      }
+    }
+  }
+  // console.log('done')
+}

package/src/types/vcfAdapter.ts

@@ -0,0 +1,86 @@
+import { createGunzip } from 'zlib'
+import readline from 'readline'
+import { Track } from '../util'
+import { getLocalOrRemoteStream } from './common'
+import { checkAbortSignal } from '@jbrowse/core/util'
+
+export async function* indexVcf(
+  config: Track,
+  attributesToIndex: string[],
+  inLocation: string,
+  outLocation: string,
+  typesToExclude: string[],
+  quiet: boolean,
+  statusCallback: (message: string) => void,
+  signal?: AbortSignal,
+) {
+  const { trackId } = config
+  let receivedBytes = 0
+  const { totalBytes, stream } = await getLocalOrRemoteStream(
+    inLocation,
+    outLocation,
+  )
+  stream.on('data', chunk => {
+    receivedBytes += chunk.length
+    const progress = Math.round((receivedBytes / totalBytes) * 100)
+    statusCallback(`${progress}`)
+  })
+
+  const gzStream = inLocation.match(/.b?gz$/)
+    ? stream.pipe(createGunzip())
+    : stream
+
+  const rl = readline.createInterface({
+    input: gzStream,
+  })
+
+  for await (const line of rl) {
+    if (line.startsWith('#')) {
+      continue
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [ref, pos, id, _ref, _alt, _qual, _filter, info] = line.split('\t')
+
+    // turns vcf info attrs into a map, and converts the arrays into space
+    // separated strings
+    const fields = Object.fromEntries(
+      info
+        .split(';')
+        .map(f => f.trim())
+        .filter(f => !!f)
+        .map(f => f.split('='))
+        .map(([key, val]) => [
+          key.trim(),
+          val ? decodeURIComponent(val).trim().split(',').join(' ') : undefined,
+        ]),
+    )
+
+    const end = fields.END
+
+    const locStr = `${ref}:${pos}..${end || +pos + 1}`
+    if (id === '.') {
+      continue
+    }
+
+    const infoAttrs = attributesToIndex
+      .map(attr => fields[attr])
+      .filter((f): f is string => !!f)
+
+    const ids = id.split(',')
+    for (let i = 0; i < ids.length; i++) {
+      const id = ids[i]
+      const attrs = [id]
+      const record = JSON.stringify([
+        encodeURIComponent(locStr),
+        encodeURIComponent(trackId),
+        encodeURIComponent(id || ''),
+        ...infoAttrs.map(a => encodeURIComponent(a || '')),
+      ]).replaceAll(',', '|')
+
+      // Check abort signal
+      checkAbortSignal(signal)
+      yield `${record} ${[...new Set(attrs)].join(' ')}\n`
+    }
+  }
+}

package/src/util.ts

@@ -0,0 +1,177 @@
+export interface UriLocation {
+  uri: string
+  locationType: 'UriLocation'
+}
+export interface LocalPathLocation {
+  localPath: string
+  locationType: 'LocalPathLocation'
+}
+export interface IndexedFastaAdapter {
+  type: 'IndexedFastaAdapter'
+  fastaLocation: UriLocation
+  faiLocation: UriLocation
+}
+
+export interface BgzipFastaAdapter {
+  type: 'BgzipFastaAdapter'
+  fastaLocation: UriLocation
+  faiLocation: UriLocation
+  gziLocation: UriLocation
+}
+
+export interface TwoBitAdapter {
+  type: 'TwoBitAdapter'
+  twoBitLocation: UriLocation
+}
+
+export interface ChromeSizesAdapter {
+  type: 'ChromSizesAdapter'
+  chromSizesLocation: UriLocation
+}
+
+export interface CustomSequenceAdapter {
+  type: string
+}
+
+export interface RefNameAliasAdapter {
+  type: 'RefNameAliasAdapter'
+  location: UriLocation
+}
+
+export interface CustomRefNameAliasAdapter {
+  type: string
+}
+export interface Assembly {
+  displayName?: string
+  name: string
+  aliases?: string[]
+  sequence: Sequence
+  refNameAliases?: {
+    adapter: RefNameAliasAdapter | CustomRefNameAliasAdapter
+  }
+  refNameColors?: string[]
+}
+
+export interface Sequence {
+  type: 'ReferenceSequenceTrack'
+  trackId: string
+  adapter:
+    | IndexedFastaAdapter
+    | BgzipFastaAdapter
+    | TwoBitAdapter
+    | ChromeSizesAdapter
+    | CustomSequenceAdapter
+}
+
+export interface Gff3TabixAdapter {
+  type: 'Gff3TabixAdapter'
+  gffGzLocation: UriLocation | LocalPathLocation
+}
+
+export interface Gff3Adapter {
+  type: 'Gff3Adapter'
+  gffLocation: UriLocation | LocalPathLocation
+}
+export interface GtfAdapter {
+  type: 'GtfAdapter'
+  gtfLocation: UriLocation | LocalPathLocation
+}
+
+export interface VcfTabixAdapter {
+  type: 'VcfTabixAdapter'
+  vcfGzLocation: UriLocation | LocalPathLocation
+}
+export interface VcfAdapter {
+  type: 'VcfAdapter'
+  vcfLocation: UriLocation | LocalPathLocation
+}
+export interface Track {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any
+}
+export interface TextSearching {
+  indexingFeatureTypesToExclude?: string[]
+  indexingAttributes?: string[]
+  textSearchAdapter: TrixTextSearchAdapter
+}
+
+export interface TrixTextSearchAdapter {
+  type: string
+  textSearchAdapterId: string
+  ixFilePath: UriLocation
+  ixxFilePath: UriLocation
+  metaFilePath: UriLocation
+  assemblyNames: string[]
+}
+
+export interface Config {
+  assemblies?: Assembly[]
+  assembly?: Assembly
+  configuration?: {}
+  aggregateTextSearchAdapters?: TrixTextSearchAdapter[]
+  connections?: unknown[]
+  defaultSession?: {}
+  tracks?: Track[]
+}
+
+export type indexType = 'aggregate' | 'perTrack'
+
+// supported adapter types by text indexer
+//  ensure that this matches the method found in @jbrowse/core/util
+export function supportedIndexingAdapters(type: string) {
+  return [
+    'Gff3TabixAdapter',
+    'VcfTabixAdapter',
+    'Gff3Adapter',
+    'VcfAdapter',
+  ].includes(type)
+}
+
+export function createTextSearchConf(
+  name: string,
+  trackIds: string[],
+  assemblyNames: string[],
+  locationPath: string,
+) {
+  // const locationPath = self.sessionPath.substring(
+  //   0,
+  //   self.sessionPath.lastIndexOf('/'),
+  // )
+  return {
+    type: 'TrixTextSearchAdapter',
+    textSearchAdapterId: name,
+    ixFilePath: {
+      localPath: locationPath + `/trix/${name}.ix`,
+      locationType: 'LocalPathLocation',
+    },
+    ixxFilePath: {
+      localPath: locationPath + `/trix/${name}.ixx`,
+      locationType: 'LocalPathLocation',
+    },
+    metaFilePath: {
+      localPath: locationPath + `/trix/${name}.json`,
+      locationType: 'LocalPathLocation',
+    },
+    tracks: trackIds,
+    assemblyNames,
+  }
+}
+
+export function findTrackConfigsToIndex(
+  tracks: Track[],
+  trackIds: string[],
+  assemblyName?: string,
+) {
+  return trackIds
+    .map(trackId => {
+      const currentTrack = tracks.find(t => trackId === t.trackId)
+      if (!currentTrack) {
+        throw new Error(`Track not found in session for trackId ${trackId}`)
+      }
+      return currentTrack
+    })
+    .filter(track =>
+      assemblyName ? track.assemblyNames.includes(assemblyName) : true,
+    )
+    .filter(track => supportedIndexingAdapters(track.adapter.type))
+}