@jbrowse/text-indexing 2.6.1

package/esm/util.d.ts

@@ -0,0 +1,118 @@
+export interface UriLocation {
+    uri: string;
+    locationType: 'UriLocation';
+}
+export interface LocalPathLocation {
+    localPath: string;
+    locationType: 'LocalPathLocation';
+}
+export interface IndexedFastaAdapter {
+    type: 'IndexedFastaAdapter';
+    fastaLocation: UriLocation;
+    faiLocation: UriLocation;
+}
+export interface BgzipFastaAdapter {
+    type: 'BgzipFastaAdapter';
+    fastaLocation: UriLocation;
+    faiLocation: UriLocation;
+    gziLocation: UriLocation;
+}
+export interface TwoBitAdapter {
+    type: 'TwoBitAdapter';
+    twoBitLocation: UriLocation;
+}
+export interface ChromeSizesAdapter {
+    type: 'ChromSizesAdapter';
+    chromSizesLocation: UriLocation;
+}
+export interface CustomSequenceAdapter {
+    type: string;
+}
+export interface RefNameAliasAdapter {
+    type: 'RefNameAliasAdapter';
+    location: UriLocation;
+}
+export interface CustomRefNameAliasAdapter {
+    type: string;
+}
+export interface Assembly {
+    displayName?: string;
+    name: string;
+    aliases?: string[];
+    sequence: Sequence;
+    refNameAliases?: {
+        adapter: RefNameAliasAdapter | CustomRefNameAliasAdapter;
+    };
+    refNameColors?: string[];
+}
+export interface Sequence {
+    type: 'ReferenceSequenceTrack';
+    trackId: string;
+    adapter: IndexedFastaAdapter | BgzipFastaAdapter | TwoBitAdapter | ChromeSizesAdapter | CustomSequenceAdapter;
+}
+export interface Gff3TabixAdapter {
+    type: 'Gff3TabixAdapter';
+    gffGzLocation: UriLocation | LocalPathLocation;
+}
+export interface Gff3Adapter {
+    type: 'Gff3Adapter';
+    gffLocation: UriLocation | LocalPathLocation;
+}
+export interface GtfAdapter {
+    type: 'GtfAdapter';
+    gtfLocation: UriLocation | LocalPathLocation;
+}
+export interface VcfTabixAdapter {
+    type: 'VcfTabixAdapter';
+    vcfGzLocation: UriLocation | LocalPathLocation;
+}
+export interface VcfAdapter {
+    type: 'VcfAdapter';
+    vcfLocation: UriLocation | LocalPathLocation;
+}
+export interface Track {
+    [key: string]: any;
+}
+export interface TextSearching {
+    indexingFeatureTypesToExclude?: string[];
+    indexingAttributes?: string[];
+    textSearchAdapter: TrixTextSearchAdapter;
+}
+export interface TrixTextSearchAdapter {
+    type: string;
+    textSearchAdapterId: string;
+    ixFilePath: UriLocation;
+    ixxFilePath: UriLocation;
+    metaFilePath: UriLocation;
+    assemblyNames: string[];
+}
+export interface Config {
+    assemblies?: Assembly[];
+    assembly?: Assembly;
+    configuration?: {};
+    aggregateTextSearchAdapters?: TrixTextSearchAdapter[];
+    connections?: unknown[];
+    defaultSession?: {};
+    tracks?: Track[];
+}
+export type indexType = 'aggregate' | 'perTrack';
+export declare function supportedIndexingAdapters(type: string): boolean;
+export declare function createTextSearchConf(name: string, trackIds: string[], assemblyNames: string[], locationPath: string): {
+    type: string;
+    textSearchAdapterId: string;
+    ixFilePath: {
+        localPath: string;
+        locationType: string;
+    };
+    ixxFilePath: {
+        localPath: string;
+        locationType: string;
+    };
+    metaFilePath: {
+        localPath: string;
+        locationType: string;
+    };
+    tracks: string[];
+    assemblyNames: string[];
+};
+export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[], assemblyName?: string): Track[];

package/esm/util.js

@@ -0,0 +1,46 @@
+// supported adapter types by text indexer
+//  ensure that this matches the method found in @jbrowse/core/util
+export function supportedIndexingAdapters(type) {
+    return [
+        'Gff3TabixAdapter',
+        'VcfTabixAdapter',
+        'Gff3Adapter',
+        'VcfAdapter',
+    ].includes(type);
+}
+export function createTextSearchConf(name, trackIds, assemblyNames, locationPath) {
+    // const locationPath = self.sessionPath.substring(
+    //   0,
+    //   self.sessionPath.lastIndexOf('/'),
+    // )
+    return {
+        type: 'TrixTextSearchAdapter',
+        textSearchAdapterId: name,
+        ixFilePath: {
+            localPath: locationPath + `/trix/${name}.ix`,
+            locationType: 'LocalPathLocation',
+        },
+        ixxFilePath: {
+            localPath: locationPath + `/trix/${name}.ixx`,
+            locationType: 'LocalPathLocation',
+        },
+        metaFilePath: {
+            localPath: locationPath + `/trix/${name}.json`,
+            locationType: 'LocalPathLocation',
+        },
+        tracks: trackIds,
+        assemblyNames,
+    };
+}
+export function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
+    return trackIds
+        .map(trackId => {
+        const currentTrack = tracks.find(t => trackId === t.trackId);
+        if (!currentTrack) {
+            throw new Error(`Track not found in session for trackId ${trackId}`);
+        }
+        return currentTrack;
+    })
+        .filter(track => assemblyName ? track.assemblyNames.includes(assemblyName) : true)
+        .filter(track => supportedIndexingAdapters(track.adapter.type));
+}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@jbrowse/text-indexing",
+  "version": "2.6.1",
+  "description": "JBrowse 2 text indexing for desktop",
+  "keywords": [
+    "jbrowse",
+    "jbrowse2",
+    "bionode",
+    "biojs",
+    "genomics"
+  ],
+  "license": "Apache-2.0",
+  "homepage": "https://jbrowse.org",
+  "bugs": "https://github.com/GMOD/jbrowse-components/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GMOD/jbrowse-components.git",
+    "directory": "packages/text-indexing"
+  },
+  "author": "JBrowse Team",
+  "distMain": "dist/index.js",
+  "distModule": "esm/index.js",
+  "srcMain": "src/index.ts",
+  "srcModule": "src/index.ts",
+  "main": "dist/index.js",
+  "module": "esm/index.js",
+  "files": [
+    "dist",
+    "esm",
+    "src"
+  ],
+  "scripts": {
+    "build:esm": "tsc --build tsconfig.build.esm.json",
+    "build:es5": "tsc --build tsconfig.build.es5.json",
+    "build": "npm run build:esm && npm run build:es5",
+    "test": "cd ../..; jest packages/text-indexing",
+    "clean": "rimraf dist esm *.tsbuildinfo",
+    "prebuild": "yarn clean",
+    "prepack": "yarn build && yarn useDist",
+    "postpack": "yarn useSrc",
+    "useDist": "node ../../scripts/useDist.js",
+    "useSrc": "node ../../scripts/useSrc.js"
+  },
+  "dependencies": {
+    "@babel/runtime": "^7.16.3",
+    "ixixx": "^2.0.1",
+    "node-fetch": "^2.6.0"
+  },
+  "peerDependencies": {
+    "mobx": "^6.0.0",
+    "mobx-react": "^7.0.0",
+    "mobx-state-tree": "^5.0.0",
+    "react": "^17.0.0",
+    "react-dom": "^17.0.0",
+    "rxjs": "^7.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "1cbe7ba097fb2d2763c776e5e429e4670cdd583c"
+}

package/src/TextIndexing.ts

@@ -0,0 +1,283 @@
+import fs from 'fs'
+import path from 'path'
+import { Readable } from 'stream'
+import { indexGff3 } from './types/gff3Adapter'
+import { indexVcf } from './types/vcfAdapter'
+import { generateMeta } from './types/common'
+import { ixIxxStream } from 'ixixx'
+import { Track, indexType, supportedIndexingAdapters } from './util'
+import { checkAbortSignal } from '@jbrowse/core/util'
+
+export async function indexTracks(args: {
+  tracks: Track[]
+  outLocation?: string
+  signal?: AbortSignal
+  attributes?: string[]
+  assemblies?: string[]
+  exclude?: string[]
+  indexType?: indexType
+  statusCallback: (message: string) => void
+}) {
+  const {
+    tracks,
+    outLocation,
+    attributes,
+    exclude,
+    assemblies,
+    indexType,
+    statusCallback,
+    signal,
+  } = args
+  const idxType = indexType || 'perTrack'
+  checkAbortSignal(signal)
+  await (idxType === 'perTrack'
+    ? perTrackIndex(
+        tracks,
+        statusCallback,
+        outLocation,
+        attributes,
+        exclude,
+        signal,
+      )
+    : aggregateIndex(
+        tracks,
+        statusCallback,
+        outLocation,
+        attributes,
+        assemblies,
+        exclude,
+        signal,
+      ))
+  checkAbortSignal(signal)
+  return []
+}
+
+async function perTrackIndex(
+  tracks: Track[],
+  statusCallback: (message: string) => void,
+  outLocation?: string,
+  attributes?: string[],
+  exclude?: string[],
+  signal?: AbortSignal,
+) {
+  const outFlag = outLocation || '.'
+
+  const isDir = fs.lstatSync(outFlag).isDirectory()
+  const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag
+  const outDir = path.dirname(confFilePath)
+  const trixDir = path.join(outDir, 'trix')
+  if (!fs.existsSync(trixDir)) {
+    fs.mkdirSync(trixDir)
+  }
+
+  // default settings
+  const attrs = attributes || ['Name', 'ID']
+  const excludeTypes = exclude || ['exon', 'CDS']
+  const force = true
+  const supportedTracks = tracks.filter(track =>
+    supportedIndexingAdapters(track.adapter?.type),
+  )
+  for (const trackConfig of supportedTracks) {
+    const { textSearching, trackId, assemblyNames } = trackConfig
+    const id = `${trackId}-index`
+    if (textSearching?.textSearchAdapter && !force) {
+      console.warn(
+        `Note: ${trackId} has already been indexed with this configuration, use --force to overwrite this track. Skipping for now`,
+      )
+      continue
+    }
+    await indexDriver(
+      [trackConfig],
+      outDir,
+      attrs,
+      id,
+      true,
+      excludeTypes,
+      assemblyNames,
+      statusCallback,
+      signal,
+    )
+  }
+}
+
+async function aggregateIndex(
+  tracks: Track[],
+  statusCallback: (message: string) => void,
+  outLocation?: string,
+  attributes?: string[],
+  assemblies?: string[],
+  exclude?: string[],
+  signal?: AbortSignal,
+) {
+  const outFlag = outLocation || '.'
+
+  const isDir = fs.lstatSync(outFlag).isDirectory()
+  const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag
+  const outDir = path.dirname(confFilePath)
+  const trixDir = path.join(outDir, 'trix')
+  if (!fs.existsSync(trixDir)) {
+    fs.mkdirSync(trixDir)
+  }
+  if (!assemblies) {
+    throw new Error(
+      'No assemblies passed. Assmeblies required for aggregate indexes',
+    )
+  }
+  for (const asm of assemblies) {
+    // console.log('Indexing assembly ' + asm + '...')
+    const id = asm + '-index'
+    // default settings
+    const attrs = attributes || ['Name', 'ID']
+    const excludeTypes = exclude || ['exon', 'CDS']
+    // const force = true
+    const quiet = true
+    // supported tracks for given assembly
+    const supportedTracks = tracks
+      .filter(track => supportedIndexingAdapters(track.adapter?.type))
+      .filter(track => (asm ? track.assemblyNames.includes(asm) : true))
+
+    await indexDriver(
+      supportedTracks,
+      outDir,
+      attrs,
+      id,
+      quiet,
+      excludeTypes,
+      [asm],
+      statusCallback,
+      signal,
+    )
+  }
+}
+
+async function indexDriver(
+  tracks: Track[],
+  idxLocation: string,
+  attributes: string[],
+  name: string,
+  quiet: boolean,
+  exclude: string[],
+  assemblyNames: string[],
+  statusCallback: (message: string) => void,
+  signal?: AbortSignal,
+) {
+  const readable = Readable.from(
+    indexFiles(
+      tracks,
+      attributes,
+      idxLocation,
+      quiet,
+      exclude,
+      statusCallback,
+      signal,
+    ),
+  )
+  statusCallback('Indexing files.')
+  try {
+    const ixIxxStream = await runIxIxx(readable, idxLocation, name)
+    checkAbortSignal(signal)
+    await generateMeta({
+      configs: tracks,
+      attributes,
+      outDir: idxLocation,
+      name,
+      exclude,
+      assemblyNames,
+    })
+    checkAbortSignal(signal)
+    return ixIxxStream
+  } catch (e) {
+    throw e
+  }
+}
+
+async function* indexFiles(
+  tracks: Track[],
+  attributes: string[],
+  outLocation: string,
+  quiet: boolean,
+  typesToExclude: string[],
+  statusCallback: (message: string) => void,
+  signal?: AbortSignal,
+) {
+  for (const track of tracks) {
+    const { adapter, textSearching } = track
+    const { type } = adapter
+    const {
+      indexingFeatureTypesToExclude: types = typesToExclude,
+      indexingAttributes: attrs = attributes,
+    } = textSearching || {}
+    // currently only supporting GFF3Tabix and VCFTabix
+    switch (type) {
+      case 'Gff3TabixAdapter': {
+        yield* indexGff3(
+          track,
+          attrs,
+          getLoc('gffGzLocation', track),
+          outLocation,
+          types,
+          quiet,
+          statusCallback,
+          signal,
+        )
+
+        break
+      }
+      case 'Gff3Adapter': {
+        yield* indexGff3(
+          track,
+          attrs,
+          getLoc('gffLocation', track),
+          outLocation,
+          types,
+          quiet,
+          statusCallback,
+          signal,
+        )
+
+        break
+      }
+      case 'VcfTabixAdapter': {
+        yield* indexVcf(
+          track,
+          attrs,
+          getLoc('vcfGzLocation', track),
+          outLocation,
+          types,
+          quiet,
+          statusCallback,
+          signal,
+        )
+
+        break
+      }
+      case 'VcfAdapter': {
+        yield* indexVcf(
+          track,
+          attrs,
+          getLoc('vcfLocation', track),
+          outLocation,
+          types,
+          quiet,
+          statusCallback,
+          signal,
+        )
+
+        break
+      }
+      // No default
+    }
+  }
+  return
+}
+
+function getLoc(attr: string, config: Track) {
+  const elt = config.adapter[attr]
+  return elt.uri || elt.localPath
+}
+
+function runIxIxx(readStream: Readable, idxLocation: string, name: string) {
+  const ixFilename = path.join(idxLocation, 'trix', `${name}.ix`)
+  const ixxFilename = path.join(idxLocation, 'trix', `${name}.ixx`)
+  return ixIxxStream(readStream, ixFilename, ixxFilename)
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './util'
+
+export { indexTracks } from './TextIndexing'

package/src/types/common.test.ts

@@ -0,0 +1,38 @@
+import { guessAdapterFromFileName, isURL, makeLocation } from './common'
+import { supportedIndexingAdapters } from '../util'
+
+describe('utils for text indexing', () => {
+  const local = './volvox.sort.gff3.gz'
+  const gff =
+    'https://jbrowse.org/genomes/CHM13/genes/chm13.draft_v1.1.gene_annotation.v4.sorted.gff.gz'
+  const gff3 =
+    'https://s3.amazonaws.com/jbrowse.org/genomes/GRCh38/gencode/gencode.v36.annotation.sort.gff3.gz'
+  const vcf =
+    'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh37/clinvar.vcf.gz'
+  const unsupported =
+    'https://s3.amazonaws.com/jbrowse.org/genomes/hg19/amplicon_deep_seq/out.marked.bam'
+  it('test isURL', () => {
+    const test1_result = isURL(local)
+    const test2_result = isURL(gff3)
+    expect(test1_result).toBe(false)
+    expect(test2_result).toBeTruthy()
+  })
+  it('test makeLocation', () => {
+    const location1 = makeLocation(local, 'localPath')
+    const location2 = makeLocation(gff3, 'uri')
+    expect(location1.locationType).toBe('LocalPathLocation')
+    expect(location2.locationType).toBe('UriLocation')
+  })
+  it('test guess adapter from file name', () => {
+    const conf1 = guessAdapterFromFileName(gff3)
+    expect(conf1.adapter.type).toBe('Gff3TabixAdapter')
+    expect(supportedIndexingAdapters(conf1.adapter.type)).toBe(true)
+    const conf2 = guessAdapterFromFileName(gff)
+    expect(conf2.adapter.type).toBe('Gff3TabixAdapter')
+    const conf3 = guessAdapterFromFileName(vcf)
+    expect(conf3.adapter.type).toBe('VcfTabixAdapter')
+    expect(() => {
+      guessAdapterFromFileName(unsupported)
+    }).toThrowError(`Unsupported file type ${unsupported}`)
+  })
+})