@jbrowse/text-indexing 4.0.3 → 4.1.0

package/esm/TextIndexing.d.ts

@@ -1,4 +1,5 @@
-import type { Track, indexType } from './util.ts';
+import type { indexType } from './util.ts';
+import type { Track } from '@jbrowse/text-indexing-core';
 export declare function indexTracks(args: {
     tracks: Track[];
     outDir?: string;

package/esm/TextIndexing.js

@@ -3,10 +3,8 @@ import path from 'path';
 import { Readable } from 'stream';
 import { isSupportedIndexingAdapter } from '@jbrowse/core/util';
 import { checkStopToken } from '@jbrowse/core/util/stopToken';
+import { generateMeta, indexGff3, indexVcf, sanitizeForFilename, } from '@jbrowse/text-indexing-core';
 import { ixIxxStream } from 'ixixx';
-import { generateMeta } from "./types/common.js";
-import { indexGff3 } from "./types/gff3Adapter.js";
-import { indexVcf } from "./types/vcfAdapter.js";
 export async function indexTracks(args) {
     const { tracks, outDir, attributesToIndex, featureTypesToExclude, assemblyNames, indexType, statusCallback, stopToken, } = args;
     const idxType = indexType || 'perTrack';
@@ -183,7 +181,8 @@ function getLoc(attr, config) {
     return elt.uri || elt.localPath;
 }
 function runIxIxx(readStream, idxLocation, name) {
-    const ixFilename = path.join(idxLocation, 'trix', `${name}.ix`);
-    const ixxFilename = path.join(idxLocation, 'trix', `${name}.ixx`);
+    const safeName = sanitizeForFilename(name);
+    const ixFilename = path.join(idxLocation, 'trix', `${safeName}.ix`);
+    const ixxFilename = path.join(idxLocation, 'trix', `${safeName}.ixx`);
     return ixIxxStream(readStream, ixFilename, ixxFilename);
 }

package/esm/index.d.ts

@@ -1,5 +1,3 @@
+export * from '@jbrowse/text-indexing-core';
 export * from './util.ts';
-export * from './types/common.ts';
 export { indexTracks } from './TextIndexing.ts';
-export { indexGff3 } from './types/gff3Adapter.ts';
-export { indexVcf } from './types/vcfAdapter.ts';

package/esm/index.js

@@ -1,5 +1,3 @@
+export * from '@jbrowse/text-indexing-core';
 export * from "./util.js";
-export * from "./types/common.js";
 export { indexTracks } from "./TextIndexing.js";
-export { indexGff3 } from "./types/gff3Adapter.js";
-export { indexVcf } from "./types/vcfAdapter.js";

package/esm/util.d.ts

@@ -1,11 +1,4 @@
-export interface UriLocation {
-    uri: string;
-    locationType: 'UriLocation';
-}
-export interface LocalPathLocation {
-    localPath: string;
-    locationType: 'LocalPathLocation';
-}
+import type { LocalPathLocation, Track, UriLocation } from '@jbrowse/text-indexing-core';
 export interface IndexedFastaAdapter {
     type: 'IndexedFastaAdapter';
     fastaLocation: UriLocation;
@@ -50,49 +43,19 @@ export interface Sequence {
     trackId: string;
     adapter: IndexedFastaAdapter | BgzipFastaAdapter | TwoBitAdapter | ChromeSizesAdapter | CustomSequenceAdapter;
 }
-type Loc = UriLocation | LocalPathLocation;
-export interface Gff3TabixAdapter {
-    type: 'Gff3TabixAdapter';
-    gffGzLocation: Loc;
-}
-export interface Gff3Adapter {
-    type: 'Gff3Adapter';
-    gffLocation: Loc;
-}
-export interface GtfAdapter {
-    type: 'GtfAdapter';
-    gtfLocation: Loc;
-}
-export interface VcfTabixAdapter {
-    type: 'VcfTabixAdapter';
-    vcfGzLocation: Loc;
-}
-export interface VcfAdapter {
-    type: 'VcfAdapter';
-    vcfLocation: Loc;
-}
-export interface Track {
-    adapter?: {
-        type: string;
-        [key: string]: unknown;
-    };
-    textSearching?: TextSearching;
-    name: string;
+export interface TrixTextSearchAdapter {
+    type: string;
+    textSearchAdapterId: string;
+    ixFilePath: UriLocation | LocalPathLocation;
+    ixxFilePath: UriLocation | LocalPathLocation;
+    metaFilePath: UriLocation | LocalPathLocation;
     assemblyNames: string[];
-    trackId: string;
 }
 export interface TextSearching {
     indexingFeatureTypesToExclude?: string[];
     indexingAttributes?: string[];
-    textSearchAdapter: TrixTextSearchAdapter;
-}
-export interface TrixTextSearchAdapter {
-    type: string;
-    textSearchAdapterId: string;
-    ixFilePath: UriLocation;
-    ixxFilePath: UriLocation;
-    metaFilePath: UriLocation;
-    assemblyNames: string[];
+    textSearchAdapter?: TrixTextSearchAdapter;
+    [key: string]: unknown;
 }
 export interface Config {
     assemblies?: Assembly[];
@@ -109,19 +72,17 @@ export declare function createTextSearchConf(name: string, trackIds: string[], a
     textSearchAdapterId: string;
     ixFilePath: {
         localPath: string;
-        locationType: string;
+        locationType: "LocalPathLocation";
     };
     ixxFilePath: {
         localPath: string;
-        locationType: string;
+        locationType: "LocalPathLocation";
     };
     metaFilePath: {
         localPath: string;
-        locationType: string;
+        locationType: "LocalPathLocation";
     };
     tracks: string[];
     assemblyNames: string[];
 };
 export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[], assemblyName?: string): Track[];
-export declare function decodeURIComponentNoThrow(uri: string): string;
-export {};

package/esm/util.js

@@ -35,11 +35,3 @@ export function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
         .filter(track => assemblyName ? track.assemblyNames.includes(assemblyName) : true)
         .filter(track => isSupportedIndexingAdapter(track.adapter?.type));
 }
-export function decodeURIComponentNoThrow(uri) {
-    try {
-        return decodeURIComponent(uri);
-    }
-    catch (e) {
-        return uri;
-    }
-}

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@jbrowse/text-indexing",
-  "version": "4.0.3",
+  "version": "4.1.0",
+  "type": "module",
   "description": "JBrowse 2 text indexing for desktop",
   "keywords": [
     "jbrowse",
@@ -23,10 +24,10 @@
     "esm"
   ],
   "dependencies": {
-    "ixixx": "^3.0.2",
-    "node-fetch": "^2.7.0",
+    "ixixx": "^3.0.3",
     "sanitize-filename": "^1.6.3",
-    "@jbrowse/core": "^4.0.3"
+    "@jbrowse/text-indexing-core": "^4.1.0",
+    "@jbrowse/core": "^4.1.0"
   },
   "peerDependencies": {
     "react": ">=18.0.0",

package/esm/types/common.d.ts

@@ -1,21 +0,0 @@
-import fs from 'fs';
-import fetch from 'node-fetch';
-import type { LocalPathLocation, Track, UriLocation } from '../util.ts';
-export declare function createRemoteStream(urlIn: string): Promise<fetch.Response>;
-export declare function isURL(FileName: string): boolean;
-export declare function getLocalOrRemoteStream({ file, out, onBytesReceived, onTotalBytes, }: {
-    file: string;
-    out: string;
-    onBytesReceived: (totalBytesReceived: number) => void;
-    onTotalBytes: (totalBytes: number) => void;
-}): Promise<NodeJS.ReadableStream | fs.ReadStream>;
-export declare function makeLocation(location: string, protocol: string): UriLocation | LocalPathLocation;
-export declare function guessAdapterFromFileName(filePath: string): Track;
-export declare function generateMeta({ configs, attributesToIndex, outDir, name, featureTypesToExclude, assemblyNames, }: {
-    configs: Track[];
-    attributesToIndex: string[];
-    outDir: string;
-    name: string;
-    featureTypesToExclude: string[];
-    assemblyNames: string[];
-}): Promise<void>;

package/esm/types/common.js

@@ -1,132 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import fetch from 'node-fetch';
-export async function createRemoteStream(urlIn) {
-    const res = await fetch(urlIn);
-    if (!res.ok) {
-        throw new Error(`Failed to fetch ${urlIn} status ${res.status} ${await res.text()}`);
-    }
-    return res;
-}
-export function isURL(FileName) {
-    let url;
-    try {
-        url = new URL(FileName);
-    }
-    catch (_) {
-        return false;
-    }
-    return url.protocol === 'http:' || url.protocol === 'https:';
-}
-export async function getLocalOrRemoteStream({ file, out, onBytesReceived, onTotalBytes, }) {
-    let receivedBytes = 0;
-    if (isURL(file)) {
-        const result = await createRemoteStream(file);
-        result.body.on('data', chunk => {
-            receivedBytes += chunk.length;
-            onBytesReceived(receivedBytes);
-        });
-        onTotalBytes(+(result.headers.get('Content-Length') || 0));
-        return result.body;
-    }
-    else {
-        const filename = path.isAbsolute(file) ? file : path.join(out, file);
-        const stream = fs.createReadStream(filename);
-        stream.on('data', chunk => {
-            receivedBytes += chunk.length;
-            onBytesReceived(receivedBytes);
-        });
-        onTotalBytes(fs.statSync(filename).size);
-        return stream;
-    }
-}
-export function makeLocation(location, protocol) {
-    if (protocol === 'uri') {
-        return { uri: location, locationType: 'UriLocation' };
-    }
-    if (protocol === 'localPath') {
-        return {
-            localPath: path.resolve(location),
-            locationType: 'LocalPathLocation',
-        };
-    }
-    throw new Error(`invalid protocol ${protocol}`);
-}
-export function guessAdapterFromFileName(filePath) {
-    const protocol = isURL(filePath) ? 'uri' : 'localPath';
-    const name = path.basename(filePath);
-    if (/\.vcf\.b?gz$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name: name,
-            assemblyNames: [],
-            adapter: {
-                type: 'VcfTabixAdapter',
-                vcfGzLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.gff3?\.b?gz$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'Gff3TabixAdapter',
-                gffGzLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.gtf?$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'GtfAdapter',
-                gtfLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.vcf$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'VcfAdapter',
-                vcfLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.gff3?$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'Gff3Adapter',
-                gffLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else {
-        throw new Error(`Unsupported file type ${filePath}`);
-    }
-}
-export async function generateMeta({ configs, attributesToIndex, outDir, name, featureTypesToExclude, assemblyNames, }) {
-    fs.writeFileSync(path.join(outDir, 'trix', `${name}_meta.json`), JSON.stringify({
-        dateCreated: new Date().toISOString(),
-        tracks: configs.map(config => {
-            const { trackId, textSearching, adapter } = config;
-            return {
-                trackId,
-                attributesIndexed: textSearching?.indexingAttributes || attributesToIndex,
-                excludedTypes: textSearching?.indexingFeatureTypesToExclude ||
-                    featureTypesToExclude,
-                adapterConf: adapter,
-            };
-        }),
-        assemblyNames,
-    }, null, 2));
-}

package/esm/types/gff3Adapter.d.ts

@@ -1,11 +0,0 @@
-export declare function indexGff3({ config, attributesToIndex, inLocation, outDir, featureTypesToExclude, onStart, onUpdate, }: {
-    config: {
-        trackId: string;
-    };
-    attributesToIndex: string[];
-    inLocation: string;
-    outDir: string;
-    featureTypesToExclude: string[];
-    onStart: (totalBytes: number) => void;
-    onUpdate: (progressBytes: number) => void;
-}): AsyncGenerator<string, void, unknown>;

package/esm/types/gff3Adapter.js

@@ -1,53 +0,0 @@
-import readline from 'readline';
-import { createGunzip } from 'zlib';
-import { decodeURIComponentNoThrow } from "../util.js";
-import { getLocalOrRemoteStream } from "./common.js";
-export async function* indexGff3({ config, attributesToIndex, inLocation, outDir, featureTypesToExclude, onStart, onUpdate, }) {
-    const { trackId } = config;
-    const stream = await getLocalOrRemoteStream({
-        file: inLocation,
-        out: outDir,
-        onTotalBytes: onStart,
-        onBytesReceived: onUpdate,
-    });
-    const rl = readline.createInterface({
-        input: /.b?gz$/.exec(inLocation) ? stream.pipe(createGunzip()) : stream,
-    });
-    for await (const line of rl) {
-        if (!line.trim()) {
-            continue;
-        }
-        else if (line.startsWith('#')) {
-            continue;
-        }
-        else if (line.startsWith('>')) {
-            break;
-        }
-        const [seq_id, , type, start, end, , , , col9] = line.split('\t');
-        const locStr = `${seq_id}:${start}..${end}`;
-        if (!featureTypesToExclude.includes(type)) {
-            const col9attrs = Object.fromEntries(col9
-                .split(';')
-                .map(f => f.trim())
-                .filter(f => !!f)
-                .map(f => f.split('='))
-                .map(([key, val]) => [
-                key.trim(),
-                val
-                    ? decodeURIComponentNoThrow(val).trim().split(',').join(' ')
-                    : undefined,
-            ]));
-            const attrs = attributesToIndex
-                .map(attr => col9attrs[attr])
-                .filter((f) => !!f);
-            if (attrs.length) {
-                const record = JSON.stringify([
-                    encodeURIComponent(locStr),
-                    encodeURIComponent(trackId),
-                    ...attrs.map(a => encodeURIComponent(a)),
-                ]).replaceAll(',', '|');
-                yield `${record} ${[...new Set(attrs)].join(' ')}\n`;
-            }
-        }
-    }
-}

package/esm/types/vcfAdapter.d.ts

@@ -1,8 +0,0 @@
-export declare function indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, }: {
-    config: any;
-    attributesToIndex: string[];
-    inLocation: string;
-    outDir: string;
-    onStart: (totalBytes: number) => void;
-    onUpdate: (progressBytes: number) => void;
-}): AsyncGenerator<string, void, unknown>;

package/esm/types/vcfAdapter.js

@@ -1,55 +0,0 @@
-import readline from 'readline';
-import { createGunzip } from 'zlib';
-import { decodeURIComponentNoThrow } from "../util.js";
-import { getLocalOrRemoteStream } from "./common.js";
-export async function* indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, }) {
-    const { trackId } = config;
-    const stream = await getLocalOrRemoteStream({
-        file: inLocation,
-        out: outDir,
-        onTotalBytes: onStart,
-        onBytesReceived: onUpdate,
-    });
-    const gzStream = /.b?gz$/.exec(inLocation)
-        ? stream.pipe(createGunzip())
-        : stream;
-    const rl = readline.createInterface({
-        input: gzStream,
-    });
-    for await (const line of rl) {
-        if (line.startsWith('#')) {
-            continue;
-        }
-        const [ref, pos, id, _ref, _alt, _qual, _filter, info] = line.split('\t');
-        const fields = Object.fromEntries(info
-            .split(';')
-            .map(f => f.trim())
-            .filter(f => !!f)
-            .map(f => f.split('='))
-            .map(([key, val]) => [
-            key.trim(),
-            val
-                ? decodeURIComponentNoThrow(val).trim().split(',').join(' ')
-                : undefined,
-        ]));
-        const end = fields.END;
-        const locStr = `${ref}:${pos}..${end || +pos + 1}`;
-        if (id === '.') {
-            continue;
-        }
-        const infoAttrs = attributesToIndex
-            .map(attr => fields[attr])
-            .filter((f) => !!f);
-        const ids = id.split(',');
-        for (const id of ids) {
-            const attrs = [id];
-            const record = JSON.stringify([
-                encodeURIComponent(locStr),
-                encodeURIComponent(trackId),
-                encodeURIComponent(id || ''),
-                ...infoAttrs.map(a => encodeURIComponent(a || '')),
-            ]).replaceAll(',', '|');
-            yield `${record} ${[...new Set(attrs)].join(' ')}\n`;
-        }
-    }
-}