@jbrowse/text-indexing 4.3.0 → 5.0.0-beta.1

package/esm/TextIndexing.d.ts

@@ -1,12 +1,14 @@
 import type { indexType } from './util.ts';
+import type { StatusCallback } from '@jbrowse/core/util';
+import type { StopToken } from '@jbrowse/core/util/stopToken';
 import type { Track } from '@jbrowse/text-indexing-core';
 export declare function indexTracks(args: {
     tracks: Track[];
-    outDir?: string;
-    stopToken?: string;
+    outDir: string;
+    stopToken?: StopToken;
     attributesToIndex?: string[];
     assemblyNames?: string[];
     featureTypesToExclude?: string[];
     indexType?: indexType;
-    statusCallback: (message: string) => void;
-}): Promise<never[]>;
+    statusCallback: StatusCallback | undefined;
+}): Promise<void>;

package/esm/TextIndexing.js

@@ -1,9 +1,8 @@
-import fs from 'fs';
-import path from 'path';
-import { Readable } from 'stream';
-import { isSupportedIndexingAdapter } from '@jbrowse/core/util';
-import { checkStopToken } from '@jbrowse/core/util/stopToken';
-import { generateMeta, indexGff3, indexVcf, sanitizeForFilename, } from '@jbrowse/text-indexing-core';
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { checkStopTokenThrottled, checkStopToken, createStopTokenChecker, } from '@jbrowse/core/util/stopToken';
+import { TRIX_DIR, defaultAttributesToIndex, defaultFeatureTypesToExclude, generateMeta, indexFiles, isSupportedIndexingAdapter, trixFilePaths, } from '@jbrowse/text-indexing-core';
 import { ixIxxStream } from 'ixixx';
 export async function indexTracks(args) {
     const { tracks, outDir, attributesToIndex, featureTypesToExclude, assemblyNames, indexType, statusCallback, stopToken, } = args;
@@ -28,26 +27,20 @@ export async function indexTracks(args) {
             stopToken,
         }));
     checkStopToken(stopToken);
-    return [];
 }
-async function perTrackIndex({ tracks, statusCallback, outDir: paramOutDir, attributesToIndex = ['Name', 'ID'], featureTypesToExclude = ['exon', 'CDS'], stopToken, }) {
-    const outFlag = paramOutDir || '.';
-    const isDir = fs.lstatSync(outFlag).isDirectory();
-    const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag;
-    const outDir = path.dirname(confFilePath);
-    const trixDir = path.join(outDir, 'trix');
-    if (!fs.existsSync(trixDir)) {
-        fs.mkdirSync(trixDir);
-    }
+function ensureTrixDir(outDir) {
+    fs.mkdirSync(path.join(outDir, TRIX_DIR), { recursive: true });
+}
+async function perTrackIndex({ tracks, statusCallback, outDir, attributesToIndex = defaultAttributesToIndex, featureTypesToExclude = defaultFeatureTypesToExclude, stopToken, }) {
+    ensureTrixDir(outDir);
     const supportedTracks = tracks.filter(track => isSupportedIndexingAdapter(track.adapter?.type));
     for (const trackConfig of supportedTracks) {
         const { trackId, assemblyNames } = trackConfig;
-        const id = `${trackId}-index`;
         await indexDriver({
             tracks: [trackConfig],
             outDir,
             attributesToIndex,
-            name: id,
+            name: `${trackId}-index`,
             featureTypesToExclude,
             assemblyNames,
             statusCallback,
@@ -55,28 +48,20 @@ async function perTrackIndex({ tracks, statusCallback, outDir: paramOutDir, attr
         });
     }
 }
-async function aggregateIndex({ tracks, statusCallback, outDir: paramOutDir, attributesToIndex = ['Name', 'ID'], featureTypesToExclude = ['exon', 'CDS'], stopToken, assemblyNames, }) {
-    const outFlag = paramOutDir || '.';
-    const isDir = fs.lstatSync(outFlag).isDirectory();
-    const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag;
-    const outDir = path.dirname(confFilePath);
-    const trixDir = path.join(outDir, 'trix');
-    if (!fs.existsSync(trixDir)) {
-        fs.mkdirSync(trixDir);
-    }
+async function aggregateIndex({ tracks, statusCallback, outDir, attributesToIndex = defaultAttributesToIndex, featureTypesToExclude = defaultFeatureTypesToExclude, stopToken, assemblyNames, }) {
     if (!assemblyNames) {
         throw new Error('No assemblies passed. Assemblies required for aggregate indexes');
     }
+    ensureTrixDir(outDir);
     for (const asm of assemblyNames) {
-        const id = `${asm}-index`;
         const supportedTracks = tracks
             .filter(track => isSupportedIndexingAdapter(track.adapter?.type))
-            .filter(track => (asm ? track.assemblyNames.includes(asm) : true));
+            .filter(track => track.assemblyNames.includes(asm));
         await indexDriver({
             tracks: supportedTracks,
-            outDir: outDir,
+            outDir,
             attributesToIndex,
-            name: id,
+            name: `${asm}-index`,
             featureTypesToExclude,
             assemblyNames: [asm],
             statusCallback,
@@ -85,15 +70,47 @@ async function aggregateIndex({ tracks, statusCallback, outDir: paramOutDir, att
     }
 }
 async function indexDriver({ tracks, outDir, attributesToIndex, name, featureTypesToExclude, assemblyNames, statusCallback, stopToken, }) {
+    const checker = createStopTokenChecker(stopToken);
+    // accumulate across tracks so an aggregate index reports monotonic progress
+    // rather than resetting to zero at each track. The denominator grows as each
+    // track's size is discovered (we don't stat every file up front), but the
+    // numerator only ever increases
+    let bankedBytes = 0;
+    let cumulativeTotal = 0;
     const readable = Readable.from(indexFiles({
         tracks,
         attributesToIndex,
         outDir,
         featureTypesToExclude,
-        statusCallback,
-        stopToken,
+        checkAbort: () => {
+            checkStopTokenThrottled(checker);
+        },
+        makeProgress: () => {
+            let trackTotal = 0;
+            return {
+                onStart: bytes => {
+                    trackTotal = bytes;
+                    cumulativeTotal += bytes;
+                },
+                onUpdate: bytes => {
+                    statusCallback?.({
+                        message: 'Indexing files',
+                        current: bankedBytes + bytes,
+                        total: cumulativeTotal,
+                    });
+                },
+                onDone: () => {
+                    bankedBytes += trackTotal;
+                },
+            };
+        },
     }));
-    statusCallback('Indexing files.');
+    // records stream straight into ixIxx, so 'end' is where the byte counts stop
+    // and the sort's own tail begins — otherwise minutes of a bar stuck at 100%
+    readable.on('end', () => {
+        statusCallback?.('Sorting and writing index');
+    });
+    statusCallback?.('Indexing files');
     await runIxIxx(readable, outDir, name);
     checkStopToken(stopToken);
     generateMeta({
@@ -106,83 +123,7 @@ async function indexDriver({ tracks, outDir, attributesToIndex, name, featureTyp
     });
     checkStopToken(stopToken);
 }
-async function* indexFiles({ tracks, attributesToIndex: idx1, outDir, featureTypesToExclude: edx1, statusCallback, }) {
-    for (const track of tracks) {
-        const { adapter, textSearching } = track;
-        const { type } = adapter || {};
-        const { indexingFeatureTypesToExclude: featureTypesToExclude = edx1, indexingAttributes: attributesToIndex = idx1, } = textSearching || {};
-        let myTotalBytes;
-        if (type === 'Gff3Adapter') {
-            yield* indexGff3({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('gffLocation', track),
-                outDir,
-                featureTypesToExclude,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
-                },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-        else if (type === 'Gff3TabixAdapter') {
-            yield* indexGff3({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('gffGzLocation', track),
-                outDir,
-                featureTypesToExclude,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
-                },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-        else if (type === 'VcfAdapter') {
-            yield* indexVcf({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('vcfLocation', track),
-                outDir,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
-                },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-        else if (type === 'VcfTabixAdapter') {
-            yield* indexVcf({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('vcfGzLocation', track),
-                outDir,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
-                },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-    }
-    return;
-}
-function getLoc(attr, config) {
-    const elt = config.adapter?.[attr];
-    if (!elt) {
-        throw new Error(`can't get find ${attr} from ${config.adapter} for text indexing`);
-    }
-    return elt.uri || elt.localPath;
-}
 function runIxIxx(readStream, idxLocation, name) {
-    const safeName = sanitizeForFilename(name);
-    const ixFilename = path.join(idxLocation, 'trix', `${safeName}.ix`);
-    const ixxFilename = path.join(idxLocation, 'trix', `${safeName}.ixx`);
-    return ixIxxStream(readStream, ixFilename, ixxFilename);
+    const { ix, ixx } = trixFilePaths(idxLocation, name);
+    return ixIxxStream(readStream, ix, ixx);
 }

package/esm/index.d.ts

@@ -1,3 +1,3 @@
-export * from '@jbrowse/text-indexing-core';
-export * from './util.ts';
+export { createTextSearchConf, findTrackConfigsToIndex } from './util.ts';
+export type { indexType } from './util.ts';
 export { indexTracks } from './TextIndexing.ts';

package/esm/index.js

@@ -1,3 +1,6 @@
-export * from '@jbrowse/text-indexing-core';
-export * from "./util.js";
-export { indexTracks } from "./TextIndexing.js";
+// This package is the desktop-facing wrapper around `@jbrowse/text-indexing-core`
+// — it adds `indexTracks` and the config helpers below, and nothing else. It
+// deliberately re-exports none of core's surface: a symbol gets one import path,
+// so the two packages cannot drift into disagreeing about the same name.
+export { createTextSearchConf, findTrackConfigsToIndex } from './util.js';
+export { indexTracks } from './TextIndexing.js';

package/esm/util.d.ts

@@ -1,88 +1,21 @@
-import type { LocalPathLocation, Track, UriLocation } from '@jbrowse/text-indexing-core';
-export interface IndexedFastaAdapter {
-    type: 'IndexedFastaAdapter';
-    fastaLocation: UriLocation;
-    faiLocation: UriLocation;
-}
-export interface BgzipFastaAdapter {
-    type: 'BgzipFastaAdapter';
-    fastaLocation: UriLocation;
-    faiLocation: UriLocation;
-    gziLocation: UriLocation;
-}
-export interface TwoBitAdapter {
-    type: 'TwoBitAdapter';
-    twoBitLocation: UriLocation;
-}
-export interface ChromeSizesAdapter {
-    type: 'ChromSizesAdapter';
-    chromSizesLocation: UriLocation;
-}
-export interface CustomSequenceAdapter {
-    type: string;
-}
-export interface RefNameAliasAdapter {
-    type: 'RefNameAliasAdapter';
-    location: UriLocation;
-}
-export interface CustomRefNameAliasAdapter {
-    type: string;
-}
-export interface Assembly {
-    displayName?: string;
-    name: string;
-    aliases?: string[];
-    sequence: Sequence;
-    refNameAliases?: {
-        adapter: RefNameAliasAdapter | CustomRefNameAliasAdapter;
-    };
-    refNameColors?: string[];
-}
-export interface Sequence {
-    type: 'ReferenceSequenceTrack';
-    trackId: string;
-    adapter: IndexedFastaAdapter | BgzipFastaAdapter | TwoBitAdapter | ChromeSizesAdapter | CustomSequenceAdapter;
-}
-export interface TrixTextSearchAdapter {
-    type: string;
-    textSearchAdapterId: string;
-    ixFilePath: UriLocation | LocalPathLocation;
-    ixxFilePath: UriLocation | LocalPathLocation;
-    metaFilePath: UriLocation | LocalPathLocation;
-    assemblyNames: string[];
-}
-export interface TextSearching {
-    indexingFeatureTypesToExclude?: string[];
-    indexingAttributes?: string[];
-    textSearchAdapter?: TrixTextSearchAdapter;
-    [key: string]: unknown;
-}
-export interface Config {
-    assemblies?: Assembly[];
-    assembly?: Assembly;
-    configuration?: Record<string, unknown>;
-    aggregateTextSearchAdapters?: TrixTextSearchAdapter[];
-    connections?: unknown[];
-    defaultSession?: Record<string, unknown>;
-    tracks?: Track[];
-}
+import type { Track } from '@jbrowse/text-indexing-core';
 export type indexType = 'aggregate' | 'perTrack';
 export declare function createTextSearchConf(name: string, trackIds: string[], assemblyNames: string[], locationPath: string): {
     type: string;
     textSearchAdapterId: string;
     ixFilePath: {
         localPath: string;
-        locationType: "LocalPathLocation";
+        locationType: 'LocalPathLocation';
     };
     ixxFilePath: {
         localPath: string;
-        locationType: "LocalPathLocation";
+        locationType: 'LocalPathLocation';
     };
     metaFilePath: {
         localPath: string;
-        locationType: "LocalPathLocation";
+        locationType: 'LocalPathLocation';
     };
     tracks: string[];
     assemblyNames: string[];
 };
-export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[], assemblyName?: string): Track[];
+export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[]): Track[];

package/esm/util.js

@@ -1,29 +1,26 @@
-import path from 'path';
-import { isSupportedIndexingAdapter } from '@jbrowse/core/util';
-import { sanitizeForFilename } from '@jbrowse/text-indexing-core';
+import { isSupportedIndexingAdapter, trixFilePaths, } from '@jbrowse/text-indexing-core';
 export function createTextSearchConf(name, trackIds, assemblyNames, locationPath) {
-    const base = path.join(locationPath, 'trix');
-    const n = sanitizeForFilename(name);
+    const paths = trixFilePaths(locationPath, name);
     return {
         type: 'TrixTextSearchAdapter',
         textSearchAdapterId: name,
         ixFilePath: {
-            localPath: path.join(base, `${n}.ix`),
+            localPath: paths.ix,
             locationType: 'LocalPathLocation',
         },
         ixxFilePath: {
-            localPath: path.join(base, `${n}.ixx`),
+            localPath: paths.ixx,
             locationType: 'LocalPathLocation',
         },
         metaFilePath: {
-            localPath: path.join(base, `${n}_meta.json`),
+            localPath: paths.meta,
             locationType: 'LocalPathLocation',
         },
         tracks: trackIds,
         assemblyNames,
     };
 }
-export function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
+export function findTrackConfigsToIndex(tracks, trackIds) {
     return trackIds
         .map(trackId => {
         const currentTrack = tracks.find(t => trackId === t.trackId);
@@ -32,6 +29,5 @@ export function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
         }
         return currentTrack;
     })
-        .filter(track => assemblyName ? track.assemblyNames.includes(assemblyName) : true)
         .filter(track => isSupportedIndexingAdapter(track.adapter?.type));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/text-indexing",
-  "version": "4.3.0",
+  "version": "5.0.0-beta.1",
   "type": "module",
   "description": "JBrowse 2 text indexing for desktop",
   "keywords": [
@@ -24,20 +24,20 @@
     "esm"
   ],
   "dependencies": {
-    "ixixx": "^3.0.4",
-    "@jbrowse/text-indexing-core": "^4.3.0",
-    "@jbrowse/core": "^4.3.0"
+    "ixixx": "^3.0.13",
+    "@jbrowse/text-indexing-core": "^5.0.0-beta.1",
+    "@jbrowse/core": "^5.0.0-beta.1"
   },
   "peerDependencies": {
-    "react": ">=18.0.0",
-    "react-dom": ">=18.0.0"
+    "react": ">=19.0.0",
+    "react-dom": ">=19.0.0"
   },
   "publishConfig": {
     "access": "public"
   },
   "sideEffects": false,
   "scripts": {
-    "build:esm": "tsc -p tsconfig.build.esm.json",
+    "build:esm": "node ../../node_modules/typescript7/bin/tsc --build tsconfig.build.esm.json",
     "build": "pnpm build:esm",
     "test": "cd ../..; jest --passWithNoTests packages/text-indexing",
     "clean": "rimraf esm *.tsbuildinfo",
@@ -45,9 +45,6 @@
   },
   "types": "esm/index.d.ts",
   "exports": {
-    ".": {
-      "types": "./esm/index.d.ts",
-      "import": "./esm/index.js"
-    }
+    ".": "./esm/index.js"
   }
 }