@jbrowse/text-indexing 4.3.0 → 5.0.0-beta.10

package/esm/TextIndexing.d.ts

@@ -1,12 +1,12 @@
 import type { indexType } from './util.ts';
-import type { Track } from '@jbrowse/text-indexing-core';
-export declare function indexTracks(args: {
+import type { StatusCallback } from '@jbrowse/core/util';
+import type { IndexingPolicy, Track } from '@jbrowse/text-indexing-core';
+export declare function indexTracks({ tracks, outDir, policy, assemblyNames, indexType, statusCallback, signal, }: {
     tracks: Track[];
-    outDir?: string;
-    stopToken?: string;
-    attributesToIndex?: string[];
+    outDir: string;
+    signal?: AbortSignal;
+    policy?: IndexingPolicy;
     assemblyNames?: string[];
-    featureTypesToExclude?: string[];
     indexType?: indexType;
-    statusCallback: (message: string) => void;
-}): Promise<never[]>;
+    statusCallback: StatusCallback | undefined;
+}): Promise<void>;

package/esm/TextIndexing.js

@@ -1,188 +1,70 @@
-import fs from 'fs';
-import path from 'path';
-import { Readable } from 'stream';
-import { isSupportedIndexingAdapter } from '@jbrowse/core/util';
-import { checkStopToken } from '@jbrowse/core/util/stopToken';
-import { generateMeta, indexGff3, indexVcf, sanitizeForFilename, } from '@jbrowse/text-indexing-core';
-import { ixIxxStream } from 'ixixx';
-export async function indexTracks(args) {
-    const { tracks, outDir, attributesToIndex, featureTypesToExclude, assemblyNames, indexType, statusCallback, stopToken, } = args;
-    const idxType = indexType || 'perTrack';
-    checkStopToken(stopToken);
-    await (idxType === 'perTrack'
-        ? perTrackIndex({
-            tracks,
-            statusCallback,
-            outDir,
-            attributesToIndex,
-            featureTypesToExclude,
-            stopToken,
-        })
-        : aggregateIndex({
-            tracks,
-            statusCallback,
-            outDir,
-            attributesToIndex,
-            assemblyNames,
-            featureTypesToExclude,
-            stopToken,
-        }));
-    checkStopToken(stopToken);
-    return [];
-}
-async function perTrackIndex({ tracks, statusCallback, outDir: paramOutDir, attributesToIndex = ['Name', 'ID'], featureTypesToExclude = ['exon', 'CDS'], stopToken, }) {
-    const outFlag = paramOutDir || '.';
-    const isDir = fs.lstatSync(outFlag).isDirectory();
-    const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag;
-    const outDir = path.dirname(confFilePath);
-    const trixDir = path.join(outDir, 'trix');
-    if (!fs.existsSync(trixDir)) {
-        fs.mkdirSync(trixDir);
-    }
-    const supportedTracks = tracks.filter(track => isSupportedIndexingAdapter(track.adapter?.type));
-    for (const trackConfig of supportedTracks) {
-        const { trackId, assemblyNames } = trackConfig;
-        const id = `${trackId}-index`;
-        await indexDriver({
-            tracks: [trackConfig],
-            outDir,
-            attributesToIndex,
-            name: id,
-            featureTypesToExclude,
-            assemblyNames,
-            statusCallback,
-            stopToken,
-        });
+import { checkAbortSignal } from '@jbrowse/core/util/aborting';
+import { defaultIndexingPolicy, isSupportedIndexingAdapter, writeTrixIndex, } from '@jbrowse/text-indexing-core';
+export async function indexTracks({ tracks, outDir, policy = defaultIndexingPolicy, assemblyNames, indexType = 'perTrack', statusCallback, signal, }) {
+    checkAbortSignal(signal);
+    const supported = tracks.filter(track => isSupportedIndexingAdapter(track.adapter?.type));
+    const write = (indexed, name, names) => indexDriver({
+        tracks: indexed,
+        outDir,
+        name,
+        policy,
+        assemblyNames: names,
+        statusCallback,
+        signal,
+    });
+    if (indexType === 'perTrack') {
+        for (const track of supported) {
+            await write([track], `${track.trackId}-index`, track.assemblyNames);
+        }
     }
-}
-async function aggregateIndex({ tracks, statusCallback, outDir: paramOutDir, attributesToIndex = ['Name', 'ID'], featureTypesToExclude = ['exon', 'CDS'], stopToken, assemblyNames, }) {
-    const outFlag = paramOutDir || '.';
-    const isDir = fs.lstatSync(outFlag).isDirectory();
-    const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag;
-    const outDir = path.dirname(confFilePath);
-    const trixDir = path.join(outDir, 'trix');
-    if (!fs.existsSync(trixDir)) {
-        fs.mkdirSync(trixDir);
+    else if (assemblyNames) {
+        for (const asm of assemblyNames) {
+            await write(supported.filter(track => track.assemblyNames.includes(asm)), `${asm}-index`, [asm]);
+        }
     }
-    if (!assemblyNames) {
+    else {
         throw new Error('No assemblies passed. Assemblies required for aggregate indexes');
     }
-    for (const asm of assemblyNames) {
-        const id = `${asm}-index`;
-        const supportedTracks = tracks
-            .filter(track => isSupportedIndexingAdapter(track.adapter?.type))
-            .filter(track => (asm ? track.assemblyNames.includes(asm) : true));
-        await indexDriver({
-            tracks: supportedTracks,
-            outDir: outDir,
-            attributesToIndex,
-            name: id,
-            featureTypesToExclude,
-            assemblyNames: [asm],
-            statusCallback,
-            stopToken,
-        });
-    }
+    checkAbortSignal(signal);
 }
-async function indexDriver({ tracks, outDir, attributesToIndex, name, featureTypesToExclude, assemblyNames, statusCallback, stopToken, }) {
-    const readable = Readable.from(indexFiles({
+function indexDriver({ tracks, outDir, name, policy, assemblyNames, statusCallback, signal, }) {
+    // accumulate across tracks so an aggregate index reports monotonic progress
+    // rather than resetting to zero at each track. The denominator grows as each
+    // track's size is discovered (we don't stat every file up front), but the
+    // numerator only ever increases
+    let bankedBytes = 0;
+    let cumulativeTotal = 0;
+    statusCallback?.('Indexing files');
+    return writeTrixIndex({
         tracks,
-        attributesToIndex,
-        outDir,
-        featureTypesToExclude,
-        statusCallback,
-        stopToken,
-    }));
-    statusCallback('Indexing files.');
-    await runIxIxx(readable, outDir, name);
-    checkStopToken(stopToken);
-    generateMeta({
-        configs: tracks,
-        attributesToIndex,
         outDir,
         name,
-        featureTypesToExclude,
+        policy,
         assemblyNames,
-    });
-    checkStopToken(stopToken);
-}
-async function* indexFiles({ tracks, attributesToIndex: idx1, outDir, featureTypesToExclude: edx1, statusCallback, }) {
-    for (const track of tracks) {
-        const { adapter, textSearching } = track;
-        const { type } = adapter || {};
-        const { indexingFeatureTypesToExclude: featureTypesToExclude = edx1, indexingAttributes: attributesToIndex = idx1, } = textSearching || {};
-        let myTotalBytes;
-        if (type === 'Gff3Adapter') {
-            yield* indexGff3({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('gffLocation', track),
-                outDir,
-                featureTypesToExclude,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
+        checkAbort: () => {
+            checkAbortSignal(signal);
+        },
+        onSort: () => {
+            statusCallback?.('Sorting and writing index');
+        },
+        makeProgress: () => {
+            let trackTotal = 0;
+            return {
+                onStart: bytes => {
+                    trackTotal = bytes;
+                    cumulativeTotal += bytes;
                 },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
+                onUpdate: bytes => {
+                    statusCallback?.({
+                        message: 'Indexing files',
+                        current: bankedBytes + bytes,
+                        total: cumulativeTotal,
+                    });
                 },
-            });
-        }
-        else if (type === 'Gff3TabixAdapter') {
-            yield* indexGff3({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('gffGzLocation', track),
-                outDir,
-                featureTypesToExclude,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
-                },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-        else if (type === 'VcfAdapter') {
-            yield* indexVcf({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('vcfLocation', track),
-                outDir,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
-                },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-        else if (type === 'VcfTabixAdapter') {
-            yield* indexVcf({
-                config: track,
-                attributesToIndex,
-                inLocation: getLoc('vcfGzLocation', track),
-                outDir,
-                onStart: totalBytes => {
-                    myTotalBytes = totalBytes;
+                onDone: () => {
+                    bankedBytes += trackTotal;
                 },
-                onUpdate: progressBytes => {
-                    statusCallback(`${progressBytes}/${myTotalBytes}`);
-                },
-            });
-        }
-    }
-    return;
-}
-function getLoc(attr, config) {
-    const elt = config.adapter?.[attr];
-    if (!elt) {
-        throw new Error(`can't get find ${attr} from ${config.adapter} for text indexing`);
-    }
-    return elt.uri || elt.localPath;
-}
-function runIxIxx(readStream, idxLocation, name) {
-    const safeName = sanitizeForFilename(name);
-    const ixFilename = path.join(idxLocation, 'trix', `${safeName}.ix`);
-    const ixxFilename = path.join(idxLocation, 'trix', `${safeName}.ixx`);
-    return ixIxxStream(readStream, ixFilename, ixxFilename);
+            };
+        },
+    });
 }

package/esm/index.d.ts

@@ -1,3 +1,3 @@
-export * from '@jbrowse/text-indexing-core';
-export * from './util.ts';
+export { createTextSearchConf, findTrackConfigsToIndex } from './util.ts';
+export type { indexType } from './util.ts';
 export { indexTracks } from './TextIndexing.ts';

package/esm/index.js

@@ -1,3 +1,6 @@
-export * from '@jbrowse/text-indexing-core';
-export * from "./util.js";
-export { indexTracks } from "./TextIndexing.js";
+// This package is the desktop-facing wrapper around `@jbrowse/text-indexing-core`
+// — it adds `indexTracks` and the config helpers below, and nothing else. It
+// deliberately re-exports none of core's surface: a symbol gets one import path,
+// so the two packages cannot drift into disagreeing about the same name.
+export { createTextSearchConf, findTrackConfigsToIndex } from './util.js';
+export { indexTracks } from './TextIndexing.js';

package/esm/util.d.ts

@@ -1,88 +1,15 @@
-import type { LocalPathLocation, Track, UriLocation } from '@jbrowse/text-indexing-core';
-export interface IndexedFastaAdapter {
-    type: 'IndexedFastaAdapter';
-    fastaLocation: UriLocation;
-    faiLocation: UriLocation;
-}
-export interface BgzipFastaAdapter {
-    type: 'BgzipFastaAdapter';
-    fastaLocation: UriLocation;
-    faiLocation: UriLocation;
-    gziLocation: UriLocation;
-}
-export interface TwoBitAdapter {
-    type: 'TwoBitAdapter';
-    twoBitLocation: UriLocation;
-}
-export interface ChromeSizesAdapter {
-    type: 'ChromSizesAdapter';
-    chromSizesLocation: UriLocation;
-}
-export interface CustomSequenceAdapter {
-    type: string;
-}
-export interface RefNameAliasAdapter {
-    type: 'RefNameAliasAdapter';
-    location: UriLocation;
-}
-export interface CustomRefNameAliasAdapter {
-    type: string;
-}
-export interface Assembly {
-    displayName?: string;
-    name: string;
-    aliases?: string[];
-    sequence: Sequence;
-    refNameAliases?: {
-        adapter: RefNameAliasAdapter | CustomRefNameAliasAdapter;
-    };
-    refNameColors?: string[];
-}
-export interface Sequence {
-    type: 'ReferenceSequenceTrack';
-    trackId: string;
-    adapter: IndexedFastaAdapter | BgzipFastaAdapter | TwoBitAdapter | ChromeSizesAdapter | CustomSequenceAdapter;
-}
-export interface TrixTextSearchAdapter {
-    type: string;
-    textSearchAdapterId: string;
-    ixFilePath: UriLocation | LocalPathLocation;
-    ixxFilePath: UriLocation | LocalPathLocation;
-    metaFilePath: UriLocation | LocalPathLocation;
-    assemblyNames: string[];
-}
-export interface TextSearching {
-    indexingFeatureTypesToExclude?: string[];
-    indexingAttributes?: string[];
-    textSearchAdapter?: TrixTextSearchAdapter;
-    [key: string]: unknown;
-}
-export interface Config {
-    assemblies?: Assembly[];
-    assembly?: Assembly;
-    configuration?: Record<string, unknown>;
-    aggregateTextSearchAdapters?: TrixTextSearchAdapter[];
-    connections?: unknown[];
-    defaultSession?: Record<string, unknown>;
-    tracks?: Track[];
-}
+import type { Track } from '@jbrowse/text-indexing-core';
 export type indexType = 'aggregate' | 'perTrack';
-export declare function createTextSearchConf(name: string, trackIds: string[], assemblyNames: string[], locationPath: string): {
-    type: string;
-    textSearchAdapterId: string;
+export declare function createTextSearchConf(name: string, assemblyNames: string[], locationPath: string): {
+    type: 'TrixTextSearchAdapter';
     ixFilePath: {
         localPath: string;
-        locationType: "LocalPathLocation";
+        locationType: 'LocalPathLocation';
     };
     ixxFilePath: {
         localPath: string;
-        locationType: "LocalPathLocation";
-    };
-    metaFilePath: {
-        localPath: string;
-        locationType: "LocalPathLocation";
+        locationType: 'LocalPathLocation';
     };
-    tracks: string[];
     assemblyNames: string[];
 };
-export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[], assemblyName?: string): Track[];
+export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[]): Track[];

package/esm/util.js

@@ -1,29 +1,20 @@
-import path from 'path';
-import { isSupportedIndexingAdapter } from '@jbrowse/core/util';
-import { sanitizeForFilename } from '@jbrowse/text-indexing-core';
-export function createTextSearchConf(name, trackIds, assemblyNames, locationPath) {
-    const base = path.join(locationPath, 'trix');
-    const n = sanitizeForFilename(name);
+import { isSupportedIndexingAdapter, trixFilePaths, } from '@jbrowse/text-indexing-core';
+export function createTextSearchConf(name, assemblyNames, locationPath) {
+    const paths = trixFilePaths(locationPath, name);
     return {
         type: 'TrixTextSearchAdapter',
-        textSearchAdapterId: name,
         ixFilePath: {
-            localPath: path.join(base, `${n}.ix`),
+            localPath: paths.ix,
             locationType: 'LocalPathLocation',
         },
         ixxFilePath: {
-            localPath: path.join(base, `${n}.ixx`),
+            localPath: paths.ixx,
             locationType: 'LocalPathLocation',
         },
-        metaFilePath: {
-            localPath: path.join(base, `${n}_meta.json`),
-            locationType: 'LocalPathLocation',
-        },
-        tracks: trackIds,
         assemblyNames,
     };
 }
-export function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
+export function findTrackConfigsToIndex(tracks, trackIds) {
     return trackIds
         .map(trackId => {
         const currentTrack = tracks.find(t => trackId === t.trackId);
@@ -32,6 +23,5 @@ export function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
         }
         return currentTrack;
     })
-        .filter(track => assemblyName ? track.assemblyNames.includes(assemblyName) : true)
         .filter(track => isSupportedIndexingAdapter(track.adapter?.type));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/text-indexing",
-  "version": "4.3.0",
+  "version": "5.0.0-beta.10",
   "type": "module",
   "description": "JBrowse 2 text indexing for desktop",
   "keywords": [
@@ -20,34 +20,31 @@
   },
   "author": "JBrowse Team",
   "main": "esm/index.js",
+  "exports": {
+    ".": "./esm/index.js",
+    "./util": "./esm/util.js"
+  },
   "files": [
     "esm"
   ],
   "dependencies": {
-    "ixixx": "^3.0.4",
-    "@jbrowse/text-indexing-core": "^4.3.0",
-    "@jbrowse/core": "^4.3.0"
+    "@jbrowse/core": "^5.0.0-beta.10",
+    "@jbrowse/text-indexing-core": "^5.0.0-beta.10"
   },
   "peerDependencies": {
-    "react": ">=18.0.0",
-    "react-dom": ">=18.0.0"
+    "react": ">=19.0.0",
+    "react-dom": ">=19.0.0"
   },
   "publishConfig": {
     "access": "public"
   },
   "sideEffects": false,
   "scripts": {
-    "build:esm": "tsc -p tsconfig.build.esm.json",
+    "build:esm": "node ../../node_modules/typescript7/bin/tsc --build tsconfig.build.esm.json",
     "build": "pnpm build:esm",
     "test": "cd ../..; jest --passWithNoTests packages/text-indexing",
     "clean": "rimraf esm *.tsbuildinfo",
     "prebuild": "pnpm clean"
   },
-  "types": "esm/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./esm/index.d.ts",
-      "import": "./esm/index.js"
-    }
-  }
+  "types": "esm/index.d.ts"
 }