@jbrowse/text-indexing-core 4.3.0 → 5.0.0-beta.10

package/esm/index.d.ts

@@ -1,4 +1,9 @@
-export * from './util.ts';
-export * from './types/common.ts';
+export { decodeURIComponentNoThrow, defaultAttributesToIndex, defaultFeatureTypesToExclude, defaultIndexingPolicy, indexableAdapters, isSupportedIndexingAdapter, trackIndexingPolicy, } from './util.ts';
+export type { Gff3IndexerOptions, Gff3Adapter, Gff3TabixAdapter, GtfAdapter, IndexableFormat, IndexerOptions, IndexingPolicy, LocalPathLocation, Track, UriLocation, VcfAdapter, VcfTabixAdapter, } from './util.ts';
+export { createReadlineInterface, getLocalOrRemoteStream, guessAdapterFromFileName, isURL, makeLocation, parseAttributes, } from './types/common.ts';
+export { TRIX_DIR, sanitizeForFilename, trixFileNames, trixFilePaths, trixFileUris, } from './trixPaths.ts';
 export { indexGff3 } from './types/gff3Adapter.ts';
+export { indexGtf } from './types/gtfAdapter.ts';
 export { indexVcf } from './types/vcfAdapter.ts';
+export type { TrackIndexProgress } from './indexFiles.ts';
+export { writeTrixIndex } from './writeTrixIndex.ts';

package/esm/index.js

@@ -1,4 +1,7 @@
-export * from "./util.js";
-export * from "./types/common.js";
-export { indexGff3 } from "./types/gff3Adapter.js";
-export { indexVcf } from "./types/vcfAdapter.js";
+export { decodeURIComponentNoThrow, defaultAttributesToIndex, defaultFeatureTypesToExclude, defaultIndexingPolicy, indexableAdapters, isSupportedIndexingAdapter, trackIndexingPolicy, } from './util.js';
+export { createReadlineInterface, getLocalOrRemoteStream, guessAdapterFromFileName, isURL, makeLocation, parseAttributes, } from './types/common.js';
+export { TRIX_DIR, sanitizeForFilename, trixFileNames, trixFilePaths, trixFileUris, } from './trixPaths.js';
+export { indexGff3 } from './types/gff3Adapter.js';
+export { indexGtf } from './types/gtfAdapter.js';
+export { indexVcf } from './types/vcfAdapter.js';
+export { writeTrixIndex } from './writeTrixIndex.js';

package/esm/indexFiles.d.ts

@@ -0,0 +1,13 @@
+import type { IndexingPolicy, Track } from './util.ts';
+export interface TrackIndexProgress {
+    onStart?: (totalBytes: number) => void;
+    onUpdate?: (processedBytes: number) => void;
+    onDone?: () => void;
+}
+export declare function indexFiles({ tracks, policy, outDir, makeProgress, checkAbort, }: {
+    tracks: Track[];
+    policy: IndexingPolicy;
+    outDir: string;
+    makeProgress?: (trackId: string) => TrackIndexProgress;
+    checkAbort?: () => void;
+}): AsyncGenerator<string, void, unknown>;

package/esm/indexFiles.js

@@ -0,0 +1,51 @@
+import { indexGff3 } from './types/gff3Adapter.js';
+import { indexGtf } from './types/gtfAdapter.js';
+import { indexVcf } from './types/vcfAdapter.js';
+import { indexableAdapters, trackIndexingPolicy } from './util.js';
+function getIndexingLocation(track, locationKey) {
+    const { adapter } = track;
+    // adapter[locationKey] is the canonical location object, e.g. gffGzLocation;
+    // fall back to the adapter itself to support the shorthand where a bare `uri`
+    // sits directly on the adapter
+    const loc = (adapter?.[locationKey] ?? adapter);
+    return loc?.locationType === 'LocalPathLocation' ? loc.localPath : loc?.uri;
+}
+const noop = () => { };
+// Streams index records for a set of tracks, each read under its own policy
+export async function* indexFiles({ tracks, policy, outDir, makeProgress, checkAbort, }) {
+    for (const track of tracks) {
+        checkAbort?.();
+        const { adapter, trackId } = track;
+        const indexable = indexableAdapters[adapter?.type ?? ''];
+        const inLocation = indexable
+            ? getIndexingLocation(track, indexable.locationKey)
+            : undefined;
+        if (indexable && inLocation) {
+            const progress = makeProgress?.(trackId);
+            const { attributes, exclude, include } = trackIndexingPolicy(track, policy);
+            const common = {
+                config: track,
+                attributesToIndex: attributes,
+                inLocation,
+                outDir,
+                onStart: progress?.onStart ?? noop,
+                onUpdate: progress?.onUpdate ?? noop,
+                checkAbort,
+            };
+            if (indexable.format === 'gff3') {
+                yield* indexGff3({
+                    ...common,
+                    featureTypesToExclude: exclude,
+                    featureTypesToInclude: include,
+                });
+            }
+            else if (indexable.format === 'gtf') {
+                yield* indexGtf(common);
+            }
+            else {
+                yield* indexVcf(common);
+            }
+            progress?.onDone?.();
+        }
+    }
+}

package/esm/trixPaths.d.ts

@@ -0,0 +1,31 @@
+/** Every trix artifact lands in this directory under the output directory. */
+export declare const TRIX_DIR = "trix";
+export declare function sanitizeForFilename(name: string): string;
+/**
+ * The three files an index called `name` writes, sanitized once.
+ *
+ * Everything that writes a trix artifact or points a config at one derives its
+ * paths from here, so a trackId holding a `/` or another Windows-invalid
+ * character names the same file for the writer as for the search reading it.
+ */
+export declare function trixFileNames(name: string): {
+    ix: string;
+    ixx: string;
+    meta: string;
+};
+/** {@link trixFileNames} as filesystem paths under `outDir`, for a writer. */
+export declare function trixFilePaths(outDir: string, name: string): {
+    ix: string;
+    ixx: string;
+    meta: string;
+};
+/**
+ * {@link trixFileNames} as URIs relative to the config, for an adapter conf.
+ * Always `/`-joined — a URI is not a filesystem path, so this one does not go
+ * through `path.join`, which would emit `\` on Windows.
+ */
+export declare function trixFileUris(name: string): {
+    ix: string;
+    ixx: string;
+    meta: string;
+};

package/esm/trixPaths.js

@@ -0,0 +1,52 @@
+import path from 'node:path';
+/** Every trix artifact lands in this directory under the output directory. */
+export const TRIX_DIR = 'trix';
+// Windows reserves these device names even with an extension, so a track or
+// assembly literally named e.g. NUL would otherwise yield an unusable NUL.ix
+const windowsReservedName = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
+// makes `name` safe as a filename on Windows: replaces the invalid characters
+// \ / : * ? " < > |, drops trailing dots/spaces (Windows silently strips them),
+// and escapes reserved device names
+export function sanitizeForFilename(name) {
+    const cleaned = name
+        .replaceAll(/[\\/:*?"<>|]/g, '_')
+        .replace(/(?<![. ])[. ]+$/, '');
+    return windowsReservedName.test(cleaned) ? `_${cleaned}` : cleaned;
+}
+/**
+ * The three files an index called `name` writes, sanitized once.
+ *
+ * Everything that writes a trix artifact or points a config at one derives its
+ * paths from here, so a trackId holding a `/` or another Windows-invalid
+ * character names the same file for the writer as for the search reading it.
+ */
+export function trixFileNames(name) {
+    const safeName = sanitizeForFilename(name);
+    return {
+        ix: `${safeName}.ix`,
+        ixx: `${safeName}.ixx`,
+        meta: `${safeName}_meta.json`,
+    };
+}
+/** {@link trixFileNames} as filesystem paths under `outDir`, for a writer. */
+export function trixFilePaths(outDir, name) {
+    const names = trixFileNames(name);
+    return {
+        ix: path.join(outDir, TRIX_DIR, names.ix),
+        ixx: path.join(outDir, TRIX_DIR, names.ixx),
+        meta: path.join(outDir, TRIX_DIR, names.meta),
+    };
+}
+/**
+ * {@link trixFileNames} as URIs relative to the config, for an adapter conf.
+ * Always `/`-joined — a URI is not a filesystem path, so this one does not go
+ * through `path.join`, which would emit `\` on Windows.
+ */
+export function trixFileUris(name) {
+    const names = trixFileNames(name);
+    return {
+        ix: `${TRIX_DIR}/${names.ix}`,
+        ixx: `${TRIX_DIR}/${names.ixx}`,
+        meta: `${TRIX_DIR}/${names.meta}`,
+    };
+}

package/esm/types/common.d.ts

@@ -1,7 +1,7 @@
-import readline from 'readline';
-import { Readable } from 'stream';
+import readline from 'node:readline';
+import { Readable } from 'node:stream';
 import type { LocalPathLocation, Track, UriLocation } from '../util.ts';
-export declare function isURL(FileName: string): boolean;
+export declare function isURL(fileName: string): boolean;
 export declare function getLocalOrRemoteStream({ file, out, onStart, onUpdate, }: {
     file: string;
     out: string;
@@ -12,12 +12,3 @@ export declare function createReadlineInterface(stream: Readable, inLocation: st
 export declare function parseAttributes(infoString: string, decodeFunc: (s: string) => string): Record<string, string | undefined>;
 export declare function makeLocation(location: string, protocol: string): UriLocation | LocalPathLocation;
 export declare function guessAdapterFromFileName(filePath: string): Track;
-export declare function sanitizeForFilename(name: string): string;
-export declare function generateMeta({ configs, attributesToIndex, outDir, name, featureTypesToExclude, assemblyNames, }: {
-    configs: Track[];
-    attributesToIndex: string[];
-    outDir: string;
-    name: string;
-    featureTypesToExclude: string[];
-    assemblyNames: string[];
-}): void;

package/esm/types/common.js

@@ -1,18 +1,19 @@
-import fs from 'fs';
-import path from 'path';
-import readline from 'readline';
-import { Readable } from 'stream';
-import { fileURLToPath } from 'url';
-import { createGunzip } from 'zlib';
-export function isURL(FileName) {
-    let url;
+import { Buffer } from 'node:buffer';
+import fs from 'node:fs';
+import path from 'node:path';
+import readline from 'node:readline';
+import { Readable } from 'node:stream';
+import { fileURLToPath } from 'node:url';
+import { createGunzip } from 'node:zlib';
+import { indexableAdapters } from '../util.js';
+export function isURL(fileName) {
     try {
-        url = new URL(FileName);
+        const url = new URL(fileName);
+        return url.protocol === 'http:' || url.protocol === 'https:';
     }
-    catch (_) {
+    catch {
         return false;
     }
-    return url.protocol === 'http:' || url.protocol === 'https:';
 }
 function convertFileUrlToPath(fileUrl) {
     try {
@@ -22,9 +23,49 @@ function convertFileUrlToPath(fileUrl) {
         }
     }
     catch {
+        // not a valid URL
     }
     return undefined;
 }
+// Turn a WHATWG ReadableStream into a node Readable by driving its reader.
+//
+// DO NOT replace this with `Readable.fromWeb(body)`. It looks equivalent and
+// passes tests (undici's fetch body IS a node:stream/web stream), but the
+// desktop indexing worker runs on Chromium's global fetch, whose body is a DOM
+// ReadableStream from a different realm. `Readable.fromWeb`'s internal
+// `instanceof` check rejects it with the baffling "must be an instance of
+// ReadableStream. Received an instance of ReadableStream", and remote (URL)
+// track indexing breaks. `getReader()` is standard on every WHATWG stream
+// regardless of realm, so this works everywhere. See common.test.ts
+// "foreign realm" regression test.
+function webStreamToNodeReadable(reader) {
+    return new Readable({
+        read() {
+            reader
+                .read()
+                .then(({ done, value }) => {
+                this.push(done ? null : Buffer.from(value));
+            })
+                .catch((e) => {
+                this.destroy(e);
+            });
+        },
+        // Cancel the source when the Readable is destroyed early (e.g. an aborted
+        // index) so the underlying fetch connection is released rather than left
+        // hanging — the cleanup Readable.fromWeb would do for us if it accepted
+        // this stream.
+        destroy(error, callback) {
+            reader
+                .cancel(error ?? undefined)
+                .then(() => {
+                callback(error);
+            })
+                .catch(() => {
+                callback(error);
+            });
+        },
+    });
+}
 export async function getLocalOrRemoteStream({ file, out, onStart, onUpdate, }) {
     let receivedBytes = 0;
     if (isURL(file)) {
@@ -38,19 +79,12 @@ export async function getLocalOrRemoteStream({ file, out, onStart, onUpdate, })
         if (!body) {
             throw new Error(`Failed to fetch ${file}: no response body`);
         }
+        // node-fetch gives a node Readable directly; global fetch gives a web
+        // ReadableStream we adapt (see webStreamToNodeReadable — do not swap in
+        // Readable.fromWeb here).
         const nodeStream = body instanceof Readable
             ? body
-            : new Readable({
-                async read() {
-                    const reader = body.getReader();
-                    let result = await reader.read();
-                    while (!result.done) {
-                        this.push(result.value);
-                        result = await reader.read();
-                    }
-                    this.push(null);
-                },
-            });
+            : webStreamToNodeReadable(body.getReader());
         nodeStream.on('data', chunk => {
             receivedBytes += chunk.length;
             onUpdate(receivedBytes);
@@ -58,6 +92,7 @@ export async function getLocalOrRemoteStream({ file, out, onStart, onUpdate, })
         return nodeStream;
     }
     else {
+        // Handle file:// URLs by converting to local path
         const localPath = convertFileUrlToPath(file) ?? file;
         const filename = path.isAbsolute(localPath)
             ? localPath
@@ -73,19 +108,10 @@ export async function getLocalOrRemoteStream({ file, out, onStart, onUpdate, })
     }
 }
 export function createReadlineInterface(stream, inLocation) {
-    const isGzipped = /.b?gz$/.exec(inLocation);
-    let inputStream;
-    if (isGzipped) {
-        const gunzip = createGunzip();
-        inputStream = stream.pipe(gunzip);
-    }
-    else {
-        inputStream = stream;
-    }
-    const rl = readline.createInterface({
-        input: inputStream,
-    });
-    return rl;
+    const inputStream = /\.b?gz$/i.test(inLocation)
+        ? stream.pipe(createGunzip())
+        : stream;
+    return readline.createInterface({ input: inputStream });
 }
 export function parseAttributes(infoString, decodeFunc) {
     const result = {};
@@ -114,61 +140,33 @@ export function makeLocation(location, protocol) {
     }
     throw new Error(`invalid protocol ${protocol}`);
 }
+// ordered most-specific-first so e.g. `.vcf.gz` matches before `.vcf`.
+//
+// `.gtf.gz` maps to GtfAdapter rather than GtfTabixAdapter on purpose: this
+// guesser only invents a throwaway track for `text-index --file`, which streams
+// the file start to finish and never opens the .tbi, and GtfAdapter's
+// gtfLocation takes a gzipped path (the reader gunzips on the suffix). A track
+// in a config.json is the other case — `jbrowse add-track` gives a bgzipped GTF
+// a GtfTabixAdapter, which is indexable too, from its own gtfGzLocation.
+const adapterGuesses = [
+    { regex: /\.vcf\.b?gz$/i, type: 'VcfTabixAdapter' },
+    { regex: /\.gff3?\.b?gz$/i, type: 'Gff3TabixAdapter' },
+    { regex: /\.gtf(\.b?gz)?$/i, type: 'GtfAdapter' },
+    { regex: /\.vcf$/i, type: 'VcfAdapter' },
+    { regex: /\.gff3?$/i, type: 'Gff3Adapter' },
+];
 export function guessAdapterFromFileName(filePath) {
-    const protocol = isURL(filePath) ? 'uri' : 'localPath';
-    const name = path.basename(filePath);
-    if (/\.vcf\.b?gz$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name: name,
-            assemblyNames: [],
-            adapter: {
-                type: 'VcfTabixAdapter',
-                vcfGzLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.gff3?\.b?gz$/i.test(filePath)) {
+    const guess = adapterGuesses.find(g => g.regex.test(filePath));
+    if (guess) {
+        const name = path.basename(filePath);
+        const protocol = isURL(filePath) ? 'uri' : 'localPath';
         return {
             trackId: name,
             name,
             assemblyNames: [],
             adapter: {
-                type: 'Gff3TabixAdapter',
-                gffGzLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.gtf?$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'GtfAdapter',
-                gtfLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.vcf$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'VcfAdapter',
-                vcfLocation: makeLocation(filePath, protocol),
-            },
-        };
-    }
-    else if (/\.gff3?$/i.test(filePath)) {
-        return {
-            trackId: name,
-            name,
-            assemblyNames: [],
-            adapter: {
-                type: 'Gff3Adapter',
-                gffLocation: makeLocation(filePath, protocol),
+                type: guess.type,
+                [indexableAdapters[guess.type].locationKey]: makeLocation(filePath, protocol),
             },
         };
     }
@@ -176,23 +174,3 @@ export function guessAdapterFromFileName(filePath) {
         throw new Error(`Unsupported file type ${filePath}`);
     }
 }
-export function sanitizeForFilename(name) {
-    return name.replace(/[\\/:*?"<>|]/g, '_');
-}
-export function generateMeta({ configs, attributesToIndex, outDir, name, featureTypesToExclude, assemblyNames, }) {
-    const safeName = sanitizeForFilename(name);
-    fs.writeFileSync(path.join(outDir, 'trix', `${safeName}_meta.json`), JSON.stringify({
-        dateCreated: new Date().toISOString(),
-        tracks: configs.map(config => {
-            const { trackId, textSearching, adapter } = config;
-            return {
-                trackId,
-                attributesIndexed: textSearching?.indexingAttributes || attributesToIndex,
-                excludedTypes: textSearching?.indexingFeatureTypesToExclude ||
-                    featureTypesToExclude,
-                adapterConf: adapter,
-            };
-        }),
-        assemblyNames,
-    }, null, 2));
-}

package/esm/types/gff3Adapter.d.ts

@@ -1,11 +1,20 @@
-export declare function indexGff3({ config, attributesToIndex, inLocation, outDir, featureTypesToExclude, onStart, onUpdate, }: {
-    config: {
-        trackId: string;
-    };
-    attributesToIndex: string[];
-    inLocation: string;
-    outDir: string;
-    featureTypesToExclude: string[];
-    onStart: (totalBytes: number) => void;
-    onUpdate: (progressBytes: number) => void;
-}): AsyncGenerator<string, void, unknown>;
+import type { Gff3IndexerOptions } from '../util.ts';
+/**
+ * Stream one GFF3 into trix records, one line per indexed feature.
+ *
+ * Two type gates, and they answer different questions. `featureTypesToExclude`
+ * names types this file is known to carry and nobody would search for — it is
+ * the right shape when you own the vocabulary, which is why the default is
+ * `exon,CDS`. `featureTypesToInclude` names the types worth indexing and drops
+ * everything else, which is the only shape that stays correct against a
+ * vocabulary someone else controls: an NCBI RefSeq GFF3 draws from 115 feature
+ * types, 80 of which are leaf regulatory/repeat/alignment records carrying no
+ * name (a `match` labels itself with a bare UUID, a `cDNA_match` with an MD5,
+ * every `biological_region` with the literal string "biological region"), and a
+ * deny list written against the types seen so far leaks the next one NCBI adds.
+ * The allow list for that file is ~33 names — gene, pseudogene, and the
+ * transcript-level types — and does not grow.
+ *
+ * Both apply when both are given: include admits, exclude then narrows.
+ */
+export declare function indexGff3({ config, attributesToIndex, inLocation, outDir, featureTypesToExclude, featureTypesToInclude, onStart, onUpdate, checkAbort, }: Gff3IndexerOptions): AsyncGenerator<string, void, unknown>;

package/esm/types/gff3Adapter.js

@@ -1,6 +1,38 @@
-import { decodeURIComponentNoThrow } from "../util.js";
-import { createReadlineInterface, getLocalOrRemoteStream, parseAttributes, } from "./common.js";
-export async function* indexGff3({ config, attributesToIndex, inLocation, outDir, featureTypesToExclude, onStart, onUpdate, }) {
+import { decodeURIComponentNoThrow, trixLine } from '../util.js';
+import { createReadlineInterface, getLocalOrRemoteStream, parseAttributes, } from './common.js';
+// GENCODE writes no `Name`: a gene row carries `gene_name`, a transcript row
+// `transcript_name`, and every row repeats its parents' names too. So `Name`
+// falls back to the name whose id is this row's own ID, which leaves a UTR or
+// codon row unnamed rather than answering a search for its transcript.
+function ownName(attrs) {
+    const id = attrs.ID;
+    return id === undefined
+        ? undefined
+        : id === attrs.transcript_id
+            ? attrs.transcript_name
+            : id === attrs.gene_id
+                ? attrs.gene_name
+                : undefined;
+}
+/**
+ * Stream one GFF3 into trix records, one line per indexed feature.
+ *
+ * Two type gates, and they answer different questions. `featureTypesToExclude`
+ * names types this file is known to carry and nobody would search for — it is
+ * the right shape when you own the vocabulary, which is why the default is
+ * `exon,CDS`. `featureTypesToInclude` names the types worth indexing and drops
+ * everything else, which is the only shape that stays correct against a
+ * vocabulary someone else controls: an NCBI RefSeq GFF3 draws from 115 feature
+ * types, 80 of which are leaf regulatory/repeat/alignment records carrying no
+ * name (a `match` labels itself with a bare UUID, a `cDNA_match` with an MD5,
+ * every `biological_region` with the literal string "biological region"), and a
+ * deny list written against the types seen so far leaks the next one NCBI adds.
+ * The allow list for that file is ~33 names — gene, pseudogene, and the
+ * transcript-level types — and does not grow.
+ *
+ * Both apply when both are given: include admits, exclude then narrows.
+ */
+export async function* indexGff3({ config, attributesToIndex, inLocation, outDir, featureTypesToExclude, featureTypesToInclude, onStart, onUpdate, checkAbort, }) {
     const { trackId } = config;
     const stream = await getLocalOrRemoteStream({
         file: inLocation,
@@ -10,29 +42,41 @@ export async function* indexGff3({ config, attributesToIndex, inLocation, outDir
     });
     const rl = createReadlineInterface(stream, inLocation);
     const excludeSet = new Set(featureTypesToExclude);
-    const encodedTrackId = encodeURIComponent(trackId);
+    // Absent rather than empty, so an empty list means "no allow list" and not
+    // "index nothing" — a caller that reads the list out of a config gets the
+    // former when the slot is unset, and silently indexing zero features would
+    // look exactly like a successful run.
+    const includeSet = featureTypesToInclude?.length
+        ? new Set(featureTypesToInclude)
+        : undefined;
     for await (const line of rl) {
-        if (!line.trim()) {
-            continue;
-        }
-        else if (line.startsWith('#')) {
+        checkAbort?.();
+        if (!line.trim() || line.startsWith('#')) {
             continue;
         }
         else if (line.startsWith('>')) {
             break;
         }
         const [seq_id, , type, start, end, , , , col9] = line.split('\t');
-        if (!excludeSet.has(type)) {
+        // a valid GFF3 feature row has 9 tab-delimited columns; skip malformed or
+        // truncated lines that would otherwise make parseAttributes throw on an
+        // undefined column 9 (mirrors the guard in the vcf indexer)
+        if (seq_id === undefined ||
+            type === undefined ||
+            start === undefined ||
+            end === undefined ||
+            col9 === undefined) {
+            continue;
+        }
+        if (!excludeSet.has(type) &&
+            (includeSet === undefined || includeSet.has(type))) {
             const col9attrs = parseAttributes(col9, decodeURIComponentNoThrow);
             const attrs = attributesToIndex
-                .map(attr => col9attrs[attr])
+                .map(attr => col9attrs[attr] ??
+                (attr === 'Name' ? ownName(col9attrs) : undefined))
                 .filter((f) => !!f);
             if (attrs.length > 0) {
-                const locStr = `${seq_id}:${start}..${end}`;
-                const encodedAttrs = attrs.map(a => `"${encodeURIComponent(a)}"`);
-                const record = `["${encodeURIComponent(locStr)}"|"${encodedTrackId}"|${encodedAttrs.join('|')}]`;
-                const uniqueAttrs = [...new Set(attrs)];
-                yield `${record} ${uniqueAttrs.join(' ')}\n`;
+                yield trixLine(`${seq_id}:${start}..${end}`, trackId, attrs);
             }
         }
     }

package/esm/types/gtfAdapter.d.ts

@@ -0,0 +1,25 @@
+import type { IndexerOptions } from '../util.ts';
+/**
+ * Parse the GTF 9th column (`gene_id "X"; transcript_id "Y";`), mirroring
+ * plugins/gtf so the indexed text matches what the track displays: entries are
+ * `key value` separated by a space (not `key=value`), values may be quoted, a
+ * repeated key expresses multiple values, and a quoted value may contain the
+ * ';' the entries are split on (`note "a; b"`), so its pieces are rejoined
+ * before the value is read.
+ */
+export declare function parseGtfAttributes(attrString: string): Record<string, string[]>;
+/**
+ * GTF, unlike GFF3, has no gene or transcript rows: a gene exists only as the
+ * exon/CDS/UTR rows repeating its `gene_id`. Indexing per row would point every
+ * hit at a single exon, so rows are grouped by `gene_id` and `transcript_id`
+ * and each group is emitted once spanning all of its rows.
+ *
+ * Grouping is what keeps one entry per gene rather than one per exon, so
+ * `featureTypesToExclude` does not apply here — dropping rows would only
+ * truncate the spans.
+ *
+ * The group map is held until EOF because GTF rows for one locus are not
+ * guaranteed contiguous. It is keyed by gene/transcript id, so it stays far
+ * smaller than the whole-file parse plugins/gtf already does to display a GTF.
+ */
+export declare function indexGtf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, checkAbort, }: IndexerOptions): AsyncGenerator<string, void, unknown>;

package/esm/types/gtfAdapter.js

@@ -0,0 +1,146 @@
+import { trixLine } from '../util.js';
+import { createReadlineInterface, getLocalOrRemoteStream } from './common.js';
+// GFF3-style attribute names have no GTF equivalent, so the shared defaults
+// (`Name`, `ID`, `symbol`) would index nothing. Each requested name also pulls
+// in its GTF spellings, so `jbrowse text-index` works on a GTF out of the box
+// while an explicit `--attributes gene_biotype` still works verbatim.
+const gtfAttributeAliases = {
+    Name: ['gene_name', 'transcript_name'],
+    ID: ['gene_id', 'transcript_id'],
+    symbol: ['gene_name'],
+};
+function expandAttributeNames(attributesToIndex) {
+    return [
+        ...new Set(attributesToIndex.flatMap(a => [a, ...(gtfAttributeAliases[a] ?? [])])),
+    ];
+}
+/**
+ * True when an entry opens a double quote it never closes, meaning the ';' it
+ * was split on fell inside the value instead of ending the entry.
+ */
+function hasUnclosedQuote(entry) {
+    const first = entry.indexOf('"');
+    return first !== -1 && !entry.includes('"', first + 1);
+}
+/**
+ * Parse the GTF 9th column (`gene_id "X"; transcript_id "Y";`), mirroring
+ * plugins/gtf so the indexed text matches what the track displays: entries are
+ * `key value` separated by a space (not `key=value`), values may be quoted, a
+ * repeated key expresses multiple values, and a quoted value may contain the
+ * ';' the entries are split on (`note "a; b"`), so its pieces are rejoined
+ * before the value is read.
+ */
+export function parseGtfAttributes(attrString) {
+    const attrs = {};
+    const entries = attrString.split(';');
+    for (let i = 0; i < entries.length; i++) {
+        let entry = entries[i];
+        while (hasUnclosedQuote(entry) && i + 1 < entries.length) {
+            entry += `;${entries[++i]}`;
+        }
+        const trimmed = entry.trim();
+        const sp = trimmed.indexOf(' ');
+        if (sp !== -1) {
+            const key = trimmed.slice(0, sp);
+            const value = trimmed
+                .slice(sp + 1)
+                .trim()
+                .replaceAll(/^"|"$/g, '')
+                // a comma separates record ids in the trix .ix format, so it can never
+                // survive into an indexed word
+                .replaceAll(',', ' ');
+            if (value) {
+                ;
+                (attrs[key] ??= []).push(value);
+            }
+        }
+    }
+    return attrs;
+}
+// composite map keys join refName + id; a separator that cannot occur in
+// either keeps two loci from colliding
+const SEP = '\u0000';
+/**
+ * GTF, unlike GFF3, has no gene or transcript rows: a gene exists only as the
+ * exon/CDS/UTR rows repeating its `gene_id`. Indexing per row would point every
+ * hit at a single exon, so rows are grouped by `gene_id` and `transcript_id`
+ * and each group is emitted once spanning all of its rows.
+ *
+ * Grouping is what keeps one entry per gene rather than one per exon, so
+ * `featureTypesToExclude` does not apply here — dropping rows would only
+ * truncate the spans.
+ *
+ * The group map is held until EOF because GTF rows for one locus are not
+ * guaranteed contiguous. It is keyed by gene/transcript id, so it stays far
+ * smaller than the whole-file parse plugins/gtf already does to display a GTF.
+ */
+export async function* indexGtf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, checkAbort, }) {
+    const { trackId } = config;
+    const stream = await getLocalOrRemoteStream({
+        file: inLocation,
+        out: outDir,
+        onStart,
+        onUpdate,
+    });
+    const rl = createReadlineInterface(stream, inLocation);
+    const attributeNames = expandAttributeNames(attributesToIndex);
+    const groups = new Map();
+    function accumulate(key, refName, start, end, values) {
+        if (values.length > 0) {
+            const group = groups.get(key);
+            if (group) {
+                group.start = Math.min(group.start, start);
+                group.end = Math.max(group.end, end);
+                for (const value of values) {
+                    group.attrs.add(value);
+                }
+            }
+            else {
+                groups.set(key, { refName, start, end, attrs: new Set(values) });
+            }
+        }
+    }
+    for await (const line of rl) {
+        checkAbort?.();
+        if (line.trim() && !line.startsWith('#')) {
+            const [seq_id, , , start, end, , , , col9] = line.split('\t');
+            if (seq_id !== undefined &&
+                start !== undefined &&
+                end !== undefined &&
+                col9 !== undefined) {
+                const parsed = parseGtfAttributes(col9);
+                const geneId = parsed.gene_id?.[0];
+                const transcriptId = parsed.transcript_id?.[0];
+                const s = Number(start);
+                const e = Number(end);
+                // `gene_*` names describe the gene, everything else the transcript, so
+                // a gene entry is labeled by its gene_name rather than by whichever
+                // transcript happened to come first
+                const geneValues = [];
+                const otherValues = [];
+                for (const name of attributeNames) {
+                    for (const value of parsed[name] ?? []) {
+                        ;
+                        (name.startsWith('gene') ? geneValues : otherValues).push(value);
+                    }
+                }
+                if (transcriptId === undefined) {
+                    const values = [...geneValues, ...otherValues];
+                    // no id to group on: merge rows that carry identical values, which
+                    // still collapses the repeated rows of one feature into one entry
+                    const key = geneId ?? values.join(SEP);
+                    accumulate(`gene${SEP}${seq_id}${SEP}${key}`, seq_id, s, e, values);
+                }
+                else {
+                    accumulate(`tx${SEP}${seq_id}${SEP}${transcriptId}`, seq_id, s, e, otherValues);
+                    if (geneId !== undefined) {
+                        accumulate(`gene${SEP}${seq_id}${SEP}${geneId}`, seq_id, s, e, geneValues);
+                    }
+                }
+            }
+        }
+    }
+    for (const { refName, start, end, attrs } of groups.values()) {
+        yield trixLine(`${refName}:${start}..${end}`, trackId, [...attrs]);
+    }
+}

package/esm/types/vcfAdapter.d.ts

@@ -1,10 +1,2 @@
-export declare function indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, }: {
-    config: {
-        trackId: string;
-    };
-    attributesToIndex: string[];
-    inLocation: string;
-    outDir: string;
-    onStart: (totalBytes: number) => void;
-    onUpdate: (progressBytes: number) => void;
-}): AsyncGenerator<string, void, unknown>;
+import type { IndexerOptions } from '../util.ts';
+export declare function indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, checkAbort, }: IndexerOptions): AsyncGenerator<string, void, unknown>;

package/esm/types/vcfAdapter.js

@@ -1,6 +1,6 @@
-import { decodeURIComponentNoThrow } from "../util.js";
-import { createReadlineInterface, getLocalOrRemoteStream, parseAttributes, } from "./common.js";
-export async function* indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, }) {
+import { decodeURIComponentNoThrow, trixLine } from '../util.js';
+import { createReadlineInterface, getLocalOrRemoteStream, parseAttributes, } from './common.js';
+export async function* indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, checkAbort, }) {
     const { trackId } = config;
     const stream = await getLocalOrRemoteStream({
         file: inLocation,
@@ -9,27 +9,42 @@ export async function* indexVcf({ config, attributesToIndex, inLocation, outDir,
         onUpdate,
     });
     const rl = createReadlineInterface(stream, inLocation);
-    const encodedTrackId = encodeURIComponent(trackId);
     for await (const line of rl) {
-        if (line.startsWith('#')) {
+        checkAbort?.();
+        if (!line.trim() || line.startsWith('#')) {
             continue;
         }
-        const [ref, pos, id, _ref, _alt, _qual, _filter, info] = line.split('\t');
-        if (id === '.') {
+        const [ref, pos, id, , , , , info] = line.split('\t');
+        // a valid VCF data row has at least 8 tab-delimited columns; skip blank or
+        // malformed/short lines that would otherwise emit garbage records or throw
+        if (ref === undefined ||
+            pos === undefined ||
+            id === undefined ||
+            info === undefined) {
             continue;
         }
         const fields = parseAttributes(info, decodeURIComponentNoThrow);
-        const end = fields.END;
-        const locStr = `${ref}:${pos}..${end || +pos + 1}`;
-        const encodedLocStr = encodeURIComponent(locStr);
+        // an empty `END=` parses to 0 via Number(''), which is finite; require a
+        // non-empty value before trusting it, else fall back to pos+1
+        const endNum = Number(fields.END);
+        const end = fields.END && Number.isFinite(endNum) ? endNum : Number(pos) + 1;
+        const locStr = `${ref}:${pos}..${end}`;
         const infoAttrs = attributesToIndex
             .map(attr => fields[attr])
             .filter((f) => !!f);
-        const encodedInfoAttrs = infoAttrs.map(a => `"${encodeURIComponent(a)}"`);
-        for (const variantId of id.split(',')) {
-            const encodedId = encodeURIComponent(variantId);
-            const record = `["${encodedLocStr}"|"${encodedTrackId}"|"${encodedId}"${encodedInfoAttrs.length > 0 ? `|${encodedInfoAttrs.join('|')}` : ''}]`;
-            yield `${record} ${variantId}\n`;
+        // VCF separates multiple IDs with ';' (matching @gmod/vcf), not ','. '.' is
+        // the no-id sentinel; such a row is still worth indexing by its INFO
+        // attributes, it just has no id to search by
+        const variantIds = id === '.' ? [] : id.split(';').filter(Boolean);
+        // one record per id so each is separately searchable, plus an id-less
+        // record when the row carries only attributes
+        const attrGroups = variantIds.length
+            ? variantIds.map(variantId => [variantId, ...infoAttrs])
+            : [infoAttrs];
+        for (const attrs of attrGroups) {
+            if (attrs.length > 0) {
+                yield trixLine(locStr, trackId, attrs);
+            }
         }
     }
 }

package/esm/util.d.ts

@@ -28,18 +28,70 @@ export interface VcfAdapter {
     vcfLocation: Loc;
 }
 export interface Track {
+    type?: string;
     adapter?: {
         type: string;
         [key: string]: unknown;
     };
     textSearching?: {
         indexingFeatureTypesToExclude?: string[];
+        indexingFeatureTypesToInclude?: string[];
         indexingAttributes?: string[];
         [key: string]: unknown;
     };
+    metadata?: {
+        skipTextIndex?: boolean;
+        [key: string]: unknown;
+    };
     name: string;
     assemblyNames: string[];
     trackId: string;
+    category?: string[];
+    description?: string;
+}
+export declare const defaultAttributesToIndex: string[];
+export declare const defaultFeatureTypesToExclude: string[];
+/** What an index reads out of a track's features. */
+export interface IndexingPolicy {
+    attributes: string[];
+    /** feature types to skip (GFF3 only) */
+    exclude: string[];
+    /**
+     * the only feature types to index, narrowed by `exclude` (GFF3 only). Absent
+     * or empty means no allow list, never "index nothing". See indexGff3
+     */
+    include?: string[];
+}
+export declare const defaultIndexingPolicy: IndexingPolicy;
+/** The policy `track` is indexed under: its own settings, else the run's. */
+export declare function trackIndexingPolicy({ textSearching }: Track, run: IndexingPolicy): IndexingPolicy;
+export type IndexableFormat = 'gff3' | 'gtf' | 'vcf';
+export declare const indexableAdapters: Record<string, {
+    locationKey: string;
+    format: IndexableFormat;
+}>;
+export declare function isSupportedIndexingAdapter(type?: string): boolean;
+export interface IndexerOptions {
+    config: {
+        trackId: string;
+    };
+    attributesToIndex: string[];
+    inLocation: string;
+    outDir: string;
+    onStart: (totalBytes: number) => void;
+    onUpdate: (progressBytes: number) => void;
+    checkAbort?: () => void;
+}
+export interface Gff3IndexerOptions extends IndexerOptions {
+    featureTypesToExclude: string[];
+    featureTypesToInclude?: string[];
 }
+/**
+ * One line of ixIxx input: the record a search hit hands back —
+ * `["loc"|"trackId"|"attr"|...]`, fields URI-encoded so neither a comma nor a
+ * space can split it — then the words the hit is found by.
+ * TrixTextSearchAdapter reads the record back.
+ */
+export declare function trixLine(loc: string, trackId: string, attrs: string[]): string;
 export declare function decodeURIComponentNoThrow(uri: string): string;
 export {};

package/esm/util.js

@@ -1,8 +1,47 @@
+export const defaultAttributesToIndex = ['Name', 'ID', 'symbol'];
+export const defaultFeatureTypesToExclude = ['exon', 'CDS'];
+export const defaultIndexingPolicy = {
+    attributes: defaultAttributesToIndex,
+    exclude: defaultFeatureTypesToExclude,
+};
+/** The policy `track` is indexed under: its own settings, else the run's. */
+export function trackIndexingPolicy({ textSearching }, run) {
+    return {
+        attributes: textSearching?.indexingAttributes ?? run.attributes,
+        exclude: textSearching?.indexingFeatureTypesToExclude ?? run.exclude,
+        include: textSearching?.indexingFeatureTypesToInclude ?? run.include,
+    };
+}
+// single source of truth for what `jbrowse text-index` can index: which adapter
+// types, where each keeps its file location, and which line parser to use.
+// Everything else (the adapter dispatch, the CLI's supported() gate, the
+// filename guesser) derives from this table.
+export const indexableAdapters = {
+    Gff3Adapter: { locationKey: 'gffLocation', format: 'gff3' },
+    Gff3TabixAdapter: { locationKey: 'gffGzLocation', format: 'gff3' },
+    GtfAdapter: { locationKey: 'gtfLocation', format: 'gtf' },
+    GtfTabixAdapter: { locationKey: 'gtfGzLocation', format: 'gtf' },
+    VcfAdapter: { locationKey: 'vcfLocation', format: 'vcf' },
+    VcfTabixAdapter: { locationKey: 'vcfGzLocation', format: 'vcf' },
+};
+export function isSupportedIndexingAdapter(type = '') {
+    return type in indexableAdapters;
+}
+/**
+ * One line of ixIxx input: the record a search hit hands back —
+ * `["loc"|"trackId"|"attr"|...]`, fields URI-encoded so neither a comma nor a
+ * space can split it — then the words the hit is found by.
+ * TrixTextSearchAdapter reads the record back.
+ */
+export function trixLine(loc, trackId, attrs) {
+    const fields = [loc, trackId, ...attrs].map(f => `"${encodeURIComponent(f)}"`);
+    return `[${fields.join('|')}] ${[...new Set(attrs)].join(' ')}\n`;
+}
 export function decodeURIComponentNoThrow(uri) {
     try {
         return decodeURIComponent(uri);
     }
-    catch (e) {
+    catch {
         return uri;
     }
 }

package/esm/writeTrixIndex.d.ts

@@ -0,0 +1,19 @@
+import type { TrackIndexProgress } from './indexFiles.ts';
+import type { IndexingPolicy, Track } from './util.ts';
+/**
+ * Index `tracks` into `<outDir>/trix/<name>.ix`, `.ixx` and `_meta.json`.
+ *
+ * `onSort` fires once every file has been read, when the sort that writes the
+ * index begins — minutes on a large index, with no byte count to report.
+ */
+export declare function writeTrixIndex({ tracks, outDir, name, policy, assemblyNames, prefixSize, makeProgress, checkAbort, onSort, }: {
+    tracks: Track[];
+    outDir: string;
+    name: string;
+    policy: IndexingPolicy;
+    assemblyNames: string[];
+    prefixSize?: number;
+    makeProgress?: (trackId: string) => TrackIndexProgress;
+    checkAbort?: () => void;
+    onSort?: () => void;
+}): Promise<void>;

package/esm/writeTrixIndex.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { ixIxxStream } from 'ixixx';
+import { indexFiles } from './indexFiles.js';
+import { TRIX_DIR, trixFilePaths } from './trixPaths.js';
+import { trackIndexingPolicy } from './util.js';
+/**
+ * Index `tracks` into `<outDir>/trix/<name>.ix`, `.ixx` and `_meta.json`.
+ *
+ * `onSort` fires once every file has been read, when the sort that writes the
+ * index begins — minutes on a large index, with no byte count to report.
+ */
+export async function writeTrixIndex({ tracks, outDir, name, policy, assemblyNames, prefixSize, makeProgress, checkAbort, onSort, }) {
+    fs.mkdirSync(path.join(outDir, TRIX_DIR), { recursive: true });
+    const records = Readable.from(indexFiles({ tracks, policy, outDir, makeProgress, checkAbort }));
+    if (onSort) {
+        records.on('end', onSort);
+    }
+    const paths = trixFilePaths(outDir, name);
+    await ixIxxStream(records, paths.ix, paths.ixx, prefixSize);
+    checkAbort?.();
+    fs.writeFileSync(paths.meta, JSON.stringify({
+        dateCreated: new Date().toISOString(),
+        tracks: tracks.map(track => {
+            const { attributes, exclude, include } = trackIndexingPolicy(track, policy);
+            return {
+                trackId: track.trackId,
+                attributesIndexed: attributes,
+                excludedTypes: exclude,
+                includedTypes: include,
+                adapterConf: track.adapter,
+            };
+        }),
+        assemblyNames,
+    }, null, 2));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/text-indexing-core",
-  "version": "4.3.0",
+  "version": "5.0.0-beta.10",
   "type": "module",
   "description": "JBrowse 2 core text indexing routines for parsing GFF3 and VCF files",
   "keywords": [
@@ -22,21 +22,20 @@
   "main": "esm/index.js",
   "types": "esm/index.d.ts",
   "exports": {
-    ".": {
-      "types": "./esm/index.d.ts",
-      "import": "./esm/index.js"
-    }
+    ".": "./esm/index.js"
   },
   "files": [
     "esm"
   ],
-  "dependencies": {},
+  "dependencies": {
+    "ixixx": "^3.0.13"
+  },
   "publishConfig": {
     "access": "public"
   },
   "sideEffects": false,
   "scripts": {
-    "build:esm": "tsc -p tsconfig.build.esm.json",
+    "build:esm": "node ../../node_modules/typescript7/bin/tsc --build tsconfig.build.esm.json",
     "build": "pnpm build:esm",
     "test": "cd ../..; jest --passWithNoTests packages/text-indexing-core",
     "clean": "rimraf esm *.tsbuildinfo",