@jbrowse/text-indexing-core 5.0.0-beta.1 → 5.0.0-beta.10

package/esm/index.d.ts

@@ -1,9 +1,9 @@
-export { decodeURIComponentNoThrow, defaultAttributesToIndex, defaultFeatureTypesToExclude, indexableAdapters, isSupportedIndexingAdapter, } from './util.ts';
-export type { Gff3IndexerOptions, Gff3Adapter, Gff3TabixAdapter, GtfAdapter, IndexableFormat, IndexerOptions, LocalPathLocation, Track, UriLocation, VcfAdapter, VcfTabixAdapter, } from './util.ts';
-export { createReadlineInterface, generateMeta, getLocalOrRemoteStream, guessAdapterFromFileName, isURL, makeLocation, parseAttributes, } from './types/common.ts';
+export { decodeURIComponentNoThrow, defaultAttributesToIndex, defaultFeatureTypesToExclude, defaultIndexingPolicy, indexableAdapters, isSupportedIndexingAdapter, trackIndexingPolicy, } from './util.ts';
+export type { Gff3IndexerOptions, Gff3Adapter, Gff3TabixAdapter, GtfAdapter, IndexableFormat, IndexerOptions, IndexingPolicy, LocalPathLocation, Track, UriLocation, VcfAdapter, VcfTabixAdapter, } from './util.ts';
+export { createReadlineInterface, getLocalOrRemoteStream, guessAdapterFromFileName, isURL, makeLocation, parseAttributes, } from './types/common.ts';
 export { TRIX_DIR, sanitizeForFilename, trixFileNames, trixFilePaths, trixFileUris, } from './trixPaths.ts';
 export { indexGff3 } from './types/gff3Adapter.ts';
 export { indexGtf } from './types/gtfAdapter.ts';
 export { indexVcf } from './types/vcfAdapter.ts';
-export { indexFiles } from './indexFiles.ts';
 export type { TrackIndexProgress } from './indexFiles.ts';
+export { writeTrixIndex } from './writeTrixIndex.ts';

package/esm/index.js

@@ -1,7 +1,7 @@
-export { decodeURIComponentNoThrow, defaultAttributesToIndex, defaultFeatureTypesToExclude, indexableAdapters, isSupportedIndexingAdapter, } from './util.js';
-export { createReadlineInterface, generateMeta, getLocalOrRemoteStream, guessAdapterFromFileName, isURL, makeLocation, parseAttributes, } from './types/common.js';
+export { decodeURIComponentNoThrow, defaultAttributesToIndex, defaultFeatureTypesToExclude, defaultIndexingPolicy, indexableAdapters, isSupportedIndexingAdapter, trackIndexingPolicy, } from './util.js';
+export { createReadlineInterface, getLocalOrRemoteStream, guessAdapterFromFileName, isURL, makeLocation, parseAttributes, } from './types/common.js';
 export { TRIX_DIR, sanitizeForFilename, trixFileNames, trixFilePaths, trixFileUris, } from './trixPaths.js';
 export { indexGff3 } from './types/gff3Adapter.js';
 export { indexGtf } from './types/gtfAdapter.js';
 export { indexVcf } from './types/vcfAdapter.js';
-export { indexFiles } from './indexFiles.js';
+export { writeTrixIndex } from './writeTrixIndex.js';

package/esm/indexFiles.d.ts

@@ -1,15 +1,13 @@
-import type { Track } from './util.ts';
+import type { IndexingPolicy, Track } from './util.ts';
 export interface TrackIndexProgress {
     onStart?: (totalBytes: number) => void;
     onUpdate?: (processedBytes: number) => void;
     onDone?: () => void;
 }
-export declare function indexFiles({ tracks, attributesToIndex, outDir, featureTypesToExclude, featureTypesToInclude, makeProgress, checkAbort, }: {
+export declare function indexFiles({ tracks, policy, outDir, makeProgress, checkAbort, }: {
     tracks: Track[];
-    attributesToIndex: string[];
+    policy: IndexingPolicy;
     outDir: string;
-    featureTypesToExclude: string[];
-    featureTypesToInclude?: string[];
     makeProgress?: (trackId: string) => TrackIndexProgress;
     checkAbort?: () => void;
 }): AsyncGenerator<string, void, unknown>;

package/esm/indexFiles.js

@@ -1,7 +1,7 @@
 import { indexGff3 } from './types/gff3Adapter.js';
 import { indexGtf } from './types/gtfAdapter.js';
 import { indexVcf } from './types/vcfAdapter.js';
-import { indexableAdapters } from './util.js';
+import { indexableAdapters, trackIndexingPolicy } from './util.js';
 function getIndexingLocation(track, locationKey) {
     const { adapter } = track;
     // adapter[locationKey] is the canonical location object, e.g. gffGzLocation;
@@ -11,23 +11,21 @@ function getIndexingLocation(track, locationKey) {
     return loc?.locationType === 'LocalPathLocation' ? loc.localPath : loc?.uri;
 }
 const noop = () => { };
-// shared generator that streams index records for a set of tracks. Dispatches
-// to the gff3/vcf indexers based on adapter type; per-track attribute and
-// exclude/include overrides from textSearching take precedence over the
-// defaults.
-export async function* indexFiles({ tracks, attributesToIndex, outDir, featureTypesToExclude, featureTypesToInclude, makeProgress, checkAbort, }) {
+// Streams index records for a set of tracks, each read under its own policy
+export async function* indexFiles({ tracks, policy, outDir, makeProgress, checkAbort, }) {
     for (const track of tracks) {
         checkAbort?.();
-        const { adapter, textSearching, trackId } = track;
+        const { adapter, trackId } = track;
         const indexable = indexableAdapters[adapter?.type ?? ''];
         const inLocation = indexable
             ? getIndexingLocation(track, indexable.locationKey)
             : undefined;
         if (indexable && inLocation) {
             const progress = makeProgress?.(trackId);
+            const { attributes, exclude, include } = trackIndexingPolicy(track, policy);
             const common = {
                 config: track,
-                attributesToIndex: textSearching?.indexingAttributes ?? attributesToIndex,
+                attributesToIndex: attributes,
                 inLocation,
                 outDir,
                 onStart: progress?.onStart ?? noop,
@@ -37,10 +35,8 @@ export async function* indexFiles({ tracks, attributesToIndex, outDir, featureTy
             if (indexable.format === 'gff3') {
                 yield* indexGff3({
                     ...common,
-                    featureTypesToExclude: textSearching?.indexingFeatureTypesToExclude ??
-                        featureTypesToExclude,
-                    featureTypesToInclude: textSearching?.indexingFeatureTypesToInclude ??
-                        featureTypesToInclude,
+                    featureTypesToExclude: exclude,
+                    featureTypesToInclude: include,
                 });
             }
             else if (indexable.format === 'gtf') {

package/esm/trixPaths.d.ts

@@ -5,11 +5,8 @@ export declare function sanitizeForFilename(name: string): string;
  * The three files an index called `name` writes, sanitized once.
  *
  * Everything that writes a trix artifact or points a config at one derives its
- * paths from here. They used to sanitize independently and one of them didn't:
- * the CLI wrote `.ix`/`.ixx` under the raw name while `generateMeta` and
- * `createTrixAdapter` sanitized, so a trackId holding a `/` aimed the write at
- * a `trix/` subdirectory that does not exist, and the rest of the
- * Windows-invalid set wrote a file no search would look for.
+ * paths from here, so a trackId holding a `/` or another Windows-invalid
+ * character names the same file for the writer as for the search reading it.
  */
 export declare function trixFileNames(name: string): {
     ix: string;

package/esm/trixPaths.js

@@ -17,11 +17,8 @@ export function sanitizeForFilename(name) {
  * The three files an index called `name` writes, sanitized once.
  *
  * Everything that writes a trix artifact or points a config at one derives its
- * paths from here. They used to sanitize independently and one of them didn't:
- * the CLI wrote `.ix`/`.ixx` under the raw name while `generateMeta` and
- * `createTrixAdapter` sanitized, so a trackId holding a `/` aimed the write at
- * a `trix/` subdirectory that does not exist, and the rest of the
- * Windows-invalid set wrote a file no search would look for.
+ * paths from here, so a trackId holding a `/` or another Windows-invalid
+ * character names the same file for the writer as for the search reading it.
  */
 export function trixFileNames(name) {
     const safeName = sanitizeForFilename(name);

package/esm/types/common.d.ts

@@ -12,12 +12,3 @@ export declare function createReadlineInterface(stream: Readable, inLocation: st
 export declare function parseAttributes(infoString: string, decodeFunc: (s: string) => string): Record<string, string | undefined>;
 export declare function makeLocation(location: string, protocol: string): UriLocation | LocalPathLocation;
 export declare function guessAdapterFromFileName(filePath: string): Track;
-export declare function generateMeta({ configs, attributesToIndex, outDir, name, featureTypesToExclude, featureTypesToInclude, assemblyNames, }: {
-    configs: Track[];
-    attributesToIndex: string[];
-    outDir: string;
-    name: string;
-    featureTypesToExclude: string[];
-    featureTypesToInclude?: string[];
-    assemblyNames: string[];
-}): void;

package/esm/types/common.js

@@ -5,7 +5,6 @@ import readline from 'node:readline';
 import { Readable } from 'node:stream';
 import { fileURLToPath } from 'node:url';
 import { createGunzip } from 'node:zlib';
-import { trixFilePaths } from '../trixPaths.js';
 import { indexableAdapters } from '../util.js';
 export function isURL(fileName) {
     try {
@@ -175,21 +174,3 @@ export function guessAdapterFromFileName(filePath) {
         throw new Error(`Unsupported file type ${filePath}`);
     }
 }
-export function generateMeta({ configs, attributesToIndex, outDir, name, featureTypesToExclude, featureTypesToInclude, assemblyNames, }) {
-    fs.writeFileSync(trixFilePaths(outDir, name).meta, JSON.stringify({
-        dateCreated: new Date().toISOString(),
-        tracks: configs.map(config => {
-            const { trackId, textSearching, adapter } = config;
-            return {
-                trackId,
-                attributesIndexed: textSearching?.indexingAttributes ?? attributesToIndex,
-                excludedTypes: textSearching?.indexingFeatureTypesToExclude ??
-                    featureTypesToExclude,
-                includedTypes: textSearching?.indexingFeatureTypesToInclude ??
-                    featureTypesToInclude,
-                adapterConf: adapter,
-            };
-        }),
-        assemblyNames,
-    }, null, 2));
-}

package/esm/types/gff3Adapter.js

@@ -1,5 +1,19 @@
-import { decodeURIComponentNoThrow } from '../util.js';
+import { decodeURIComponentNoThrow, trixLine } from '../util.js';
 import { createReadlineInterface, getLocalOrRemoteStream, parseAttributes, } from './common.js';
+// GENCODE writes no `Name`: a gene row carries `gene_name`, a transcript row
+// `transcript_name`, and every row repeats its parents' names too. So `Name`
+// falls back to the name whose id is this row's own ID, which leaves a UTR or
+// codon row unnamed rather than answering a search for its transcript.
+function ownName(attrs) {
+    const id = attrs.ID;
+    return id === undefined
+        ? undefined
+        : id === attrs.transcript_id
+            ? attrs.transcript_name
+            : id === attrs.gene_id
+                ? attrs.gene_name
+                : undefined;
+}
 /**
  * Stream one GFF3 into trix records, one line per indexed feature.
  *
@@ -35,7 +49,6 @@ export async function* indexGff3({ config, attributesToIndex, inLocation, outDir
     const includeSet = featureTypesToInclude?.length
         ? new Set(featureTypesToInclude)
         : undefined;
-    const encodedTrackId = encodeURIComponent(trackId);
     for await (const line of rl) {
         checkAbort?.();
         if (!line.trim() || line.startsWith('#')) {
@@ -59,14 +72,11 @@ export async function* indexGff3({ config, attributesToIndex, inLocation, outDir
             (includeSet === undefined || includeSet.has(type))) {
             const col9attrs = parseAttributes(col9, decodeURIComponentNoThrow);
             const attrs = attributesToIndex
-                .map(attr => col9attrs[attr])
+                .map(attr => col9attrs[attr] ??
+                (attr === 'Name' ? ownName(col9attrs) : undefined))
                 .filter((f) => !!f);
             if (attrs.length > 0) {
-                const locStr = `${seq_id}:${start}..${end}`;
-                const encodedAttrs = attrs.map(a => `"${encodeURIComponent(a)}"`);
-                const record = `["${encodeURIComponent(locStr)}"|"${encodedTrackId}"|${encodedAttrs.join('|')}]`;
-                const uniqueAttrs = [...new Set(attrs)];
-                yield `${record} ${uniqueAttrs.join(' ')}\n`;
+                yield trixLine(`${seq_id}:${start}..${end}`, trackId, attrs);
             }
         }
     }

package/esm/types/gtfAdapter.js

@@ -1,3 +1,4 @@
+import { trixLine } from '../util.js';
 import { createReadlineInterface, getLocalOrRemoteStream } from './common.js';
 // GFF3-style attribute names have no GTF equivalent, so the shared defaults
 // (`Name`, `ID`, `symbol`) would index nothing. Each requested name also pulls
@@ -82,7 +83,6 @@ export async function* indexGtf({ config, attributesToIndex, inLocation, outDir,
         onUpdate,
     });
     const rl = createReadlineInterface(stream, inLocation);
-    const encodedTrackId = encodeURIComponent(trackId);
     const attributeNames = expandAttributeNames(attributesToIndex);
     const groups = new Map();
     function accumulate(key, refName, start, end, values) {
@@ -141,9 +141,6 @@ export async function* indexGtf({ config, attributesToIndex, inLocation, outDir,
         }
     }
     for (const { refName, start, end, attrs } of groups.values()) {
-        const locStr = `${refName}:${start}..${end}`;
-        const encodedAttrs = [...attrs].map(a => `"${encodeURIComponent(a)}"`);
-        const record = `["${encodeURIComponent(locStr)}"|"${encodedTrackId}"|${encodedAttrs.join('|')}]`;
-        yield `${record} ${[...attrs].join(' ')}\n`;
+        yield trixLine(`${refName}:${start}..${end}`, trackId, [...attrs]);
     }
 }

package/esm/types/vcfAdapter.js

@@ -1,4 +1,4 @@
-import { decodeURIComponentNoThrow } from '../util.js';
+import { decodeURIComponentNoThrow, trixLine } from '../util.js';
 import { createReadlineInterface, getLocalOrRemoteStream, parseAttributes, } from './common.js';
 export async function* indexVcf({ config, attributesToIndex, inLocation, outDir, onStart, onUpdate, checkAbort, }) {
     const { trackId } = config;
@@ -9,7 +9,6 @@ export async function* indexVcf({ config, attributesToIndex, inLocation, outDir,
         onUpdate,
     });
     const rl = createReadlineInterface(stream, inLocation);
-    const encodedTrackId = encodeURIComponent(trackId);
     for await (const line of rl) {
         checkAbort?.();
         if (!line.trim() || line.startsWith('#')) {
@@ -30,7 +29,6 @@ export async function* indexVcf({ config, attributesToIndex, inLocation, outDir,
         const endNum = Number(fields.END);
         const end = fields.END && Number.isFinite(endNum) ? endNum : Number(pos) + 1;
         const locStr = `${ref}:${pos}..${end}`;
-        const encodedLocStr = encodeURIComponent(locStr);
         const infoAttrs = attributesToIndex
             .map(attr => fields[attr])
             .filter((f) => !!f);
@@ -45,11 +43,7 @@ export async function* indexVcf({ config, attributesToIndex, inLocation, outDir,
             : [infoAttrs];
         for (const attrs of attrGroups) {
             if (attrs.length > 0) {
-                // attributes go in the record so the adapter can display them, and in
-                // the trailing word list so trix indexes them as searchable terms
-                const encodedAttrs = attrs.map(a => `"${encodeURIComponent(a)}"`);
-                const record = `["${encodedLocStr}"|"${encodedTrackId}"|${encodedAttrs.join('|')}]`;
-                yield `${record} ${[...new Set(attrs)].join(' ')}\n`;
+                yield trixLine(locStr, trackId, attrs);
             }
         }
     }

package/esm/util.d.ts

@@ -51,6 +51,20 @@ export interface Track {
 }
 export declare const defaultAttributesToIndex: string[];
 export declare const defaultFeatureTypesToExclude: string[];
+/** What an index reads out of a track's features. */
+export interface IndexingPolicy {
+    attributes: string[];
+    /** feature types to skip (GFF3 only) */
+    exclude: string[];
+    /**
+     * the only feature types to index, narrowed by `exclude` (GFF3 only). Absent
+     * or empty means no allow list, never "index nothing". See indexGff3
+     */
+    include?: string[];
+}
+export declare const defaultIndexingPolicy: IndexingPolicy;
+/** The policy `track` is indexed under: its own settings, else the run's. */
+export declare function trackIndexingPolicy({ textSearching }: Track, run: IndexingPolicy): IndexingPolicy;
 export type IndexableFormat = 'gff3' | 'gtf' | 'vcf';
 export declare const indexableAdapters: Record<string, {
     locationKey: string;
@@ -72,5 +86,12 @@ export interface Gff3IndexerOptions extends IndexerOptions {
     featureTypesToExclude: string[];
     featureTypesToInclude?: string[];
 }
+/**
+ * One line of ixIxx input: the record a search hit hands back —
+ * `["loc"|"trackId"|"attr"|...]`, fields URI-encoded so neither a comma nor a
+ * space can split it — then the words the hit is found by.
+ * TrixTextSearchAdapter reads the record back.
+ */
+export declare function trixLine(loc: string, trackId: string, attrs: string[]): string;
 export declare function decodeURIComponentNoThrow(uri: string): string;
 export {};

package/esm/util.js

@@ -1,5 +1,17 @@
 export const defaultAttributesToIndex = ['Name', 'ID', 'symbol'];
 export const defaultFeatureTypesToExclude = ['exon', 'CDS'];
+export const defaultIndexingPolicy = {
+    attributes: defaultAttributesToIndex,
+    exclude: defaultFeatureTypesToExclude,
+};
+/** The policy `track` is indexed under: its own settings, else the run's. */
+export function trackIndexingPolicy({ textSearching }, run) {
+    return {
+        attributes: textSearching?.indexingAttributes ?? run.attributes,
+        exclude: textSearching?.indexingFeatureTypesToExclude ?? run.exclude,
+        include: textSearching?.indexingFeatureTypesToInclude ?? run.include,
+    };
+}
 // single source of truth for what `jbrowse text-index` can index: which adapter
 // types, where each keeps its file location, and which line parser to use.
 // Everything else (the adapter dispatch, the CLI's supported() gate, the
@@ -15,6 +27,16 @@ export const indexableAdapters = {
 export function isSupportedIndexingAdapter(type = '') {
     return type in indexableAdapters;
 }
+/**
+ * One line of ixIxx input: the record a search hit hands back —
+ * `["loc"|"trackId"|"attr"|...]`, fields URI-encoded so neither a comma nor a
+ * space can split it — then the words the hit is found by.
+ * TrixTextSearchAdapter reads the record back.
+ */
+export function trixLine(loc, trackId, attrs) {
+    const fields = [loc, trackId, ...attrs].map(f => `"${encodeURIComponent(f)}"`);
+    return `[${fields.join('|')}] ${[...new Set(attrs)].join(' ')}\n`;
+}
 export function decodeURIComponentNoThrow(uri) {
     try {
         return decodeURIComponent(uri);

package/esm/writeTrixIndex.d.ts

@@ -0,0 +1,19 @@
+import type { TrackIndexProgress } from './indexFiles.ts';
+import type { IndexingPolicy, Track } from './util.ts';
+/**
+ * Index `tracks` into `<outDir>/trix/<name>.ix`, `.ixx` and `_meta.json`.
+ *
+ * `onSort` fires once every file has been read, when the sort that writes the
+ * index begins — minutes on a large index, with no byte count to report.
+ */
+export declare function writeTrixIndex({ tracks, outDir, name, policy, assemblyNames, prefixSize, makeProgress, checkAbort, onSort, }: {
+    tracks: Track[];
+    outDir: string;
+    name: string;
+    policy: IndexingPolicy;
+    assemblyNames: string[];
+    prefixSize?: number;
+    makeProgress?: (trackId: string) => TrackIndexProgress;
+    checkAbort?: () => void;
+    onSort?: () => void;
+}): Promise<void>;

package/esm/writeTrixIndex.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { ixIxxStream } from 'ixixx';
+import { indexFiles } from './indexFiles.js';
+import { TRIX_DIR, trixFilePaths } from './trixPaths.js';
+import { trackIndexingPolicy } from './util.js';
+/**
+ * Index `tracks` into `<outDir>/trix/<name>.ix`, `.ixx` and `_meta.json`.
+ *
+ * `onSort` fires once every file has been read, when the sort that writes the
+ * index begins — minutes on a large index, with no byte count to report.
+ */
+export async function writeTrixIndex({ tracks, outDir, name, policy, assemblyNames, prefixSize, makeProgress, checkAbort, onSort, }) {
+    fs.mkdirSync(path.join(outDir, TRIX_DIR), { recursive: true });
+    const records = Readable.from(indexFiles({ tracks, policy, outDir, makeProgress, checkAbort }));
+    if (onSort) {
+        records.on('end', onSort);
+    }
+    const paths = trixFilePaths(outDir, name);
+    await ixIxxStream(records, paths.ix, paths.ixx, prefixSize);
+    checkAbort?.();
+    fs.writeFileSync(paths.meta, JSON.stringify({
+        dateCreated: new Date().toISOString(),
+        tracks: tracks.map(track => {
+            const { attributes, exclude, include } = trackIndexingPolicy(track, policy);
+            return {
+                trackId: track.trackId,
+                attributesIndexed: attributes,
+                excludedTypes: exclude,
+                includedTypes: include,
+                adapterConf: track.adapter,
+            };
+        }),
+        assemblyNames,
+    }, null, 2));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/text-indexing-core",
-  "version": "5.0.0-beta.1",
+  "version": "5.0.0-beta.10",
   "type": "module",
   "description": "JBrowse 2 core text indexing routines for parsing GFF3 and VCF files",
   "keywords": [
@@ -27,7 +27,9 @@
   "files": [
     "esm"
   ],
-  "dependencies": {},
+  "dependencies": {
+    "ixixx": "^3.0.13"
+  },
   "publishConfig": {
     "access": "public"
   },