@jbrowse/text-indexing 2.6.1

package/dist/types/vcfAdapter.js

@@ -0,0 +1,67 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.indexVcf = void 0;
+const zlib_1 = require("zlib");
+const readline_1 = __importDefault(require("readline"));
+const common_1 = require("./common");
+const util_1 = require("@jbrowse/core/util");
+async function* indexVcf(config, attributesToIndex, inLocation, outLocation, typesToExclude, quiet, statusCallback, signal) {
+    const { trackId } = config;
+    let receivedBytes = 0;
+    const { totalBytes, stream } = await (0, common_1.getLocalOrRemoteStream)(inLocation, outLocation);
+    stream.on('data', chunk => {
+        receivedBytes += chunk.length;
+        const progress = Math.round((receivedBytes / totalBytes) * 100);
+        statusCallback(`${progress}`);
+    });
+    const gzStream = inLocation.match(/.b?gz$/)
+        ? stream.pipe((0, zlib_1.createGunzip)())
+        : stream;
+    const rl = readline_1.default.createInterface({
+        input: gzStream,
+    });
+    for await (const line of rl) {
+        if (line.startsWith('#')) {
+            continue;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const [ref, pos, id, _ref, _alt, _qual, _filter, info] = line.split('\t');
+        // turns vcf info attrs into a map, and converts the arrays into space
+        // separated strings
+        const fields = Object.fromEntries(info
+            .split(';')
+            .map(f => f.trim())
+            .filter(f => !!f)
+            .map(f => f.split('='))
+            .map(([key, val]) => [
+            key.trim(),
+            val ? decodeURIComponent(val).trim().split(',').join(' ') : undefined,
+        ]));
+        const end = fields.END;
+        const locStr = `${ref}:${pos}..${end || +pos + 1}`;
+        if (id === '.') {
+            continue;
+        }
+        const infoAttrs = attributesToIndex
+            .map(attr => fields[attr])
+            .filter((f) => !!f);
+        const ids = id.split(',');
+        for (let i = 0; i < ids.length; i++) {
+            const id = ids[i];
+            const attrs = [id];
+            const record = JSON.stringify([
+                encodeURIComponent(locStr),
+                encodeURIComponent(trackId),
+                encodeURIComponent(id || ''),
+                ...infoAttrs.map(a => encodeURIComponent(a || '')),
+            ]).replaceAll(',', '|');
+            // Check abort signal
+            (0, util_1.checkAbortSignal)(signal);
+            yield `${record} ${[...new Set(attrs)].join(' ')}\n`;
+        }
+    }
+}
+exports.indexVcf = indexVcf;

package/dist/util.d.ts

@@ -0,0 +1,118 @@
+export interface UriLocation {
+    uri: string;
+    locationType: 'UriLocation';
+}
+export interface LocalPathLocation {
+    localPath: string;
+    locationType: 'LocalPathLocation';
+}
+export interface IndexedFastaAdapter {
+    type: 'IndexedFastaAdapter';
+    fastaLocation: UriLocation;
+    faiLocation: UriLocation;
+}
+export interface BgzipFastaAdapter {
+    type: 'BgzipFastaAdapter';
+    fastaLocation: UriLocation;
+    faiLocation: UriLocation;
+    gziLocation: UriLocation;
+}
+export interface TwoBitAdapter {
+    type: 'TwoBitAdapter';
+    twoBitLocation: UriLocation;
+}
+export interface ChromeSizesAdapter {
+    type: 'ChromSizesAdapter';
+    chromSizesLocation: UriLocation;
+}
+export interface CustomSequenceAdapter {
+    type: string;
+}
+export interface RefNameAliasAdapter {
+    type: 'RefNameAliasAdapter';
+    location: UriLocation;
+}
+export interface CustomRefNameAliasAdapter {
+    type: string;
+}
+export interface Assembly {
+    displayName?: string;
+    name: string;
+    aliases?: string[];
+    sequence: Sequence;
+    refNameAliases?: {
+        adapter: RefNameAliasAdapter | CustomRefNameAliasAdapter;
+    };
+    refNameColors?: string[];
+}
+export interface Sequence {
+    type: 'ReferenceSequenceTrack';
+    trackId: string;
+    adapter: IndexedFastaAdapter | BgzipFastaAdapter | TwoBitAdapter | ChromeSizesAdapter | CustomSequenceAdapter;
+}
+export interface Gff3TabixAdapter {
+    type: 'Gff3TabixAdapter';
+    gffGzLocation: UriLocation | LocalPathLocation;
+}
+export interface Gff3Adapter {
+    type: 'Gff3Adapter';
+    gffLocation: UriLocation | LocalPathLocation;
+}
+export interface GtfAdapter {
+    type: 'GtfAdapter';
+    gtfLocation: UriLocation | LocalPathLocation;
+}
+export interface VcfTabixAdapter {
+    type: 'VcfTabixAdapter';
+    vcfGzLocation: UriLocation | LocalPathLocation;
+}
+export interface VcfAdapter {
+    type: 'VcfAdapter';
+    vcfLocation: UriLocation | LocalPathLocation;
+}
+export interface Track {
+    [key: string]: any;
+}
+export interface TextSearching {
+    indexingFeatureTypesToExclude?: string[];
+    indexingAttributes?: string[];
+    textSearchAdapter: TrixTextSearchAdapter;
+}
+export interface TrixTextSearchAdapter {
+    type: string;
+    textSearchAdapterId: string;
+    ixFilePath: UriLocation;
+    ixxFilePath: UriLocation;
+    metaFilePath: UriLocation;
+    assemblyNames: string[];
+}
+export interface Config {
+    assemblies?: Assembly[];
+    assembly?: Assembly;
+    configuration?: {};
+    aggregateTextSearchAdapters?: TrixTextSearchAdapter[];
+    connections?: unknown[];
+    defaultSession?: {};
+    tracks?: Track[];
+}
+export type indexType = 'aggregate' | 'perTrack';
+export declare function supportedIndexingAdapters(type: string): boolean;
+export declare function createTextSearchConf(name: string, trackIds: string[], assemblyNames: string[], locationPath: string): {
+    type: string;
+    textSearchAdapterId: string;
+    ixFilePath: {
+        localPath: string;
+        locationType: string;
+    };
+    ixxFilePath: {
+        localPath: string;
+        locationType: string;
+    };
+    metaFilePath: {
+        localPath: string;
+        locationType: string;
+    };
+    tracks: string[];
+    assemblyNames: string[];
+};
+export declare function findTrackConfigsToIndex(tracks: Track[], trackIds: string[], assemblyName?: string): Track[];

package/dist/util.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findTrackConfigsToIndex = exports.createTextSearchConf = exports.supportedIndexingAdapters = void 0;
+// supported adapter types by text indexer
+//  ensure that this matches the method found in @jbrowse/core/util
+function supportedIndexingAdapters(type) {
+    return [
+        'Gff3TabixAdapter',
+        'VcfTabixAdapter',
+        'Gff3Adapter',
+        'VcfAdapter',
+    ].includes(type);
+}
+exports.supportedIndexingAdapters = supportedIndexingAdapters;
+function createTextSearchConf(name, trackIds, assemblyNames, locationPath) {
+    // const locationPath = self.sessionPath.substring(
+    //   0,
+    //   self.sessionPath.lastIndexOf('/'),
+    // )
+    return {
+        type: 'TrixTextSearchAdapter',
+        textSearchAdapterId: name,
+        ixFilePath: {
+            localPath: locationPath + `/trix/${name}.ix`,
+            locationType: 'LocalPathLocation',
+        },
+        ixxFilePath: {
+            localPath: locationPath + `/trix/${name}.ixx`,
+            locationType: 'LocalPathLocation',
+        },
+        metaFilePath: {
+            localPath: locationPath + `/trix/${name}.json`,
+            locationType: 'LocalPathLocation',
+        },
+        tracks: trackIds,
+        assemblyNames,
+    };
+}
+exports.createTextSearchConf = createTextSearchConf;
+function findTrackConfigsToIndex(tracks, trackIds, assemblyName) {
+    return trackIds
+        .map(trackId => {
+        const currentTrack = tracks.find(t => trackId === t.trackId);
+        if (!currentTrack) {
+            throw new Error(`Track not found in session for trackId ${trackId}`);
+        }
+        return currentTrack;
+    })
+        .filter(track => assemblyName ? track.assemblyNames.includes(assemblyName) : true)
+        .filter(track => supportedIndexingAdapters(track.adapter.type));
+}
+exports.findTrackConfigsToIndex = findTrackConfigsToIndex;

package/esm/TextIndexing.d.ts

@@ -0,0 +1,11 @@
+import { Track, indexType } from './util';
+export declare function indexTracks(args: {
+    tracks: Track[];
+    outLocation?: string;
+    signal?: AbortSignal;
+    attributes?: string[];
+    assemblies?: string[];
+    exclude?: string[];
+    indexType?: indexType;
+    statusCallback: (message: string) => void;
+}): Promise<never[]>;

package/esm/TextIndexing.js

@@ -0,0 +1,128 @@
+import fs from 'fs';
+import path from 'path';
+import { Readable } from 'stream';
+import { indexGff3 } from './types/gff3Adapter';
+import { indexVcf } from './types/vcfAdapter';
+import { generateMeta } from './types/common';
+import { ixIxxStream } from 'ixixx';
+import { supportedIndexingAdapters } from './util';
+import { checkAbortSignal } from '@jbrowse/core/util';
+export async function indexTracks(args) {
+    const { tracks, outLocation, attributes, exclude, assemblies, indexType, statusCallback, signal, } = args;
+    const idxType = indexType || 'perTrack';
+    checkAbortSignal(signal);
+    await (idxType === 'perTrack'
+        ? perTrackIndex(tracks, statusCallback, outLocation, attributes, exclude, signal)
+        : aggregateIndex(tracks, statusCallback, outLocation, attributes, assemblies, exclude, signal));
+    checkAbortSignal(signal);
+    return [];
+}
+async function perTrackIndex(tracks, statusCallback, outLocation, attributes, exclude, signal) {
+    const outFlag = outLocation || '.';
+    const isDir = fs.lstatSync(outFlag).isDirectory();
+    const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag;
+    const outDir = path.dirname(confFilePath);
+    const trixDir = path.join(outDir, 'trix');
+    if (!fs.existsSync(trixDir)) {
+        fs.mkdirSync(trixDir);
+    }
+    // default settings
+    const attrs = attributes || ['Name', 'ID'];
+    const excludeTypes = exclude || ['exon', 'CDS'];
+    const force = true;
+    const supportedTracks = tracks.filter(track => { var _a; return supportedIndexingAdapters((_a = track.adapter) === null || _a === void 0 ? void 0 : _a.type); });
+    for (const trackConfig of supportedTracks) {
+        const { textSearching, trackId, assemblyNames } = trackConfig;
+        const id = `${trackId}-index`;
+        if ((textSearching === null || textSearching === void 0 ? void 0 : textSearching.textSearchAdapter) && !force) {
+            console.warn(`Note: ${trackId} has already been indexed with this configuration, use --force to overwrite this track. Skipping for now`);
+            continue;
+        }
+        await indexDriver([trackConfig], outDir, attrs, id, true, excludeTypes, assemblyNames, statusCallback, signal);
+    }
+}
+async function aggregateIndex(tracks, statusCallback, outLocation, attributes, assemblies, exclude, signal) {
+    const outFlag = outLocation || '.';
+    const isDir = fs.lstatSync(outFlag).isDirectory();
+    const confFilePath = isDir ? path.join(outFlag, 'config.json') : outFlag;
+    const outDir = path.dirname(confFilePath);
+    const trixDir = path.join(outDir, 'trix');
+    if (!fs.existsSync(trixDir)) {
+        fs.mkdirSync(trixDir);
+    }
+    if (!assemblies) {
+        throw new Error('No assemblies passed. Assmeblies required for aggregate indexes');
+    }
+    for (const asm of assemblies) {
+        // console.log('Indexing assembly ' + asm + '...')
+        const id = asm + '-index';
+        // default settings
+        const attrs = attributes || ['Name', 'ID'];
+        const excludeTypes = exclude || ['exon', 'CDS'];
+        // const force = true
+        const quiet = true;
+        // supported tracks for given assembly
+        const supportedTracks = tracks
+            .filter(track => { var _a; return supportedIndexingAdapters((_a = track.adapter) === null || _a === void 0 ? void 0 : _a.type); })
+            .filter(track => (asm ? track.assemblyNames.includes(asm) : true));
+        await indexDriver(supportedTracks, outDir, attrs, id, quiet, excludeTypes, [asm], statusCallback, signal);
+    }
+}
+async function indexDriver(tracks, idxLocation, attributes, name, quiet, exclude, assemblyNames, statusCallback, signal) {
+    const readable = Readable.from(indexFiles(tracks, attributes, idxLocation, quiet, exclude, statusCallback, signal));
+    statusCallback('Indexing files.');
+    try {
+        const ixIxxStream = await runIxIxx(readable, idxLocation, name);
+        checkAbortSignal(signal);
+        await generateMeta({
+            configs: tracks,
+            attributes,
+            outDir: idxLocation,
+            name,
+            exclude,
+            assemblyNames,
+        });
+        checkAbortSignal(signal);
+        return ixIxxStream;
+    }
+    catch (e) {
+        throw e;
+    }
+}
+async function* indexFiles(tracks, attributes, outLocation, quiet, typesToExclude, statusCallback, signal) {
+    for (const track of tracks) {
+        const { adapter, textSearching } = track;
+        const { type } = adapter;
+        const { indexingFeatureTypesToExclude: types = typesToExclude, indexingAttributes: attrs = attributes, } = textSearching || {};
+        // currently only supporting GFF3Tabix and VCFTabix
+        switch (type) {
+            case 'Gff3TabixAdapter': {
+                yield* indexGff3(track, attrs, getLoc('gffGzLocation', track), outLocation, types, quiet, statusCallback, signal);
+                break;
+            }
+            case 'Gff3Adapter': {
+                yield* indexGff3(track, attrs, getLoc('gffLocation', track), outLocation, types, quiet, statusCallback, signal);
+                break;
+            }
+            case 'VcfTabixAdapter': {
+                yield* indexVcf(track, attrs, getLoc('vcfGzLocation', track), outLocation, types, quiet, statusCallback, signal);
+                break;
+            }
+            case 'VcfAdapter': {
+                yield* indexVcf(track, attrs, getLoc('vcfLocation', track), outLocation, types, quiet, statusCallback, signal);
+                break;
+            }
+            // No default
+        }
+    }
+    return;
+}
+function getLoc(attr, config) {
+    const elt = config.adapter[attr];
+    return elt.uri || elt.localPath;
+}
+function runIxIxx(readStream, idxLocation, name) {
+    const ixFilename = path.join(idxLocation, 'trix', `${name}.ix`);
+    const ixxFilename = path.join(idxLocation, 'trix', `${name}.ixx`);
+    return ixIxxStream(readStream, ixFilename, ixxFilename);
+}

package/esm/index.d.ts

@@ -0,0 +1,2 @@
+export * from './util';
+export { indexTracks } from './TextIndexing';

package/esm/index.js

@@ -0,0 +1,2 @@
+export * from './util';
+export { indexTracks } from './TextIndexing';

package/esm/types/common.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import fs from 'fs';
+import { LocalPathLocation, UriLocation, Track } from '../util';
+export declare function createRemoteStream(urlIn: string): Promise<import("node-fetch").Response>;
+export declare function isURL(FileName: string): boolean;
+export declare function getLocalOrRemoteStream(uri: string, out: string): Promise<{
+    totalBytes: number;
+    stream: NodeJS.ReadableStream | fs.ReadStream;
+}>;
+export declare function makeLocation(location: string, protocol: string): UriLocation | LocalPathLocation;
+export declare function guessAdapterFromFileName(filePath: string): Track;
+/**
+ * Generates metadata of index given a filename (trackId or assembly)
+ * @param name -  assembly name or trackId
+ * @param attributes -  attributes indexed
+ * @param include -  feature types included from index
+ * @param exclude -  feature types excluded from index
+ * @param configs - list of track
+ */
+export declare function generateMeta({ configs, attributes, outDir, name, exclude, assemblyNames, }: {
+    configs: Track[];
+    attributes: string[];
+    outDir: string;
+    name: string;
+    exclude: string[];
+    assemblyNames: string[];
+}): Promise<void>;

package/esm/types/common.js

@@ -0,0 +1,142 @@
+import fs from 'fs';
+import fetch from 'node-fetch';
+import path from 'path';
+// Method for handing off the parsing of a gff3 file URL.
+// Calls the proper parser depending on if it is gzipped or not.
+// Returns a @gmod/gff stream.
+export async function createRemoteStream(urlIn) {
+    const response = await fetch(urlIn);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch ${urlIn} status ${response.status} ${response.statusText}`);
+    }
+    return response;
+}
+// Checks if the passed in string is a valid URL.
+// Returns a boolean.
+export function isURL(FileName) {
+    let url;
+    try {
+        url = new URL(FileName);
+    }
+    catch (_) {
+        return false;
+    }
+    return url.protocol === 'http:' || url.protocol === 'https:';
+}
+export async function getLocalOrRemoteStream(uri, out) {
+    var _a;
+    let stream;
+    let totalBytes = 0;
+    if (isURL(uri)) {
+        const result = await createRemoteStream(uri);
+        totalBytes = +(((_a = result.headers) === null || _a === void 0 ? void 0 : _a.get('Content-Length')) || 0);
+        stream = result.body;
+    }
+    else {
+        const filename = path.isAbsolute(uri) ? uri : path.join(out, uri);
+        totalBytes = fs.statSync(filename).size;
+        stream = fs.createReadStream(filename);
+    }
+    return { totalBytes, stream };
+}
+export function makeLocation(location, protocol) {
+    if (protocol === 'uri') {
+        return { uri: location, locationType: 'UriLocation' };
+    }
+    if (protocol === 'localPath') {
+        return {
+            localPath: path.resolve(location),
+            locationType: 'LocalPathLocation',
+        };
+    }
+    throw new Error(`invalid protocol ${protocol}`);
+}
+export function guessAdapterFromFileName(filePath) {
+    // const uri = isURL(filePath) ? filePath : path.resolve(filePath)
+    const protocol = isURL(filePath) ? 'uri' : 'localPath';
+    const name = path.basename(filePath);
+    if (/\.vcf\.b?gz$/i.test(filePath)) {
+        return {
+            trackId: name,
+            name: name,
+            assemblyNames: [],
+            adapter: {
+                type: 'VcfTabixAdapter',
+                vcfGzLocation: makeLocation(filePath, protocol),
+            },
+        };
+    }
+    else if (/\.gff3?\.b?gz$/i.test(filePath)) {
+        return {
+            trackId: name,
+            name,
+            assemblyNames: [],
+            adapter: {
+                type: 'Gff3TabixAdapter',
+                gffGzLocation: makeLocation(filePath, protocol),
+            },
+        };
+    }
+    else if (/\.gtf?$/i.test(filePath)) {
+        return {
+            trackId: name,
+            name,
+            assemblyNames: [],
+            adapter: {
+                type: 'GtfAdapter',
+                gtfLocation: makeLocation(filePath, protocol),
+            },
+        };
+    }
+    else if (/\.vcf$/i.test(filePath)) {
+        return {
+            trackId: name,
+            name,
+            assemblyNames: [],
+            adapter: {
+                type: 'VcfAdapter',
+                vcfLocation: makeLocation(filePath, protocol),
+            },
+        };
+    }
+    else if (/\.gff3?$/i.test(filePath)) {
+        return {
+            trackId: name,
+            name,
+            assemblyNames: [],
+            adapter: {
+                type: 'Gff3Adapter',
+                gffLocation: makeLocation(filePath, protocol),
+            },
+        };
+    }
+    else {
+        throw new Error(`Unsupported file type ${filePath}`);
+    }
+}
+/**
+ * Generates metadata of index given a filename (trackId or assembly)
+ * @param name -  assembly name or trackId
+ * @param attributes -  attributes indexed
+ * @param include -  feature types included from index
+ * @param exclude -  feature types excluded from index
+ * @param configs - list of track
+ */
+export async function generateMeta({ configs, attributes, outDir, name, exclude, assemblyNames, }) {
+    const tracks = configs.map(config => {
+        const { trackId, textSearching, adapter } = config;
+        const includeExclude = (textSearching === null || textSearching === void 0 ? void 0 : textSearching.indexingFeatureTypesToExclude) || exclude;
+        const metaAttrs = (textSearching === null || textSearching === void 0 ? void 0 : textSearching.indexingAttributes) || attributes;
+        return {
+            trackId: trackId,
+            attributesIndexed: metaAttrs,
+            excludedTypes: includeExclude,
+            adapterConf: adapter,
+        };
+    });
+    fs.writeFileSync(path.join(outDir, 'trix', `${name}_meta.json`), JSON.stringify({
+        dateCreated: new Date().toISOString(),
+        tracks,
+        assemblyNames,
+    }, null, 2));
+}

package/esm/types/gff3Adapter.d.ts

@@ -0,0 +1,2 @@
+import { Track } from '../util';
+export declare function indexGff3(config: Track, attributes: string[], inLocation: string, outLocation: string, typesToExclude: string[], quiet: boolean, statusCallback: (message: string) => void, signal?: AbortSignal): AsyncGenerator<string, void, unknown>;

package/esm/types/gff3Adapter.js

@@ -0,0 +1,55 @@
+import { createGunzip } from 'zlib';
+import readline from 'readline';
+import { getLocalOrRemoteStream } from './common';
+import { checkAbortSignal } from '@jbrowse/core/util';
+export async function* indexGff3(config, attributes, inLocation, outLocation, typesToExclude, quiet, statusCallback, signal) {
+    const { trackId } = config;
+    let receivedBytes = 0;
+    const { totalBytes, stream } = await getLocalOrRemoteStream(inLocation, outLocation);
+    stream.on('data', chunk => {
+        receivedBytes += chunk.length;
+        // send an update?
+        const progress = Math.round((receivedBytes / totalBytes) * 100);
+        statusCallback(`${progress}`);
+    });
+    const rl = readline.createInterface({
+        input: inLocation.match(/.b?gz$/) ? stream.pipe(createGunzip()) : stream,
+    });
+    for await (const line of rl) {
+        if (line.startsWith('#')) {
+            continue;
+        }
+        else if (line.startsWith('>')) {
+            break;
+        }
+        const [seq_id, , type, start, end, , , , col9] = line.split('\t');
+        const locStr = `${seq_id}:${start}..${end}`;
+        if (!typesToExclude.includes(type)) {
+            // turns gff3 attrs into a map, and converts the arrays into space
+            // separated strings
+            const col9attrs = Object.fromEntries(col9
+                .split(';')
+                .map(f => f.trim())
+                .filter(f => !!f)
+                .map(f => f.split('='))
+                .map(([key, val]) => [
+                key.trim(),
+                decodeURIComponent(val).trim().split(',').join(' '),
+            ]));
+            const attrs = attributes
+                .map(attr => col9attrs[attr])
+                .filter((f) => !!f);
+            if (attrs.length) {
+                const record = JSON.stringify([
+                    encodeURIComponent(locStr),
+                    encodeURIComponent(trackId),
+                    ...attrs.map(a => encodeURIComponent(a)),
+                ]).replaceAll(',', '|');
+                // Check abort signal
+                checkAbortSignal(signal);
+                yield `${record} ${[...new Set(attrs)].join(' ')}\n`;
+            }
+        }
+    }
+    // console.log('done')
+}

package/esm/types/vcfAdapter.d.ts

@@ -0,0 +1,2 @@
+import { Track } from '../util';
+export declare function indexVcf(config: Track, attributesToIndex: string[], inLocation: string, outLocation: string, typesToExclude: string[], quiet: boolean, statusCallback: (message: string) => void, signal?: AbortSignal): AsyncGenerator<string, void, unknown>;

package/esm/types/vcfAdapter.js

@@ -0,0 +1,60 @@
+import { createGunzip } from 'zlib';
+import readline from 'readline';
+import { getLocalOrRemoteStream } from './common';
+import { checkAbortSignal } from '@jbrowse/core/util';
+export async function* indexVcf(config, attributesToIndex, inLocation, outLocation, typesToExclude, quiet, statusCallback, signal) {
+    const { trackId } = config;
+    let receivedBytes = 0;
+    const { totalBytes, stream } = await getLocalOrRemoteStream(inLocation, outLocation);
+    stream.on('data', chunk => {
+        receivedBytes += chunk.length;
+        const progress = Math.round((receivedBytes / totalBytes) * 100);
+        statusCallback(`${progress}`);
+    });
+    const gzStream = inLocation.match(/.b?gz$/)
+        ? stream.pipe(createGunzip())
+        : stream;
+    const rl = readline.createInterface({
+        input: gzStream,
+    });
+    for await (const line of rl) {
+        if (line.startsWith('#')) {
+            continue;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const [ref, pos, id, _ref, _alt, _qual, _filter, info] = line.split('\t');
+        // turns vcf info attrs into a map, and converts the arrays into space
+        // separated strings
+        const fields = Object.fromEntries(info
+            .split(';')
+            .map(f => f.trim())
+            .filter(f => !!f)
+            .map(f => f.split('='))
+            .map(([key, val]) => [
+            key.trim(),
+            val ? decodeURIComponent(val).trim().split(',').join(' ') : undefined,
+        ]));
+        const end = fields.END;
+        const locStr = `${ref}:${pos}..${end || +pos + 1}`;
+        if (id === '.') {
+            continue;
+        }
+        const infoAttrs = attributesToIndex
+            .map(attr => fields[attr])
+            .filter((f) => !!f);
+        const ids = id.split(',');
+        for (let i = 0; i < ids.length; i++) {
+            const id = ids[i];
+            const attrs = [id];
+            const record = JSON.stringify([
+                encodeURIComponent(locStr),
+                encodeURIComponent(trackId),
+                encodeURIComponent(id || ''),
+                ...infoAttrs.map(a => encodeURIComponent(a || '')),
+            ]).replaceAll(',', '|');
+            // Check abort signal
+            checkAbortSignal(signal);
+            yield `${record} ${[...new Set(attrs)].join(' ')}\n`;
+        }
+    }
+}