@jbrowse/plugin-gtf 2.13.1 → 2.14.0

package/dist/GtfAdapter/GtfAdapter.d.ts

@@ -2,16 +2,18 @@ import { BaseFeatureDataAdapter, BaseOptions } from '@jbrowse/core/data_adapters
 import { NoAssemblyRegion } from '@jbrowse/core/util/types';
 import IntervalTree from '@flatten-js/interval-tree';
 import { Feature } from '@jbrowse/core/util';
+type StatusCallback = (arg: string) => void;
 export default class GtfAdapter extends BaseFeatureDataAdapter {
-    protected gtfFeatures?: Promise<{
-        feats: Record<string, string[]>;
+    calculatedIntervalTreeMap: Record<string, IntervalTree>;
+    gtfFeatures?: Promise<{
+        header: string;
+        intervalTreeMap: Record<string, (sc?: StatusCallback) => IntervalTree>;
     }>;
-    protected intervalTrees: Record<string, Promise<IntervalTree | undefined> | undefined>;
     private loadDataP;
     private loadData;
     getRefNames(opts?: BaseOptions): Promise<string[]>;
-    private loadFeatureIntervalTreeHelper;
-    private loadFeatureIntervalTree;
+    getHeader(opts?: BaseOptions): Promise<string>;
     getFeatures(query: NoAssemblyRegion, opts?: BaseOptions): import("rxjs").Observable<Feature>;
     freeResources(): void;
 }
+export {};

package/dist/GtfAdapter/GtfAdapter.js

@@ -12,43 +12,79 @@ const bgzf_filehandle_1 = require("@gmod/bgzf-filehandle");
 const gtf_1 = __importDefault(require("@gmod/gtf"));
 // locals
 const util_2 = require("../util");
-function isGzip(buf) {
-    return buf[0] === 31 && buf[1] === 139 && buf[2] === 8;
-}
+const decoder = typeof TextDecoder !== 'undefined' ? new TextDecoder('utf8') : undefined;
 class GtfAdapter extends BaseAdapter_1.BaseFeatureDataAdapter {
     constructor() {
         super(...arguments);
-        this.intervalTrees = {};
+        this.calculatedIntervalTreeMap = {};
     }
-    async loadDataP(opts = {}) {
-        const gtfLoc = this.getConf('gtfLocation');
-        const buffer = await (0, io_1.openLocation)(gtfLoc, this.pluginManager).readFile(opts);
-        const buf = isGzip(buffer) ? await (0, bgzf_filehandle_1.unzip)(buffer) : buffer;
-        // 512MB  max chrome string length is 512MB
-        if (buf.length > 536870888) {
-            throw new Error('Data exceeds maximum string length (512MB)');
-        }
-        const data = new TextDecoder('utf8', { fatal: true }).decode(buf);
-        const lines = data
-            .split(/\n|\r\n|\r/)
-            .filter(f => !!f && !f.startsWith('#'));
-        const feats = {};
-        for (const line of lines) {
-            if (line.startsWith('#')) {
-                continue;
+    async loadDataP(opts) {
+        const { statusCallback = () => { } } = opts || {};
+        const buf = (await (0, io_1.openLocation)(this.getConf('gtfLocation'), this.pluginManager).readFile(opts));
+        const buffer = (0, util_1.isGzip)(buf)
+            ? await (0, util_1.updateStatus)('Unzipping', statusCallback, () => (0, bgzf_filehandle_1.unzip)(buf))
+            : buf;
+        const headerLines = [];
+        const featureMap = {};
+        let blockStart = 0;
+        let i = 0;
+        while (blockStart < buffer.length) {
+            const n = buffer.indexOf('\n', blockStart);
+            // could be a non-newline ended file, so slice to end of file if n===-1
+            const b = n === -1 ? buffer.slice(blockStart) : buffer.slice(blockStart, n);
+            const line = ((decoder === null || decoder === void 0 ? void 0 : decoder.decode(b)) || b.toString()).trim();
+            if (line) {
+                if (line.startsWith('#')) {
+                    headerLines.push(line);
+                }
+                else if (line.startsWith('>')) {
+                    break;
+                }
+                else {
+                    const ret = line.indexOf('\t');
+                    const refName = line.slice(0, ret);
+                    if (!featureMap[refName]) {
+                        featureMap[refName] = '';
+                    }
+                    featureMap[refName] += `${line}\n`;
+                }
             }
-            const tab = line.indexOf('\t');
-            const refName = line.slice(0, tab);
-            if (!feats[refName]) {
-                feats[refName] = [];
+            if (i++ % 10000 === 0) {
+                statusCallback(`Loading ${Math.floor(blockStart / 1000000).toLocaleString('en-US')}/${Math.floor(buffer.length / 1000000).toLocaleString('en-US')} MB`);
             }
-            feats[refName].push(line);
+            blockStart = n + 1;
         }
-        return { feats };
+        const intervalTreeMap = Object.fromEntries(Object.entries(featureMap).map(([refName, lines]) => [
+            refName,
+            (sc) => {
+                if (!this.calculatedIntervalTreeMap[refName]) {
+                    sc === null || sc === void 0 ? void 0 : sc('Parsing GTF data');
+                    const intervalTree = new interval_tree_1.default();
+                    gtf_1.default.parseStringSync(lines, {
+                        parseFeatures: true,
+                        parseComments: false,
+                        parseDirectives: false,
+                        parseSequences: false,
+                    })
+                        .flat()
+                        .map((f, i) => new util_1.SimpleFeature({
+                        data: (0, util_2.featureData)(f),
+                        id: `${this.id}-${refName}-${i}`,
+                    }))
+                        .forEach(obj => intervalTree.insert([obj.get('start'), obj.get('end')], obj));
+                    this.calculatedIntervalTreeMap[refName] = intervalTree;
+                }
+                return this.calculatedIntervalTreeMap[refName];
+            },
+        ]));
+        return {
+            header: headerLines.join('\n'),
+            intervalTreeMap,
+        };
     }
     async loadData(opts = {}) {
         if (!this.gtfFeatures) {
-            this.gtfFeatures = this.loadDataP(opts).catch(e => {
+            this.gtfFeatures = this.loadDataP(opts).catch((e) => {
                 this.gtfFeatures = undefined;
                 throw e;
             });
@@ -56,46 +92,22 @@ class GtfAdapter extends BaseAdapter_1.BaseFeatureDataAdapter {
         return this.gtfFeatures;
     }
     async getRefNames(opts = {}) {
-        const { feats } = await this.loadData(opts);
-        return Object.keys(feats);
-    }
-    async loadFeatureIntervalTreeHelper(refName) {
-        const { feats } = await this.loadData();
-        const lines = feats[refName];
-        if (!lines) {
-            return undefined;
-        }
-        const data = gtf_1.default.parseStringSync(lines.join('\n'), {
-            parseFeatures: true,
-            parseComments: false,
-            parseDirectives: false,
-            parseSequences: false,
-        });
-        const intervalTree = new interval_tree_1.default();
-        const ret = data.flat().map((f, i) => new util_1.SimpleFeature({
-            data: (0, util_2.featureData)(f),
-            id: `${this.id}-${refName}-${i}`,
-        }));
-        for (const obj of ret) {
-            intervalTree.insert([obj.get('start'), obj.get('end')], obj);
-        }
-        return intervalTree;
+        const { intervalTreeMap } = await this.loadData(opts);
+        return Object.keys(intervalTreeMap);
     }
-    async loadFeatureIntervalTree(refName) {
-        if (!this.intervalTrees[refName]) {
-            this.intervalTrees[refName] = this.loadFeatureIntervalTreeHelper(refName).catch(e => {
-                this.intervalTrees[refName] = undefined;
-                throw e;
-            });
-        }
-        return this.intervalTrees[refName];
+    async getHeader(opts = {}) {
+        const { header } = await this.loadData(opts);
+        return header;
     }
     getFeatures(query, opts = {}) {
         return (0, rxjs_1.ObservableCreate)(async (observer) => {
+            var _a;
             try {
                 const { start, end, refName } = query;
-                const intervalTree = await this.loadFeatureIntervalTree(refName);
-                intervalTree === null || intervalTree === void 0 ? void 0 : intervalTree.search([start, end]).forEach(f => observer.next(f));
+                const { intervalTreeMap } = await this.loadData(opts);
+                (_a = intervalTreeMap[refName]) === null || _a === void 0 ? void 0 : _a.call(intervalTreeMap, opts.statusCallback).search([start, end]).forEach(f => {
+                    observer.next(f);
+                });
                 observer.complete();
             }
             catch (e) {

package/dist/util.d.ts

@@ -5,7 +5,7 @@ export interface FeatureLoc {
     end: number;
     strand: Strand;
     seq_name: string;
-    child_features: FeatureLoc[][];
+    child_features?: FeatureLoc[][];
     data: unknown;
     derived_features: unknown;
     attributes: Record<string, unknown[]>;

package/dist/util.js

@@ -8,10 +8,10 @@ function featureData(data) {
     f.phase = Number(data.frame);
     f.refName = data.seq_name;
     if (data.score === null) {
-        delete f.score;
+        f.score = undefined;
     }
     if (data.frame === null) {
-        delete f.score;
+        f.score = undefined;
     }
     const defaultFields = new Set([
         'start',
@@ -30,12 +30,12 @@ function featureData(data) {
             // reproduces behavior of NCList
             b += '2';
         }
-        if (data.attributes[a] !== null) {
+        if (data.attributes[a]) {
             let attr = data.attributes[a];
             if (Array.isArray(attr) && attr.length === 1) {
                 // gtf uses double quotes for text values in the attributes column,
                 // remove them
-                attr = `${attr[0]}`.replaceAll(/^"|"$/g, '');
+                attr = attr[0].replaceAll(/^"|"$/g, '');
             }
             f[b] = attr;
         }
@@ -46,15 +46,14 @@ function featureData(data) {
     if (data.child_features && data.child_features.length > 0) {
         f.subfeatures = data.child_features.flatMap(childLocs => childLocs.map(childLoc => featureData(childLoc)));
     }
-    delete f.child_features;
-    delete f.data;
-    delete f.derived_features;
-    // eslint-disable-next-line no-underscore-dangle
-    delete f._linehash;
-    delete f.attributes;
-    delete f.seq_name;
-    delete f.featureType;
-    delete f.frame;
+    f.child_features = undefined;
+    f.data = undefined;
+    f.derived_features = undefined;
+    f._linehash = undefined;
+    f.attributes = undefined;
+    f.seq_name = undefined;
+    f.featureType = undefined;
+    f.frame = undefined;
     if (f.transcript_id) {
         f.name = f.transcript_id;
     }

package/esm/GtfAdapter/GtfAdapter.d.ts

@@ -2,16 +2,18 @@ import { BaseFeatureDataAdapter, BaseOptions } from '@jbrowse/core/data_adapters
 import { NoAssemblyRegion } from '@jbrowse/core/util/types';
 import IntervalTree from '@flatten-js/interval-tree';
 import { Feature } from '@jbrowse/core/util';
+type StatusCallback = (arg: string) => void;
 export default class GtfAdapter extends BaseFeatureDataAdapter {
-    protected gtfFeatures?: Promise<{
-        feats: Record<string, string[]>;
+    calculatedIntervalTreeMap: Record<string, IntervalTree>;
+    gtfFeatures?: Promise<{
+        header: string;
+        intervalTreeMap: Record<string, (sc?: StatusCallback) => IntervalTree>;
     }>;
-    protected intervalTrees: Record<string, Promise<IntervalTree | undefined> | undefined>;
     private loadDataP;
     private loadData;
     getRefNames(opts?: BaseOptions): Promise<string[]>;
-    private loadFeatureIntervalTreeHelper;
-    private loadFeatureIntervalTree;
+    getHeader(opts?: BaseOptions): Promise<string>;
     getFeatures(query: NoAssemblyRegion, opts?: BaseOptions): import("rxjs").Observable<Feature>;
     freeResources(): void;
 }
+export {};

package/esm/GtfAdapter/GtfAdapter.js

@@ -2,48 +2,84 @@ import { BaseFeatureDataAdapter, } from '@jbrowse/core/data_adapters/BaseAdapter
 import { openLocation } from '@jbrowse/core/util/io';
 import { ObservableCreate } from '@jbrowse/core/util/rxjs';
 import IntervalTree from '@flatten-js/interval-tree';
-import { SimpleFeature } from '@jbrowse/core/util';
+import { SimpleFeature, updateStatus, isGzip, } from '@jbrowse/core/util';
 import { unzip } from '@gmod/bgzf-filehandle';
 import gtf from '@gmod/gtf';
 // locals
 import { featureData } from '../util';
-function isGzip(buf) {
-    return buf[0] === 31 && buf[1] === 139 && buf[2] === 8;
-}
+const decoder = typeof TextDecoder !== 'undefined' ? new TextDecoder('utf8') : undefined;
 export default class GtfAdapter extends BaseFeatureDataAdapter {
     constructor() {
         super(...arguments);
-        this.intervalTrees = {};
+        this.calculatedIntervalTreeMap = {};
     }
-    async loadDataP(opts = {}) {
-        const gtfLoc = this.getConf('gtfLocation');
-        const buffer = await openLocation(gtfLoc, this.pluginManager).readFile(opts);
-        const buf = isGzip(buffer) ? await unzip(buffer) : buffer;
-        // 512MB  max chrome string length is 512MB
-        if (buf.length > 536870888) {
-            throw new Error('Data exceeds maximum string length (512MB)');
-        }
-        const data = new TextDecoder('utf8', { fatal: true }).decode(buf);
-        const lines = data
-            .split(/\n|\r\n|\r/)
-            .filter(f => !!f && !f.startsWith('#'));
-        const feats = {};
-        for (const line of lines) {
-            if (line.startsWith('#')) {
-                continue;
+    async loadDataP(opts) {
+        const { statusCallback = () => { } } = opts || {};
+        const buf = (await openLocation(this.getConf('gtfLocation'), this.pluginManager).readFile(opts));
+        const buffer = isGzip(buf)
+            ? await updateStatus('Unzipping', statusCallback, () => unzip(buf))
+            : buf;
+        const headerLines = [];
+        const featureMap = {};
+        let blockStart = 0;
+        let i = 0;
+        while (blockStart < buffer.length) {
+            const n = buffer.indexOf('\n', blockStart);
+            // could be a non-newline ended file, so slice to end of file if n===-1
+            const b = n === -1 ? buffer.slice(blockStart) : buffer.slice(blockStart, n);
+            const line = ((decoder === null || decoder === void 0 ? void 0 : decoder.decode(b)) || b.toString()).trim();
+            if (line) {
+                if (line.startsWith('#')) {
+                    headerLines.push(line);
+                }
+                else if (line.startsWith('>')) {
+                    break;
+                }
+                else {
+                    const ret = line.indexOf('\t');
+                    const refName = line.slice(0, ret);
+                    if (!featureMap[refName]) {
+                        featureMap[refName] = '';
+                    }
+                    featureMap[refName] += `${line}\n`;
+                }
             }
-            const tab = line.indexOf('\t');
-            const refName = line.slice(0, tab);
-            if (!feats[refName]) {
-                feats[refName] = [];
+            if (i++ % 10000 === 0) {
+                statusCallback(`Loading ${Math.floor(blockStart / 1000000).toLocaleString('en-US')}/${Math.floor(buffer.length / 1000000).toLocaleString('en-US')} MB`);
             }
-            feats[refName].push(line);
+            blockStart = n + 1;
         }
-        return { feats };
+        const intervalTreeMap = Object.fromEntries(Object.entries(featureMap).map(([refName, lines]) => [
+            refName,
+            (sc) => {
+                if (!this.calculatedIntervalTreeMap[refName]) {
+                    sc === null || sc === void 0 ? void 0 : sc('Parsing GTF data');
+                    const intervalTree = new IntervalTree();
+                    gtf.parseStringSync(lines, {
+                        parseFeatures: true,
+                        parseComments: false,
+                        parseDirectives: false,
+                        parseSequences: false,
+                    })
+                        .flat()
+                        .map((f, i) => new SimpleFeature({
+                        data: featureData(f),
+                        id: `${this.id}-${refName}-${i}`,
+                    }))
+                        .forEach(obj => intervalTree.insert([obj.get('start'), obj.get('end')], obj));
+                    this.calculatedIntervalTreeMap[refName] = intervalTree;
+                }
+                return this.calculatedIntervalTreeMap[refName];
+            },
+        ]));
+        return {
+            header: headerLines.join('\n'),
+            intervalTreeMap,
+        };
     }
     async loadData(opts = {}) {
         if (!this.gtfFeatures) {
-            this.gtfFeatures = this.loadDataP(opts).catch(e => {
+            this.gtfFeatures = this.loadDataP(opts).catch((e) => {
                 this.gtfFeatures = undefined;
                 throw e;
             });
@@ -51,46 +87,22 @@ export default class GtfAdapter extends BaseFeatureDataAdapter {
         return this.gtfFeatures;
     }
     async getRefNames(opts = {}) {
-        const { feats } = await this.loadData(opts);
-        return Object.keys(feats);
-    }
-    async loadFeatureIntervalTreeHelper(refName) {
-        const { feats } = await this.loadData();
-        const lines = feats[refName];
-        if (!lines) {
-            return undefined;
-        }
-        const data = gtf.parseStringSync(lines.join('\n'), {
-            parseFeatures: true,
-            parseComments: false,
-            parseDirectives: false,
-            parseSequences: false,
-        });
-        const intervalTree = new IntervalTree();
-        const ret = data.flat().map((f, i) => new SimpleFeature({
-            data: featureData(f),
-            id: `${this.id}-${refName}-${i}`,
-        }));
-        for (const obj of ret) {
-            intervalTree.insert([obj.get('start'), obj.get('end')], obj);
-        }
-        return intervalTree;
+        const { intervalTreeMap } = await this.loadData(opts);
+        return Object.keys(intervalTreeMap);
     }
-    async loadFeatureIntervalTree(refName) {
-        if (!this.intervalTrees[refName]) {
-            this.intervalTrees[refName] = this.loadFeatureIntervalTreeHelper(refName).catch(e => {
-                this.intervalTrees[refName] = undefined;
-                throw e;
-            });
-        }
-        return this.intervalTrees[refName];
+    async getHeader(opts = {}) {
+        const { header } = await this.loadData(opts);
+        return header;
     }
     getFeatures(query, opts = {}) {
         return ObservableCreate(async (observer) => {
+            var _a;
             try {
                 const { start, end, refName } = query;
-                const intervalTree = await this.loadFeatureIntervalTree(refName);
-                intervalTree === null || intervalTree === void 0 ? void 0 : intervalTree.search([start, end]).forEach(f => observer.next(f));
+                const { intervalTreeMap } = await this.loadData(opts);
+                (_a = intervalTreeMap[refName]) === null || _a === void 0 ? void 0 : _a.call(intervalTreeMap, opts.statusCallback).search([start, end]).forEach(f => {
+                    observer.next(f);
+                });
                 observer.complete();
             }
             catch (e) {

package/esm/util.d.ts

@@ -5,7 +5,7 @@ export interface FeatureLoc {
     end: number;
     strand: Strand;
     seq_name: string;
-    child_features: FeatureLoc[][];
+    child_features?: FeatureLoc[][];
     data: unknown;
     derived_features: unknown;
     attributes: Record<string, unknown[]>;

package/esm/util.js

@@ -5,10 +5,10 @@ export function featureData(data) {
     f.phase = Number(data.frame);
     f.refName = data.seq_name;
     if (data.score === null) {
-        delete f.score;
+        f.score = undefined;
     }
     if (data.frame === null) {
-        delete f.score;
+        f.score = undefined;
     }
     const defaultFields = new Set([
         'start',
@@ -27,12 +27,12 @@ export function featureData(data) {
             // reproduces behavior of NCList
             b += '2';
         }
-        if (data.attributes[a] !== null) {
+        if (data.attributes[a]) {
             let attr = data.attributes[a];
             if (Array.isArray(attr) && attr.length === 1) {
                 // gtf uses double quotes for text values in the attributes column,
                 // remove them
-                attr = `${attr[0]}`.replaceAll(/^"|"$/g, '');
+                attr = attr[0].replaceAll(/^"|"$/g, '');
             }
             f[b] = attr;
         }
@@ -43,15 +43,14 @@ export function featureData(data) {
     if (data.child_features && data.child_features.length > 0) {
         f.subfeatures = data.child_features.flatMap(childLocs => childLocs.map(childLoc => featureData(childLoc)));
     }
-    delete f.child_features;
-    delete f.data;
-    delete f.derived_features;
-    // eslint-disable-next-line no-underscore-dangle
-    delete f._linehash;
-    delete f.attributes;
-    delete f.seq_name;
-    delete f.featureType;
-    delete f.frame;
+    f.child_features = undefined;
+    f.data = undefined;
+    f.derived_features = undefined;
+    f._linehash = undefined;
+    f.attributes = undefined;
+    f.seq_name = undefined;
+    f.featureType = undefined;
+    f.frame = undefined;
     if (f.transcript_id) {
         f.name = f.transcript_id;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/plugin-gtf",
-  "version": "2.13.1",
+  "version": "2.14.0",
   "description": "JBrowse 2 gtf feature adapter",
   "keywords": [
     "jbrowse",
@@ -24,7 +24,7 @@
   ],
   "scripts": {
     "build": "npm-run-all build:*",
-    "test": "cd ../..; jest plugins/gtf --passWithNoTests",
+    "test": "cd ../..; jest --passWithNoTests plugins/gtf --passWithNoTests",
     "prepublishOnly": "yarn test",
     "prepack": "yarn build && yarn useDist",
     "postpack": "yarn useSrc",
@@ -56,5 +56,5 @@
   "distModule": "esm/index.js",
   "srcModule": "src/index.ts",
   "module": "esm/index.js",
-  "gitHead": "fcebca71cc1d066654603e1a9accfa6c6d4f764d"
+  "gitHead": "9fb8231d932db40adf0a283081765431756c66ff"
 }