@jbrowse/plugin-gff3 4.3.0 → 5.0.0-beta.2

package/esm/Gff3Adapter/Gff3Adapter.d.ts

@@ -1,19 +1,11 @@
 import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter';
-import { IntervalTree } from '@jbrowse/core/util';
+import type { Gff3AdapterConfig } from './configSchema.ts';
 import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter';
 import type { Feature } from '@jbrowse/core/util/simpleFeature';
 import type { NoAssemblyRegion } from '@jbrowse/core/util/types';
-type StatusCallback = (arg: string) => void;
-export default class Gff3Adapter extends BaseFeatureDataAdapter {
-    calculatedIntervalTreeMap: Record<string, IntervalTree<Feature>>;
-    gffFeatures?: Promise<{
-        header: string;
-        intervalTreeMap: Record<string, (sc?: StatusCallback) => IntervalTree<Feature>>;
-    }>;
-    private loadDataP;
+export default class Gff3Adapter extends BaseFeatureDataAdapter<Gff3AdapterConfig> {
     private loadData;
     getRefNames(opts?: BaseOptions): Promise<string[]>;
     getHeader(opts?: BaseOptions): Promise<string>;
     getFeatures(query: NoAssemblyRegion, opts?: BaseOptions): import("rxjs").Observable<Feature>;
 }
-export {};

package/esm/Gff3Adapter/Gff3Adapter.js

@@ -1,51 +1,40 @@
-import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter';
-import { IntervalTree, fetchAndMaybeUnzip } from '@jbrowse/core/util';
+import { BaseFeatureDataAdapter, cachedSetup, } from '@jbrowse/core/data_adapters/BaseAdapter';
+import { fetchAndMaybeUnzip } from '@jbrowse/core/util';
 import { openLocation } from '@jbrowse/core/util/io';
+import { groupLinesByRef, makeFeatureIntervalTreeMap, } from '@jbrowse/core/util/parseLineByLine';
 import { ObservableCreate } from '@jbrowse/core/util/rxjs';
-import SimpleFeature from '@jbrowse/core/util/simpleFeature';
-import { parseStringSyncJBrowse } from 'gff-nostream';
-import { parseGffBuffer } from "./gffParser.js";
+import { parseLinesLazy } from 'gff-nostream';
+import { Gff3Feature } from '../Gff3Feature.js';
 export default class Gff3Adapter extends BaseFeatureDataAdapter {
-    calculatedIntervalTreeMap = {};
-    gffFeatures;
-    async loadDataP(opts) {
-        const { statusCallback = () => { } } = opts || {};
-        const buffer = await fetchAndMaybeUnzip(openLocation(this.getConf('gffLocation'), this.pluginManager), opts);
-        const { header, featureMap } = parseGffBuffer(buffer, statusCallback);
-        const intervalTreeMap = Object.fromEntries(Object.entries(featureMap).map(([refName, lines]) => [
-            refName,
-            (sc) => {
-                if (!this.calculatedIntervalTreeMap[refName]) {
-                    sc?.('Parsing GFF data');
-                    const intervalTree = new IntervalTree();
-                    const features = parseStringSyncJBrowse(lines);
-                    for (let i = 0, l = features.length; i < l; i++) {
-                        const f = features[i];
-                        const obj = new SimpleFeature({
-                            data: f,
-                            id: `${this.id}-${refName}-${i}`,
-                        });
-                        intervalTree.insert([f.start, f.end], obj);
-                    }
-                    this.calculatedIntervalTreeMap[refName] = intervalTree;
-                }
-                return this.calculatedIntervalTreeMap[refName];
-            },
-        ]));
-        return {
-            header,
-            intervalTreeMap,
-        };
-    }
-    async loadData(opts) {
-        if (!this.gffFeatures) {
-            this.gffFeatures = this.loadDataP(opts).catch((e) => {
-                this.gffFeatures = undefined;
-                throw e;
-            });
-        }
-        return this.gffFeatures;
-    }
+    // the whole file is resident after one load, so the fetch/parse status comes
+    // from inside the load itself rather than a label wrapped around it
+    loadData = cachedSetup({
+        setup: async (opts) => {
+            const buffer = await fetchAndMaybeUnzip(openLocation(this.getConf('gffLocation'), this.pluginManager), opts);
+            const { headerLines, linesByRef } = groupLinesByRef(buffer, opts.statusCallback);
+            const intervalTreeMap = makeFeatureIntervalTreeMap(linesByRef, 
+            // lines are already split and comment/FASTA-filtered by
+            // groupLinesByRef, so feed them straight to parseLinesLazy rather than
+            // re-joining and re-splitting through parseStringSync.
+            //
+            // Lazy because the whole file stays resident for the session: leaving
+            // column 9 as text rather than an object per attribute is what keeps
+            // that resident set small (8.5x on GENCODE-shaped input), and the
+            // render path reads only a handful of attributes anyway — see
+            // Gff3Feature.
+            //
+            // The id the tree entry carries is minted here, beside the feature
+            // rather than stamped onto it, so the parsed feature is the library's
+            // shape and Gff3Feature serializes it identically for both adapters.
+            (lines, refName) => parseLinesLazy(lines).map((feature, i) => ({
+                start: feature.start,
+                end: feature.end,
+                feature,
+                uniqueId: `${this.id}-${refName}-${i}`,
+            })), 'Parsing GFF data');
+            return { header: headerLines.join('\n'), intervalTreeMap };
+        },
+    });
     async getRefNames(opts = {}) {
         const { intervalTreeMap } = await this.loadData(opts);
         return Object.keys(intervalTreeMap);
@@ -55,21 +44,21 @@ export default class Gff3Adapter extends BaseFeatureDataAdapter {
         return header;
     }
     getFeatures(query, opts = {}) {
+        // no try/catch: ObservableCreate forwards a rejected callback to
+        // observer.error itself
         return ObservableCreate(async (observer) => {
-            try {
-                const { start, end, refName } = query;
-                const { intervalTreeMap } = await this.loadData(opts);
-                for (const f of intervalTreeMap[refName]?.(opts.statusCallback).search([
+            const { start, end, refName } = query;
+            const { intervalTreeMap } = await this.loadData(opts);
+            const tree = intervalTreeMap[refName];
+            if (tree) {
+                for (const { feature, uniqueId } of tree(opts.statusCallback).search([
                     start,
                     end,
-                ]) || []) {
-                    observer.next(f);
+                ])) {
+                    observer.next(new Gff3Feature(feature, uniqueId));
                 }
-                observer.complete();
-            }
-            catch (e) {
-                observer.error(e);
             }
+            observer.complete();
         }, opts.stopToken);
     }
 }

package/esm/Gff3Adapter/configSchema.d.ts

@@ -1,10 +1,34 @@
+import type { Instance } from '@jbrowse/mobx-state-tree';
+export declare function normalizeSnapshot(snap: Record<string, unknown>): Record<string, unknown>;
+/**
+ * #config Gff3Adapter
+ * #category adapter
+ * #trackType FeatureTrack
+ * #fileFormat feature | GFF3 (plain) | Loaded entirely into memory; for small files
+ * used to load plain-text GFF3 files. Loads the whole file into memory, so
+ * prefer the Gff3TabixAdapter for large files.
+ *
+ * #example
+ * ```js
+ * {
+ *   type: 'Gff3Adapter',
+ *   uri: 'https://example.com/genes.gff3',
+ * }
+ * ```
+ */
 declare const Gff3Adapter: import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaType<{
-    gffLocation: {
-        type: string;
-        defaultValue: {
-            uri: string;
-            locationType: string;
+    /**
+     * #slot
+     * location of the GFF3 file. May be gzipped; it is decompressed and parsed
+     * in full on first use, so the file has to fit in memory.
+     */
+    readonly gffLocation: {
+        readonly type: "fileLocation";
+        readonly defaultValue: {
+            readonly uri: "/path/to/my.gff";
+            readonly locationType: "UriLocation";
         };
     };
 }, import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaOptions<undefined, undefined>>;
+export type Gff3AdapterConfig = Instance<typeof Gff3Adapter>;
 export default Gff3Adapter;

package/esm/Gff3Adapter/configSchema.js

@@ -1,6 +1,37 @@
 import { ConfigurationSchema } from '@jbrowse/core/configuration';
-function x() { }
+export function normalizeSnapshot(snap) {
+    return snap.uri
+        ? {
+            ...snap,
+            gffLocation: {
+                uri: snap.uri,
+                baseUri: snap.baseUri,
+            },
+        }
+        : snap;
+}
+/**
+ * #config Gff3Adapter
+ * #category adapter
+ * #trackType FeatureTrack
+ * #fileFormat feature | GFF3 (plain) | Loaded entirely into memory; for small files
+ * used to load plain-text GFF3 files. Loads the whole file into memory, so
+ * prefer the Gff3TabixAdapter for large files.
+ *
+ * #example
+ * ```js
+ * {
+ *   type: 'Gff3Adapter',
+ *   uri: 'https://example.com/genes.gff3',
+ * }
+ * ```
+ */
 const Gff3Adapter = ConfigurationSchema('Gff3Adapter', {
+    /**
+     * #slot
+     * location of the GFF3 file. May be gzipped; it is decompressed and parsed
+     * in full on first use, so the file has to fit in memory.
+     */
     gffLocation: {
         type: 'fileLocation',
         defaultValue: {
@@ -10,16 +41,19 @@ const Gff3Adapter = ConfigurationSchema('Gff3Adapter', {
     },
 }, {
     explicitlyTyped: true,
-    preProcessSnapshot: snap => {
-        return snap.uri
-            ? {
-                ...snap,
-                gffLocation: {
-                    uri: snap.uri,
-                    baseUri: snap.baseUri,
-                },
-            }
-            : snap;
-    },
+    /**
+     * #preProcessSnapshot
+     *
+     *
+     * preprocessor to allow minimal config:
+     *
+     * ```json
+     * {
+     *   "type": "Gff3Adapter",
+     *   "uri": "yourfile.gff3",
+     * }
+     * ```
+     */
+    preProcessSnapshot: normalizeSnapshot,
 });
 export default Gff3Adapter;

package/esm/Gff3Adapter/index.js

@@ -1,10 +1,11 @@
 import { AdapterType } from '@jbrowse/core/pluggableElementTypes';
-import configSchema from "./configSchema.js";
+import configSchema, { normalizeSnapshot } from './configSchema.js';
 export default function Gff3AdapterF(pluginManager) {
     pluginManager.addAdapterType(() => new AdapterType({
         name: 'Gff3Adapter',
         displayName: 'GFF3 adapter',
+        normalizeSnapshot,
         configSchema,
-        getAdapterClass: () => import("./Gff3Adapter.js").then(r => r.default),
+        getAdapterClass: () => import('./Gff3Adapter.js').then(r => r.default),
     }));
 }

package/esm/Gff3Feature.d.ts

@@ -0,0 +1,51 @@
+import type { Feature, SimpleFeatureSerialized } from '@jbrowse/core/util/simpleFeature';
+import type { LazyGffFeature } from 'gff-nostream';
+/**
+ * A {@link Feature} over a GFF3 feature whose attributes are still raw column-9
+ * text, resolving each one only when something asks for it.
+ *
+ * This exists because the two things that read a GFF3 feature want opposite
+ * amounts of it. Rendering reads a fixed handful of keys — `start`, `end`,
+ * `strand`, `type`, `phase`, `subfeatures`, plus `name`/`id` for labels and
+ * `gbkey` for the stock NCBI filter — and then packs the result into typed
+ * arrays, so the feature objects are dropped and never cross the RPC boundary.
+ * The feature details panel wants everything, and re-fetches the region from
+ * scratch to get it (`GetCanvasFeatureDetails`). Parsing every attribute during
+ * a render therefore did work that rendering did not use and that the details
+ * path did not reuse.
+ *
+ * `toJSON` still materializes the lot, which is what the details path
+ * serializes, so nothing downstream sees a smaller feature than it used to.
+ */
+export declare class Gff3Feature implements Feature {
+    private data;
+    private uniqueId;
+    private parentFeature?;
+    private inflated?;
+    constructor(data: LazyGffFeature, uniqueId: string, parentFeature?: Feature | undefined);
+    /**
+     * A switch rather than a lookup table or a `data[name]` fallthrough: the
+     * fixed columns are read tens of times per feature by the layout pass, and
+     * they must not pay for an attribute scan. Anything not named here is an
+     * attribute, resolved on demand.
+     */
+    get(name: 'refName'): string;
+    get(name: 'name' | 'type' | 'id' | 'source'): string | undefined;
+    get(name: 'start' | 'end'): number;
+    get(name: 'phase'): 0 | 1 | 2 | undefined;
+    get(name: 'strand'): -1 | 0 | 1 | undefined;
+    get(name: 'score'): number | undefined;
+    get(name: 'subfeatures'): Feature[] | undefined;
+    get(name: string): unknown;
+    id(): string;
+    parent(): Feature | undefined;
+    /**
+     * Wrapped lazily rather than in the constructor: a feature the layout pass
+     * never reaches never allocates a wrapper per descendant. (SimpleFeature
+     * inflates its whole subtree eagerly on construction.)
+     */
+    children(): Feature[];
+    tags(): string[];
+    private columns;
+    toJSON(): SimpleFeatureSerialized;
+}

package/esm/Gff3Feature.js

@@ -0,0 +1,110 @@
+import { getAttribute, getAttributes } from 'gff-nostream';
+/**
+ * A {@link Feature} over a GFF3 feature whose attributes are still raw column-9
+ * text, resolving each one only when something asks for it.
+ *
+ * This exists because the two things that read a GFF3 feature want opposite
+ * amounts of it. Rendering reads a fixed handful of keys — `start`, `end`,
+ * `strand`, `type`, `phase`, `subfeatures`, plus `name`/`id` for labels and
+ * `gbkey` for the stock NCBI filter — and then packs the result into typed
+ * arrays, so the feature objects are dropped and never cross the RPC boundary.
+ * The feature details panel wants everything, and re-fetches the region from
+ * scratch to get it (`GetCanvasFeatureDetails`). Parsing every attribute during
+ * a render therefore did work that rendering did not use and that the details
+ * path did not reuse.
+ *
+ * `toJSON` still materializes the lot, which is what the details path
+ * serializes, so nothing downstream sees a smaller feature than it used to.
+ */
+export class Gff3Feature {
+    data;
+    uniqueId;
+    parentFeature;
+    inflated;
+    constructor(data, uniqueId, parentFeature) {
+        this.data = data;
+        this.uniqueId = uniqueId;
+        this.parentFeature = parentFeature;
+    }
+    get(name) {
+        switch (name) {
+            case 'start': {
+                return this.data.start;
+            }
+            case 'end': {
+                return this.data.end;
+            }
+            case 'type': {
+                return this.data.type;
+            }
+            case 'refName': {
+                return this.data.refName;
+            }
+            case 'source': {
+                return this.data.source;
+            }
+            case 'score': {
+                return this.data.score;
+            }
+            case 'phase': {
+                return this.data.phase;
+            }
+            case 'strand': {
+                // a subfeature with no strand column of its own inherits its parent's,
+                // which is what SimpleFeature does too — it resolves the same way here,
+                // through the parent handle rather than by copying the field down
+                return this.data.strand ?? this.parentFeature?.get('strand');
+            }
+            case 'subfeatures': {
+                return this.children();
+            }
+            case 'parent': {
+                return this.parentFeature;
+            }
+            default: {
+                return getAttribute(this.data, name);
+            }
+        }
+    }
+    id() {
+        return this.uniqueId;
+    }
+    parent() {
+        return this.parentFeature;
+    }
+    /**
+     * Wrapped lazily rather than in the constructor: a feature the layout pass
+     * never reaches never allocates a wrapper per descendant. (SimpleFeature
+     * inflates its whole subtree eagerly on construction.)
+     */
+    children() {
+        this.inflated ??= this.data.subfeatures.map((f, i) => new Gff3Feature(f, `${this.uniqueId}-${i}`, this));
+        return this.inflated;
+    }
+    tags() {
+        return [
+            ...Object.keys(this.columns()),
+            'subfeatures',
+            ...Object.keys(getAttributes(this.data)),
+        ];
+    }
+    // the eight parsed columns, without the raw attribute text or the nested
+    // features, both of which the serialized form carries differently
+    columns() {
+        const { attributeString, subfeatures, ...rest } = this.data;
+        return rest;
+    }
+    toJSON() {
+        const d = {
+            ...this.columns(),
+            ...getAttributes(this.data),
+            strand: this.get('strand'),
+            uniqueId: this.uniqueId,
+            subfeatures: this.children().map(f => f.toJSON()),
+        };
+        if (this.parentFeature) {
+            d.parentId = this.parentFeature.id();
+        }
+        return d;
+    }
+}

package/esm/Gff3TabixAdapter/Gff3TabixAdapter.d.ts

@@ -1,21 +1,12 @@
-import { TabixIndexedFile } from '@gmod/tabix';
 import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter';
+import type { Gff3TabixAdapterConfig } from './configSchema.ts';
 import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter';
 import type { Feature } from '@jbrowse/core/util/simpleFeature';
 import type { Region } from '@jbrowse/core/util/types';
-export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
-    private configured?;
-    private configurePre;
-    protected configurePre2(): Promise<{
-        gff: TabixIndexedFile;
-        dontRedispatchSet: Set<string>;
-    }>;
-    configure(opts?: BaseOptions): Promise<{
-        gff: TabixIndexedFile;
-        dontRedispatchSet: Set<string>;
-    }>;
+export default class Gff3TabixAdapter extends BaseFeatureDataAdapter<Gff3TabixAdapterConfig> {
+    private configure;
     getRefNames(opts?: BaseOptions): Promise<string[]>;
     getHeader(opts?: BaseOptions): Promise<string>;
+    getRegionByteSize(regions: Region[], opts?: BaseOptions): Promise<number>;
     getFeatures(query: Region, opts?: BaseOptions): import("rxjs").Observable<Feature>;
-    private getFeaturesHelper;
 }

package/esm/Gff3TabixAdapter/Gff3TabixAdapter.js

@@ -1,108 +1,85 @@
 import { TabixIndexedFile } from '@gmod/tabix';
-import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter';
-import { updateStatus } from '@jbrowse/core/util';
-import { openLocation } from '@jbrowse/core/util/io';
+import { BaseFeatureDataAdapter, cachedSetup, } from '@jbrowse/core/data_adapters/BaseAdapter';
+import { downloadStatus } from '@jbrowse/core/util';
+import { sharedBgzfWorkerPool } from '@jbrowse/core/util/bgzfWorkerPool';
+import { decompressedBytesBudget } from '@jbrowse/core/util/cacheBudgets';
+import { openLocation, openTabixIndexFilehandle } from '@jbrowse/core/util/io';
 import { doesIntersect2 } from '@jbrowse/core/util/range';
 import { ObservableCreate } from '@jbrowse/core/util/rxjs';
-import SimpleFeature from '@jbrowse/core/util/simpleFeature';
-import { parseRecordsJBrowse } from 'gff-nostream';
+import { readTabixLinesRedispatched } from '@jbrowse/core/util/tabix';
+import { hasIdAttribute, parseRecordsLazy } from 'gff-nostream';
+import { Gff3Feature } from '../Gff3Feature.js';
 export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
-    configured;
-    async configurePre(_opts) {
-        const gffGzLocation = this.getConf('gffGzLocation');
-        const indexType = this.getConf(['index', 'indexType']);
-        const loc = this.getConf(['index', 'location']);
-        const dontRedispatch = this.getConf('dontRedispatch');
-        const gff = new TabixIndexedFile({
-            filehandle: openLocation(gffGzLocation, this.pluginManager),
-            csiFilehandle: indexType === 'CSI' ? openLocation(loc, this.pluginManager) : undefined,
-            tbiFilehandle: indexType !== 'CSI' ? openLocation(loc, this.pluginManager) : undefined,
-            chunkCacheSize: 50 * 2 ** 20,
-        });
-        return {
-            gff,
-            dontRedispatchSet: new Set(dontRedispatch),
-            header: await gff.getHeader(),
-        };
-    }
-    async configurePre2() {
-        if (!this.configured) {
-            this.configured = this.configurePre().catch((e) => {
-                this.configured = undefined;
-                throw e;
+    configure = cachedSetup({
+        label: 'Downloading index',
+        setup: async (_opts, onProgress) => {
+            const gff = new TabixIndexedFile({
+                filehandle: openLocation(this.getConf('gffGzLocation'), this.pluginManager),
+                ...openTabixIndexFilehandle(this.getConf(['index', 'location']), this.getConf(['index', 'indexType']), this.pluginManager),
+                chunkCacheBudget: decompressedBytesBudget,
+                bgzfWorkerPool: sharedBgzfWorkerPool(),
             });
-        }
-        return this.configured;
-    }
-    async configure(opts) {
-        const { statusCallback = () => { } } = opts || {};
-        return updateStatus('Downloading index', statusCallback, () => this.configurePre2());
-    }
+            return {
+                gff,
+                dontRedispatchSet: new Set(this.getConf('dontRedispatch')),
+                // the index is a whole-file read, so its byte ticks turn the
+                // "Downloading index" label into a determinate bar
+                header: await gff.getHeader({ onProgress }),
+            };
+        },
+    });
     async getRefNames(opts = {}) {
         const { gff } = await this.configure(opts);
-        return gff.getReferenceSequenceNames(opts);
+        return downloadStatus('Downloading index', opts.statusCallback, onProgress => gff.getReferenceSequenceNames({ ...opts, onProgress }));
     }
     async getHeader(opts = {}) {
+        const { header } = await this.configure(opts);
+        return header;
+    }
+    // Index-only compressed-byte estimate (no feature download), used by the
+    // feature-fetch RPC to short-circuit an over-budget region before pulling
+    // every line — see executeRenderFeatureData.
+    async getRegionByteSize(regions, opts = {}) {
         const { gff } = await this.configure(opts);
-        return gff.getHeader();
+        return gff.bytesForRegions(regions, opts);
     }
     getFeatures(query, opts = {}) {
         return ObservableCreate(async (observer) => {
-            const { gff } = await this.configure(opts);
-            const metadata = await gff.getMetadata();
-            await this.getFeaturesHelper(query, opts, metadata, observer, true);
-        }, opts.stopToken);
-    }
-    async getFeaturesHelper(query, opts, metadata, observer, allowRedispatch, originalQuery = query) {
-        const { statusCallback = () => { } } = opts;
-        try {
-            const lines = [];
-            const { dontRedispatchSet, gff } = await this.configure(opts);
-            await updateStatus('Downloading features', statusCallback, () => gff.getLines(query.refName, query.start, query.end, (line, fileOffset, start, end) => {
-                const t1 = line.indexOf('\t');
-                const t2 = line.indexOf('\t', t1 + 1);
-                const t3 = line.indexOf('\t', t2 + 1);
-                const type = line.slice(t2 + 1, t3);
-                lines.push({
-                    line,
-                    lineHash: fileOffset,
-                    start,
-                    end,
-                    hasEscapes: line.includes('%'),
-                    type,
-                });
-            }));
-            if (allowRedispatch && lines.length) {
-                let minStart = Number.POSITIVE_INFINITY;
-                let maxEnd = Number.NEGATIVE_INFINITY;
-                for (const rec of lines) {
-                    if (!dontRedispatchSet.has(rec.type)) {
-                        const start = rec.start - 1;
-                        if (start < minStart) {
-                            minStart = start;
-                        }
-                        if (rec.end > maxEnd) {
-                            maxEnd = rec.end;
-                        }
+            try {
+                const { gff, dontRedispatchSet } = await this.configure(opts);
+                // The bound exists to reach a record's children, so what disqualifies a
+                // record is having none. `ID` answers that exactly — nothing can name a
+                // parent that has no name — and gff-nostream decides it with the scan
+                // its linker uses, so a record the tree would attach children to is
+                // never one the bound leaves out. `dontRedispatch` can only approximate
+                // the same question by type: every NCBI `GCF_*_genomic.gff.gz` opens
+                // each reference with a chromosome-long `match` record no list named,
+                // and none of them has an `ID`. The list covers what `ID` cannot see: a
+                // record carrying an `ID` that nothing references, which hosted hg19
+                // RefSeq's chromosome-long `region` is.
+                const canHaveChildren = (line) => hasIdAttribute(line.line) && !dontRedispatchSet.has(line.type);
+                const lines = await readTabixLinesRedispatched(gff, query, 
+                // A caller reading only top-level features is owed nothing by the
+                // flanks and pays for them anyway: they complete SUBFEATURE lists, and
+                // a top-level feature overlapping the query is in the query's own read
+                // already, since tabix returns every overlapping line and a child is
+                // contained in its parent. On an NCBI `GCF_*_genomic.gff.gz`, whose
+                // chromosome-long `match` widens the bound at every window, that is
+                // 193,008 lines parsed to keep 3 features — 2734 ms against 8 ms.
+                opts.topLevelOnly ? () => false : canHaveChildren, opts);
+                // emit only top-level features intersecting the original query. the
+                // byte offset stays on our own record and is used purely to mint a
+                // stable id, so it never pollutes the feature's data
+                for (const { feature, record } of parseRecordsLazy(lines)) {
+                    if (doesIntersect2(feature.start, feature.end, query.start, query.end)) {
+                        observer.next(new Gff3Feature(feature, `${this.id}-offset-${record.offset}`));
                     }
                 }
-                if (maxEnd > query.end || minStart < query.start) {
-                    await this.getFeaturesHelper({ ...query, start: minStart, end: maxEnd }, opts, metadata, observer, false, query);
-                    return;
-                }
+                observer.complete();
             }
-            for (const feature of parseRecordsJBrowse(lines)) {
-                if (doesIntersect2(feature.start, feature.end, originalQuery.start, originalQuery.end)) {
-                    observer.next(new SimpleFeature({
-                        data: feature,
-                        id: `${this.id}-offset-${feature._lineHash}`,
-                    }));
-                }
+            catch (e) {
+                observer.error(e);
             }
-            observer.complete();
-        }
-        catch (e) {
-            observer.error(e);
-        }
+        }, opts.stopToken);
     }
 }

package/esm/Gff3TabixAdapter/configSchema.d.ts

@@ -1,28 +1,73 @@
+import type { Instance } from '@jbrowse/mobx-state-tree';
+export declare function normalizeSnapshot(snap: Record<string, unknown>): Record<string, unknown>;
+/**
+ * #config Gff3TabixAdapter
+ * #category adapter
+ * #trackType FeatureTrack
+ * #fileFormat feature | GFF3 (tabix)
+ * used to load bgzip-compressed, tabix-indexed GFF3 files
+ *
+ * #example
+ * The `uri` shorthand auto-resolves the `.tbi` index; add `csi: true` for a
+ * `.csi` index instead:
+ * ```js
+ * {
+ *   type: 'Gff3TabixAdapter',
+ *   uri: 'https://example.com/genes.gff3.gz',
+ * }
+ * ```
+ */
 declare const Gff3TabixAdapter: import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaType<{
-    gffGzLocation: {
-        type: string;
-        defaultValue: {
-            uri: string;
-            locationType: string;
+    readonly densityAdapter: {
+        readonly type: 'frozen';
+        readonly defaultValue: null;
+        readonly description: 'optional quantitative sub-adapter (e.g. a BigWigAdapter over a features-per-bin bigWig) drawn as a density band where the region is too large to fetch features; null disables it';
+    };
+    /**
+     * #slot
+     * location of the bgzip-compressed GFF3, sorted by position. Must be bgzip
+     * rather than plain gzip, which tabix cannot index.
+     */
+    readonly gffGzLocation: {
+        readonly type: "fileLocation";
+        readonly defaultValue: {
+            readonly uri: "/path/to/my.gff.gz";
+            readonly locationType: "UriLocation";
         };
     };
-    index: import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaType<{
-        indexType: {
-            model: import("@jbrowse/mobx-state-tree").ISimpleType<"TBI" | "CSI">;
-            type: string;
-            defaultValue: string;
+    readonly index: import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaType<{
+        readonly indexType: {
+            readonly model: import("@jbrowse/mobx-state-tree").ISimpleType<"CSI" | "TBI">;
+            readonly type: 'stringEnum';
+            readonly defaultValue: 'TBI';
+            readonly description: '`TBI` is the usual `tabix` output. `CSI` is required for a reference longer than 512 Mb, which TBI cannot address.';
         };
-        location: {
-            type: string;
-            defaultValue: {
-                uri: string;
-                locationType: string;
+        readonly location: {
+            readonly type: 'fileLocation';
+            readonly defaultValue: {
+                readonly uri: '/path/to/my.gz.tbi';
+                readonly locationType: 'UriLocation';
             };
+            readonly description: 'location of the tabix index. Only needed when it is not named `<file>.tbi` (or `.csi`), which is what the `uri` shorthand assumes.';
         };
     }, import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaOptions<undefined, undefined>>;
-    dontRedispatch: {
-        type: string;
-        defaultValue: string[];
+    /**
+     * #slot
+     * the Gff3TabixAdapter has to "redispatch" if it fetches a region and
+     * features it finds inside that region extend outside the region we requested.
+     * you can disable this for certain feature types to avoid fetching e.g. the
+     * entire chromosome
+     *
+     * the defaults are the whole-sequence records the common GFF3 sources emit:
+     * `region` (NCBI), `supercontig`/`scaffold` (Ensembl, for non-chromosomal
+     * sequences), plus `chromosome` and `contig`. They span an entire reference
+     * and have no children, so letting one expand the fetch would pull a whole
+     * chromosome to gain nothing
+     */
+    readonly dontRedispatch: {
+        readonly type: "stringArray";
+        readonly defaultValue: readonly ["chromosome", "region", "contig", "supercontig", "scaffold"];
     };
 }, import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaOptions<undefined, undefined>>;
+export type Gff3TabixAdapterConfig = Instance<typeof Gff3TabixAdapter>;
 export default Gff3TabixAdapter;

package/esm/Gff3TabixAdapter/configSchema.js

@@ -1,7 +1,31 @@
-import { ConfigurationSchema } from '@jbrowse/core/configuration';
-import { types } from '@jbrowse/mobx-state-tree';
-function x() { }
+import { ConfigurationSchema, expandTabixShorthand, tabixIndexFields, } from '@jbrowse/core/configuration';
+import { densityAdapterConfigSchemaFields } from '@jbrowse/core/data_adapters/BaseAdapter';
+export function normalizeSnapshot(snap) {
+    return expandTabixShorthand(snap, 'gffGzLocation');
+}
+/**
+ * #config Gff3TabixAdapter
+ * #category adapter
+ * #trackType FeatureTrack
+ * #fileFormat feature | GFF3 (tabix)
+ * used to load bgzip-compressed, tabix-indexed GFF3 files
+ *
+ * #example
+ * The `uri` shorthand auto-resolves the `.tbi` index; add `csi: true` for a
+ * `.csi` index instead:
+ * ```js
+ * {
+ *   type: 'Gff3TabixAdapter',
+ *   uri: 'https://example.com/genes.gff3.gz',
+ * }
+ * ```
+ */
 const Gff3TabixAdapter = ConfigurationSchema('Gff3TabixAdapter', {
+    /**
+     * #slot
+     * location of the bgzip-compressed GFF3, sorted by position. Must be bgzip
+     * rather than plain gzip, which tabix cannot index.
+     */
     gffGzLocation: {
         type: 'fileLocation',
         defaultValue: {
@@ -9,42 +33,48 @@ const Gff3TabixAdapter = ConfigurationSchema('Gff3TabixAdapter', {
             locationType: 'UriLocation',
         },
     },
-    index: ConfigurationSchema('Gff3TabixIndex', {
-        indexType: {
-            model: types.enumeration('IndexType', ['TBI', 'CSI']),
-            type: 'stringEnum',
-            defaultValue: 'TBI',
-        },
-        location: {
-            type: 'fileLocation',
-            defaultValue: {
-                uri: '/path/to/my.gff.gz.tbi',
-                locationType: 'UriLocation',
-            },
-        },
-    }),
+    index: ConfigurationSchema('TabixIndex', { ...tabixIndexFields }),
+    /**
+     * #slot
+     * the Gff3TabixAdapter has to "redispatch" if it fetches a region and
+     * features it finds inside that region extend outside the region we requested.
+     * you can disable this for certain feature types to avoid fetching e.g. the
+     * entire chromosome
+     *
+     * the defaults are the whole-sequence records the common GFF3 sources emit:
+     * `region` (NCBI), `supercontig`/`scaffold` (Ensembl, for non-chromosomal
+     * sequences), plus `chromosome` and `contig`. They span an entire reference
+     * and have no children, so letting one expand the fetch would pull a whole
+     * chromosome to gain nothing
+     */
     dontRedispatch: {
         type: 'stringArray',
-        defaultValue: ['chromosome', 'region', 'contig'],
+        defaultValue: [
+            'chromosome',
+            'region',
+            'contig',
+            'supercontig',
+            'scaffold',
+        ],
     },
+    ...densityAdapterConfigSchemaFields,
 }, {
     explicitlyTyped: true,
-    preProcessSnapshot: snap => {
-        return snap.uri
-            ? {
-                ...snap,
-                gffGzLocation: {
-                    uri: snap.uri,
-                    baseUri: snap.baseUri,
-                },
-                index: {
-                    location: {
-                        uri: `${snap.uri}.tbi`,
-                        baseUri: snap.baseUri,
-                    },
-                },
-            }
-            : snap;
-    },
+    /**
+     * #preProcessSnapshot
+     *
+     *
+     * preprocessor to allow minimal config, assumes tbi index at
+     * yourfile.gff3.gz.tbi (or .csi if csi:true):
+     *
+     * ```json
+     * {
+     *   "type": "Gff3TabixAdapter",
+     *   "uri": "yourfile.gff3.gz",
+     *   "csi": true
+     * }
+     * ```
+     */
+    preProcessSnapshot: normalizeSnapshot,
 });
 export default Gff3TabixAdapter;

package/esm/Gff3TabixAdapter/index.js

@@ -1,10 +1,11 @@
 import { AdapterType } from '@jbrowse/core/pluggableElementTypes';
-import configSchema from "./configSchema.js";
+import configSchema, { normalizeSnapshot } from './configSchema.js';
 export default function Gff3TabixAdapterF(pluginManager) {
     pluginManager.addAdapterType(() => new AdapterType({
         name: 'Gff3TabixAdapter',
         displayName: 'GFF3 tabix adapter',
+        normalizeSnapshot,
         configSchema,
-        getAdapterClass: () => import("./Gff3TabixAdapter.js").then(r => r.default),
+        getAdapterClass: () => import('./Gff3TabixAdapter.js').then(r => r.default),
     }));
 }

package/esm/index.js

@@ -1,12 +1,10 @@
 import Plugin from '@jbrowse/core/Plugin';
-import Gff3AdapterF from "./Gff3Adapter/index.js";
-import Gff3TabixAdapterF from "./Gff3TabixAdapter/index.js";
-import GuessGff3F from "./GuessGff3/index.js";
+import Gff3AdapterF from './Gff3Adapter/index.js';
+import Gff3TabixAdapterF from './Gff3TabixAdapter/index.js';
 export default class GFF3Plugin extends Plugin {
     name = 'GFF3Plugin';
     install(pluginManager) {
         Gff3TabixAdapterF(pluginManager);
         Gff3AdapterF(pluginManager);
-        GuessGff3F(pluginManager);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/plugin-gff3",
-  "version": "4.3.0",
+  "version": "5.0.0-beta.2",
   "type": "module",
   "description": "JBrowse 2 gff3.",
   "keywords": [
@@ -21,13 +21,12 @@
     "esm"
   ],
   "dependencies": {
-    "@flatten-js/interval-tree": "^2.0.3",
-    "@gmod/tabix": "^3.3.3",
-    "@jbrowse/mobx-state-tree": "^5.6.0",
-    "gff-nostream": "^3.0.5",
-    "mobx": "^6.15.3",
+    "@gmod/tabix": "^3.8.2",
+    "@jbrowse/mobx-state-tree": "^6.5.1",
+    "gff-nostream": "^5.4.0",
+    "mobx": "^7.0.3",
     "rxjs": "^7.8.2",
-    "@jbrowse/core": "^4.3.0"
+    "@jbrowse/core": "^5.0.0-beta.2"
   },
   "publishConfig": {
     "access": "public"
@@ -35,16 +34,13 @@
   "sideEffects": false,
   "scripts": {
     "build": "pnpm run /^build:/",
-    "test": "cd ../..; jest --passWithNoTests plugins/gff3 --passWithNoTests",
+    "test": "cd ../..; jest --passWithNoTests plugins/gff3",
     "prebuild": "pnpm clean",
-    "build:esm": "tsc -p tsconfig.build.esm.json",
+    "build:esm": "node ../../node_modules/typescript7/bin/tsc --build tsconfig.build.esm.json",
     "clean": "rimraf esm *.tsbuildinfo"
   },
   "types": "esm/index.d.ts",
   "exports": {
-    ".": {
-      "types": "./esm/index.d.ts",
-      "import": "./esm/index.js"
-    }
+    ".": "./esm/index.js"
   }
 }

package/esm/Gff3Adapter/gffParser.d.ts

@@ -1,7 +0,0 @@
-import type { StatusCallback } from '@jbrowse/core/util/parseLineByLine';
-interface GffParseResult {
-    header: string;
-    featureMap: Record<string, string>;
-}
-export declare function parseGffBuffer(buffer: Uint8Array, statusCallback?: StatusCallback): GffParseResult;
-export {};

package/esm/Gff3Adapter/gffParser.js

@@ -1,26 +0,0 @@
-import { parseLineByLine } from '@jbrowse/core/util/parseLineByLine';
-export function parseGffBuffer(buffer, statusCallback = () => { }) {
-    const headerLines = [];
-    const featureMap = {};
-    parseLineByLine(buffer, line => {
-        if (line.startsWith('#')) {
-            headerLines.push(line);
-        }
-        else if (line.startsWith('>')) {
-            return false;
-        }
-        else {
-            const ret = line.indexOf('\t');
-            const refName = line.slice(0, ret);
-            if (!featureMap[refName]) {
-                featureMap[refName] = '';
-            }
-            featureMap[refName] += `${line}\n`;
-        }
-        return true;
-    }, statusCallback);
-    return {
-        header: headerLines.join('\n'),
-        featureMap,
-    };
-}

package/esm/GuessGff3/index.d.ts

@@ -1,2 +0,0 @@
-import type PluginManager from '@jbrowse/core/PluginManager';
-export default function GuessGff3F(pluginManager: PluginManager): void;

package/esm/GuessGff3/index.js

@@ -1,30 +0,0 @@
-import { testAdapter } from '@jbrowse/core/util';
-import { getFileName, makeIndex, makeIndexType, } from '@jbrowse/core/util/tracks';
-export default function GuessGff3F(pluginManager) {
-    pluginManager.addToExtensionPoint('Core-guessAdapterForLocation', (adapterGuesser) => {
-        return (file, index, adapterHint) => {
-            const fileName = getFileName(file);
-            const indexName = index && getFileName(index);
-            if (testAdapter(fileName, /\.gff3?\.b?gz$/i, adapterHint, 'Gff3TabixAdapter')) {
-                return {
-                    type: 'Gff3TabixAdapter',
-                    bamLocation: file,
-                    gffGzLocation: file,
-                    index: {
-                        location: index || makeIndex(file, '.tbi'),
-                        indexType: makeIndexType(indexName, 'CSI', 'TBI'),
-                    },
-                };
-            }
-            else if (testAdapter(fileName, /\.gff3?$/i, adapterHint, 'Gff3Adapter')) {
-                return {
-                    type: 'Gff3Adapter',
-                    gffLocation: file,
-                };
-            }
-            else {
-                return adapterGuesser(file, index, adapterHint);
-            }
-        };
-    });
-}

package/esm/featureData.d.ts

@@ -1,16 +0,0 @@
-import type { GFF3FeatureLineWithRefs } from 'gff-nostream';
-interface GFF3Feature {
-    start: number;
-    end: number;
-    strand?: number;
-    type: string | null;
-    source: string | null;
-    refName: string;
-    derived_features: unknown[] | null;
-    phase?: number;
-    score?: number;
-    subfeatures: GFF3Feature[] | undefined;
-    [key: string]: unknown;
-}
-export declare function featureData(data: GFF3FeatureLineWithRefs): GFF3Feature;
-export {};

package/esm/featureData.js

@@ -1,52 +0,0 @@
-const DEFAULT_FIELDS = new Set([
-    'start',
-    'end',
-    'seq_id',
-    'score',
-    'type',
-    'source',
-    'phase',
-    'strand',
-]);
-export function featureData(data) {
-    const { end, start, child_features, derived_features, attributes, type, source, phase, seq_id, score, strand, } = data;
-    let strand2;
-    if (strand === '+') {
-        strand2 = 1;
-    }
-    else if (strand === '-') {
-        strand2 = -1;
-    }
-    else if (strand === '.') {
-        strand2 = 0;
-    }
-    const dataAttributes = attributes || {};
-    const resultAttributes = {};
-    for (const a of Object.keys(dataAttributes)) {
-        let b = a.toLowerCase();
-        if (DEFAULT_FIELDS.has(b)) {
-            b += '2';
-        }
-        if (dataAttributes[a] && a !== '_lineHash') {
-            let attr = dataAttributes[a];
-            if (Array.isArray(attr) && attr.length === 1) {
-                ;
-                [attr] = attr;
-            }
-            resultAttributes[b] = attr;
-        }
-    }
-    return {
-        ...resultAttributes,
-        start: start - 1,
-        end: end,
-        strand: strand2,
-        type,
-        source,
-        refName: seq_id,
-        derived_features,
-        phase: phase === null ? undefined : Number(phase),
-        score: score === null ? undefined : score,
-        subfeatures: child_features.flatMap(childLocs => childLocs.map(childLoc => featureData(childLoc))),
-    };
-}