@jbrowse/plugin-gff3 5.0.0-beta.11 → 5.0.0-beta.14

package/esm/Gff3Adapter/Gff3Adapter.js

@@ -1,10 +1,11 @@
+import { parseLinesLazy } from '@gmod/gff-nostream';
 import { BaseFeatureDataAdapter, cachedSetup, } from '@jbrowse/core/data_adapters/BaseAdapter';
 import { fetchAndMaybeUnzip } from '@jbrowse/core/util';
 import { openLocation } from '@jbrowse/core/util/io';
 import { groupLinesByRef, makeFeatureIntervalTreeMap, } from '@jbrowse/core/util/parseLineByLine';
 import { ObservableCreate } from '@jbrowse/core/util/rxjs';
-import { parseLinesLazy } from 'gff-nostream';
 import { Gff3Feature } from '../Gff3Feature.js';
+import { cutAtOrigin, sequenceLengths } from '../originSpanning.js';
 export default class Gff3Adapter extends BaseFeatureDataAdapter {
     // the whole file is resident after one load, so the fetch/parse status comes
     // from inside the load itself rather than a label wrapped around it
@@ -32,7 +33,8 @@ export default class Gff3Adapter extends BaseFeatureDataAdapter {
                 feature,
                 uniqueId: `${this.id}-${refName}-${i}`,
             })), 'Parsing GFF data');
-            return { header: headerLines.join('\n'), intervalTreeMap };
+            const header = headerLines.join('\n');
+            return { header, intervalTreeMap, lengths: sequenceLengths(header) };
         },
     });
     async getRefNames(opts = {}) {
@@ -48,13 +50,19 @@ export default class Gff3Adapter extends BaseFeatureDataAdapter {
         // observer.error itself
         return ObservableCreate(async (observer) => {
             const { start, end, refName } = query;
-            const { intervalTreeMap } = await this.loadData(opts);
-            const tree = intervalTreeMap[refName];
+            const { intervalTreeMap, lengths } = await this.loadData(opts);
+            const tree = intervalTreeMap[refName]?.(opts.statusCallback);
             if (tree) {
-                for (const { feature, uniqueId } of tree(opts.statusCallback).search([
-                    start,
-                    end,
-                ])) {
+                // a feature crossing a circular sequence's origin is stored from its
+                // start near the end, so a query short of the end looks there too
+                const length = lengths.get(refName);
+                const candidates = [
+                    ...tree.search([start, end]),
+                    ...(length !== undefined && end < length
+                        ? tree.search([length - 1, length])
+                        : []),
+                ];
+                for (const { feature, uniqueId } of cutAtOrigin(candidates, query, length)) {
                     observer.next(new Gff3Feature(feature, uniqueId));
                 }
             }

package/esm/Gff3Adapter/configSchema.d.ts

@@ -5,8 +5,8 @@ export declare function normalizeSnapshot(snap: Record<string, unknown>): Record
  * #category adapter
  * #trackType FeatureTrack
  * #fileFormat feature | GFF3 (plain) | Loaded entirely into memory; for small files
- * used to load plain-text GFF3 files. Loads the whole file into memory, so
- * prefer the Gff3TabixAdapter for large files.
+ * Reads plain-text GFF3 files. The adapter loads the whole file into memory,
+ * so use `Gff3TabixAdapter` for large files.
  *
  * #example
  * ```js

package/esm/Gff3Adapter/configSchema.js

@@ -7,8 +7,8 @@ export function normalizeSnapshot(snap) {
  * #category adapter
  * #trackType FeatureTrack
  * #fileFormat feature | GFF3 (plain) | Loaded entirely into memory; for small files
- * used to load plain-text GFF3 files. Loads the whole file into memory, so
- * prefer the Gff3TabixAdapter for large files.
+ * Reads plain-text GFF3 files. The adapter loads the whole file into memory,
+ * so use `Gff3TabixAdapter` for large files.
  *
  * #example
  * ```js
@@ -33,6 +33,7 @@ const Gff3Adapter = ConfigurationSchema('Gff3Adapter', {
     },
 }, {
     explicitlyTyped: true,
+    closed: true,
     /**
      * #preProcessSnapshot
      *

package/esm/Gff3Feature.d.ts

@@ -1,5 +1,5 @@
+import type { LazyGffFeature } from '@gmod/gff-nostream';
 import type { Feature, SimpleFeatureSerialized } from '@jbrowse/core/util/simpleFeature';
-import type { LazyGffFeature } from 'gff-nostream';
 /**
  * A {@link Feature} over a GFF3 feature whose attributes are still raw column-9
  * text, resolving each one only when something asks for it.
@@ -38,6 +38,8 @@ export declare class Gff3Feature implements Feature {
     get(name: 'subfeatures'): Feature[] | undefined;
     get(name: string): unknown;
     id(): string;
+    /** The feature's column 9 keyed as the file writes it (`fileAttributes`). */
+    fileAttributes(): Record<string, string[]>;
     parent(): Feature | undefined;
     /**
      * Wrapped lazily rather than in the constructor: a feature the layout pass

package/esm/Gff3Feature.js

@@ -1,4 +1,5 @@
-import { getAttribute, getAttributes } from 'gff-nostream';
+import { getAttribute, getAttributes } from '@gmod/gff-nostream';
+import { fileAttributes } from './fileAttributes.js';
 /**
  * A {@link Feature} over a GFF3 feature whose attributes are still raw column-9
  * text, resolving each one only when something asks for it.
@@ -69,6 +70,10 @@ export class Gff3Feature {
     id() {
         return this.uniqueId;
     }
+    /** The feature's column 9 keyed as the file writes it (`fileAttributes`). */
+    fileAttributes() {
+        return fileAttributes(this.data.attributeString);
+    }
     parent() {
         return this.parentFeature;
     }

package/esm/Gff3TabixAdapter/Gff3TabixAdapter.d.ts

@@ -4,7 +4,10 @@ import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter';
 import type { Feature } from '@jbrowse/core/util/simpleFeature';
 import type { Region } from '@jbrowse/core/util/types';
 export default class Gff3TabixAdapter extends BaseFeatureDataAdapter<Gff3TabixAdapterConfig> {
+    private file?;
+    freeResources(): void;
     private configure;
+    private lengths;
     getRefNames(opts?: BaseOptions): Promise<string[]>;
     getHeader(opts?: BaseOptions): Promise<string>;
     getRegionByteSize(regions: Region[], opts?: BaseOptions): Promise<number>;

package/esm/Gff3TabixAdapter/Gff3TabixAdapter.js

@@ -1,14 +1,19 @@
+import { hasIdAttribute, parseRecordsLazy } from '@gmod/gff-nostream';
 import { TabixIndexedFile } from '@gmod/tabix';
 import { BaseFeatureDataAdapter, cachedSetup, } from '@jbrowse/core/data_adapters/BaseAdapter';
 import { downloadStatus } from '@jbrowse/core/util';
 import { decompressedBytesBudget } from '@jbrowse/core/util/cacheBudgets';
 import { openLocation, openTabixIndexFilehandle } from '@jbrowse/core/util/io';
-import { doesIntersect2 } from '@jbrowse/core/util/range';
 import { ObservableCreate } from '@jbrowse/core/util/rxjs';
 import { readTabixLinesRedispatched } from '@jbrowse/core/util/tabix';
-import { hasIdAttribute, parseRecordsLazy } from 'gff-nostream';
 import { Gff3Feature } from '../Gff3Feature.js';
+import { cutAtOrigin, sequenceLengths } from '../originSpanning.js';
 export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
+    // set once the file is open, for dataAdapterCache's eviction to clear
+    file;
+    freeResources() {
+        this.file?.clearChunkCache();
+    }
     configure = cachedSetup({
         label: 'Downloading index',
         setup: async (opts, onProgress) => {
@@ -17,6 +22,7 @@ export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
                 ...openTabixIndexFilehandle(this.getConf(['index', 'location']), this.getConf(['index', 'indexType']), this.pluginManager),
                 chunkCacheBudget: decompressedBytesBudget,
             });
+            this.file = gff;
             // the index is a whole-file read, so its byte ticks turn the
             // "Downloading index" label into a determinate bar
             await gff.getReferenceSequenceNames({ onProgress, signal: opts.signal });
@@ -26,6 +32,15 @@ export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
             };
         },
     });
+    // the header read is sized from the index, so it can start only once the
+    // index is in; the region read needs the same index and nothing else, so
+    // getFeatures runs the two together
+    lengths = cachedSetup({
+        setup: async (opts) => {
+            const { gff } = await this.configure(opts);
+            return sequenceLengths(await gff.getHeader(opts));
+        },
+    });
     async getRefNames(opts = {}) {
         const { gff } = await this.configure(opts);
         return downloadStatus('Downloading index', opts.statusCallback, onProgress => gff.getReferenceSequenceNames({ ...opts, onProgress }));
@@ -47,7 +62,7 @@ export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
                 const { gff, dontRedispatchSet } = await this.configure(opts);
                 // The bound exists to reach a record's children, so what disqualifies a
                 // record is having none. `ID` answers that exactly — nothing can name a
-                // parent that has no name — and gff-nostream decides it with the scan
+                // parent that has no name — and @gmod/gff-nostream decides it with the scan
                 // its linker uses, so a record the tree would attach children to is
                 // never one the bound leaves out. `dontRedispatch` can only approximate
                 // the same question by type: every NCBI `GCF_*_genomic.gff.gz` opens
@@ -56,22 +71,39 @@ export default class Gff3TabixAdapter extends BaseFeatureDataAdapter {
                 // record carrying an `ID` that nothing references, which hosted hg19
                 // RefSeq's chromosome-long `region` is.
                 const canHaveChildren = (line) => hasIdAttribute(line.line) && !dontRedispatchSet.has(line.type);
-                const lines = await readTabixLinesRedispatched(gff, query, 
+                const read = (region) => readTabixLinesRedispatched(gff, region, 
                 // A caller reading only top-level features is owed nothing by the
-                // flanks and pays for them anyway: they complete SUBFEATURE lists, and
-                // a top-level feature overlapping the query is in the query's own read
-                // already, since tabix returns every overlapping line and a child is
-                // contained in its parent. On an NCBI `GCF_*_genomic.gff.gz`, whose
-                // chromosome-long `match` widens the bound at every window, that is
-                // 193,008 lines parsed to keep 3 features — 2734 ms against 8 ms.
+                // flanks and pays for them anyway: they complete SUBFEATURE lists,
+                // and a top-level feature overlapping the query is in the query's
+                // own read already, since tabix returns every overlapping line and
+                // a child is contained in its parent. On an NCBI
+                // `GCF_*_genomic.gff.gz`, whose chromosome-long `match` widens the
+                // bound at every window, that is 193,008 lines parsed to keep 3
+                // features — 2734 ms against 8 ms.
                 opts.topLevelOnly ? () => false : canHaveChildren, opts);
-                // emit only top-level features intersecting the original query. the
-                // byte offset stays on our own record and is used purely to mint a
-                // stable id, so it never pollutes the feature's data
-                for (const { feature, record } of parseRecordsLazy(lines)) {
-                    if (doesIntersect2(feature.start, feature.end, query.start, query.end)) {
-                        observer.next(new Gff3Feature(feature, `${this.id}-offset-${record.offset}`));
-                    }
+                // a feature crossing a circular sequence's origin is indexed from its
+                // start near the end, so a query short of the end reads the last base
+                // too, for the part such a feature carries past the origin. That read
+                // needs the length from the header, so it starts when the header
+                // arrives and the query's own read does not wait for it.
+                const tailRead = this.lengths(opts).then(async (lengths) => {
+                    const length = lengths.get(query.refName);
+                    return {
+                        length,
+                        lines: length !== undefined && query.end < length
+                            ? await read({ ...query, start: length - 1, end: length })
+                            : [],
+                    };
+                });
+                // the byte offset stays on our own record and is used purely to mint
+                // a stable id, so it never pollutes the feature's data
+                const [lines, tail] = await Promise.all([read(query), tailRead]);
+                const candidates = [lines, tail.lines].flatMap(group => parseRecordsLazy(group).map(({ feature, record }) => ({
+                    feature,
+                    uniqueId: `${this.id}-offset-${record.offset}`,
+                })));
+                for (const { feature, uniqueId } of cutAtOrigin(candidates, query, tail.length)) {
+                    observer.next(new Gff3Feature(feature, uniqueId));
                 }
                 observer.complete();
             }

package/esm/Gff3TabixAdapter/configSchema.d.ts

@@ -5,7 +5,7 @@ export declare function normalizeSnapshot(snap: Record<string, unknown>): Record
  * #category adapter
  * #trackType FeatureTrack
  * #fileFormat feature | GFF3 (tabix)
- * used to load bgzip-compressed, tabix-indexed GFF3 files
+ * Reads bgzip-compressed, tabix-indexed GFF3 files.
  *
  * #example
  * The `uri` shorthand auto-resolves the `.tbi` index; add `csi: true` for a
@@ -18,9 +18,6 @@ export declare function normalizeSnapshot(snap: Record<string, unknown>): Record
  * ```
  */
 declare const Gff3TabixAdapter: import("@jbrowse/core/configuration/configurationSchema").ConfigurationSchemaType<{
-    readonly densityAdapter: {
-        type: "maybeFrozen";
-    };
     type: {
         type: "string";
     } & {

package/esm/Gff3TabixAdapter/configSchema.js

@@ -1,5 +1,4 @@
 import { ConfigurationSchema, expandTabixShorthand, tabixIndexSchema, } from '@jbrowse/core/configuration';
-import { densityAdapterConfigSchemaFields } from '@jbrowse/core/data_adapters/BaseAdapter';
 export function normalizeSnapshot(snap) {
     return expandTabixShorthand(snap, 'gffGzLocation');
 }
@@ -8,7 +7,7 @@ export function normalizeSnapshot(snap) {
  * #category adapter
  * #trackType FeatureTrack
  * #fileFormat feature | GFF3 (tabix)
- * used to load bgzip-compressed, tabix-indexed GFF3 files
+ * Reads bgzip-compressed, tabix-indexed GFF3 files.
  *
  * #example
  * The `uri` shorthand auto-resolves the `.tbi` index; add `csi: true` for a
@@ -35,7 +34,7 @@ const Gff3TabixAdapter = ConfigurationSchema('Gff3TabixAdapter', {
     },
     /**
      * #slot
-     * where the tabix index is and which kind it is. The `uri` shorthand derives
+     * The tabix index location and its kind. The `uri` shorthand derives
      * both, so a config using it states neither.
      */
     index: tabixIndexSchema(),
@@ -62,9 +61,9 @@ const Gff3TabixAdapter = ConfigurationSchema('Gff3TabixAdapter', {
             'scaffold',
         ],
     },
-    ...densityAdapterConfigSchemaFields,
 }, {
     explicitlyTyped: true,
+    closed: true,
     /**
      * #preProcessSnapshot
      *

package/esm/fileAttributes.d.ts

@@ -0,0 +1,7 @@
+/**
+ * A GFF3 column 9 keyed as the file writes it (`ID`, `Name`, `gene_name`),
+ * each value the comma-separated list the column gives, %-escapes decoded.
+ * `@gmod/gff-nostream` lowercases the keys for JBrowse's own reads, which a
+ * host reading the file's fields should not see.
+ */
+export declare function fileAttributes(column: string): Record<string, string[]>;

package/esm/fileAttributes.js

@@ -0,0 +1,38 @@
+/**
+ * A GFF3 column 9 keyed as the file writes it (`ID`, `Name`, `gene_name`),
+ * each value the comma-separated list the column gives, %-escapes decoded.
+ * `@gmod/gff-nostream` lowercases the keys for JBrowse's own reads, which a
+ * host reading the file's fields should not see.
+ */
+export function fileAttributes(column) {
+    const out = {};
+    if (column === '.' || column === '') {
+        return out;
+    }
+    for (const entry of column.split(';')) {
+        const eq = entry.indexOf('=');
+        const key = unescape(entry.slice(0, eq).trim());
+        const values = entry
+            .slice(eq + 1)
+            .trimEnd()
+            .split(',')
+            .filter(v => v !== '')
+            .map(unescape);
+        if (eq > 0 && key && values.length) {
+            out[key] = values;
+        }
+    }
+    return out;
+}
+function unescape(s) {
+    return s.includes('%')
+        ? s.replaceAll(/(?:%[\dA-Fa-f]{2})+/g, run => {
+            try {
+                return decodeURIComponent(run);
+            }
+            catch {
+                return run;
+            }
+        })
+        : s;
+}

package/esm/originSpanning.d.ts

@@ -0,0 +1,24 @@
+import type { LazyGffFeature } from '@gmod/gff-nostream';
+/** Each sequence's length, as the file's `##sequence-region` pragmas state it. */
+export declare function sequenceLengths(header: string): Map<string, number>;
+export interface Candidate {
+    feature: LazyGffFeature;
+    uniqueId: string;
+}
+/**
+ * The candidates `query` overlaps, a feature crossing the origin of a
+ * sequence `length` long cut there into the part before it and the part after.
+ *
+ * GFF3 writes such a feature on a circular sequence with its end past the
+ * sequence's length, in "virtual" coordinates, so one record covers both
+ * sides. Each half keeps the children that fall in it, clipped; the half
+ * after the origin, shifted back onto the sequence, takes `originTailId` of
+ * the id.
+ */
+export declare function cutAtOrigin(candidates: Iterable<Candidate>, query: {
+    start: number;
+    end: number;
+}, length: number | undefined): Generator<{
+    feature: LazyGffFeature;
+    uniqueId: string;
+}, void, unknown>;

package/esm/originSpanning.js

@@ -0,0 +1,59 @@
+import { originTailId } from '@jbrowse/core/util/originCut';
+import { doesIntersect2 } from '@jbrowse/core/util/range';
+/** Each sequence's length, as the file's `##sequence-region` pragmas state it. */
+export function sequenceLengths(header) {
+    const lengths = new Map();
+    for (const line of header.split('\n')) {
+        const match = /^##sequence-region\s+(\S+)\s+\d+\s+(\d+)/.exec(line);
+        if (match) {
+            lengths.set(match[1], Number(match[2]));
+        }
+    }
+    return lengths;
+}
+function clip(feature, start, end, shift) {
+    const from = Math.max(feature.start, start);
+    const to = Math.min(feature.end, end);
+    return to > from
+        ? {
+            ...feature,
+            start: from - shift,
+            end: to - shift,
+            subfeatures: feature.subfeatures.flatMap(child => clip(child, start, end, shift) ?? []),
+        }
+        : undefined;
+}
+/**
+ * The candidates `query` overlaps, a feature crossing the origin of a
+ * sequence `length` long cut there into the part before it and the part after.
+ *
+ * GFF3 writes such a feature on a circular sequence with its end past the
+ * sequence's length, in "virtual" coordinates, so one record covers both
+ * sides. Each half keeps the children that fall in it, clipped; the half
+ * after the origin, shifted back onto the sequence, takes `originTailId` of
+ * the id.
+ */
+export function* cutAtOrigin(candidates, query, length) {
+    const seen = new Set();
+    for (const { feature, uniqueId } of candidates) {
+        if (seen.has(uniqueId)) {
+            continue;
+        }
+        seen.add(uniqueId);
+        const halves = length !== undefined && feature.end > length
+            ? [
+                { half: clip(feature, feature.start, length, 0), uniqueId },
+                {
+                    half: clip(feature, length, feature.end, length),
+                    uniqueId: originTailId(uniqueId),
+                },
+            ]
+            : [{ half: feature, uniqueId }];
+        for (const { half, uniqueId: id } of halves) {
+            if (half &&
+                doesIntersect2(half.start, half.end, query.start, query.end)) {
+                yield { feature: half, uniqueId: id };
+            }
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/plugin-gff3",
-  "version": "5.0.0-beta.11",
+  "version": "5.0.0-beta.14",
   "type": "module",
   "description": "JBrowse 2 gff3.",
   "keywords": [
@@ -21,12 +21,12 @@
     "esm"
   ],
   "dependencies": {
-    "@gmod/tabix": "^3.9.1",
-    "@jbrowse/mobx-state-tree": "^6.7.0",
-    "gff-nostream": "^5.6.0",
-    "mobx": "^7.0.5",
+    "@gmod/gff-nostream": "^5.6.2",
+    "@gmod/tabix": "^3.11.0",
+    "@jbrowse/mobx-state-tree": "^6.9.0",
+    "mobx": "^7.0.6",
     "rxjs": "^7.8.2",
-    "@jbrowse/core": "^5.0.0-beta.11"
+    "@jbrowse/core": "^5.0.0-beta.14"
   },
   "peerDependencies": {
     "react": ">=19.0.0",