@apollo-annotation/shared 0.3.8 → 0.3.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apollo-annotation/shared",
-  "version": "0.3.8",
+  "version": "0.3.9",
   "main": "./dist/index.js",
   "scripts": {
     "build": "yarn clean && tsc --build",
@@ -10,10 +10,10 @@
     "test:ci": "NODE_V8_COVERAGE=./coverage glob -c \"tsx --test --test-reporter spec --experimental-test-coverage \" \"**/*.test.ts\""
   },
   "dependencies": {
-    "@apollo-annotation/common": "^0.3.8",
-    "@apollo-annotation/mst": "^0.3.8",
-    "@apollo-annotation/schemas": "^0.3.8",
-    "@gmod/gff": "1.2.0",
+    "@apollo-annotation/common": "^0.3.9",
+    "@apollo-annotation/mst": "^0.3.9",
+    "@apollo-annotation/schemas": "^0.3.9",
+    "@gmod/gff": "^2.0.0",
     "@gmod/indexedfasta": "^2.0.4",
     "@jbrowse/core": "^3.6.5",
     "bson-objectid": "^2.0.4",
@@ -25,7 +25,7 @@
     "@nestjs/common": "^10.1.0",
     "@nestjs/core": "^10.1.0",
     "@types/chai": "^4.3.19",
-    "@types/node": "^18.14.2",
+    "@types/node": "^20.19.15",
     "chai": "^5.1.1",
     "chai-exclude": "^3.0.0",
     "glob": "^11.0.0",

package/src/Changes/AddAssemblyAndFeaturesFromFileChange.ts

@@ -19,6 +19,7 @@ export interface SerializedAddAssemblyAndFeaturesFromFileChangeBase
 export interface AddAssemblyAndFeaturesFromFileChangeDetails {
   assemblyName: string
   fileIds: { fa: string }
+  parseOptions?: { bufferSize: number }
 }
 
 export interface SerializedAddAssemblyAndFeaturesFromFileChangeSingle
@@ -68,7 +69,7 @@ export class AddAssemblyAndFeaturesFromFileChange extends FromFileBaseChange {
     const { assemblyModel, checkModel, fileModel, filesService, user } = backend
     const { assembly, changes, logger } = this
     for (const change of changes) {
-      const { assemblyName, fileIds } = change
+      const { assemblyName, fileIds, parseOptions } = change
       const fileId = fileIds.fa
 
       const { FILE_UPLOAD_FOLDER } = process.env
@@ -110,13 +111,13 @@ export class AddAssemblyAndFeaturesFromFileChange extends FromFileBaseChange {
         backend,
       )
 
-      // Loop all features
-      logger.debug?.(
-        `**************** LOOPATAAN KAIKKI FEATURET SEURAAVAKSI File type: "${fileDoc.type}"`,
-      )
+      const { bufferSize = 10_000 } = parseOptions ?? {}
       const featureStream = filesService.parseGFF3(
         filesService.getFileStream(fileDoc),
+        { bufferSize },
       )
+      // @ts-expect-error type is wrong here
+      // eslint-disable-next-line @typescript-eslint/await-thenable
       for await (const f of featureStream) {
         const gff3Feature = f as GFF3Feature
         logger.verbose?.(`ENTRY=${JSON.stringify(gff3Feature)}`)

package/src/Changes/AddFeatureChange.ts

@@ -167,9 +167,23 @@ export class AddFeatureChange extends FeatureChange {
     for (const change of changes) {
       const { addedFeature, parentFeatureId } = change
       if (parentFeatureId) {
-        const parentFeature = dataStore.getFeature(parentFeatureId)
+        let parentFeature = dataStore.getFeature(parentFeatureId)
+        // maybe the parent feature hasn't been loaded yet
         if (!parentFeature) {
-          throw new Error(`Could not find parent feature "${parentFeatureId}"`)
+          await dataStore.loadFeatures([
+            {
+              assemblyName: assembly,
+              refName: addedFeature.refSeq,
+              start: addedFeature.min,
+              end: addedFeature.max,
+            },
+          ])
+          parentFeature = dataStore.getFeature(parentFeatureId)
+          if (!parentFeature) {
+            throw new Error(
+              `Could not find parent feature "${parentFeatureId}"`,
+            )
+          }
         }
         // create an ID for the parent feature if it does not have one
         if (!parentFeature.attributes.get('_id')) {

package/src/Changes/AddFeaturesFromFileChange.ts

@@ -18,6 +18,7 @@ export interface SerializedAddFeaturesFromFileChangeBase
 
 export interface AddFeaturesFromFileChangeDetails {
   fileId: string
+  parseOptions?: { bufferSize: number }
 }
 
 export interface SerializedAddFeaturesFromFileChangeSingle
@@ -75,7 +76,7 @@ export class AddFeaturesFromFileChange extends FromFileBaseChange {
     }
 
     for (const change of changes) {
-      const { fileId } = change
+      const { fileId, parseOptions } = change
 
       const { FILE_UPLOAD_FOLDER } = process.env
       if (!FILE_UPLOAD_FOLDER) {
@@ -89,10 +90,14 @@ export class AddFeaturesFromFileChange extends FromFileBaseChange {
       logger.debug?.(`FileId "${fileId}", checksum "${fileDoc.checksum}"`)
 
       // Read data from compressed file and parse the content
+      const { bufferSize = 10_000 } = parseOptions ?? {}
       const featureStream = filesService.parseGFF3(
         filesService.getFileStream(fileDoc),
+        { bufferSize },
       )
       let featureCount = 0
+      // @ts-expect-error type is wrong here
+      // eslint-disable-next-line @typescript-eslint/await-thenable
       for await (const f of featureStream) {
         const gff3Feature = f as GFF3Feature
 

package/src/Changes/AddRefSeqAliasesChange.ts

@@ -35,11 +35,7 @@ export class AddRefSeqAliasesChange extends AssemblySpecificChange {
     if (!assembly) {
       throw new Error(`assembly ${this.assembly} not found`)
     }
-    const sessionAliases = assembly.refNameAliases
-
-    if (!sessionAliases) {
-      throw new Error('Session refNameAliases not found in assembly')
-    }
+    const sessionAliases = assembly.refNameAliases ?? {}
 
     for (const refSeqAlias of this.refSeqAliases) {
       const { aliases, refName } = refSeqAlias

package/src/Changes/FromFileBaseChange.ts

@@ -1,7 +1,4 @@
-/* eslint-disable @typescript-eslint/restrict-plus-operands */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
 import {
   AssemblySpecificChange,
@@ -38,8 +35,11 @@ export abstract class FromFileBaseChange extends AssemblySpecificChange {
     let parsingStarted = false
     logger.debug?.('starting sequence stream')
     let lineCount = 0
+    const decoder = new TextDecoder()
+    // @ts-expect-error type is wrong here
+    // eslint-disable-next-line @typescript-eslint/await-thenable
     for await (const data of sequenceStream) {
-      const chunk = data.toString()
+      const chunk = decoder.decode(data)
       lastLineIsIncomplete = !chunk.endsWith('\n')
       // chunk is small enough that you can split the whole thing into lines without having to make it into smaller chunks first.
       const lines = chunk.split(/\r?\n/)
@@ -48,9 +48,9 @@ export abstract class FromFileBaseChange extends AssemblySpecificChange {
         incompleteLine = ''
       }
       if (lastLineIsIncomplete) {
-        incompleteLine = lines.pop() || ''
+        incompleteLine = lines.pop() ?? ''
       }
-      for await (const line of lines) {
+      for (const line of lines) {
         lineCount++
         if (lineCount % 1_000_000 === 0) {
           logger.debug?.(`Processed ${lineCount} lines`)

package/src/Checks/CDSCheck.ts

@@ -6,6 +6,8 @@ import {
 import { intersection2 } from '@jbrowse/core/util'
 import ObjectID from 'bson-objectid'
 
+import { getPrintableId } from './util'
+
 enum STOP_CODONS {
   'TAG',
   'TAA',
@@ -175,7 +177,7 @@ async function checkMRNA(
           refSeq: refSeq.toString(),
           start: cdsStart,
           end: cdsStart,
-          message: `Unexpected start codon in feature "${_id}": ${start_codon}`,
+          message: `Unexpected start codon "${start_codon}" in feature "${getPrintableId(feature)}": `,
         })
       }
 
@@ -189,7 +191,7 @@ async function checkMRNA(
           refSeq: refSeq.toString(),
           start: cdsEnd,
           end: cdsEnd,
-          message: `Missing stop codon for feature "${_id}"`,
+          message: `Missing stop codon in feature "${getPrintableId(feature)}"`,
         })
       }
     } else {
@@ -201,7 +203,7 @@ async function checkMRNA(
         refSeq: refSeq.toString(),
         start: cdsEnd,
         end: cdsEnd,
-        message: `Missing stop codon: The coding sequence for feature "${_id}" is not a multiple of three`,
+        message: `Missing stop codon in feature "${getPrintableId(feature)}"`,
       })
     }
     for (const [idx, codon] of codons.entries()) {
@@ -220,7 +222,7 @@ async function checkMRNA(
           refSeq: refSeq.toString(),
           start: codonStart,
           end: codonEnd,
-          message: `The coding sequence for feature "${_id}" has an internal stop codon`,
+          message: `Internal stop codon in feature "${getPrintableId(feature)}"`,
         })
       }
     }

package/src/Checks/TranscriptCheck.ts

@@ -6,6 +6,8 @@ import {
 import { revcom } from '@jbrowse/core/util'
 import ObjectID from 'bson-objectid'
 
+import { getPrintableId } from './util'
+
 interface SpliceSequence {
   fivePrimeSeq: string
   fivePrimeMin: number
@@ -35,15 +37,17 @@ async function getSpliceSequences(
   if (exons.length < 2) {
     return []
   }
+  exons.sort((a, b) => (a.min < b.min ? -1 : 1))
 
   const spliceSeq: SpliceSequence[] = []
   for (let i = 0; i < exons.length - 1; i++) {
     let fivePrimeMin = exons[i].max
-    let threePrimeMin = exons[i + 1].min - 2
+    let threePrimeMin = exons[i + 1].min
     if (transcript.strand === -1) {
-      const _fivePrimeMin = fivePrimeMin
-      fivePrimeMin = threePrimeMin
-      threePrimeMin = _fivePrimeMin
+      ;[threePrimeMin, fivePrimeMin] = [fivePrimeMin, threePrimeMin]
+      fivePrimeMin -= 2
+    } else {
+      threePrimeMin -= 2
     }
 
     let fivePrimeSeq = await getSequence(fivePrimeMin, fivePrimeMin + 2)
@@ -53,12 +57,7 @@ async function getSpliceSequences(
       fivePrimeSeq = revcom(fivePrimeSeq)
     }
 
-    spliceSeq.push({
-      fivePrimeSeq,
-      fivePrimeMin,
-      threePrimeSeq,
-      threePrimeMin,
-    })
+    spliceSeq.push({ fivePrimeSeq, fivePrimeMin, threePrimeSeq, threePrimeMin })
   }
   return spliceSeq
 }
@@ -82,7 +81,7 @@ async function checkTranscript(
         refSeq: feature.refSeq.toString(),
         start: spliceSequence.fivePrimeMin,
         end: spliceSequence.fivePrimeMin + 2,
-        message: `Unexpected 5' splice site in "${feature._id}". Expected: ${[...VALID_FIVE_PRIME_SEQ].join('|')}, got: ${spliceSequence.fivePrimeSeq}`,
+        message: `Unexpected 5′ splice site in "${getPrintableId(feature)}". Expected: ${[...VALID_FIVE_PRIME_SEQ].join('|')}, got: ${spliceSequence.fivePrimeSeq}`,
       })
     }
     if (
@@ -96,7 +95,7 @@ async function checkTranscript(
         refSeq: feature.refSeq.toString(),
         start: spliceSequence.threePrimeMin,
         end: spliceSequence.threePrimeMin + 2,
-        message: `Unexpected 3' splice site in "${feature._id}". Expected: ${[...VALID_THREE_PRIME_SEQ].join('|')}, got: ${spliceSequence.threePrimeSeq}`,
+        message: `Unexpected 3′ splice site in "${getPrintableId(feature)}". Expected: ${[...VALID_THREE_PRIME_SEQ].join('|')}, got: ${spliceSequence.threePrimeSeq}`,
       })
     }
   }

package/src/Checks/util.ts

@@ -0,0 +1,13 @@
+import { type AnnotationFeatureSnapshot } from '@apollo-annotation/mst'
+
+export function getPrintableId(feature: AnnotationFeatureSnapshot): string {
+  const gff_id = feature.attributes?.gff_id?.join(', ')
+  if (gff_id) {
+    return `${gff_id} (_id: ${feature._id.toString()})`
+  }
+  const gff_name = feature.attributes?.gff_name?.join(', ')
+  if (gff_name) {
+    return `${gff_name} (_id: ${feature._id.toString()})`
+  }
+  return `_id: ${feature._id.toString()}`
+}

package/src/GFF3/annotationFeatureToGFF3.ts

@@ -101,7 +101,10 @@ export function annotationFeatureToGFF3(
     score,
     strand: feature.strand ? (feature.strand === 1 ? '+' : '-') : null,
     phase: null,
-    attributes: Object.keys(attributes).length > 0 ? attributes : null,
+    attributes:
+      Object.keys(attributes).length > 0
+        ? (attributes as Record<string, string[]>)
+        : null,
     derived_features: [],
     child_features: prepareChildFeatures(
       feature,

package/src/GFF3/gff3ToAnnotationFeature.test.ts

@@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'
 import { describe, it } from 'node:test'
 
 import { type AnnotationFeatureSnapshot } from '@apollo-annotation/mst'
-import gff, { type GFF3Feature } from '@gmod/gff'
+import { type GFF3Feature, parseStringSync } from '@gmod/gff'
 import { assert, use } from 'chai'
 import chaiExclude from 'chai-exclude'
 
@@ -116,7 +116,7 @@ function readFeatureFile(fn: string): GFF3Feature[] {
       feature.push(line)
     }
   }
-  const inGff = gff.parseStringSync(feature.join('\n')) as GFF3Feature[]
+  const inGff = parseStringSync(feature.join('\n')) as GFF3Feature[]
   return inGff
 }
 
@@ -232,7 +232,7 @@ describe('gff3ToAnnotationFeature', () => {
   for (const testCase of testCases) {
     const [description, featureLine, convertedFeature] = testCase
     it(`converts ${description}`, () => {
-      const gff3Feature = gff.parseStringSync(featureLine, {
+      const gff3Feature = parseStringSync(featureLine, {
         parseSequences: false,
       })
       const feature = gff3ToAnnotationFeature(gff3Feature[0])

package/src/GFF3/gff3ToAnnotationFeature.ts

@@ -115,7 +115,7 @@ function convertFeatureAttributes(
   if (attributesCollections.length > 0) {
     for (const attributesCollection of attributesCollections) {
       for (const [key, val] of Object.entries(attributesCollection)) {
-        if (!val || key === 'Parent') {
+        if (key === 'Parent') {
           continue
         }
         const newKey = isGFFReservedAttribute(key) ? gffToInternal[key] : key