@apollo-annotation/shared 0.1.19 → 0.1.20

package/dist/util.d.ts

@@ -0,0 +1,4 @@
+import { AnnotationFeatureSnapshot } from '@apollo-annotation/mst';
+import { GFF3Feature } from '@gmod/gff';
+export declare function makeGFF3Feature(feature: AnnotationFeatureSnapshot, parentId?: string, refSeqNames?: Record<string, string | undefined>): GFF3Feature;
+export declare function splitStringIntoChunks(input: string, chunkSize: number): string[];

package/dist/util.js

@@ -0,0 +1,102 @@
+"use strict";
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeGFF3Feature = makeGFF3Feature;
+exports.splitStringIntoChunks = splitStringIntoChunks;
+function makeGFF3Feature(feature, parentId, refSeqNames) {
+    const locations = [{ start: feature.min, end: feature.max }];
+    // const locations = feature.discontinuousLocations?.length
+    //   ? feature.discontinuousLocations
+    //   : [{ start: feature.start, end: feature.end, phase: feature.phase }]
+    const attributes = JSON.parse(JSON.stringify(feature.attributes));
+    const ontologyTerms = [];
+    const source = feature.attributes?.source?.[0] ?? null;
+    delete attributes.source;
+    if (parentId) {
+        attributes.Parent = [parentId];
+    }
+    if (attributes._id) {
+        attributes.ID = attributes._id;
+        delete attributes._id;
+    }
+    if (attributes.gff_name) {
+        attributes.Name = attributes.gff_name;
+        delete attributes.gff_name;
+    }
+    if (attributes.gff_alias) {
+        attributes.Alias = attributes.gff_alias;
+        delete attributes.gff_alias;
+    }
+    if (attributes.gff_target) {
+        attributes.Target = attributes.gff_target;
+        delete attributes.gff_target;
+    }
+    if (attributes.gff_gap) {
+        attributes.Gap = attributes.gff_gap;
+        delete attributes.gff_gap;
+    }
+    if (attributes.gff_derives_from) {
+        attributes.Derives_from = attributes.gff_derives_from;
+        delete attributes.gff_derives_from;
+    }
+    if (attributes.gff_note) {
+        attributes.Note = attributes.gff_note;
+        delete attributes.gff_note;
+    }
+    if (attributes.gff_dbxref) {
+        attributes.Dbxref = attributes.gff_dbxref;
+        delete attributes.gff_dbxref;
+    }
+    if (attributes.gff_is_circular) {
+        attributes.Is_circular = attributes.gff_is_circular;
+        delete attributes.gff_is_circular;
+    }
+    if (attributes.gff_ontology_term) {
+        ontologyTerms.push(...attributes.gff_ontology_term);
+        delete attributes.gff_ontology_term;
+    }
+    if (attributes['Gene Ontology']) {
+        ontologyTerms.push(...attributes['Gene Ontology']);
+        delete attributes['Gene Ontology'];
+    }
+    if (attributes['Sequence Ontology']) {
+        ontologyTerms.push(...attributes['Sequence Ontology']);
+        delete attributes['Sequence Ontology'];
+    }
+    if (ontologyTerms.length > 0) {
+        attributes.Ontology_term = ontologyTerms;
+    }
+    return locations.map((location) => ({
+        start: location.start + 1,
+        end: location.end,
+        seq_id: refSeqNames ? refSeqNames[feature.refSeq] ?? null : feature.refSeq,
+        source,
+        type: feature.type,
+        score: null,
+        // score: feature.score ?? null,
+        strand: feature.strand ? (feature.strand === 1 ? '+' : '-') : null,
+        phase: null,
+        // phase:
+        //   location.phase === 0
+        //     ? '0'
+        //     : location.phase === 1
+        //       ? '1'
+        //       : location.phase === 2
+        //         ? '2'
+        //         : null,
+        attributes: Object.keys(attributes).length > 0 ? attributes : null,
+        derived_features: [],
+        child_features: feature.children
+            ? Object.values(feature.children).map((child) => makeGFF3Feature(child, attributes.ID?.[0], refSeqNames))
+            : [],
+    }));
+}
+function splitStringIntoChunks(input, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < input.length; i += chunkSize) {
+        const chunk = input.slice(i, i + chunkSize);
+        chunks.push(chunk);
+    }
+    return chunks;
+}
+//# sourceMappingURL=util.js.map

package/dist/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";AAAA,4DAA4D;;AAK5D,0CA+FC;AAED,sDAUC;AA3GD,SAAgB,eAAe,CAC7B,OAAkC,EAClC,QAAiB,EACjB,WAAgD;IAEhD,MAAM,SAAS,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAC5D,2DAA2D;IAC3D,qCAAqC;IACrC,yEAAyE;IACzE,MAAM,UAAU,GAAyC,IAAI,CAAC,KAAK,CACjE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CACnC,CAAA;IACD,MAAM,aAAa,GAAa,EAAE,CAAA;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;IACtD,OAAO,UAAU,CAAC,MAAM,CAAA;IACxB,IAAI,QAAQ,EAAE,CAAC;QACb,UAAU,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;IACD,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC;QACnB,UAAU,CAAC,EAAE,GAAG,UAAU,CAAC,GAAG,CAAA;QAC9B,OAAO,UAAU,CAAC,GAAG,CAAA;IACvB,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAA;QACrC,OAAO,UAAU,CAAC,QAAQ,CAAA;IAC5B,CAAC;IACD,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,SAAS,CAAA;QACvC,OAAO,UAAU,CAAC,SAAS,CAAA;IAC7B,CAAC;IACD,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1B,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,UAAU,CAAA;QACzC,OAAO,UAAU,CAAC,UAAU,CAAA;IAC9B,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QACvB,UAAU,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAA;QACnC,OAAO,UAAU,CAAC,OAAO,CAAA;IAC3B,CAAC;IACD,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAChC,UAAU,CAAC,YAAY,GAAG,UAAU,CAAC,gBAAgB,CAAA;QACrD,OAAO,UAAU,CAAC,gBAAgB,CAAA;IACpC,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAA;QACrC,OAAO,UAAU,CAAC,QAAQ,CAAA;IAC5B,CAAC;IACD,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1B,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,UAAU,CAAA;QACzC,OAAO,UAAU,CAAC,UAAU,CAAA;IAC9B,CAAC;IACD,IAAI,UAAU,CAAC,eAAe,EAAE,CAAC;QAC/B,UAAU,CAAC,WAAW,GAAG,UAAU,CAAC,eAAe,CAAA;QACnD,OAAO,UAAU,CAAC,eAAe,CAAA;IACnC,CAAC;IACD,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;QACjC,aAAa,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAA;QACnD,OAAO,UAAU,CAAC,iBAAiB,CAAA;IACrC,CAAC;IACD,IAAI,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QAChC,aAAa,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC,CAAA;QAClD,OAAO,UAAU,CAAC,eAAe,CAAC,CAAA;IACpC,CAAC;IACD,IAAI,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACpC,aAAa,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAA;QACtD,OAAO,UAAU,CAAC,mBAAmB,CAAC,CAAA;IACxC,CAAC;IACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,UAAU,CAAC,aAAa,GAAG,aAAa,CAAA;IAC1C,CAAC;IACD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,KAAK,EAAE,QAAQ,CAAC,KAAK,GAAG,CAAC;QACzB,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;QAC1E,MAAM;QACN,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,KAAK,EAAE,IAAI;QACX,gCAAgC;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAClE,KAAK,EAAE,IAAI;QACX,SAAS;QACT,yBAAyB;QACzB,YAAY;QACZ,6BAA6B;QAC7B,cAAc;QACd,+BAA+B;QAC/B,gBAAgB;QAChB,kBAAkB;QAClB,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;QAClE,gBAAgB,EAAE,EAAE;QACpB,cAAc,EAAE,OAAO,CAAC,QAAQ;YAC9B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC5C,eAAe,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CACxD;YACH,CAAC,CAAC,EAAE;KACP,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAgB,qBAAqB,CACnC,KAAa,EACb,SAAiB;IAEjB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAA;QAC3C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACpB,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC"}

package/package.json

@@ -1,18 +1,18 @@
 {
   "name": "@apollo-annotation/shared",
-  "version": "0.1.19",
+  "version": "0.1.20",
   "main": "./dist/index.js",
   "scripts": {
-    "start": "tsc --build --watch",
     "build": "yarn clean && tsc --build",
     "clean": "rimraf dist",
+    "start": "tsc --build --watch",
     "test": "glob -c \"tsx --test --test-reporter spec \" \"**/*.test.ts\"",
     "test:ci": "NODE_V8_COVERAGE=./coverage glob -c \"tsx --test --test-reporter spec --experimental-test-coverage \" \"**/*.test.ts\""
   },
   "dependencies": {
-    "@apollo-annotation/common": "^0.1.19",
-    "@apollo-annotation/mst": "^0.1.19",
-    "@apollo-annotation/schemas": "^0.1.19",
+    "@apollo-annotation/common": "^0.1.20",
+    "@apollo-annotation/mst": "^0.1.20",
+    "@apollo-annotation/schemas": "^0.1.20",
     "@gmod/gff": "1.2.0",
     "@gmod/indexedfasta": "^2.0.4",
     "@jbrowse/core": "^2.13.1",
@@ -26,8 +26,11 @@
   "devDependencies": {
     "@nestjs/common": "^10.1.0",
     "@nestjs/core": "^10.1.0",
+    "@types/chai": "^4.3.19",
     "@types/node": "^18.14.2",
     "@types/rimraf": "^3",
+    "chai": "^5.1.1",
+    "chai-exclude": "^3.0.0",
     "glob": "^11.0.0",
     "mobx": "^6.6.1",
     "mobx-state-tree": "^5.1.7",

package/src/Changes/AddAssemblyAndFeaturesFromFileChange.ts

@@ -18,7 +18,7 @@ export interface SerializedAddAssemblyAndFeaturesFromFileChangeBase
 
 export interface AddAssemblyAndFeaturesFromFileChangeDetails {
   assemblyName: string
-  fileId: string
+  fileIds: { fa: string }
 }
 
 export interface SerializedAddAssemblyAndFeaturesFromFileChangeSingle
@@ -53,8 +53,8 @@ export class AddAssemblyAndFeaturesFromFileChange extends FromFileBaseChange {
   toJSON(): SerializedAddAssemblyAndFeaturesFromFileChange {
     const { assembly, changes, typeName } = this
     if (changes.length === 1) {
-      const [{ assemblyName, fileId }] = changes
-      return { typeName, assembly, assemblyName, fileId }
+      const [{ assemblyName, fileIds }] = changes
+      return { typeName, assembly, assemblyName, fileIds }
     }
     return { typeName, assembly, changes }
   }
@@ -68,7 +68,8 @@ export class AddAssemblyAndFeaturesFromFileChange extends FromFileBaseChange {
     const { assemblyModel, fileModel, filesService, user } = backend
     const { assembly, changes, logger } = this
     for (const change of changes) {
-      const { assemblyName, fileId } = change
+      const { assemblyName, fileIds } = change
+      const fileId = fileIds.fa
 
       const { FILE_UPLOAD_FOLDER } = process.env
       if (!FILE_UPLOAD_FOLDER) {
@@ -90,7 +91,7 @@ export class AddAssemblyAndFeaturesFromFileChange extends FromFileBaseChange {
       }
       // Add assembly
       const [newAssemblyDoc] = await assemblyModel.create([
-        { _id: assembly, name: assemblyName, user, status: -1 },
+        { _id: assembly, name: assemblyName, user, status: -1, fileId },
       ])
       logger.debug?.(
         `Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id.toHexString()}"`,

package/src/Changes/AddAssemblyFromFileChange.ts

@@ -7,6 +7,7 @@ import {
   SerializedAssemblySpecificChange,
   ServerDataStore,
 } from '@apollo-annotation/common'
+import { BgzipIndexedFasta } from '@gmod/indexedfasta'
 
 import { FromFileBaseChange } from './FromFileBaseChange'
 
@@ -17,7 +18,7 @@ export interface SerializedAddAssemblyFromFileChangeBase
 
 export interface AddAssemblyFromFileChangeDetails {
   assemblyName: string
-  fileId: string
+  fileIds: { fa: string } | { fa: string; fai: string; gzi: string }
 }
 
 export interface SerializedAddAssemblyFromFileChangeSingle
@@ -52,8 +53,8 @@ export class AddAssemblyFromFileChange extends FromFileBaseChange {
   toJSON(): SerializedAddAssemblyFromFileChange {
     const { assembly, changes, typeName } = this
     if (changes.length === 1) {
-      const [{ assemblyName, fileId }] = changes
-      return { typeName, assembly, assemblyName, fileId }
+      const [{ assemblyName, fileIds }] = changes
+      return { typeName, assembly, assemblyName, fileIds }
     }
     return { typeName, assembly, changes }
   }
@@ -64,49 +65,133 @@ export class AddAssemblyFromFileChange extends FromFileBaseChange {
    * @returns
    */
   async executeOnServer(backend: ServerDataStore) {
-    const { assemblyModel, fileModel, user } = backend
-    const { assembly, changes, logger } = this
-
+    const { changes } = this
     for (const change of changes) {
-      const { assemblyName, fileId } = change
-
-      const { FILE_UPLOAD_FOLDER } = process.env
-      if (!FILE_UPLOAD_FOLDER) {
-        throw new Error('No FILE_UPLOAD_FOLDER found in .env file')
-      }
-      // Get file checksum
-      const fileDoc = await fileModel.findById(fileId).exec()
-      if (!fileDoc) {
-        throw new Error(`File "${fileId}" not found in Mongo`)
-      }
-      logger.debug?.(`FileId "${fileId}", checksum "${fileDoc.checksum}"`)
-
-      // Check and add new assembly
-      const assemblyDoc = await assemblyModel
-        .findOne({ name: assemblyName })
-        .exec()
-      if (assemblyDoc) {
-        throw new Error(`Assembly "${assemblyName}" already exists`)
-      }
-      // Add assembly
-      const [newAssemblyDoc] = await assemblyModel.create([
-        { _id: assembly, name: assemblyName, user, status: -1 },
+      const { assemblyName, fileIds } = change
+      await ('gzi' in fileIds
+        ? this.executeOnServerIndexed(backend, assemblyName, fileIds)
+        : this.executeOnServerFasta(backend, assemblyName, fileIds.fa))
+    }
+  }
+
+  async executeOnServerIndexed(
+    backend: ServerDataStore,
+    assemblyName: string,
+    fileIds: { fa: string; fai: string; gzi: string },
+  ) {
+    const { CHUNK_SIZE } = process.env
+    const customChunkSize = CHUNK_SIZE && Number(CHUNK_SIZE)
+
+    const { FILE_UPLOAD_FOLDER } = process.env
+    if (!FILE_UPLOAD_FOLDER) {
+      throw new Error('No FILE_UPLOAD_FOLDER found in .env file')
+    }
+
+    const { fa, fai, gzi } = fileIds
+    const { assemblyModel, fileModel, filesService, refSeqModel, user } =
+      backend
+
+    const faDoc = await fileModel.findById(fa)
+    const faChecksum = faDoc?.checksum
+    if (!faChecksum) {
+      throw new Error(`No checksum for file document ${faDoc}`)
+    }
+
+    const faiDoc = await fileModel.findById(fai)
+    const faiChecksum = faiDoc?.checksum
+    if (!faiChecksum) {
+      throw new Error(`No checksum for file document ${faiDoc}`)
+    }
+
+    const gziDoc = await fileModel.findById(gzi)
+    const gziChecksum = gziDoc?.checksum
+    if (!gziChecksum) {
+      throw new Error(`No checksum for file document ${gziDoc}`)
+    }
+
+    const sequenceAdapter = new BgzipIndexedFasta({
+      fasta: filesService.getFileHandle(faDoc),
+      fai: filesService.getFileHandle(faiDoc),
+      gzi: filesService.getFileHandle(gziDoc),
+    })
+    const allSequenceSizes = await sequenceAdapter.getSequenceSizes()
+
+    const assemblyDoc = await assemblyModel
+      .findOne({ name: assemblyName })
+      .exec()
+    if (assemblyDoc) {
+      throw new Error(`Assembly "${assemblyName}" already exists`)
+    }
+    const [newAssemblyDoc] = await assemblyModel.create([
+      {
+        _id: this.assembly,
+        name: assemblyName,
+        user,
+        status: -1,
+        fileIds,
+      },
+    ])
+    this.logger.debug?.(
+      `Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id}"`,
+    )
+
+    for (const sequenceName in allSequenceSizes) {
+      const [newRefSeqDoc] = await refSeqModel.create([
+        {
+          name: sequenceName,
+          assembly: newAssemblyDoc._id,
+          length: allSequenceSizes[sequenceName],
+          ...(customChunkSize ? { chunkSize: customChunkSize } : null),
+          user,
+          status: -1,
+        },
       ])
-      logger.debug?.(
-        `Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id}"`,
-      )
-      logger.debug?.(
-        `File type: "${fileDoc.type}", assemblyId: "${newAssemblyDoc._id}"`,
+      this.logger.debug?.(
+        `Added new refSeq "${sequenceName}", docId "${newRefSeqDoc._id}"`,
       )
+    }
+  }
 
-      // Add refSeqs
-      // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
-      await this.addRefSeqIntoDb(
-        fileDoc,
-        newAssemblyDoc._id.toString(),
-        backend,
-      )
+  async executeOnServerFasta(
+    backend: ServerDataStore,
+    assemblyName: string,
+    fileId: string,
+  ) {
+    const { assemblyModel, fileModel, user } = backend
+    // Get file checksum
+    const fileDoc = await fileModel.findById(fileId).exec()
+    if (!fileDoc) {
+      throw new Error(`File "${fileId}" not found in Mongo`)
+    }
+    this.logger.debug?.(`FileId "${fileId}", checksum "${fileDoc.checksum}"`)
+
+    // Check and add new assembly
+    const assemblyDoc = await assemblyModel
+      .findOne({ name: assemblyName })
+      .exec()
+    if (assemblyDoc) {
+      throw new Error(`Assembly "${assemblyName}" already exists`)
     }
+    // Add assembly
+    const [newAssemblyDoc] = await assemblyModel.create([
+      {
+        _id: this.assembly,
+        name: assemblyName,
+        user,
+        status: -1,
+        fileIds: { fa: fileId },
+      },
+    ])
+    this.logger.debug?.(
+      `Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id}"`,
+    )
+    this.logger.debug?.(
+      `File type: "${fileDoc.type}", assemblyId: "${newAssemblyDoc._id}"`,
+    )
+
+    // Add refSeqs
+    // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
+    await this.addRefSeqIntoDb(fileDoc, newAssemblyDoc._id.toString(), backend)
   }
 
   async executeOnLocalGFF3(_backend: LocalGFF3DataStore) {