@apollo-annotation/shared 0.1.18 → 0.1.20

package/src/Changes/FromFileBaseChange.ts

@@ -0,0 +1,221 @@
+/* eslint-disable @typescript-eslint/restrict-plus-operands */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-argument */
+import {
+  AssemblySpecificChange,
+  ServerDataStore,
+} from '@apollo-annotation/common'
+import { FileDocument, RefSeqDocument } from '@apollo-annotation/schemas'
+import { GFF3Feature } from '@gmod/gff'
+
+import { gff3ToAnnotationFeature } from '../GFF3'
+
+export abstract class FromFileBaseChange extends AssemblySpecificChange {
+  async addRefSeqIntoDb(
+    fileDoc: FileDocument,
+    assembly: string,
+    backend: ServerDataStore,
+  ) {
+    const { logger } = this
+    const { filesService, refSeqChunkModel, refSeqModel, user } = backend
+    const { CHUNK_SIZE } = process.env
+    const customChunkSize = CHUNK_SIZE && Number(CHUNK_SIZE)
+    let chunkIndex = 0
+    let refSeqLen = 0
+    let refSeqDoc: RefSeqDocument | undefined
+    let fastaInfoStarted = fileDoc.type !== 'text/x-gff3'
+
+    // Read data from compressed file and parse the content
+    const sequenceStream = filesService.getFileStream(fileDoc)
+    let sequenceBuffer = ''
+    let incompleteLine = ''
+    let lastLineIsIncomplete = true
+    let parsingStarted = false
+    logger.debug?.('starting sequence stream')
+    let lineCount = 0
+    for await (const data of sequenceStream) {
+      const chunk = data.toString()
+      lastLineIsIncomplete = !chunk.endsWith('\n')
+      // chunk is small enough that you can split the whole thing into lines without having to make it into smaller chunks first.
+      const lines = chunk.split(/\r?\n/)
+      if (incompleteLine) {
+        lines[0] = `${incompleteLine}${lines[0]}`
+        incompleteLine = ''
+      }
+      if (lastLineIsIncomplete) {
+        incompleteLine = lines.pop() || ''
+      }
+      for await (const line of lines) {
+        lineCount++
+        if (lineCount % 1_000_000 === 0) {
+          logger.debug?.(`Processed ${lineCount} lines`)
+        }
+        // In case of GFF3 file we start to read sequence after '##FASTA' is found
+        if (!fastaInfoStarted) {
+          if (line.trim() === '##FASTA') {
+            fastaInfoStarted = true
+          }
+          continue
+        }
+        const refSeqInfoLine = /^>\s*(\S+)\s*(.*)/.exec(line)
+        // Add new ref sequence info if we are reference seq info line
+        if (refSeqInfoLine) {
+          parsingStarted = true
+          logger.debug?.(
+            `Reference sequence information line "${refSeqInfoLine[0]}"`,
+          )
+
+          // If there is sequence from previous reference sequence then we need to add it to previous ref seq
+          if (sequenceBuffer !== '') {
+            if (!refSeqDoc) {
+              throw new Error('No refSeq document found')
+            }
+            refSeqLen += sequenceBuffer.length
+            // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
+            await refSeqChunkModel.create([
+              {
+                refSeq: refSeqDoc._id,
+                n: chunkIndex,
+                sequence: sequenceBuffer,
+                user,
+                status: -1,
+              },
+            ])
+            sequenceBuffer = ''
+          }
+          await refSeqDoc?.updateOne({ length: refSeqLen })
+          // await refSeqDoc?.updateOne({ length: refSeqLen }, { session })
+          refSeqLen = 0
+          chunkIndex = 0
+
+          const name = refSeqInfoLine[1].trim()
+          const description = refSeqInfoLine[2] ? refSeqInfoLine[2].trim() : ''
+
+          // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
+          const [newRefSeqDoc] = await refSeqModel.create([
+            {
+              name,
+              description,
+              assembly,
+              length: 0,
+              ...(customChunkSize ? { chunkSize: customChunkSize } : null),
+              user,
+              status: -1,
+            },
+          ])
+          logger.debug?.(
+            `Added new refSeq "${name}", desc "${description}", docId "${newRefSeqDoc._id}"`,
+          )
+          refSeqDoc = newRefSeqDoc
+        } else if (/\S/.test(line)) {
+          if (!refSeqDoc) {
+            throw new Error('No refSeq document found')
+          }
+          const { _id, chunkSize } = refSeqDoc
+          sequenceBuffer += line.replaceAll(/\s/g, '')
+          // If sequence block > chunk size then save chunk into Mongo
+          while (sequenceBuffer.length >= chunkSize) {
+            const sequence = sequenceBuffer.slice(0, chunkSize)
+            refSeqLen += sequence.length
+            // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
+            await refSeqChunkModel.create([
+              { refSeq: _id, n: chunkIndex, sequence, user, status: -1 },
+            ])
+            chunkIndex++
+            // Set remaining sequence
+            sequenceBuffer = sequenceBuffer.slice(chunkSize)
+          }
+        }
+      }
+    }
+    if (!parsingStarted) {
+      throw new Error('No reference sequences found in file')
+    }
+
+    if (sequenceBuffer || lastLineIsIncomplete) {
+      if (!refSeqDoc) {
+        throw new Error('No refSeq document found')
+      }
+      // If the file did not end with line break so the last line is incomplete
+      if (lastLineIsIncomplete) {
+        sequenceBuffer += incompleteLine
+      }
+      refSeqLen += sequenceBuffer.length
+      logger.verbose?.(
+        `*** Add the very last chunk to ref seq ("${refSeqDoc._id}", index ${chunkIndex} and total length for ref seq is ${refSeqLen}): "${sequenceBuffer}"`,
+      )
+      logger.debug?.(
+        `Creating refSeq chunk number ${chunkIndex} of "${refSeqDoc._id}"`,
+      )
+      // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
+      await refSeqChunkModel.create([
+        {
+          refSeq: refSeqDoc._id,
+          n: chunkIndex,
+          sequence: sequenceBuffer,
+          user,
+          status: -1,
+        },
+      ])
+      await refSeqDoc.updateOne({ length: refSeqLen })
+    }
+  }
+
+  private refSeqCache = new Map<string, RefSeqDocument>()
+
+  async removeExistingFeatures(backend: ServerDataStore) {
+    const { featureModel, refSeqModel } = backend
+    const { assembly, logger } = this
+    logger.debug?.(`Removing existing features for assembly = ${assembly}`)
+
+    const refSeqs: RefSeqDocument[] = await refSeqModel
+      .find({ assembly })
+      .exec()
+
+    for (const refSeq of refSeqs) {
+      await featureModel.deleteMany({ refSeq: refSeq._id })
+    }
+  }
+
+  async addFeatureIntoDb(gff3Feature: GFF3Feature, backend: ServerDataStore) {
+    const { featureModel, refSeqModel, user } = backend
+    const { assembly, refSeqCache } = this
+
+    const [{ seq_id: refName }] = gff3Feature
+    if (!refName) {
+      throw new Error(
+        `Valid seq_id not found in feature ${JSON.stringify(gff3Feature)}`,
+      )
+    }
+    let refSeqDoc = refSeqCache.get(refName)
+    if (!refSeqDoc) {
+      refSeqDoc =
+        (await refSeqModel.findOne({ assembly, name: refName }).exec()) ??
+        undefined
+      if (refSeqDoc) {
+        refSeqCache.set(refName, refSeqDoc)
+      }
+    }
+    if (!refSeqDoc) {
+      throw new Error(
+        `RefSeq was not found by assembly "${assembly}" and seq_id "${refName}" not found`,
+      )
+    }
+    // Let's add featureId to parent feature
+    const featureIds: string[] = []
+
+    const newFeature = gff3ToAnnotationFeature(
+      gff3Feature,
+      refSeqDoc._id,
+      featureIds,
+    )
+
+    // Add into Mongo
+    // We cannot use Mongo 'session' / transaction here because Mongo has 16 MB limit for transaction
+    await featureModel.create([
+      { allIds: featureIds, ...newFeature, user, status: -1 },
+    ])
+  }
+}

package/src/Changes/ImportJBrowseConfigChange.ts

@@ -0,0 +1,116 @@
+/* eslint-disable @typescript-eslint/require-await */
+import {
+  Change,
+  ChangeOptions,
+  ClientDataStore,
+  LocalGFF3DataStore,
+  ServerDataStore,
+} from '@apollo-annotation/common'
+
+interface JBrowseAssembly {
+  sequence: { adapter: { type: string } }
+}
+
+interface JBrowseTrack {
+  type: string
+  trackId: string
+}
+
+interface JBrowsePlugin {
+  name: string
+}
+
+interface JBrowseInternetAccount {
+  type: string
+}
+
+export interface JBrowseConfig {
+  assemblies?: JBrowseAssembly[]
+  tracks?: JBrowseTrack[]
+  plugins?: JBrowsePlugin[]
+  internetAccounts?: JBrowseInternetAccount[]
+  [key: string]: unknown
+}
+
+export interface SerializedImportJBrowseConfigChange {
+  typeName: 'ImportJBrowseConfigChange'
+  oldJBrowseConfig?: JBrowseConfig
+  newJBrowseConfig?: JBrowseConfig
+}
+
+export function filterJBrowseConfig(config: JBrowseConfig): JBrowseConfig {
+  const { assemblies, internetAccounts, plugins, tracks, ...rest } = config
+  const filteredAssemblies = assemblies?.filter(
+    (a) => a.sequence.adapter.type !== 'ApolloSequenceAdapter',
+  )
+  const filteredTracks = tracks?.filter((t) => t.type !== 'ApolloTrack')
+  const filteredPlugins = plugins?.filter((p) => p.name !== 'Apollo')
+  const filteredInternetAccounts = internetAccounts?.filter(
+    (i) => i.type !== 'ApolloInternetAccount',
+  )
+  const filteredConfig = rest as JBrowseConfig
+  if (filteredAssemblies) {
+    filteredConfig.assemblies = filteredAssemblies
+  }
+  if (filteredTracks) {
+    filteredConfig.tracks = filteredTracks
+  }
+  if (filteredPlugins) {
+    filteredConfig.plugins = filteredPlugins
+  }
+  if (filteredInternetAccounts) {
+    filteredConfig.internetAccounts = filteredInternetAccounts
+  }
+  return filteredConfig
+}
+
+export class ImportJBrowseConfigChange extends Change {
+  typeName = 'ImportJBrowseConfigChange' as const
+  oldJBrowseConfig?: JBrowseConfig
+  newJBrowseConfig?: JBrowseConfig
+
+  constructor(
+    json: SerializedImportJBrowseConfigChange,
+    options?: ChangeOptions,
+  ) {
+    super(json, options)
+    this.oldJBrowseConfig = json.oldJBrowseConfig
+    this.newJBrowseConfig = json.newJBrowseConfig
+  }
+
+  toJSON(): SerializedImportJBrowseConfigChange {
+    const { newJBrowseConfig, oldJBrowseConfig, typeName } = this
+    return { typeName, oldJBrowseConfig, newJBrowseConfig }
+  }
+
+  async executeOnServer(backend: ServerDataStore) {
+    const { jbrowseConfigModel } = backend
+    const { logger, newJBrowseConfig } = this
+    await jbrowseConfigModel.deleteMany()
+    if (!newJBrowseConfig) {
+      return
+    }
+    const filteredConfig = filterJBrowseConfig(newJBrowseConfig)
+    await jbrowseConfigModel.create(filteredConfig)
+    logger.debug?.('Stored new JBrowse Config')
+  }
+
+  async executeOnLocalGFF3(_backend: LocalGFF3DataStore) {
+    throw new Error('executeOnLocalGFF3 not implemented')
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async executeOnClient(_dataStore: ClientDataStore) {}
+
+  getInverse() {
+    const { logger, newJBrowseConfig, oldJBrowseConfig } = this
+    return new ImportJBrowseConfigChange(
+      {
+        typeName: 'ImportJBrowseConfigChange',
+        oldJBrowseConfig: newJBrowseConfig,
+        newJBrowseConfig: oldJBrowseConfig,
+      },
+      { logger },
+    )
+  }
+}

package/src/Changes/LocationEndChange.ts

@@ -30,7 +30,7 @@ interface SerializedLocationEndChangeMultiple
   changes: LocationEndChangeDetails[]
 }
 
-type SerializedLocationEndChange =
+export type SerializedLocationEndChange =
   | SerializedLocationEndChangeSingle
   | SerializedLocationEndChangeMultiple
 
@@ -80,7 +80,9 @@ export class LocationEndChange extends FeatureChange {
         throw new Error(errMsg)
         // throw new NotFoundException(errMsg)  -- This is causing runtime error because Exception comes from @nestjs/common!!!
       }
-      logger.debug?.(`*** Feature found: ${JSON.stringify(topLevelFeature)}`)
+      logger.debug?.(
+        `*** TOP level feature found: ${JSON.stringify(topLevelFeature)}`,
+      )
 
       const foundFeature = this.getFeatureFromId(topLevelFeature, featureId)
       if (!foundFeature) {
@@ -89,28 +91,33 @@ export class LocationEndChange extends FeatureChange {
         throw new Error(errMsg)
       }
       logger.debug?.(`*** Found feature: ${JSON.stringify(foundFeature)}`)
-      if (
-        foundFeature.discontinuousLocations &&
-        foundFeature.discontinuousLocations.length > 0
-      ) {
-        const errMsg =
-          'Must use "DiscontinuousLocationEndChange" to change a feature end that has discontinuous locations'
-        logger.error(errMsg)
-        throw new Error(errMsg)
-      }
-      if (foundFeature.end !== oldEnd) {
-        const errMsg = `*** ERROR: Feature's current end value ${foundFeature.end} doesn't match with expected value ${oldEnd}`
-        logger.error(errMsg)
-        throw new Error(errMsg)
+      if (foundFeature.max === oldEnd) {
+        featuresForChanges.push({ feature: foundFeature, topLevelFeature })
+      } else {
+        if (foundFeature.children) {
+          for (const [, childFeature] of foundFeature.children) {
+            if (childFeature.max === oldEnd) {
+              logger.debug?.(
+                `*************** UPDATE CHILD FEATURE ID= ${featureId}, CHILD: ${JSON.stringify(
+                  childFeature,
+                )}`,
+              )
+              featuresForChanges.push({
+                feature: childFeature,
+                topLevelFeature,
+              })
+              break
+            }
+          }
+        }
       }
-      featuresForChanges.push({ feature: foundFeature, topLevelFeature })
     }
 
     // Let's update objects.
     for (const [idx, change] of changes.entries()) {
       const { newEnd } = change
       const { feature, topLevelFeature } = featuresForChanges[idx]
-      feature.end = newEnd
+      feature.max = newEnd
       if (topLevelFeature._id.equals(feature._id)) {
         topLevelFeature.markModified('end') // Mark as modified. Without this save() -method is not updating data in database
       } else {
@@ -144,7 +151,7 @@ export class LocationEndChange extends FeatureChange {
       if (!feature) {
         throw new Error(`Could not find feature with identifier "${changedId}"`)
       }
-      feature.setEnd(this.changes[idx].newEnd)
+      feature.setMax(this.changes[idx].newEnd)
     }
   }
 

package/src/Changes/LocationStartChange.ts

@@ -30,7 +30,7 @@ interface SerializedLocationStartChangeMultiple
   changes: LocationStartChangeDetails[]
 }
 
-type SerializedLocationStartChange =
+export type SerializedLocationStartChange =
   | SerializedLocationStartChangeSingle
   | SerializedLocationStartChangeMultiple
 
@@ -80,7 +80,9 @@ export class LocationStartChange extends FeatureChange {
         throw new Error(errMsg)
         // throw new NotFoundException(errMsg)  -- This is causing runtime error because Exception comes from @nestjs/common!!!
       }
-      logger.debug?.(`*** Feature found: ${JSON.stringify(topLevelFeature)}`)
+      logger.debug?.(
+        `*** TOP level feature found: ${JSON.stringify(topLevelFeature)}`,
+      )
 
       const foundFeature = this.getFeatureFromId(topLevelFeature, featureId)
       if (!foundFeature) {
@@ -89,28 +91,33 @@ export class LocationStartChange extends FeatureChange {
         throw new Error(errMsg)
       }
       logger.debug?.(`*** Found feature: ${JSON.stringify(foundFeature)}`)
-      if (
-        foundFeature.discontinuousLocations &&
-        foundFeature.discontinuousLocations.length > 0
-      ) {
-        const errMsg =
-          'Must use "DiscontinuousLocationStartChange" to change a feature start that has discontinuous locations'
-        logger.error(errMsg)
-        throw new Error(errMsg)
-      }
-      if (foundFeature.start !== oldStart) {
-        const errMsg = `*** ERROR: Feature's current start value ${foundFeature.start} doesn't match with expected value ${oldStart}`
-        logger.error(errMsg)
-        throw new Error(errMsg)
+      if (foundFeature.min === oldStart) {
+        featuresForChanges.push({ feature: foundFeature, topLevelFeature })
+      } else {
+        if (foundFeature.children) {
+          for (const [, childFeature] of foundFeature.children) {
+            if (childFeature.min === oldStart) {
+              logger.debug?.(
+                `*** UPDATE CHILD FEATURE ID= ${featureId}, CHILD: ${JSON.stringify(
+                  childFeature,
+                )}`,
+              )
+              featuresForChanges.push({
+                feature: childFeature,
+                topLevelFeature,
+              })
+              break
+            }
+          }
+        }
       }
-      featuresForChanges.push({ feature: foundFeature, topLevelFeature })
     }
 
     // Let's update objects.
     for (const [idx, change] of changes.entries()) {
       const { newStart } = change
       const { feature, topLevelFeature } = featuresForChanges[idx]
-      feature.start = newStart
+      feature.min = newStart
       if (topLevelFeature._id.equals(feature._id)) {
         topLevelFeature.markModified('start') // Mark as modified. Without this save() -method is not updating data in database
       } else {
@@ -144,7 +151,7 @@ export class LocationStartChange extends FeatureChange {
       if (!feature) {
         throw new Error(`Could not find feature with identifier "${changedId}"`)
       }
-      feature.setStart(this.changes[idx].newStart)
+      feature.setMin(this.changes[idx].newStart)
     }
   }
 

package/src/Changes/TypeChange.ts

@@ -29,7 +29,7 @@ interface SerializedTypeChangeMultiple extends SerializedTypeChangeBase {
   changes: TypeChangeDetails[]
 }
 
-type SerializedTypeChange =
+export type SerializedTypeChange =
   | SerializedTypeChangeSingle
   | SerializedTypeChangeMultiple
 

package/src/Changes/index.ts

@@ -3,12 +3,12 @@ import { AddAssemblyFromExternalChange } from './AddAssemblyFromExternalChange'
 import { AddAssemblyFromFileChange } from './AddAssemblyFromFileChange'
 import { AddFeatureChange } from './AddFeatureChange'
 import { AddFeaturesFromFileChange } from './AddFeaturesFromFileChange'
+import { AddRefSeqAliasesChange } from './AddRefSeqAliasesChange'
 import { DeleteAssemblyChange } from './DeleteAssemblyChange'
 import { DeleteFeatureChange } from './DeleteFeatureChange'
 import { DeleteUserChange } from './DeleteUserChange'
-import { DiscontinuousLocationEndChange } from './DiscontinuousLocationEndChange'
-import { DiscontinuousLocationStartChange } from './DiscontinuousLocationStartChange'
 import { FeatureAttributeChange } from './FeatureAttributeChange'
+import { ImportJBrowseConfigChange } from './ImportJBrowseConfigChange'
 import { LocationEndChange } from './LocationEndChange'
 import { LocationStartChange } from './LocationStartChange'
 import { StrandChange } from './StrandChange'
@@ -24,14 +24,14 @@ export const changes = {
   DeleteAssemblyChange,
   DeleteFeatureChange,
   DeleteUserChange,
-  DiscontinuousLocationEndChange,
-  DiscontinuousLocationStartChange,
   FeatureAttributeChange,
+  ImportJBrowseConfigChange,
   LocationEndChange,
   LocationStartChange,
   StrandChange,
   TypeChange,
   UserChange,
+  AddRefSeqAliasesChange,
 }
 
 export * from './AddAssemblyAndFeaturesFromFileChange'
@@ -42,11 +42,11 @@ export * from './AddFeaturesFromFileChange'
 export * from './DeleteAssemblyChange'
 export * from './DeleteFeatureChange'
 export * from './DeleteUserChange'
-export * from './DiscontinuousLocationEndChange'
-export * from './DiscontinuousLocationStartChange'
 export * from './FeatureAttributeChange'
+export * from './ImportJBrowseConfigChange'
 export * from './LocationEndChange'
 export * from './LocationStartChange'
 export * from './StrandChange'
 export * from './TypeChange'
 export * from './UserChange'
+export * from './AddRefSeqAliasesChange'