@apollo-annotation/jbrowse-plugin-apollo 0.1.0

package/src/ApolloTextSearchAdapter/ApolloTextSearchAdapter.ts

@@ -0,0 +1,64 @@
+import { readConfObject } from '@jbrowse/core/configuration'
+import {
+  BaseAdapter,
+  BaseTextSearchAdapter,
+  BaseTextSearchArgs,
+} from '@jbrowse/core/data_adapters/BaseAdapter'
+import BaseResult from '@jbrowse/core/TextSearch/BaseResults'
+
+export class ApolloTextSearchAdapter
+  extends BaseAdapter
+  implements BaseTextSearchAdapter
+{
+  get baseURL() {
+    return readConfObject(this.config, 'baseURL').uri
+  }
+
+  get trackId() {
+    return readConfObject(this.config, 'trackId')
+  }
+
+  get assemblyNames() {
+    return readConfObject(this.config, 'assemblyNames')
+  }
+
+  mapBaseResult(
+    features: {
+      refSeq: any // eslint-disable-line @typescript-eslint/no-explicit-any
+      start: number
+      end: number
+    }[],
+    query: string,
+  ) {
+    return features.map(
+      (feature) =>
+        new BaseResult({
+          label: query,
+          trackId: this.trackId,
+          locString: `${feature.refSeq?.name}:${feature.start + 1}..${
+            feature.end
+          }`,
+        }),
+    )
+  }
+
+  async searchIndex(args: BaseTextSearchArgs): Promise<BaseResult[]> {
+    const results = []
+    for (const assemblyName of this.assemblyNames) {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const session = this.pluginManager?.rootModel?.session as any
+      const backendDriver =
+        session?.apolloDataStore.getBackendDriver(assemblyName)
+      const r = await backendDriver.searchFeatures(args.queryString, [
+        assemblyName,
+      ])
+      results.push(...r)
+    }
+
+    const query = args.queryString
+    return this.mapBaseResult(results, query)
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  freeResources() {}
+}

package/src/ApolloTextSearchAdapter/configSchema.ts

@@ -0,0 +1,24 @@
+import { ConfigurationSchema } from '@jbrowse/core/configuration'
+
+export default ConfigurationSchema(
+  'ApolloTextSearchAdapter',
+  {
+    assemblyNames: {
+      type: 'stringArray',
+      defaultValue: [],
+      description: 'List of assemblies covered by text search adapter',
+    },
+    trackId: {
+      type: 'string',
+      defaultValue: '',
+    },
+    baseURL: {
+      type: 'fileLocation',
+      defaultValue: {
+        uri: '',
+        locationType: 'UriLocation',
+      },
+    },
+  },
+  { explicitlyTyped: true, explicitIdentifier: 'textSearchAdapterId' },
+)

package/src/ApolloTextSearchAdapter/index.ts

@@ -0,0 +1,18 @@
+import { TextSearchAdapterType } from '@jbrowse/core/pluggableElementTypes'
+import PluginManager from '@jbrowse/core/PluginManager'
+
+import { ApolloTextSearchAdapter } from './ApolloTextSearchAdapter'
+import configSchema from './configSchema'
+
+export function installApolloTextSearchAdapter(pluginManager: PluginManager) {
+  pluginManager.addTextSearchAdapterType(
+    () =>
+      new TextSearchAdapterType({
+        name: 'ApolloTextSearchAdapter',
+        displayName: 'Apollo text search adapter',
+        configSchema,
+        AdapterClass: ApolloTextSearchAdapter,
+        description: 'Apollo Text Search adapter',
+      }),
+  )
+}

package/src/BackendDrivers/BackendDriver.ts

@@ -0,0 +1,31 @@
+import { Assembly } from '@jbrowse/core/assemblyManager/assembly'
+import { Region } from '@jbrowse/core/util'
+import { Change, ClientDataStore } from 'apollo-common'
+import { AnnotationFeatureSnapshot, CheckResultSnapshot } from 'apollo-mst'
+import { ValidationResultSet } from 'apollo-shared'
+
+import { SubmitOpts } from '../ChangeManager'
+
+export abstract class BackendDriver {
+  constructor(protected clientStore: ClientDataStore) {}
+
+  abstract getFeatures(
+    region: Region,
+  ): Promise<[AnnotationFeatureSnapshot[], CheckResultSnapshot[]]>
+
+  abstract getSequence(region: Region): Promise<{ seq: string; refSeq: string }>
+
+  abstract getRegions(assemblyName: string): Promise<Region[]>
+
+  abstract getAssemblies(internetAccountConfigId?: string): Assembly[]
+
+  abstract submitChange(
+    change: Change,
+    opts: SubmitOpts,
+  ): Promise<ValidationResultSet>
+
+  abstract searchFeatures(
+    term: string,
+    assemblies: string[],
+  ): Promise<AnnotationFeatureSnapshot[]>
+}

package/src/BackendDrivers/CollaborationServerDriver.ts

@@ -0,0 +1,318 @@
+import { getConf } from '@jbrowse/core/configuration'
+import { BaseInternetAccountModel } from '@jbrowse/core/pluggableElementTypes'
+import { Region, getSession } from '@jbrowse/core/util'
+import { AssemblySpecificChange, Change } from 'apollo-common'
+import {
+  AnnotationFeatureSnapshot,
+  ApolloRefSeqI,
+  CheckResultSnapshot,
+} from 'apollo-mst'
+import { ChangeMessage, ValidationResultSet } from 'apollo-shared'
+import { Socket } from 'socket.io-client'
+
+import { ChangeManager, SubmitOpts } from '../ChangeManager'
+import { createFetchErrorMessage } from '../util'
+import { BackendDriver } from './BackendDriver'
+
+export interface ApolloInternetAccount extends BaseInternetAccountModel {
+  baseURL: string
+  socket: Socket
+  setLastChangeSequenceNumber(sequenceNumber: number): void
+  getMissingChanges(): void
+}
+
+export class CollaborationServerDriver extends BackendDriver {
+  private inFlight = new Map<string, Promise<string>>()
+
+  private async fetch(
+    internetAccount: ApolloInternetAccount,
+    info: RequestInfo,
+    init?: RequestInit,
+  ) {
+    const customFetch = internetAccount.getFetcher({
+      locationType: 'UriLocation',
+      uri: info.toString(),
+    })
+    return customFetch(info, init)
+  }
+
+  async searchFeatures(term: string, assemblies: string[]) {
+    const internetAccount = this.clientStore.getInternetAccount(
+      assemblies[0],
+    ) as ApolloInternetAccount
+    const { baseURL } = internetAccount
+
+    const url = new URL('features/searchFeatures', baseURL)
+    const searchParams = new URLSearchParams({
+      assemblies: assemblies.join(','),
+      term,
+    })
+    url.search = searchParams.toString()
+    const uri = url.toString()
+
+    const response = await this.fetch(internetAccount, uri)
+    if (!response.ok) {
+      const errorMessage = await createFetchErrorMessage(
+        response,
+        'searchFeatures failed',
+      )
+      throw new Error(errorMessage)
+    }
+    return response.json() as Promise<AnnotationFeatureSnapshot[]>
+  }
+
+  /**
+   * Call backend endpoint to get features by criteria
+   * @param region -  Searchable region containing refSeq, start and end
+   * @returns
+   */
+  async getFeatures(region: Region) {
+    const { assemblyName, end, refName, start } = region
+    const { assemblyManager } = getSession(this.clientStore)
+    const assembly = assemblyManager.get(assemblyName)
+    if (!assembly) {
+      throw new Error(`Could not find assembly with name "${assemblyName}"`)
+    }
+    const { ids } = getConf(assembly, ['sequence', 'metadata']) as {
+      ids: Record<string, string>
+    }
+    const refSeq = ids[refName]
+    if (!refSeq) {
+      throw new Error(`Could not find refSeq "${refName}"`)
+    }
+    const internetAccount = this.clientStore.getInternetAccount(
+      assemblyName,
+    ) as ApolloInternetAccount
+    const { baseURL } = internetAccount
+
+    const url = new URL('features/getFeatures', baseURL)
+    const searchParams = new URLSearchParams({
+      refSeq,
+      start: String(start),
+      end: String(end),
+    })
+    url.search = searchParams.toString()
+    const uri = url.toString()
+
+    const response = await this.fetch(internetAccount, uri)
+    if (!response.ok) {
+      const errorMessage = await createFetchErrorMessage(
+        response,
+        'getFeatures failed',
+      )
+      throw new Error(errorMessage)
+    }
+    await this.checkSocket(assemblyName, refName, internetAccount)
+    return response.json() as Promise<
+      [AnnotationFeatureSnapshot[], CheckResultSnapshot[]]
+    >
+  }
+
+  /**
+   * Checks if there is assembly-refSeq specific socket. If not, it opens one
+   * @param assembly - assemblyId
+   * @param refSeq - refSeqName
+   * @param internetAccount - internet account
+   */
+  async checkSocket(
+    assembly: string,
+    refSeq: string,
+    internetAccount: ApolloInternetAccount,
+  ) {
+    const { socket } = internetAccount
+    const token = internetAccount.retrieveToken()
+    const channel = `${assembly}-${refSeq}`
+    const changeManager = new ChangeManager(this.clientStore)
+
+    if (!socket.hasListeners(channel)) {
+      socket.on(channel, async (message: ChangeMessage) => {
+        // Save server last change sequence into session storage
+        internetAccount.setLastChangeSequenceNumber(
+          Number(message.changeSequence),
+        )
+        if (message.userSessionId !== token && message.channel === channel) {
+          const change = Change.fromJSON(message.changeInfo)
+          await changeManager.submit(change, { submitToBackend: false })
+        }
+      })
+    }
+  }
+
+  /**
+   * Call backend endpoint to get sequence by criteria
+   * @param region -  Searchable region containing refSeq, start and end
+   * @returns
+   */
+  async getSequence(region: Region): Promise<{ seq: string; refSeq: string }> {
+    const inFlightKey = `${region.refName}:${region.start}-${region.end}`
+    const inFlightPromise = this.inFlight.get(inFlightKey)
+    const { assemblyName, end, refName, start } = region
+    const { assemblyManager } = getSession(this.clientStore)
+    const assembly = assemblyManager.get(assemblyName)
+    if (!assembly) {
+      throw new Error(`Could not find assembly with name "${assemblyName}"`)
+    }
+    const { ids } = getConf(assembly, ['sequence', 'metadata']) as {
+      ids: Record<string, string>
+    }
+    const refSeq = ids[refName]
+    if (!refSeq) {
+      throw new Error(`Could not find refSeq "${refName}"`)
+    }
+    if (inFlightPromise) {
+      const seq = await inFlightPromise
+      return { seq, refSeq }
+    }
+    let apolloAssembly = this.clientStore.assemblies.get(assemblyName)
+    if (!apolloAssembly) {
+      apolloAssembly = this.clientStore.addAssembly(assemblyName)
+    }
+    let apolloRefSeq = apolloAssembly?.refSeqs.get(refSeq)
+    if (!apolloRefSeq) {
+      apolloRefSeq = apolloAssembly.addRefSeq(refSeq, refName)
+    }
+    const clientStoreSequence = apolloRefSeq.getSequence(start, end)
+    if (clientStoreSequence.length === end - start) {
+      return { seq: clientStoreSequence, refSeq }
+    }
+    const internetAccount = this.clientStore.getInternetAccount(
+      assemblyName,
+    ) as ApolloInternetAccount
+    const { baseURL } = internetAccount
+
+    const url = new URL('sequence', baseURL)
+    const searchParams = new URLSearchParams({
+      refSeq,
+      start: String(start),
+      end: String(end),
+    })
+    url.search = searchParams.toString()
+    const uri = url.toString()
+
+    const seqPromise = this.getSeqFromServer(
+      internetAccount,
+      uri,
+      apolloRefSeq,
+      start,
+      end,
+    )
+    this.inFlight.set(inFlightKey, seqPromise)
+    const seq = await seqPromise
+    await this.checkSocket(assemblyName, refName, internetAccount)
+    this.inFlight.delete(inFlightKey)
+    return { seq, refSeq }
+  }
+
+  private async getSeqFromServer(
+    internetAccount: ApolloInternetAccount,
+    uri: string,
+    apolloRefSeq: ApolloRefSeqI,
+    start: number,
+    stop: number,
+  ) {
+    const response = await this.fetch(internetAccount, uri)
+    if (!response.ok) {
+      let errorMessage
+      try {
+        errorMessage = await response.text()
+      } catch {
+        errorMessage = ''
+      }
+      throw new Error(
+        `getSequence failed: ${response.status} (${response.statusText})${
+          errorMessage ? ` (${errorMessage})` : ''
+        }`,
+      )
+    }
+    const seq = await response.text()
+    apolloRefSeq.addSequence({ sequence: seq, start, stop })
+    return seq
+  }
+
+  async getRegions(assemblyName: string): Promise<Region[]> {
+    const { assemblyManager } = getSession(this.clientStore)
+    const assembly = assemblyManager.get(assemblyName)
+    if (!assembly) {
+      throw new Error(`Could not find assembly with name "${assemblyName}"`)
+    }
+    const internetAccount = this.clientStore.getInternetAccount(
+      assemblyName,
+    ) as ApolloInternetAccount
+    const { baseURL } = internetAccount
+    const url = new URL('refSeqs', baseURL)
+    const searchParams = new URLSearchParams({ assembly: assemblyName })
+    url.search = searchParams.toString()
+    const uri = url.toString()
+
+    const response = await this.fetch(internetAccount, uri)
+    if (!response.ok) {
+      let errorMessage
+      try {
+        errorMessage = await response.text()
+      } catch {
+        errorMessage = ''
+      }
+      throw new Error(
+        `getRegions failed: ${response.status} (${response.statusText})${
+          errorMessage ? ` (${errorMessage})` : ''
+        }`,
+      )
+    }
+    const refSeqs = await response.json()
+    return refSeqs.map((refSeq: { name: string; length: number }) => ({
+      refName: refSeq.name,
+      start: 0,
+      end: refSeq.length,
+    }))
+  }
+
+  getAssemblies(internetAccountId?: string) {
+    const { assemblyManager } = getSession(this.clientStore)
+    return assemblyManager.assemblies.filter((assembly) => {
+      const sequenceMetadata = getConf(assembly, ['sequence', 'metadata']) as
+        | { apollo: boolean; internetAccountConfigId?: string }
+        | undefined
+      if (
+        sequenceMetadata &&
+        sequenceMetadata.apollo &&
+        sequenceMetadata.internetAccountConfigId
+      ) {
+        if (internetAccountId) {
+          return sequenceMetadata.internetAccountConfigId === internetAccountId
+        }
+        return true
+      }
+      return false
+    })
+  }
+
+  async submitChange(
+    change: Change | AssemblySpecificChange,
+    opts: SubmitOpts = {},
+  ) {
+    const { internetAccountId } = opts
+    const internetAccount = this.clientStore.getInternetAccount(
+      'assembly' in change ? change.assembly : undefined,
+      internetAccountId,
+    ) as ApolloInternetAccount
+    const { baseURL } = internetAccount
+    const url = new URL('changes', baseURL).href
+    const response = await this.fetch(internetAccount, url, {
+      method: 'POST',
+      body: JSON.stringify(change.toJSON()),
+      headers: { 'Content-Type': 'application/json' },
+    })
+    if (!response.ok) {
+      const errorMessage = await createFetchErrorMessage(
+        response,
+        'submitChange failed',
+      )
+      throw new Error(errorMessage)
+    }
+    const results = new ValidationResultSet()
+    if (!response.ok) {
+      results.ok = false
+    }
+    return results
+  }
+}

package/src/BackendDrivers/DesktopFileDriver.ts

@@ -0,0 +1,170 @@
+import gff, { GFF3Item } from '@gmod/gff'
+import { getConf } from '@jbrowse/core/configuration'
+import { Region, getSession } from '@jbrowse/core/util'
+import {
+  AssemblySpecificChange,
+  Change,
+  isAssemblySpecificChange,
+} from 'apollo-common'
+import { AnnotationFeatureSnapshot, CheckResultSnapshot } from 'apollo-mst'
+import { ValidationResultSet, makeGFF3Feature } from 'apollo-shared'
+import { getSnapshot } from 'mobx-state-tree'
+
+import { checkFeatures, loadAssemblyIntoClient } from '../util'
+import { BackendDriver } from './BackendDriver'
+
+function splitStringIntoChunks(input: string, chunkSize: number): string[] {
+  const chunks: string[] = []
+  for (let i = 0; i < input.length; i += chunkSize) {
+    const chunk = input.slice(i, i + chunkSize)
+    chunks.push(chunk)
+  }
+  return chunks
+}
+
+export class DesktopFileDriver extends BackendDriver {
+  async loadAssembly(assemblyName: string) {
+    const { assemblyManager } = getSession(this.clientStore)
+    const assembly = assemblyManager.get(assemblyName)
+    if (!assembly) {
+      throw new Error(`Assembly ${assemblyName} not found`)
+    }
+    const { file } = getConf(assembly, ['sequence', 'metadata']) as {
+      file: string
+    }
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require('node:fs') as typeof import('fs')
+    const fileContents = await fs.promises.readFile(file, 'utf8')
+    return loadAssemblyIntoClient(assemblyName, fileContents, this.clientStore)
+  }
+
+  async getAssembly(assemblyName: string) {
+    let assembly = this.clientStore.assemblies.get(assemblyName)
+    if (!assembly) {
+      assembly = await this.loadAssembly(assemblyName)
+    }
+    return assembly
+  }
+
+  async getFeatures(
+    region: Region,
+  ): Promise<[AnnotationFeatureSnapshot[], CheckResultSnapshot[]]> {
+    await this.getAssembly(region.assemblyName)
+    return [[], []]
+  }
+
+  async getSequence(region: Region) {
+    const { assemblyName, end, refName, start } = region
+    const assembly = await this.getAssembly(assemblyName)
+    const refSeq = assembly.refSeqs.get(refName)
+    if (!refSeq) {
+      throw new Error(`refSeq ${refName} not found in client data store`)
+    }
+    const seq = refSeq.getSequence(start, end)
+    return { seq, refSeq: refName }
+  }
+
+  async getRegions(assemblyName: string): Promise<Region[]> {
+    const assembly = await this.getAssembly(assemblyName)
+    const regions: Region[] = []
+    for (const [, refSeq] of assembly.refSeqs) {
+      regions.push({
+        assemblyName,
+        refName: refSeq.name,
+        start: refSeq.sequence[0].start,
+        end: refSeq.sequence[0].stop,
+      })
+    }
+    return regions
+  }
+
+  getAssemblies() {
+    const { assemblyManager } = getSession(this.clientStore)
+    return assemblyManager.assemblies.filter((assembly) => {
+      const sequenceMetadata = getConf(assembly, ['sequence', 'metadata']) as
+        | { apollo: boolean; internetAccountConfigId?: string; file?: string }
+        | undefined
+      return Boolean(
+        sequenceMetadata &&
+          sequenceMetadata.apollo &&
+          !sequenceMetadata.internetAccountConfigId &&
+          sequenceMetadata.file,
+      )
+    })
+  }
+
+  async submitChange(change: Change | AssemblySpecificChange) {
+    if (!isAssemblySpecificChange(change)) {
+      throw new Error(
+        `Cannot use this type of change with local file: "${change.typeName}"`,
+      )
+    }
+    const { assemblyManager } = getSession(this.clientStore)
+    const assembly = assemblyManager.get(change.assembly)
+    if (!assembly) {
+      throw new Error(`Could not find assembly with name "${change.assembly}"`)
+    }
+    const { file } = getConf(assembly, ['sequence', 'metadata']) as {
+      file: string
+    }
+    const clientAssembly = this.clientStore.assemblies.get(change.assembly)
+    if (!clientAssembly) {
+      throw new Error(
+        `Could not find assembly in client with name "${change.assembly}"`,
+      )
+    }
+    const refSeqs = new Set(...clientAssembly.refSeqs.keys())
+    const { checkResults } = this.clientStore
+    for (const checkResult of checkResults.values()) {
+      if (refSeqs.has(checkResult.refSeq)) {
+        checkResults.delete(checkResult._id)
+      }
+    }
+    const newCheckResults = await checkFeatures(clientAssembly)
+    this.clientStore.addCheckResults(newCheckResults)
+    const gff3Items: GFF3Item[] = [{ directive: 'gff-version', value: '3' }]
+    for (const [, refSeq] of clientAssembly.refSeqs) {
+      gff3Items.push({
+        directive: 'sequence-region',
+        value: `${refSeq.name} 1 ${refSeq.sequence[0].stop}`,
+      })
+    }
+    for (const comment of clientAssembly.comments) {
+      gff3Items.push({ comment })
+    }
+    for (const [, refSeq] of clientAssembly.refSeqs) {
+      const { features } = refSeq
+      for (const [, feature] of features) {
+        gff3Items.push(makeGFF3Feature(getSnapshot(feature)))
+      }
+    }
+    for (const [, refSeq] of clientAssembly.refSeqs) {
+      const [sequence] = refSeq.sequence
+      const formattedSequence = splitStringIntoChunks(
+        sequence.sequence,
+        80,
+      ).join('\n')
+      gff3Items.push({
+        id: refSeq.name,
+        description: refSeq.description,
+        sequence: formattedSequence,
+      })
+    }
+
+    const gff3Contents = gff.formatSync(gff3Items)
+
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const fs = require('node:fs') as typeof import('fs')
+    await fs.promises.writeFile(file, gff3Contents, 'utf8')
+
+    const results = new ValidationResultSet()
+    return results
+  }
+
+  async searchFeatures(
+    _term: string,
+    _assemblies: string[],
+  ): Promise<AnnotationFeatureSnapshot[]> {
+    return []
+  }
+}

package/src/BackendDrivers/InMemoryFileDriver.ts

@@ -0,0 +1,76 @@
+import { getConf } from '@jbrowse/core/configuration'
+import { Region, getSession } from '@jbrowse/core/util'
+import { AssemblySpecificChange, Change } from 'apollo-common'
+import { AnnotationFeatureSnapshot, CheckResultSnapshot } from 'apollo-mst'
+import { ValidationResultSet } from 'apollo-shared'
+
+import { SubmitOpts } from '../ChangeManager'
+import { BackendDriver } from './BackendDriver'
+
+export class InMemoryFileDriver extends BackendDriver {
+  async getFeatures(): Promise<
+    [AnnotationFeatureSnapshot[], CheckResultSnapshot[]]
+  > {
+    return [[], []]
+  }
+
+  async getSequence(region: Region) {
+    const { assemblyName, end, refName, start } = region
+    const assembly = this.clientStore.assemblies.get(assemblyName)
+    if (!assembly) {
+      return { seq: '', refSeq: refName }
+    }
+    const refSeq = assembly.refSeqs.get(refName)
+    if (!refSeq) {
+      return { seq: '', refSeq: refName }
+    }
+    const seq = refSeq.getSequence(start, end)
+    return { seq, refSeq: refName }
+  }
+
+  async getRegions(assemblyName: string): Promise<Region[]> {
+    const assembly = this.clientStore.assemblies.get(assemblyName)
+    if (!assembly) {
+      return []
+    }
+    const regions: Region[] = []
+    for (const [, refSeq] of assembly.refSeqs) {
+      regions.push({
+        assemblyName,
+        refName: refSeq.name,
+        start: refSeq.sequence[0].start,
+        end: refSeq.sequence[0].stop,
+      })
+    }
+    return regions
+  }
+
+  getAssemblies() {
+    const { assemblyManager } = getSession(this.clientStore)
+    return assemblyManager.assemblies.filter((assembly) => {
+      const sequenceMetadata = getConf(assembly, ['sequence', 'metadata']) as
+        | { apollo: boolean; internetAccountConfigId?: string; file?: string }
+        | undefined
+      return Boolean(
+        sequenceMetadata &&
+          sequenceMetadata.apollo &&
+          !sequenceMetadata.file &&
+          !sequenceMetadata.internetAccountConfigId,
+      )
+    })
+  }
+
+  async submitChange(
+    _change: Change | AssemblySpecificChange,
+    _opts: SubmitOpts = {},
+  ) {
+    return new ValidationResultSet()
+  }
+
+  async searchFeatures(
+    _term: string,
+    _assemblies: string[],
+  ): Promise<AnnotationFeatureSnapshot[]> {
+    return []
+  }
+}