@apollo-annotation/jbrowse-plugin-apollo 0.1.0

package/src/OntologyManager/OntologyStore/fulltext-stopwords.ts

@@ -0,0 +1,137 @@
+/** set of words that should be ignored by fulltext indexing */
+export const genericEnglishStopwords = new Set([
+  'i',
+  'me',
+  'my',
+  'myself',
+  'we',
+  'our',
+  'ours',
+  'ourselves',
+  'you',
+  'your',
+  'yours',
+  'yourself',
+  'yourselves',
+  'he',
+  'him',
+  'his',
+  'himself',
+  'she',
+  'her',
+  'hers',
+  'herself',
+  'it',
+  'its',
+  'itself',
+  'they',
+  'them',
+  'their',
+  'theirs',
+  'themselves',
+  'what',
+  'which',
+  'who',
+  'whom',
+  'this',
+  'that',
+  'these',
+  'those',
+  'am',
+  'is',
+  'are',
+  'was',
+  'were',
+  'be',
+  'been',
+  'being',
+  'have',
+  'has',
+  'had',
+  'having',
+  'do',
+  'does',
+  'did',
+  'doing',
+  'a',
+  'an',
+  'the',
+  'and',
+  'but',
+  'if',
+  'or',
+  'because',
+  'as',
+  'until',
+  'while',
+  'of',
+  'at',
+  'by',
+  'for',
+  'with',
+  'about',
+  'against',
+  'between',
+  'into',
+  'through',
+  'during',
+  'before',
+  'after',
+  'above',
+  'below',
+  'to',
+  'from',
+  'up',
+  'down',
+  'in',
+  'out',
+  'on',
+  'off',
+  'over',
+  'under',
+  'again',
+  'further',
+  'then',
+  'once',
+  'here',
+  'there',
+  'when',
+  'where',
+  'why',
+  'how',
+  'all',
+  'any',
+  'both',
+  'each',
+  'few',
+  'more',
+  'most',
+  'other',
+  'some',
+  'such',
+  'no',
+  'nor',
+  'not',
+  'only',
+  'own',
+  'same',
+  'so',
+  'than',
+  'too',
+  'very',
+  's',
+  't',
+  'can',
+  'will',
+  'just',
+  'don',
+  'should',
+  'now',
+  ...'1234567890',
+])
+
+/**
+ * The set of stopwords we use for fulltext indexing. Currently
+ * just generic English stopwords, but will likely be expanded over time.
+ */
+export const stopwords = genericEnglishStopwords

package/src/OntologyManager/OntologyStore/fulltext.test.ts

@@ -0,0 +1,94 @@
+import { describe, expect, it } from '@jest/globals'
+
+import {
+  PREFIXED_ID_PATH,
+  elaborateMatch,
+  extractWords,
+  getWords,
+} from './fulltext'
+import { OntologyDBNode } from './indexeddb-schema'
+
+const testNode: OntologyDBNode = {
+  id: 'http://purl.obolibrary.org/obo/SO_0000001',
+  lbl: 'region',
+  type: 'CLASS',
+  meta: {
+    definition: {
+      val: 'A sequence_feature with an extent greater than zero. A nucleotide region is composed of bases and a polypeptide region is composed of amino acids. It may also be termed a region of some number of nucleotides.',
+      xrefs: ['SO:ke'],
+    },
+    subsets: ['http://purl.obolibrary.org/obo/so#SOFA'],
+    synonyms: [{ pred: 'hasExactSynonym', val: 'sequence' }],
+    basicPropertyValues: [
+      {
+        pred: 'http://www.geneontology.org/formats/oboInOwl#hasOBONamespace',
+        val: 'sequence',
+      },
+    ],
+  },
+}
+
+const prefixes = new Map<string, string>([
+  ['SO:', 'http://purl.obolibrary.org/obo/SO_'],
+])
+
+describe('extractWords', () => {
+  it('can words from the members of objects', () => {
+    const result = extractWords(['bar baz', 'noggin'])
+    expect([...result]).toEqual(['bar', 'baz', 'noggin'])
+  })
+  it('can get the words from mix of stuff', () => {
+    const set = extractWords(['zoz-zoo', 'bar baz', 'noggin', 'twenty'])
+    expect([...set]).toEqual(['zoz', 'zoo', 'bar', 'baz', 'noggin', 'twenty'])
+  })
+})
+
+describe('getWords', () => {
+  it('can get the words from a test node', () => {
+    const result = getWords(
+      testNode,
+      [
+        PREFIXED_ID_PATH,
+        '$.lbl',
+        '$.meta.definition.val',
+        '$.meta.synonyms[*].val',
+      ],
+      prefixes,
+    )
+    expect([...result]).toMatchSnapshot()
+  })
+})
+
+describe('elaborateMatch', () => {
+  it('can do one', () => {
+    const result = elaborateMatch(
+      [
+        { displayName: 'label', jsonPath: '$.lbl' },
+        { displayName: 'synonym', jsonPath: '$.meta.synonyms[*].val' },
+        { displayName: 'definition', jsonPath: '$.meta.definition.val' },
+      ],
+      testNode,
+      new Set([1, 2]),
+      ['zonk', 'nucleotide', 'region'],
+      prefixes,
+    )
+    expect(result.length).toBe(1)
+    expect(result).toMatchSnapshot()
+  })
+  it('can do another', () => {
+    const result = elaborateMatch(
+      [
+        { displayName: 'ID', jsonPath: PREFIXED_ID_PATH },
+        { displayName: 'label', jsonPath: '$.lbl' },
+        { displayName: 'synonym', jsonPath: '$.meta.synonyms[*].val' },
+        { displayName: 'definition', jsonPath: '$.meta.definition.val' },
+      ],
+      testNode,
+      new Set([0]),
+      ['SO:0000001'],
+      prefixes,
+    )
+    expect(result.length).toBe(1)
+    expect(result).toMatchSnapshot()
+  })
+})

package/src/OntologyManager/OntologyStore/fulltext.ts

@@ -0,0 +1,264 @@
+/* eslint-disable import/no-named-as-default-member */
+// jsonpath triggers this rule for some reason. import { query } from 'jsonpath' does not work
+
+import { checkAbortSignal } from '@jbrowse/core/util'
+import jsonpath from 'jsonpath'
+
+import { stopwords } from './fulltext-stopwords'
+import { OntologyDBNode } from './indexeddb-schema'
+import { applyPrefixes } from './prefixes'
+import OntologyStore, { Transaction } from '.'
+import { TextIndexFieldDefinition } from '..'
+
+/** special value of jsonPath that gets the IRI (that is, ID) of the node with the configured prefixes applied */
+export const PREFIXED_ID_PATH = '$PREFIXED_ID'
+
+/** small wrapper for jsonpath.query that intercepts requests for the special prefixed ID path */
+function jsonPathQuery(
+  node: OntologyDBNode,
+  path: string,
+  prefixes: Map<string, string>,
+) {
+  if (path === PREFIXED_ID_PATH) {
+    return [applyPrefixes(node.id, prefixes)]
+  }
+  return jsonpath.query(node, path)
+}
+
+function wordsInString(str: string) {
+  return str
+    .toLowerCase()
+    .split(/[^\d:A-Za-z]+/)
+    .filter((word) => word && !stopwords.has(word))
+}
+
+/**
+ * recursively get the indexable words from an iterator
+ * of any objects
+ **/
+export function* extractWords(
+  strings: Iterable<string>,
+): Generator<string, void, undefined> {
+  for (const str of strings) {
+    yield* wordsInString(str)
+  }
+}
+
+export function* extractStrings(
+  things: Iterable<unknown>,
+): Generator<string, void, undefined> {
+  for (const thing of things) {
+    if (typeof thing === 'string') {
+      yield thing
+    } else if (typeof thing === 'object') {
+      const members = jsonpath.query(thing, '$..*')
+      yield* extractStrings(members)
+    }
+  }
+}
+
+/** @returns generator of tuples of [jsonpath, word] */
+export function* getWords(
+  node: OntologyDBNode,
+  jsonPaths: Iterable<string>,
+  prefixes: Map<string, string>,
+): Generator<[string, string], void, undefined> {
+  for (const path of jsonPaths) {
+    const queryResult = jsonPathQuery(node, path, prefixes) as unknown[]
+    if (queryResult.length > 0) {
+      for (const word of extractWords(extractStrings(queryResult))) {
+        yield [path, word]
+      }
+    }
+  }
+}
+
+export interface Match {
+  term: OntologyDBNode
+  field: TextIndexFieldDefinition
+  str: string
+  score: number
+}
+
+export function isMatch(thing: object): thing is Match {
+  return (
+    'term' in thing && 'field' in thing && 'str' in thing && 'score' in thing
+  )
+}
+
+/**
+ *
+ **/
+export async function textSearch(
+  this: OntologyStore,
+  text: string,
+  tx?: Transaction<['nodes']>,
+  signal?: AbortSignal,
+) {
+  const db = await this.db
+  const myTx = tx ?? db.transaction(['nodes'])
+
+  checkAbortSignal(signal)
+
+  const queryWords = [...wordsInString(text)]
+
+  const queries: Promise<void>[] = []
+
+  /**
+   * Build a structure of which terms match which words.
+   * This is a Map of term.id -\> Set\<query word number\>
+   **/
+  const initialMatches = new Map<string, [OntologyDBNode, Set<number>]>()
+
+  // find startsWith and complete matches
+  queries.push(
+    ...queryWords.map(async (queryWord, queryWordIndex) => {
+      checkAbortSignal(signal)
+      const idx = myTx.objectStore('nodes').index('full-text-words')
+      for await (const cursor of idx.iterate(
+        IDBKeyRange.bound(queryWord, `${queryWord}\uFFFF`, false, false),
+      )) {
+        checkAbortSignal(signal)
+        const term = cursor.value
+        const termMatches = initialMatches.get(term.id) ?? [
+          term,
+          new Set<number>(),
+        ]
+        termMatches[1].add(queryWordIndex)
+        initialMatches.set(term.id, termMatches)
+      }
+    }),
+  )
+
+  await Promise.all(queries)
+
+  checkAbortSignal(signal)
+
+  // now rank the term matches and add some detail
+  const results: Match[] = []
+  for (const [, [term, wordIndexes]] of initialMatches) {
+    checkAbortSignal(signal)
+    results.push(
+      ...elaborateMatch(
+        this.textIndexFields,
+        term,
+        wordIndexes,
+        queryWords,
+        this.prefixes,
+      ),
+    )
+  }
+
+  // sort the terms by score descending
+  results.sort((a, b) => b.score - a.score)
+
+  // truncate if necessary
+  return results.slice(
+    0,
+    this.options.maxSearchResults ?? this.DEFAULT_MAX_SEARCH_RESULTS,
+  )
+}
+
+export function elaborateMatch(
+  textIndexPaths: TextIndexFieldDefinition[],
+  term: OntologyDBNode,
+  queryWordIndexes: Set<number>,
+  queryWords: string[],
+  prefixes: Map<string, string>,
+): Match[] {
+  const sortedWordIndexes = [...queryWordIndexes].sort()
+  const matchedQueryWords = sortedWordIndexes.map((i) => queryWords[i])
+  const queryWordRegexps = matchedQueryWords.map((queryWord) => {
+    const escaped = queryWord.replaceAll(/[$()*+./?[\\\]^{|}-]/g, '\\$&')
+    return new RegExp(`\\b${escaped}`, 'gi')
+  })
+  // const needle = matchedQueryWords.join(' ')
+
+  // ranking weights that can be tweaked if you know what you're doing
+  const FIELD_PRIORITY_WEIGHT = 1
+  const MATCH_WORDS_CLOSENESS_WEIGHT = 0.05
+  const MATCH_ADJACENCY_BONUS = 1
+  const MATCH_RIGHT_ORDER_BONUS = 1
+  const MATCH_LENGTH_WEIGHT = 0.01
+  const PCT_OF_STRING_WEIGHT = 0.05
+  const WORD_BONUS = 100 // bonus for each of the words matched
+
+  // inspect the node at each of the index paths, because we don't know which ones matched
+  interface WordMatch {
+    wordIndex: number
+    position: number
+  }
+  let matches: (Match & { wordMatches: WordMatch[] })[] = []
+  let maxScore = 0
+  for (const [fieldIdx, field] of textIndexPaths.entries()) {
+    const wordsMatched = new Set<number>()
+    const fieldPriorityBonus = textIndexPaths.length - fieldIdx - 1
+    const termStrings = [
+      ...extractStrings(jsonPathQuery(term, field.jsonPath, prefixes)),
+    ]
+    // find occurrences of each of the words in the strings
+    for (const str of termStrings) {
+      let score = 0
+      const wordMatches: WordMatch[] = []
+      for (const [wordIndex, re] of queryWordRegexps.entries()) {
+        for (const match of str.matchAll(re)) {
+          score += 1 + fieldPriorityBonus * FIELD_PRIORITY_WEIGHT
+          wordsMatched.add(wordIndex)
+          const position = match.index
+          const queryWord = queryWords[wordIndex]
+          if (position !== undefined) {
+            score += queryWord.length * MATCH_LENGTH_WEIGHT
+            score +=
+              (queryWord.length / str.length) * 100 * PCT_OF_STRING_WEIGHT
+            wordMatches.push({ wordIndex, position })
+          }
+        }
+      }
+
+      // apply the words-matched bonus
+      score += wordsMatched.size * WORD_BONUS
+
+      if (maxScore < score) {
+        maxScore = score
+      }
+      // sort the word matches by position in the target string ascending
+      wordMatches.sort((a, b) => a.position - b.position)
+      if (wordMatches.length > 0) {
+        matches.push({ term, field, str, score, wordMatches })
+      }
+    }
+  }
+
+  // Keep only the highest-scored matches. Usually 1, but there
+  // could be multiple if there is a tie for first place.
+  matches = matches.filter((m) => m.score === maxScore)
+
+  for (const match of matches) {
+    const { wordMatches } = match
+    // re-examine the word order and spacing to give bonuses for the
+    // right order and close spacing
+    for (let i = 0; i < wordMatches.length - 1; i++) {
+      // bonus for pairs with adjacent word indexes and close spacing
+      const m1 = wordMatches[i]
+      const m2 = wordMatches[i + 1]
+      const wdiff = m2.wordIndex - m1.wordIndex
+      if (wdiff === 1 || wdiff === -1) {
+        // they are adjacent, bonus
+        match.score += MATCH_ADJACENCY_BONUS
+        if (wdiff === 1) {
+          // they are in the right order, bonus
+          match.score += MATCH_RIGHT_ORDER_BONUS
+        }
+        // give additional bonus for how close they are
+        const spacing =
+          Math.abs(
+            m2.position -
+              (m1.position + matchedQueryWords[m1.wordIndex].length),
+          ) - 1
+        match.score -= spacing * MATCH_WORDS_CLOSENESS_WEIGHT
+      }
+    }
+  }
+
+  return matches
+}

package/src/OntologyManager/OntologyStore/index.test.ts

@@ -0,0 +1,130 @@
+import path from 'node:path'
+
+import { beforeAll, describe, expect, it, jest } from '@jest/globals'
+
+import OntologyStore from '.'
+import { OntologyClass, isOntologyClass } from '..'
+
+jest.setTimeout(1_000_000_000)
+
+const prefixes = new Map([
+  ['SO:', 'http://purl.obolibrary.org/obo/SO_'],
+  ['GO:', 'http://purl.obolibrary.org/obo/GO_'],
+])
+
+// jsonpath uses an "obj instanceof Object" check in its "query", which fails in
+// tests because the mocked indexedDb uses a different scope and thus a
+// different "Object". This intercepts calls to "query" in this test and makes
+// sure the main scope "Object" is used.
+jest.mock('jsonpath', () => {
+  const original = jest.requireActual('jsonpath') as typeof import('jsonpath')
+  return {
+    ...original,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    query: jest.fn((obj: any, pathExpression: string, count?: number) => {
+      const newObj =
+        obj instanceof Object ? obj : JSON.parse(JSON.stringify(obj))
+      return original.query(newObj, pathExpression, count)
+    }),
+  }
+})
+
+let so: OntologyStore
+
+beforeAll(async () => {
+  const localPath = path.resolve(__dirname, '../../../test_data/so-v3.1.json')
+  so = new OntologyStore(
+    'Sequence Ontology',
+    'automated testing',
+    { locationType: 'LocalPathLocation', localPath },
+    { prefixes },
+  )
+  await so.db
+})
+
+describe('OntologyStore', () => {
+  it('can load goslim generic', async () => {
+    const localPath = path.resolve(
+      __dirname,
+      '../../../test_data/goslim_generic.json',
+    )
+    const goslimGeneric = new OntologyStore(
+      'Gene Ontology',
+      'automated testing',
+      { locationType: 'LocalPathLocation', localPath },
+      { prefixes },
+    )
+
+    expect(await goslimGeneric.termCount()).toMatchSnapshot()
+
+    expect(
+      await goslimGeneric.getTermsByFulltext('mitotic nuclear division'),
+    ).toMatchSnapshot()
+  })
+
+  it('can query SO', async () => {
+    expect(await so.termCount()).toMatchSnapshot()
+  })
+  it('can query SO gene terms and parts of genes', async () => {
+    const geneTerms = await so.getTermsWithLabelOrSynonym('gene')
+    expect(geneTerms).toMatchSnapshot()
+    expect(isOntologyClass(geneTerms[0])).toBe(true)
+    const geneParts = await so.getClassesThat(
+      'part_of',
+      geneTerms as OntologyClass[],
+    )
+    expect(geneParts).toMatchSnapshot()
+  })
+  it('can query SO features not part of something else', async () => {
+    const topLevelClasses = await so.getClassesWithoutPropertyLabeled(
+      'part_of',
+      { includeSubProperties: true },
+    )
+    expect(topLevelClasses.length).toMatchSnapshot()
+    expect(topLevelClasses.find((term) => term.lbl === 'mRNA')).toBeUndefined()
+    // gene is member_of gene_group, so also doesn't appear here. There doesn't seem to be
+    // clarification in SO for when a feature MUST be part_of.
+    expect(topLevelClasses.find((term) => term.lbl === 'gene')).toBeUndefined()
+  })
+  it('can expand subclasses of SO:000039 and still get SO:000039', async () => {
+    const expanded = so.expandSubclasses(
+      ['http://purl.obolibrary.org/obo/SO_0000039'],
+      'is_a',
+    )
+
+    const ex = []
+    for await (const node of expanded) {
+      ex.push(node)
+    }
+    expect(ex.length).toBeGreaterThan(0)
+    expect(ex).toEqual(['http://purl.obolibrary.org/obo/SO_0000039'])
+  })
+  it('can query valid part_of for match', async () => {
+    const parentTypeTerms = await so.getTermsWithLabelOrSynonym('match', {
+      includeSubclasses: false,
+    })
+    // eslint-disable-next-line unicorn/no-array-callback-reference
+    const parentTypeClassTerms = parentTypeTerms.filter(isOntologyClass)
+    expect(parentTypeClassTerms).toMatchSnapshot()
+    const subpartTerms = await so.getClassesThat(
+      'part_of',
+      parentTypeClassTerms,
+    )
+    expect(subpartTerms.length).toBeGreaterThan(0)
+  })
+
+  it('SO clone_insert_end is among valid subparts of BAC_cloned_genomic_insert', async () => {
+    const bcgi = await so.getTermsWithLabelOrSynonym(
+      'BAC_cloned_genomic_insert',
+      { includeSubclasses: false },
+    )
+    // eslint-disable-next-line unicorn/no-array-callback-reference
+    const bcgiClass = bcgi.filter(isOntologyClass)
+    expect(bcgiClass.length).toBe(1)
+    expect(bcgiClass[0].lbl).toBe('BAC_cloned_genomic_insert')
+    const subpartTerms = await so.getClassesThat('part_of', bcgiClass)
+    expect(subpartTerms.length).toBeGreaterThan(0)
+    expect(subpartTerms.find((t) => t.lbl === 'clone_insert_end')).toBeTruthy()
+    expect(subpartTerms).toMatchSnapshot()
+  })
+})