@opencloning/opencloningdb 1.9.1 → 2.2.1

package/src/sequencing_check/sequencingCheckUtils.test.js

@@ -0,0 +1,143 @@
+import { describe, it, expect } from 'vitest';
+import {
+  buildSortedRows,
+  fileRowStatus,
+  formatIdentity,
+  percentIdentity,
+  runWithConcurrency,
+  sequenceRowStatus,
+} from './sequencingCheckUtils';
+
+describe('percentIdentity', () => {
+  it('returns 100 for identical sequences, ignoring case', () => {
+    expect(percentIdentity('ACGTACGTAC', 'acgtACGTac')).toBe(100);
+  });
+
+  it('counts mismatches', () => {
+    expect(percentIdentity('ACGTACGTAC', 'ACGTTCGTAC')).toBe(90);
+  });
+
+  it('counts internal and terminal gaps as mismatches', () => {
+    expect(percentIdentity('ACGTACGTAC', 'ACG-ACGTAC')).toBe(90);
+    expect(percentIdentity('ACGTACGTAC', '--GTACGTAC')).toBe(80);
+    expect(percentIdentity('ACGT-CGTAC', 'ACGTACGTAC')).toBe(90);
+  });
+
+  it('ignores columns where both sequences have a gap', () => {
+    expect(percentIdentity('AC--GT', 'AC--GT')).toBe(100);
+    expect(percentIdentity('AC--GT', 'AC--GA')).toBe(75);
+  });
+
+  it('returns 0 for empty alignments', () => {
+    expect(percentIdentity('', '')).toBe(0);
+    expect(percentIdentity('--', '--')).toBe(0);
+  });
+
+  it('throws if the aligned sequences have different lengths', () => {
+    expect(() => percentIdentity('ACGT', 'ACG')).toThrow();
+  });
+});
+
+describe('formatIdentity', () => {
+  it('uses 2 decimals and never rounds up to 100', () => {
+    expect(formatIdentity(100)).toBe('100.00');
+    expect(formatIdentity(99.999)).toBe('99.99');
+    expect(formatIdentity(87.5)).toBe('87.50');
+  });
+});
+
+describe('runWithConcurrency', () => {
+  it('runs all items, never more than limit at a time, and keeps the order of results', async () => {
+    let active = 0;
+    let maxActive = 0;
+    const items = [5, 1, 8, 2, 7, 3, 6, 4, 9];
+    const results = await runWithConcurrency(items, 4, async (item) => {
+      active++;
+      maxActive = Math.max(maxActive, active);
+      await new Promise((resolve) => setTimeout(resolve, item));
+      active--;
+      return item * 2;
+    });
+    expect(maxActive).toBe(4);
+    expect(results).toEqual(items.map((item) => item * 2));
+  });
+
+  it('works with fewer items than the limit, and with no items', async () => {
+    expect(await runWithConcurrency([1], 4, async (item) => item)).toEqual([1]);
+    expect(await runWithConcurrency([], 4, async (item) => item)).toEqual([]);
+  });
+});
+
+describe('fileRowStatus and sequenceRowStatus', () => {
+  it('maps file results to StatusIcon statuses', () => {
+    expect(fileRowStatus(undefined)).toBe(null);
+    expect(fileRowStatus({ status: 'pending' })).toBe('pending');
+    expect(fileRowStatus({ status: 'error', message: 'x' })).toBe('error');
+    expect(fileRowStatus({ status: 'match', identity: 100 })).toBe('clear');
+    expect(fileRowStatus({ status: 'aligned', identity: 100 })).toBe('clear');
+    expect(fileRowStatus({ status: 'aligned', identity: 99.5 })).toBe('warning');
+  });
+
+  it('uses the worst status of the files, or info if there are no files', () => {
+    const files = [{ id: 1 }, { id: 2 }];
+    expect(sequenceRowStatus([], {})).toBe('info');
+    expect(sequenceRowStatus(files, {})).toBe(null);
+    expect(sequenceRowStatus(files, { 1: { status: 'match', identity: 100 }, 2: { status: 'match', identity: 100 } })).toBe('clear');
+    expect(sequenceRowStatus(files, { 1: { status: 'match', identity: 100 }, 2: { status: 'pending' } })).toBe('pending');
+    expect(sequenceRowStatus(files, { 1: { status: 'aligned', identity: 90 }, 2: { status: 'pending' } })).toBe('warning');
+    expect(sequenceRowStatus(files, { 1: { status: 'aligned', identity: 90 }, 2: { status: 'error' } })).toBe('error');
+  });
+});
+
+describe('buildSortedRows', () => {
+  const sequences = [
+    { databaseId: 1, name: 'perfect' },
+    { databaseId: 2, name: 'no files' },
+    { databaseId: 3, name: 'low' },
+    { databaseId: 4, name: 'loading' },
+    { databaseId: 5, name: 'error' },
+    { databaseId: 6, name: 'not checked' },
+  ];
+  const filesBySequenceId = {
+    1: [{ id: 10, name: 'a.fa' }],
+    2: [],
+    3: [{ id: 30, name: 'high.fa' }, { id: 31, name: 'low.fa' }],
+    // 4 is still loading
+    5: [{ id: 50, name: 'ok.fa' }, { id: 51, name: 'broken.fa' }],
+    6: [{ id: 60, name: 'b.fa' }],
+  };
+  const results = {
+    10: { status: 'match', identity: 100 },
+    30: { status: 'aligned', identity: 99 },
+    31: { status: 'aligned', identity: 80 },
+    50: { status: 'aligned', identity: 100 },
+    51: { status: 'error', message: 'Boom' },
+  };
+
+  it('sorts sequences: errors, lowest identity first, not checked or loading, no files last', () => {
+    const rows = buildSortedRows(sequences, filesBySequenceId, results);
+    expect(rows.map((row) => row.sequence.name)).toEqual(['error', 'low', 'perfect', 'loading', 'not checked', 'no files']);
+  });
+
+  it('sorts the files within each sequence and sets the identity of the sequence to the minimum', () => {
+    const rows = buildSortedRows(sequences, filesBySequenceId, results);
+    const low = rows.find((row) => row.sequence.name === 'low');
+    expect(low.files.map(({ file }) => file.name)).toEqual(['low.fa', 'high.fa']);
+    expect(low.identity).toBe(80);
+    expect(low.status).toBe('warning');
+    const error = rows.find((row) => row.sequence.name === 'error');
+    expect(error.files.map(({ file }) => file.name)).toEqual(['broken.fa', 'ok.fa']);
+    expect(error.status).toBe('error');
+  });
+
+  it('sets the status of sequences with no files and of sequences whose files are loading', () => {
+    const rows = buildSortedRows(sequences, filesBySequenceId, results);
+    const noFiles = rows.find((row) => row.sequence.name === 'no files');
+    expect(noFiles.status).toBe('info');
+    expect(noFiles.filesLoaded).toBe(true);
+    expect(noFiles.identity).toBeUndefined();
+    const loading = rows.find((row) => row.sequence.name === 'loading');
+    expect(loading.status).toBe('pending');
+    expect(loading.filesLoaded).toBe(false);
+  });
+});

package/src/sequencing_check/useSequencingCheck.js

@@ -0,0 +1,114 @@
+import React from 'react';
+import { useMutation, useQueries } from '@tanstack/react-query';
+import useBackendHttpClient from '@opencloning/ui/hooks/useBackendHttpClient';
+import { convertToTeselaJson } from '@opencloning/utils/readNwrite';
+import error2String from '@opencloning/utils/error2String';
+import OpenCloningDBInterface from '../OpenCloningDBInterface';
+import { MAX_CONCURRENT_CHECKS, buildSortedRows, percentIdentity, runWithConcurrency } from './sequencingCheckUtils';
+import useSequencingQueries from './useSequencingQueries';
+
+const { getSequencingFiles } = OpenCloningDBInterface;
+
+/**
+ * Compare a sequencing file to its reference sequence. Plasmids are expected to match exactly, so if the
+ * query has the same length as the reference, its seguid (computed by the backend with the topology of the
+ * reference) is compared to the one stored in the database first, and the alignment is skipped if they match.
+ *
+ * The parsed file and the alignment come from useSequencingQueries, so they are reused by the alignment viewer
+ * and the next checks (only the seguid is computed again). Failed alignments are not cached, so they are retried.
+ * reference is { circular, length } of the sequence, parsed once per sequence.
+ */
+async function checkSequencingFile({ httpClient, getParsedFile, getAlignment }, { sequence, file, reference }) {
+  const { sequence: query } = await getParsedFile(file);
+  if (!query) {
+    throw new Error('The file has no sequence');
+  }
+
+  if (sequence.seguid && query.length === reference.length) {
+    try {
+      const { data: querySeguid } = await httpClient.post('seguid', { sequence: query, circular: reference.circular });
+      if (querySeguid === sequence.seguid) {
+        return { status: 'match', identity: 100 };
+      }
+    } catch {
+      // The seguid could not be computed (e.g. unexpected characters), the alignment decides
+    }
+  }
+
+  const { alignedSequences } = await getAlignment(sequence, file);
+  return { status: 'aligned', identity: percentIdentity(alignedSequences[0], alignedSequences[1]) };
+}
+
+/**
+ * Defined outside the hook so that its reference is stable: useQueries then only re-runs it (and returns
+ * a new object) when one of the queries changes, so filesData can be used as a memo dependency.
+ * filesData[i] is the list of files of the i-th sequence, undefined while loading.
+ */
+function combineFilesQueries(queries) {
+  return {
+    filesData: queries.map((query) => query.data),
+    filesError: queries.find((query) => query.error)?.error,
+    filesLoading: queries.some((query) => query.isPending),
+  };
+}
+
+/**
+ * Check sequences (output of GetSequencesInBulkComponent) against their sequencing files.
+ *
+ * The sequencing files of each sequence are listed on mount, run() checks all the files, at most
+ * MAX_CONCURRENT_CHECKS at the same time, and the rows are updated as each check finishes.
+ */
+export default function useSequencingCheck(sequences) {
+  const httpClient = useBackendHttpClient();
+  const { getParsedFile, getAlignment } = useSequencingQueries();
+  const [results, setResults] = React.useState({});
+
+  // Same query key as the sequence detail page, so the cache is shared
+  const { filesData, filesError, filesLoading } = useQueries({
+    queries: sequences.map(({ databaseId }) => ({
+      queryKey: ['sequence', databaseId, 'sequencing_files'],
+      queryFn: () => getSequencingFiles(databaseId),
+      retry: false,
+    })),
+    combine: combineFilesQueries,
+  });
+
+  const filesBySequenceId = React.useMemo(() => Object.fromEntries(
+    sequences.map(({ databaseId }, i) => [databaseId, filesData[i]]),
+  ), [sequences, filesData]);
+
+  const references = React.useMemo(() => Object.fromEntries(sequences.map((sequence) => {
+    const { circular, sequence: bases } = convertToTeselaJson(sequence.textFileSequence);
+    return [sequence.databaseId, { circular, length: bases.length }];
+  })), [sequences]);
+
+  const jobs = React.useMemo(() => sequences.flatMap((sequence) => (filesBySequenceId[sequence.databaseId] ?? [])
+    .map((file) => ({ sequence, file, reference: references[sequence.databaseId] }))), [sequences, filesBySequenceId, references]);
+
+  const setResult = (fileId, result) => setResults((prev) => ({ ...prev, [fileId]: result }));
+
+  const mutation = useMutation({
+    mutationFn: async () => {
+      setResults(Object.fromEntries(jobs.map(({ file }) => [file.id, { status: 'pending' }])));
+      await runWithConcurrency(jobs, MAX_CONCURRENT_CHECKS, async (job) => {
+        try {
+          setResult(job.file.id, await checkSequencingFile({ httpClient, getParsedFile, getAlignment }, job));
+        } catch (error) {
+          setResult(job.file.id, { status: 'error', message: error2String(error) });
+        }
+      });
+    },
+  });
+
+  const rows = React.useMemo(() => buildSortedRows(sequences, filesBySequenceId, results), [sequences, filesBySequenceId, results]);
+  const done = jobs.filter(({ file }) => results[file.id] && results[file.id].status !== 'pending').length;
+
+  return {
+    rows,
+    filesLoading,
+    filesError,
+    run: mutation.mutate,
+    isRunning: mutation.isPending,
+    progress: { done, total: jobs.length },
+  };
+}

package/src/sequencing_check/useSequencingQueries.js

@@ -0,0 +1,51 @@
+import React from 'react';
+import { useQueryClient } from '@tanstack/react-query';
+import useBackendHttpClient from '@opencloning/ui/hooks/useBackendHttpClient';
+import { readSequencingFile } from '@opencloning/utils/readNwrite';
+
+/**
+ * The content of a sequencing file and its alignment to a sequence do not change, so they never go stale.
+ * They are kept for a while after nothing uses them (the parsed files may include chromatograms).
+ * Failed queries are not cached, so they are retried the next time.
+ */
+const cacheOptions = { staleTime: Infinity, gcTime: 30 * 60 * 1000, retry: false };
+
+/**
+ * Cached sequencing file contents and alignments, shared by the sequencing check and the alignment viewer,
+ * so that a file is downloaded and aligned at most once.
+ *
+ * - getParsedFile(file): parsed content (teselaJson) of a sequencing file (from the output of getSequencingFiles)
+ * - getAlignment(sequence, file): alignment of a sequencing file to its sequence (an item of the output of
+ *   GetSequencesInBulkComponent), { fileContent, alignedSequences } where alignedSequences is the align_sanger
+ *   response ([aligned reference, aligned file])
+ * - alignmentQuery(sequence, file): the query options of getAlignment, to use with useQuery
+ */
+export default function useSequencingQueries() {
+  const queryClient = useQueryClient();
+  const httpClient = useBackendHttpClient();
+
+  return React.useMemo(() => {
+    const getParsedFile = (file) => queryClient.ensureQueryData({
+      queryKey: ['sequencing_file', file.id, 'parsed'],
+      queryFn: async () => (await readSequencingFile(await file.getFile())).teselaJson,
+      ...cacheOptions,
+    });
+
+    const alignmentQuery = (sequence, file) => ({
+      queryKey: ['sequence', sequence.databaseId, 'sequencing_alignment', file.id],
+      queryFn: async () => {
+        const fileContent = await getParsedFile(file);
+        const { data: alignedSequences } = await httpClient.post('align_sanger', {
+          sequence: sequence.textFileSequence,
+          traces: [fileContent.sequence],
+        });
+        return { fileContent, alignedSequences };
+      },
+      ...cacheOptions,
+    });
+
+    const getAlignment = (sequence, file) => queryClient.ensureQueryData(alignmentQuery(sequence, file));
+
+    return { getParsedFile, getAlignment, alignmentQuery };
+  }, [queryClient, httpClient]);
+}

package/src/testUtils.test.js

@@ -3,7 +3,8 @@ import { http, HttpResponse } from 'msw';
 import { describe, expect, it } from 'vitest';
 import { baseUrl, setHttpClientTokenGetter } from './common';
 
-const STUB_FOLDER = `${__dirname}/../../../OpenCloning_backend/stubs/db`;
+// Generated by `opencloning-cli db stubs` in python/
+const STUB_FOLDER = `${__dirname}/../../../../python/stubs/db`;
 
 /**
  * @typedef {Object} DbStub