@openneuro/server 4.47.7 → 5.0.0

package/src/graphql/resolvers/build-search-query.ts

@@ -0,0 +1,391 @@
+/**
+ * Translate DatasetSearchInput into ElasticSearch Query DSL
+ *
+ * Port of query building logic from the React side
+ */
+
+interface DatasetSearchInput {
+  keywords?: string[]
+  modality?: string
+  ageRange?: (number | null)[]
+  subjectCountRange?: (number | null)[]
+  diagnosis?: string
+  tasks?: string[]
+  authors?: string[]
+  sex?: string
+  dateRange?: string
+  species?: string
+  studyStructure?: string
+  studyDomains?: string[]
+  bidsDatasetType?: string
+  brainInitiative?: boolean
+  bodyParts?: string[]
+  scannerManufacturers?: string[]
+  scannerManufacturersModelNames?: string[]
+  tracerNames?: string[]
+  tracerRadionuclides?: string[]
+  userId?: string
+  publicOnly?: boolean
+}
+
+const KNOWN_SPECIES = ["Human", "Rat", "Mouse"]
+
+const SECONDARY_MODALITIES: Record<
+  string,
+  { secondary: string; primary: string }
+> = {
+  Diffusion: { secondary: "mri_diffusion", primary: "mri" },
+  Structural: { secondary: "mri_structural", primary: "mri" },
+  Functional: { secondary: "mri_functional", primary: "mri" },
+  Perfusion: { secondary: "mri_perfusion", primary: "mri" },
+  Static: { secondary: "pet_static", primary: "pet" },
+  Dynamic: { secondary: "pet_dynamic", primary: "pet" },
+}
+
+interface BoolQuery {
+  bool: {
+    must?: object[]
+    filter?: object[]
+    must_not?: object[]
+    should?: object[]
+  }
+}
+
+const addClause = (
+  query: BoolQuery,
+  type: "must" | "filter" | "must_not" | "should",
+  clause: object,
+) => {
+  if (query.bool[type]) {
+    query.bool[type] = [...query.bool[type], clause]
+  } else {
+    query.bool[type] = [clause]
+  }
+}
+
+const simpleQueryString = (
+  queryString: string,
+  fields?: string[],
+  fuzzy = true,
+) => ({
+  simple_query_string: {
+    query: `${queryString}${fuzzy ? "~" : ""}`,
+    fields,
+  },
+})
+
+const matchQuery = (
+  field: string,
+  queryString: string,
+  fuzziness?: string,
+  operator?: string,
+) => ({
+  match: {
+    [field]: {
+      query: queryString,
+      fuzziness,
+      operator,
+    },
+  },
+})
+
+const multiMatchQuery = (field: string, queryStrings: string[]) => ({
+  bool: {
+    should: queryStrings.map((qs) => matchQuery(field, qs)),
+    minimum_should_match: 1,
+  },
+})
+
+const rangeQuery = (
+  field: string,
+  gte?: number | string | null,
+  lte?: number | string | null,
+  relation: string = "INTERSECTS",
+) => ({
+  range: {
+    [field]: {
+      gte,
+      lte,
+      relation,
+    },
+  },
+})
+
+const rangeListLengthQuery = (field: string, gte: number, lte: number) => ({
+  script: {
+    script: {
+      lang: "painless",
+      source: `
+          if (doc[params.field].size() != 0) {
+            return ( doc[params.field].size() >= params.gte && doc[params.field].size() <= params.lte )
+          } else return false`,
+      params: { field, gte, lte },
+    },
+  },
+})
+
+const sqsJoinWithAND = (list: string[]) =>
+  list.map((str) => `${str}`).join(" + ")
+const joinWithOR = (list: string[]) => list.map((str) => `${str}`).join(" | ")
+
+const isActiveRange = (range: (number | null)[] | undefined): boolean =>
+  Array.isArray(range) &&
+  range.length === 2 &&
+  (range[0] !== null || range[1] !== null)
+
+/**
+ * Build an ElasticSearch bool query from structured search input.
+ * Returns { query, isEmpty } where query is the ES Query DSL object.
+ */
+export const buildElasticQuery = (
+  input: DatasetSearchInput,
+): { query: BoolQuery; isEmpty: boolean } => {
+  const query: BoolQuery = { bool: {} }
+
+  if (input.keywords?.length) {
+    addClause(
+      query,
+      "must",
+      simpleQueryString(sqsJoinWithAND(input.keywords), [
+        "id^20",
+        "latestSnapshot.readme",
+        "latestSnapshot.description.Name^6",
+        "latestSnapshot.description.Authors^3",
+        "latestSnapshot.contributors.name^2",
+      ]),
+    )
+  }
+
+  if (input.modality) {
+    if (SECONDARY_MODALITIES[input.modality]) {
+      addClause(
+        query,
+        "filter",
+        matchQuery(
+          "latestSnapshot.summary.secondaryModalities",
+          SECONDARY_MODALITIES[input.modality].secondary,
+        ),
+      )
+    } else {
+      addClause(
+        query,
+        "filter",
+        matchQuery("latestSnapshot.summary.modalities", input.modality),
+      )
+    }
+  }
+
+  if (isActiveRange(input.ageRange)) {
+    addClause(
+      query,
+      "filter",
+      rangeQuery(
+        "latestSnapshot.summary.subjectMetadata.age",
+        input.ageRange[0],
+        input.ageRange[1],
+      ),
+    )
+  }
+
+  if (isActiveRange(input.subjectCountRange)) {
+    addClause(
+      query,
+      "filter",
+      rangeListLengthQuery(
+        "latestSnapshot.summary.subjects",
+        input.subjectCountRange[0] || 0,
+        input.subjectCountRange[1] || 1000000,
+      ),
+    )
+  }
+
+  if (input.diagnosis) {
+    addClause(
+      query,
+      "filter",
+      matchQuery("metadata.dxStatus", input.diagnosis),
+    )
+  }
+
+  if (input.bidsDatasetType) {
+    addClause(
+      query,
+      "filter",
+      matchQuery(
+        "latestSnapshot.description.DatasetType",
+        input.bidsDatasetType,
+      ),
+    )
+  }
+
+  if (input.brainInitiative) {
+    addClause(
+      query,
+      "filter",
+      matchQuery("brainInitiative", String(input.brainInitiative)),
+    )
+  }
+
+  if (input.tasks?.length) {
+    addClause(
+      query,
+      "must",
+      simpleQueryString(sqsJoinWithAND(input.tasks), [
+        "latestSnapshot.summary.tasks",
+      ]),
+    )
+  }
+
+  if (input.authors?.length) {
+    const authorQuery = matchQuery(
+      "latestSnapshot.contributors.name",
+      joinWithOR(input.authors),
+      "2",
+    )
+    addClause(query, "must", {
+      bool: {
+        should: [authorQuery],
+      },
+    })
+  }
+
+  if (input.sex && input.sex !== "All") {
+    let queryStrings: string[] = []
+    if (input.sex === "Male") {
+      queryStrings = ["male", "m", "M", "MALE", "Male"]
+    } else if (input.sex === "Female") {
+      queryStrings = ["female", "f", "F", "FEMALE", "Female"]
+    }
+    addClause(
+      query,
+      "filter",
+      multiMatchQuery(
+        "latestSnapshot.summary.subjectMetadata.sex",
+        queryStrings,
+      ),
+    )
+  }
+
+  if (input.dateRange && input.dateRange !== "All Time") {
+    let d: number
+    if (input.dateRange === "Last 30 days") {
+      d = 30
+    } else if (input.dateRange === "Last 180 days") {
+      d = 180
+    } else {
+      d = 365
+    }
+    addClause(query, "filter", rangeQuery("created", `now-${d}d/d`, "now/d"))
+  }
+
+  if (input.species) {
+    if (input.species === "Other") {
+      const species = KNOWN_SPECIES.join(" ")
+      addClause(
+        query,
+        "must_not",
+        matchQuery("metadata.species", species, "AUTO", "OR"),
+      )
+    } else if (input.species === "Human") {
+      query.bool["should"] = [
+        matchQuery("metadata.species", "Human", "AUTO"),
+        { term: { _content: "" } },
+      ]
+    } else {
+      addClause(
+        query,
+        "filter",
+        matchQuery("metadata.species", input.species, "AUTO"),
+      )
+    }
+  }
+
+  if (input.studyStructure) {
+    addClause(
+      query,
+      "filter",
+      matchQuery("metadata.studyLongitudinal", input.studyStructure, "AUTO"),
+    )
+  }
+
+  if (input.studyDomains?.length) {
+    addClause(
+      query,
+      "must",
+      matchQuery("metadata.studyDomain", joinWithOR(input.studyDomains)),
+    )
+  }
+
+  // PET-specific fields (only apply when modality is pet or unset)
+  if (
+    input.modality === "pet" || input.modality === null ||
+    input.modality === undefined
+  ) {
+    if (input.bodyParts?.length) {
+      addClause(
+        query,
+        "must",
+        simpleQueryString(sqsJoinWithAND(input.bodyParts), [
+          "latestSnapshot.summary.pet.BodyPart",
+        ]),
+      )
+    }
+    if (input.scannerManufacturers?.length) {
+      addClause(
+        query,
+        "must",
+        simpleQueryString(sqsJoinWithAND(input.scannerManufacturers), [
+          "latestSnapshot.summary.pet.ScannerManufacturer",
+        ]),
+      )
+    }
+    if (input.scannerManufacturersModelNames?.length) {
+      addClause(
+        query,
+        "must",
+        simpleQueryString(
+          sqsJoinWithAND(input.scannerManufacturersModelNames),
+          [
+            "latestSnapshot.summary.pet.ScannerManufacturersModelName",
+          ],
+        ),
+      )
+    }
+    if (input.tracerNames?.length) {
+      addClause(
+        query,
+        "must",
+        simpleQueryString(sqsJoinWithAND(input.tracerNames), [
+          "latestSnapshot.summary.pet.TracerName",
+        ]),
+      )
+    }
+    if (input.tracerRadionuclides?.length) {
+      addClause(
+        query,
+        "must",
+        simpleQueryString(sqsJoinWithAND(input.tracerRadionuclides), [
+          "latestSnapshot.summary.pet.TracerRadionuclide",
+        ]),
+      )
+    }
+  }
+
+  if (input.userId) {
+    addClause(query, "filter", {
+      terms: {
+        "permissions.userPermissions.user.id": [input.userId],
+      },
+    })
+  }
+
+  if (input.publicOnly) {
+    addClause(query, "filter", {
+      term: { public: { value: true } },
+    })
+  }
+
+  const isEmpty = Object.keys(query.bool).length === 0
+
+  return { query, isEmpty }
+}

package/src/graphql/resolvers/cache.ts

@@ -1,4 +1,5 @@
 import { redis } from "../../libs/redis.js"
+import { clearDatasetTrees } from "../../cache/tree"
 
 /**
  * Clear all cache entries for a given datasetId
@@ -11,8 +12,11 @@ export async function cacheClear(
   // Check for admin and validate datasetId argument
   if (userInfo?.admin && datasetId.length == 8 && datasetId.startsWith("ds")) {
     try {
+      // Clear tree cache entries via the dataset-to-trees index
+      await clearDatasetTrees(redis, datasetId)
+
+      // Also clear non-tree cache keys (descriptions, snapshots, etc.)
       const stream = redis.scanStream({
-        // Scan for any keys that include the datasetId
         match: `*${datasetId}*`,
       })
       const pipeline = redis.pipeline()

package/src/graphql/resolvers/dataset-search.ts

@@ -5,6 +5,7 @@ import Star from "../../models/stars"
 import Subscription from "../../models/subscription"
 import Permission from "../../models/permission"
 import { hashObject } from "../../libs/authentication/crypto"
+import { buildElasticQuery } from "./build-search-query"
 
 const elasticIndex = "datasets"
 
@@ -195,43 +196,56 @@ const parseQuery = async (query, datasetType, datasetStatus, userId) => {
 }
 
 /**
- * Search result cursor resolver
- * TODO this is a Relay pagination type and could use the interface
- * @param {any} obj
- * @param {object} args
- * @param {object} args.query Stringified Query (DSL) argument for ElasticSearch
- * @param {boolean} args.allDatasets Admin option for returning all datasets (overrides datasetType and datasetStatus, but keeps other search parameters) (default = false)
- * @param {string} args.datasetType Stringified Query (DSL) argument for ElasticSearch
- * @param {string} args.datasetStatus Stringified Query (DSL) argument for ElasticSearch
- * @param {object} args.sortBy Stringified Query (DSL) argument for ElasticSearch
- * @param {string} args.after Cursor for paging forward
- * @param {number} args.first Limit of entries to find
+ * Resolve the sort parameter from SearchSortOption enum to ES sort objects.
+ */
+const resolveSort = (sortBy: string | undefined, isEmpty: boolean) => {
+  if (sortBy === "newest") return { created: "desc" }
+  if (sortBy === "oldest") return { created: "asc" }
+  if (sortBy === "activity") return { "analytics.downloads": "desc" }
+  if (sortBy === "name_asc") return { "metadata.datasetName": "asc" }
+  if (sortBy === "name_desc") return { "metadata.datasetName": "desc" }
+  if (sortBy === "last_updated") {
+    return { "metadata.latestSnapshotCreatedAt": "desc" }
+  }
+  // "relevance" or undefined: if no filters are set, sort by newest
+  if (isEmpty) return { created: "desc" }
+  return null
+}
+
+/**
+ * Search result cursor resolver using typed DatasetSearchInput
  */
 export const advancedDatasetSearchConnection = async (
   obj,
   {
-    query,
+    query: searchInput,
     allDatasets = false,
     datasetType,
     datasetStatus,
-    sortBy,
     after,
     first = 25,
   },
   { user, userInfo },
 ) => {
+  // Build the ES query from structured input
+  const { query: esQuery, isEmpty } = buildElasticQuery(searchInput)
+  const sortByOption = searchInput.sortBy
+
   // Create an identity for this search (used to cache connections)
   const searchId = hashObject({
-    query,
+    searchInput,
     datasetType,
     datasetStatus,
-    sortBy,
+    sortByOption,
     user,
   })
-  const sort = [{ _score: "desc" }, { id: "desc" }]
-  if (sortBy) {
-    sort.unshift(sortBy)
+
+  const sort: Record<string, string>[] = [{ _score: "desc" }, { id: "desc" }]
+  const extraSort = resolveSort(sortByOption, isEmpty)
+  if (extraSort) {
+    sort.unshift(extraSort)
   }
+
   // Parse out the decode token and add it to our query if successful
   let search_after
   if (after) {
@@ -241,13 +255,17 @@ export const advancedDatasetSearchConnection = async (
       // Don't include search_after if parsing fails
     }
   }
+
+  // Apply dataset type/status/permission filters
+  const query = allDatasets
+    ? esQuery
+    : await parseQuery(esQuery, datasetType, datasetStatus, user)
+
   const requestBody = {
     index: elasticIndex,
     size: first,
     sort,
-    query: allDatasets
-      ? query
-      : await parseQuery(query, datasetType, datasetStatus, user),
+    query,
     search_after,
   }
   // Run the query
@@ -268,11 +286,10 @@ export const advancedDatasetSearch = {
     type: "DatasetConnection",
     resolve: advancedDatasetSearchConnection,
     args: {
-      query: { type: "JSON!" },
+      query: { type: "DatasetSearchInput!" },
       allDatasets: { type: "Boolean" },
       datasetType: { type: "String" },
       datasetStatus: { type: "String" },
-      sortBy: { type: "JSON" },
       after: { type: "String" },
       first: { type: "Int" },
     },