@openneuro/server 4.36.1 → 4.37.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.36.1",
+  "version": "4.37.0-alpha.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -20,8 +20,8 @@
     "@apollo/utils.keyvadapter": "3.0.0",
     "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/schema": "^10.0.0",
-    "@keyv/redis": "^2.7.0",
-    "@openneuro/search": "^4.36.1",
+    "@keyv/redis": "^4.5.0",
+    "@openneuro/search": "^4.37.0-alpha.0",
     "@sentry/node": "^8.25.0",
     "@sentry/profiling-node": "^8.25.0",
     "base64url": "^3.0.0",
@@ -37,10 +37,10 @@
     "graphql-iso-date": "^3.6.1",
     "graphql-tools": "9.0.0",
     "immutable": "^3.8.2",
-    "ioredis": "4.17.3",
+    "ioredis": "^5.6.1",
     "jsdom": "24.0.0",
     "jsonwebtoken": "^9.0.0",
-    "keyv": "^4.5.3",
+    "keyv": "^5.3.4",
     "mime-types": "^2.1.19",
     "mongodb-memory-server": "^9.2.0",
     "mongoose": "^8.9.5",
@@ -55,6 +55,8 @@
     "passport-orcid": "0.0.4",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "redis-smq": "^8.3.1",
+    "redis-smq-common": "^8.3.1",
     "redlock": "^4.0.0",
     "request": "^2.83.0",
     "semver": "^5.5.0",
@@ -86,5 +88,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "1b95eb2f9fcabcdca1f4fc4e1f2064ed2c4dedf7"
+  "gitHead": "01cd2b234a23c76a234dfb2c2839857041da777b"
 }

package/src/__mocks__/config.ts

@@ -4,6 +4,10 @@ const config = {
       secret: "123456",
     },
   },
+  redis: {
+    port: 6379,
+    host: "localhost",
+  },
   datalad: {
     uri: "datalad",
     workers: 4,

package/src/app.ts

@@ -65,6 +65,7 @@ export async function expressApolloSetup() {
     schema,
     // Always allow introspection - our schema is public
     introspection: true,
+    // @ts-expect-error Type mismatch for keyv and ioredis recent releases
     cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(redis) })),
     plugins: [
       ApolloServerPluginLandingPageLocalDefault(),

package/src/cache/types.ts

@@ -13,5 +13,6 @@ export enum CacheType {
   participantCount = "participantCount",
   snapshotDownload = "download",
   draftRevision = "revision",
-  brainInitiative = "brainInitiative"
+  brainInitiative = "brainInitiative",
+  validation = "validation",
 }

package/src/datalad/snapshots.ts

@@ -3,7 +3,6 @@
  */
 import * as Sentry from "@sentry/node"
 import request from "superagent"
-import { reindexDataset } from "../elasticsearch/reindex-dataset"
 import { redis, redlock } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import config from "../config"
@@ -23,6 +22,7 @@ import { updateDatasetRevision } from "./draft"
 import { getDatasetWorker } from "../libs/datalad-service"
 import { join } from "path"
 import { createEvent, updateEvent } from "../libs/events"
+import { queueIndexDataset } from "../queues/producer-methods"
 
 const lockSnapshot = (datasetId, tag) => {
   return redlock.lock(
@@ -177,7 +177,7 @@ export const createSnapshot = async (
     await updateEvent(event)
 
     // Immediate indexing for new snapshots
-    await reindexDataset(datasetId)
+    queueIndexDataset(datasetId)
 
     announceNewSnapshot(snapshot, datasetId, user)
     return snapshot

package/src/graphql/resolvers/comment.ts

@@ -105,7 +105,7 @@ export const deleteComment = async (
 const CommentFields = {
   parent: (obj) => comment(obj, { id: obj.parentId }),
   replies,
-  user: (obj) => user(obj, { id: obj.user._id }, null),
+  user: (obj) => user(obj, { id: obj.user._id }),
 }
 
 export default CommentFields

package/src/graphql/resolvers/permissions.ts

@@ -34,7 +34,7 @@ const publishPermissions = async (datasetId) => {
     userPermissions: await Promise.all(
       userPermissions.map(async (userPermission) => ({
         ...userPermission,
-        user: await user(ds, { id: userPermission.userId }, null),
+        user: await user(ds, { id: userPermission.userId }),
       })),
     ),
   }

package/src/graphql/resolvers/user.ts

@@ -7,7 +7,11 @@ function isValidOrcid(orcid: string): boolean {
   return /^[0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9]{3}[0-9X]$/.test(orcid || "")
 }
 
-export async function user(obj, { id }, { userInfo }) {
+export async function user(
+  obj,
+  { id },
+  { userInfo }: { userInfo?: Record<string, unknown> } = {},
+) {
   let user
   if (isValidOrcid(id)) {
     user = await User.findOne({

package/src/graphql/resolvers/validation.ts

@@ -2,41 +2,53 @@ import config from "../../config"
 import { generateDataladCookie } from "../../libs/authentication/jwt"
 import { getDatasetWorker } from "../../libs/datalad-service"
 import Validation from "../../models/validation"
-import { redlock } from "../../libs/redis"
+import { redis, redlock } from "../../libs/redis"
+import CacheItem from "../../cache/item"
+import { CacheType } from "../../cache/types"
 
 /**
  * Issues resolver for schema validator
  */
 export const validation = async (dataset, _, { userInfo }) => {
-  return Validation.findOne({
-    id: dataset.revision,
-    datasetId: dataset.id,
-  })
-    .exec()
-    .then((data) => {
-      if (!data && userInfo) {
-        // If no results were found, acquire a lock and run validation
-        revalidate(
-          null,
-          { datasetId: dataset.id, ref: dataset.revision },
-          { userInfo },
-        )
-      }
-      if (data) {
-        // Return with errors and warning counts appended
-        return {
-          ...data.toObject(),
-          errors: data.issues.filter((issue) =>
-            issue.severity === "error"
-          ).length,
-          warnings: data.issues.filter((issue) =>
-            issue.severity === "warning"
-          ).length,
-        }
-      } else {
-        return null
-      }
+  const cache = new CacheItem(
+    redis,
+    CacheType.validation,
+    [dataset.id, dataset.revision],
+    // This cache is valid forever but may be large, drop inaccessed values weekly
+    604800,
+  )
+  return cache.get((doNotCache) => {
+    return Validation.findOne({
+      id: dataset.revision,
+      datasetId: dataset.id,
     })
+      .exec()
+      .then((data) => {
+        if (!data && userInfo) {
+          // If no results were found, acquire a lock and run validation
+          revalidate(
+            null,
+            { datasetId: dataset.id, ref: dataset.revision },
+            { userInfo },
+          )
+        }
+        if (data) {
+          // Return with errors and warning counts appended
+          return {
+            ...data.toObject(),
+            errors: data.issues.filter((issue) =>
+              issue.severity === "error"
+            ).length,
+            warnings: data.issues.filter((issue) =>
+              issue.severity === "warning"
+            ).length,
+          }
+        } else {
+          doNotCache(true)
+          return null
+        }
+      })
+  })
 }
 
 /**

package/src/libs/authentication/orcid.ts

@@ -4,9 +4,11 @@ import * as Sentry from "@sentry/node"
 import { userMigration } from "./user-migration"
 import User from "../../models/user"
 
-export const requestAuth = passport.authenticate("orcid", {
-  session: false,
-})
+export const requestAuth = (req, res, next) =>
+  passport.authenticate("orcid", {
+    session: false,
+    state: req.query.redirectPath || null,
+  })(req, res, next)
 
 /**
  * Complete a successful login
@@ -30,41 +32,44 @@ export function completeRequestLogin(req, res, next, user) {
 }
 
 export const authCallback = (req, res, next) =>
-  passport.authenticate("orcid", async (err, user) => {
-    if (err) {
-      Sentry.captureException(err)
-      if (err.type) {
-        return res.redirect(`/error/orcid/${err.type}`)
-      } else {
-        return res.redirect("/error/orcid/unknown")
+  passport.authenticate(
+    "orcid",
+    async (err, user) => {
+      if (err) {
+        Sentry.captureException(err)
+        if (err.type) {
+          return res.redirect(`/error/orcid/${err.type}`)
+        } else {
+          return res.redirect("/error/orcid/unknown")
+        }
+      }
+      if (!user) {
+        return res.redirect("/")
       }
-    }
-    if (!user) {
-      return res.redirect("/")
-    }
 
-    try {
-      // adds new date for login/lastSeen
-      await User.findByIdAndUpdate(user._id, { lastSeen: new Date() })
-    } catch (error: unknown) {
-      if (error instanceof Error) {
-        Sentry.captureException(error)
-      } else {
-        Sentry.captureException(new Error(String(error)))
+      try {
+        // adds new date for login/lastSeen
+        await User.findByIdAndUpdate(user._id, { lastSeen: new Date() })
+      } catch (error: unknown) {
+        if (error instanceof Error) {
+          Sentry.captureException(error)
+        } else {
+          Sentry.captureException(new Error(String(error)))
+        }
+        // Don't block the login flow
       }
-      // Don't block the login flow
-    }
 
-    // Google user
-    const existingAuth = parsedJwtFromRequest(req)
-    if (existingAuth) {
-      // Migrate Google to ORCID
-      if (existingAuth.provider === "google") {
+      // Google user
+      const existingAuth = parsedJwtFromRequest(req)
+      if (
+        existingAuth && existingAuth.provider === "google" &&
+        existingAuth.exp * 1000 > Date.now()
+      ) {
         return userMigration(user.providerId, existingAuth.sub).then(() => {
           return completeRequestLogin(req, res, next, user)
         })
+      } else {
+        return completeRequestLogin(req, res, next, user)
       }
-    } else {
-      return completeRequestLogin(req, res, next, user)
-    }
-  })(req, res, next)
+    },
+  )(req, res, next)

package/src/libs/authentication/passport.ts

@@ -95,9 +95,8 @@ export const verifyORCIDUser = (
   params,
   done,
 ) => {
-  const token = `${profile.orcid}:${profile.access_token}`
   orcid
-    .getProfile(token)
+    .getProfile(profile.orcid, profile.access_token)
     .then((info) => {
       profile.info = info
       profile.provider = PROVIDERS.ORCID
@@ -166,7 +165,7 @@ export const setupPassportAuth = () => {
           config.auth.orcid.apiURI.includes("sandbox"),
         clientID: config.auth.orcid.clientID,
         clientSecret: config.auth.orcid.clientSecret,
-        scope: "/activities/update",
+        scope: ["/activities/update", "/read-limited"],
         callbackURL: `${config.url + config.apiPrefix}auth/orcid/callback`,
       },
       verifyORCIDUser,

package/src/libs/authentication/user-migration.ts

@@ -5,6 +5,7 @@ import Dataset from "../../models/dataset"
 import Permission from "../../models/permission"
 import Comment from "../../models/comment"
 import Deletion from "../../models/deletion"
+import { queueIndexDataset } from "../../queues/producer-methods"
 import * as Sentry from "@sentry/node"
 
 /**
@@ -20,6 +21,7 @@ import * as Sentry from "@sentry/node"
 export async function userMigration(orcid: string, userId: string) {
   const session = await mongoose.startSession()
   try {
+    const updateDatasets: Record<string, boolean> = {}
     await session.withTransaction(async () => {
       try {
         // Load both original records
@@ -55,6 +57,7 @@ export async function userMigration(orcid: string, userId: string) {
           // Record this dataset uploader as migrated
           migration.datasets.push(dataset.id)
           await dataset.save({ session })
+          updateDatasets[dataset.id] = true
         }
 
         // Migrate dataset permissions
@@ -70,6 +73,7 @@ export async function userMigration(orcid: string, userId: string) {
           // Record this permission as migrated
           migration.permissions.push(permission.toObject())
           await permission.save({ session })
+          updateDatasets[permission.datasetId] = true
         }
 
         // Migrate dataset deletions
@@ -110,6 +114,10 @@ export async function userMigration(orcid: string, userId: string) {
         // Save success
         migration.success = true
         await migration.save({ session })
+        // Request reindexing
+        for (const updateDataset of Object.keys(updateDatasets)) {
+          queueIndexDataset(updateDataset)
+        }
       } catch (err) {
         Sentry.captureException(err)
         throw err

package/src/libs/orcid.ts

@@ -1,110 +1,120 @@
 // Camel case rule is disabled since ORCID API uses snake case variables
-import request from "request"
 import xmldoc from "xmldoc"
 import config from "../config"
 import * as Sentry from "@sentry/node"
 
 export default {
-  getProfile(token) {
-    return new Promise((resolve, reject) => {
-      const data = token.split(":")
-      if (data.length != 2) {
-        reject("Invalid token")
-      }
-      const orcid = data[0]
-      const accessToken = data[1]
-
-      request.get(
+  async getProfile(orcid, accessToken) {
+    try {
+      const response = await fetch(
         `${config.auth.orcid.apiURI}/v2.0/${orcid}/record`,
         {
-          headers: { Authorization: `Bearer ${accessToken}` },
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+          },
         },
-        (err, res) => {
-          if (err) {
-            Sentry.captureException(err)
-            reject({
-              message:
-                "An unexpected ORCID login failure occurred, please try again later.",
-            })
-          }
-          const doc = new xmldoc.XmlDocument(res.body)
-          let name = doc.valueWithPath(
-            "person:person.person:name.personal-details:credit-name",
-          )
-          const firstname = doc.valueWithPath(
-            "person:person.person:name.personal-details:given-names",
-          )
-          const lastname = doc.valueWithPath(
-            "person:person.person:name.personal-details:family-name",
-          )
-          const email = doc.valueWithPath(
-            "person:person.email:emails.email:email.email:email",
-          )
+      )
+      const text = await response.text()
+      const doc = new xmldoc.XmlDocument(text)
+      let name = doc.valueWithPath(
+        "person:person.person:name.personal-details:credit-name",
+      )
+      const firstname = doc.valueWithPath(
+        "person:person.person:name.personal-details:given-names",
+      )
+      const lastname = doc.valueWithPath(
+        "person:person.person:name.personal-details:family-name",
+      )
+      const email = doc.valueWithPath(
+        "person:person.email:emails.email:email.email:email",
+      )
 
-          if (!name && firstname && lastname) {
-            if (firstname && lastname) {
-              name = `${firstname} ${lastname}`
-            } else {
-              name = lastname || firstname
-            }
-          }
+      if (!name && firstname && lastname) {
+        if (firstname && lastname) {
+          name = `${firstname} ${lastname}`
+        } else {
+          name = lastname || firstname
+        }
+      }
 
-          resolve({
-            name,
-            email,
-          })
+      return {
+        name,
+        email,
+      }
+    } catch (err) {
+      Sentry.captureException(err, {
+        extra: {
+          orcid,
         },
-      )
-    })
+      })
+    }
   },
 
-  refreshToken(refreshToken, callback) {
-    request.post(
-      `${config.auth.orcid.URI}/oauth/token`,
-      {
-        form: {
-          client_id: config.auth.orcid.clientID,
-          client_secret: config.auth.orcid.clientSecret,
-          redirect_uri: config.auth.orcid.redirectURI,
-          grant_type: "refresh_token",
-          refresh_token: refreshToken,
+  async refreshToken(refreshToken, callback) {
+    try {
+      const form = new URLSearchParams({
+        client_id: config.auth.orcid.clientID,
+        client_secret: config.auth.orcid.clientSecret,
+        redirect_uri: config.auth.orcid.redirectURI,
+        grant_type: "refresh_token",
+        refresh_token: refreshToken,
+      })
+      const res = await fetch(`${config.auth.orcid.URI}/oauth/token`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+          Accept: "application/json",
         },
-        json: true,
-      },
-      (err, res) => {
-        if (!err) {
-          const { orcid, access_token } = res.body
-          res.body.access_token = `${orcid}:${access_token}`
-        }
-        callback(err, res.body)
-      },
-    )
+        body: form,
+      })
+      const body = await res.json()
+      if (!res.ok) {
+        callback(
+          new Error(body.error_description || `ORCID API error: ${res.status}`),
+          body,
+        )
+      } else {
+        const { orcid, access_token } = body
+        body.access_token = `${orcid}:${access_token}`
+        callback(null, body)
+      }
+    } catch (err) {
+      Sentry.captureException(err)
+      callback(err, null)
+    }
   },
 
-  validateToken(code, callback) {
-    request.post(
-      `${config.auth.orcid.URI}/oauth/token`,
-      {
-        form: {
-          client_id: config.auth.orcid.clientID,
-          client_secret: config.auth.orcid.clientSecret,
-          redirect_uri: config.auth.orcid.redirectURI,
-          grant_type: "authorization_code",
-          code,
+  async validateToken(code, callback) {
+    try {
+      const form = new URLSearchParams({
+        client_id: config.auth.orcid.clientID,
+        client_secret: config.auth.orcid.clientSecret,
+        redirect_uri: config.auth.orcid.redirectURI,
+        grant_type: "authorization_code",
+        code,
+      })
+      const res = await fetch(`${config.auth.orcid.URI}/oauth/token`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+          Accept: "application/json",
         },
-        json: true,
-      },
-      (err, res) => {
-        if (!err) {
-          const { orcid, access_token } = res.body
-          res.body.access_token = `${orcid}:${access_token}`
-
-          // Check if user has email
-          this.getProfile(res.body.access_token)
-        } else {
-          callback(err, res.body)
-        }
-      },
-    )
+        body: form,
+      })
+      const body = await res.json()
+      if (!res.ok) {
+        callback(
+          new Error(body.error_description || `ORCID API error: ${res.status}`),
+          body,
+        )
+      } else {
+        const { orcid, access_token } = body
+        body.access_token = `${orcid}:${access_token}`
+        callback(null, body)
+      }
+    } catch (err) {
+      Sentry.captureException(err)
+      callback(err, null)
+    }
   },
 }

package/src/libs/redis.ts

@@ -3,28 +3,7 @@
 // dependencies --------------------------------------------------
 import Redis from "ioredis"
 import Redlock from "redlock"
+import config from "../config"
 
-let redis = null
-let redlock = null
-
-const connect = async (config) => {
-  return new Promise((resolve) => {
-    if (!redis) {
-      console.log(
-        'Connecting to Redis "redis://%s:%d/0"',
-        config.host,
-        config.port,
-      )
-      redis = new Redis(config)
-      redlock = new Redlock([redis])
-      redis.on("connect", () => {
-        resolve(redis)
-      })
-    } else {
-      resolve(redis)
-    }
-  })
-}
-
-export default { connect }
-export { connect, redis, redlock }
+export const redis = new Redis(config.redis)
+export const redlock = new Redlock([redis])

package/src/queues/consumer.ts

@@ -0,0 +1,18 @@
+import type { Consumer } from "redis-smq"
+import { reindexDataset } from "../elasticsearch/reindex-dataset"
+import { OpenNeuroQueues } from "./queues"
+import * as Sentry from "@sentry/node"
+
+export function startConsumer(consumer: Consumer) {
+  const reindexMessageHandler = async (msg, cb) => {
+    // Index one dataset
+    reindexDataset(msg.body.datasetId).then(cb)
+  }
+
+  consumer.consume(OpenNeuroQueues.INDEXING, reindexMessageHandler, (err) => {
+    if (err) {
+      Sentry.captureException(err)
+    }
+  })
+  return consumer
+}

package/src/queues/producer-methods.ts

@@ -0,0 +1,18 @@
+import { ProducibleMessage } from "redis-smq"
+import { producer } from "./setup"
+import { OpenNeuroQueues } from "./queues"
+import * as Sentry from "@sentry/node"
+
+/**
+ * Queue search indexing for a dataset
+ * @param datasetId Dataset to index
+ */
+export function queueIndexDataset(datasetId: string) {
+  const msg = new ProducibleMessage()
+  msg.setQueue(OpenNeuroQueues.INDEXING).setBody({ datasetId })
+  producer.produce(msg, (err) => {
+    if (err) {
+      Sentry.captureException(err)
+    }
+  })
+}

package/src/queues/queues.ts

@@ -0,0 +1,34 @@
+import { Queue } from "redis-smq"
+import { EQueueDeliveryModel, EQueueType, QueueRateLimit } from "redis-smq"
+import * as Sentry from "@sentry/node"
+
+export enum OpenNeuroQueues {
+  INDEXING = "elasticsearch_indexing",
+}
+
+export function setupQueues() {
+  const indexingQueue = new Queue()
+  indexingQueue.save(
+    OpenNeuroQueues.INDEXING,
+    EQueueType.FIFO_QUEUE,
+    EQueueDeliveryModel.POINT_TO_POINT,
+    (err) => {
+      // The queue may already exist, don't log that error
+      if (err !== "QueueQueueExistsError") {
+        Sentry.captureException(err)
+      }
+    },
+  )
+
+  // Limit indexing queue to 8 runs per minute to avoid stacking indexing excessively
+  const queueRateLimit = new QueueRateLimit()
+  queueRateLimit.set(
+    OpenNeuroQueues.INDEXING,
+    { limit: 8, interval: 60000 },
+    (err) => {
+      if (err) {
+        Sentry.captureException(err)
+      }
+    },
+  )
+}

package/src/queues/setup.ts

@@ -0,0 +1,29 @@
+import { Configuration, Consumer, Producer } from "redis-smq"
+import type { IRedisSMQConfig } from "redis-smq"
+import { ERedisConfigClient } from "redis-smq-common"
+import { startConsumer } from "./consumer"
+import { setupQueues } from "./queues"
+import config from "../config"
+
+const smqConfig: IRedisSMQConfig = {
+  redis: {
+    // Using ioredis as the Redis client
+    client: ERedisConfigClient.IOREDIS,
+    // Add any other ioredis options here
+    options: {
+      host: config.redis.host,
+      port: config.redis.port,
+    },
+  },
+}
+
+Configuration.getSetConfig(smqConfig)
+
+// Producer starts automatically
+export const producer = new Producer()
+export const consumer = new Consumer()
+
+export function initQueues() {
+  setupQueues()
+  startConsumer(consumer)
+}

package/src/server.ts

@@ -1,26 +1,19 @@
 import "./sentry"
+import "./libs/redis"
 import config from "./config"
 import { createServer } from "http"
 import mongoose from "mongoose"
-import { connect as redisConnect } from "./libs/redis"
 import { expressApolloSetup } from "./app"
-
-const redisConnectionSetup = async () => {
-  try {
-    await redisConnect(config.redis)
-  } catch (err) {
-    // eslint-disable-next-line no-console
-    console.error(err)
-    process.exit(1)
-  }
-}
+import { initQueues } from "./queues/setup"
 
 void mongoose.connect(config.mongo.url, {
   dbName: config.mongo.dbName,
   connectTimeoutMS: config.mongo.connectTimeoutMS,
 })
 
-void redisConnectionSetup().then(async () => {
+async function init() {
+  // Start redis message queues
+  initQueues()
   const app = await expressApolloSetup()
   const server = createServer(app)
   server.listen(config.port, () => {
@@ -29,4 +22,6 @@ void redisConnectionSetup().then(async () => {
     // Setup GraphQL subscription transport
     //subscriptionServerFactory(server)
   })
-})
+}
+
+init()