@openneuro/server 4.41.1 → 4.43.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.41.1",
+  "version": "4.43.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -21,7 +21,7 @@
     "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/schema": "^10.0.0",
     "@keyv/redis": "^4.5.0",
-    "@openneuro/search": "^4.41.1",
+    "@openneuro/search": "^4.43.0",
     "@sentry/node": "^8.25.0",
     "@sentry/profiling-node": "^8.25.0",
     "base64url": "^3.0.0",
@@ -89,5 +89,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "11f72adfa4edde68dc840b22522a5742af88d5ea"
+  "gitHead": "4744c0f70be2518a5660efdd54fa874f79cbe57f"
 }

package/src/datalad/draft.ts

@@ -7,17 +7,30 @@ import { getDatasetWorker } from "../libs/datalad-service"
 import CacheItem, { CacheType } from "../cache/item"
 import { redis } from "../libs/redis"
 
-export const getDraftRevision = async (datasetId) => {
+// Draft info resolver
+type DraftInfo = {
+  ref: string
+  hexsha: string // Duplicate of ref for backwards compatibility
+  tree: string
+  message: string
+  modified: Date
+}
+
+export const getDraftRevision = async (datasetId): Promise<string> => {
+  const { hexsha } = await getDraftInfo(datasetId)
+  return hexsha
+}
+
+export const getDraftInfo = async (datasetId) => {
   const cache = new CacheItem(redis, CacheType.draftRevision, [datasetId], 10)
-  return cache.get(async (_doNotCache): Promise<string> => {
+  return cache.get(async (_doNotCache): Promise<DraftInfo> => {
     const draftUrl = `http://${
       getDatasetWorker(
         datasetId,
       )
     }/datasets/${datasetId}/draft`
     const response = await fetch(draftUrl)
-    const { hexsha } = await response.json()
-    return hexsha
+    return await response.json()
   })
 }
 

package/src/datalad/files.ts

@@ -89,7 +89,7 @@ export const getFiles = (datasetId, treeish): Promise<[DatasetFile?]> => {
   const cache = new CacheItem(redis, CacheType.commitFiles, [
     datasetId,
     treeish.substring(0, 7),
-  ])
+  ], 432000)
   return cache.get(
     async (doNotCache): Promise<[DatasetFile?]> => {
       const response = await fetch(

package/src/datalad/snapshots.ts

@@ -222,7 +222,12 @@ export const getSnapshot = (
       datasetId,
     )
   }/datasets/${datasetId}/snapshots/${commitRef}`
-  const cache = new CacheItem(redis, CacheType.snapshot, [datasetId, commitRef])
+  const cache = new CacheItem(
+    redis,
+    CacheType.snapshot,
+    [datasetId, commitRef],
+    432000,
+  )
   return cache.get(() =>
     request
       .get(url)
@@ -281,7 +286,7 @@ export const downloadFiles = (datasetId, tag) => {
   const downloadCache = new CacheItem(redis, CacheType.snapshotDownload, [
     datasetId,
     tag,
-  ])
+  ], 432000)
   // Return an existing cache object if we have one
   return downloadCache.get(async () => {
     // If not, fetch all trees sequentially and cache the result (hopefully some or all trees are cached)

package/src/graphql/resolvers/fileCheck.ts

@@ -1,5 +1,9 @@
+import config from "../../config"
+import { generateDataladCookie } from "../../libs/authentication/jwt"
+import { getDatasetWorker } from "../../libs/datalad-service"
+import { redlock } from "../../libs/redis"
 import FileCheck from "../../models/fileCheck"
-import { checkDatasetAdmin } from "../permissions"
+import { checkDatasetAdmin, checkDatasetWrite } from "../permissions"
 
 export const updateFileCheck = async (
   obj,
@@ -15,3 +19,41 @@ export const updateFileCheck = async (
     .lean()
     .exec()
 }
+
+export const fsckUrl = (datasetId) => {
+  return `http://${
+    getDatasetWorker(
+      datasetId,
+    )
+  }/datasets/${datasetId}/fsck`
+}
+
+export const fsckDataset = async (_, { datasetId }, { user, userInfo }) => {
+  // Anonymous users can't trigger fsck
+  try {
+    await checkDatasetWrite(datasetId, user, userInfo)
+  } catch {
+    return false
+  }
+  try {
+    // Lock for 30 minutes to avoid stacking fsck requests
+    await redlock.lock(`openneuro:fsck-local-lock:${datasetId}`, 1800000)
+    const response = await fetch(fsckUrl(datasetId), {
+      method: "POST",
+      body: JSON.stringify({}),
+      headers: {
+        cookie: generateDataladCookie(config)(userInfo),
+      },
+    })
+    if (response.status === 200) {
+      return true
+    } else {
+      return false
+    }
+  } catch (err) {
+    // Backend unavailable or lock failed
+    if (err) {
+      return false
+    }
+  }
+}

package/src/graphql/resolvers/git.ts

@@ -1,4 +1,4 @@
-import { checkDatasetWrite } from "../permissions.js"
+import { checkDatasetRead, checkDatasetWrite } from "../permissions.js"
 import { generateRepoToken } from "../../libs/authentication/jwt"
 import { getDatasetEndpoint } from "../../libs/datalad-service.js"
 
@@ -16,9 +16,17 @@ export async function prepareRepoAccess(
   { datasetId }: { datasetId: string },
   { user, userInfo }: { user: string; userInfo: Record<string, unknown> },
 ): Promise<{ token: string; endpoint: number }> {
-  await checkDatasetWrite(datasetId, user, userInfo)
-  return {
-    token: generateRepoToken(userInfo, datasetId),
-    endpoint: getDatasetEndpoint(datasetId),
+  try {
+    await checkDatasetWrite(datasetId, user, userInfo)
+    return {
+      token: generateRepoToken(userInfo, datasetId, false),
+      endpoint: getDatasetEndpoint(datasetId),
+    }
+  } catch {
+    await checkDatasetRead(datasetId, user, userInfo)
+    return {
+      token: generateRepoToken(userInfo, datasetId),
+      endpoint: getDatasetEndpoint(datasetId),
+    }
   }
 }

package/src/graphql/resolvers/mutation.ts

@@ -52,7 +52,7 @@ import {
   updateEventStatus,
 } from "./datasetEvents"
 import { createGitEvent } from "./gitEvents"
-import { updateFileCheck } from "./fileCheck"
+import { fsckDataset, updateFileCheck } from "./fileCheck"
 import { updateContributors } from "../../datalad/contributors"
 import { updateWorkerTask } from "./worker"
 
@@ -91,6 +91,7 @@ const Mutation = {
   finishUpload,
   cacheClear,
   revalidate,
+  fsckDataset,
   prepareRepoAccess,
   reexportRemotes,
   resetDraft,

package/src/graphql/resolvers/snapshots.ts

@@ -311,7 +311,7 @@ const Snapshot = {
   analytics: (snapshot) => analytics(snapshot),
   issues: (snapshot) => snapshotIssues(snapshot),
   issuesStatus: (snapshot) => issuesSnapshotStatus(snapshot),
-  validation: (snapshot) => snapshotValidation(snapshot),
+  validation: snapshotValidation,
   contributors: (snapshot) => {
     const datasetId = snapshot.datasetId
     return contributors({

package/src/graphql/resolvers/validation.ts

@@ -14,10 +14,9 @@ export const validation = async (dataset, _, { userInfo }) => {
     redis,
     CacheType.validation,
     [dataset.id, dataset.revision],
-    // This cache is valid forever but may be large, drop inaccessed values weekly
-    604800,
   )
-  return cache.get((doNotCache) => {
+  let validationObject
+  const cacheResult = await cache.get((doNotCache) => {
     return Validation.findOne({
       id: dataset.revision,
       datasetId: dataset.id,
@@ -33,9 +32,11 @@ export const validation = async (dataset, _, { userInfo }) => {
           )
         }
         if (data) {
-          // Return with errors and warning counts appended
+          // Return with errors and warning counts
+          validationObject = data.toObject()
           return {
-            ...data.toObject(),
+            id: validationObject.id,
+            datasetId: validationObject.datasetId,
             errors: data.issues.filter((issue) =>
               issue.severity === "error"
             ).length,
@@ -49,31 +50,41 @@ export const validation = async (dataset, _, { userInfo }) => {
         }
       })
   })
+  if (!cacheResult) return null
+  // Return the cached result along with lazy-loaded issues and codeMessages
+  const result = {
+    ...cacheResult,
+    issues: async () => {
+      if (!validationObject) {
+        validationObject = (await Validation.findOne({
+          id: dataset.revision,
+          datasetId: dataset.id,
+        }, { issues: 1 }).exec()).toObject()
+      }
+      return validationObject.issues
+    },
+    codeMessages: async () => {
+      if (!validationObject) {
+        validationObject = (await Validation.findOne({
+          id: dataset.revision,
+          datasetId: dataset.id,
+        }, { codeMessages: 1 }).exec()).toObject()
+      }
+      return validationObject.codeMessages
+    },
+  }
+  return result
 }
 
 /**
  * Snapshot issues resolver for schema validator
  */
-export const snapshotValidation = async (snapshot) => {
-  const datasetId = snapshot.id.split(":")[0]
-  const validation = await Validation.findOne({
-    id: snapshot.hexsha,
-    datasetId,
-  }).exec()
-  if (validation) {
-    // Return with errors and warning counts appended
-    return {
-      ...validation.toObject(),
-      errors: validation.issues.filter((issue) =>
-        issue.severity === "error"
-      ).length,
-      warnings:
-        validation.issues.filter((issue) => issue.severity === "warning")
-          .length,
-    }
-  } else {
-    return null
+export const snapshotValidation = async (snapshot, _, context) => {
+  const dataset = {
+    id: snapshot.id.split(":")[0],
+    revision: snapshot.hexsha,
   }
+  return validation(dataset, _, context)
 }
 
 export function validationSeveritySort(a, b) {

package/src/graphql/schema.ts

@@ -178,6 +178,8 @@ export const typeDefs = `
     finishUpload(uploadId: ID!): Boolean
     # Drop cached data for a dataset - requires site admin access
     cacheClear(datasetId: ID!): Boolean
+    # Rerun fsck on a dataset if 30 minutes have passed since last request
+    fsckDataset(datasetId: ID!): Boolean
     # Rerun the latest validator on a given commit
     revalidate(datasetId: ID!, ref: String!): Boolean
     # Request a temporary token for git access

package/src/libs/authentication/jwt.ts

@@ -31,6 +31,11 @@ export const buildToken = (
     name: user.name,
     admin: user.admin,
   }
+  // Give anonymous reviewers a generic name/email for git operations
+  if (user.reviewer) {
+    fields.name = "Anonymous Reviewer"
+    fields.email = "reviewer@openneuro.org"
+  }
   // Allow extensions of the base token format
   if (options) {
     if (options && "scopes" in options) {
@@ -98,12 +103,16 @@ export function generateReviewerToken(
 export function generateRepoToken(
   user,
   datasetId,
+  readonly = true,
   expiresIn = 7 * 60 * 60 * 24,
 ) {
   const options = {
-    scopes: ["dataset:git"],
+    scopes: ["dataset:git:read"],
     dataset: datasetId,
   }
+  if (!readonly) {
+    options.scopes.push("dataset:git:write")
+  }
   return buildToken(config, user, expiresIn, options)
 }
 

package/src/sentry.ts

@@ -21,4 +21,8 @@ Sentry.init({
     }
     return event
   },
+  ignoreErrors: [
+    "You do not have access to read this dataset.",
+    "You do not have access to modify this dataset.",
+  ],
 })