@openneuro/server 4.21.0 → 4.21.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.21.0",
+  "version": "4.21.2",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -21,7 +21,7 @@
     "@elastic/elasticsearch": "7.15.0",
     "@graphql-tools/schema": "^10.0.0",
     "@keyv/redis": "^2.7.0",
-    "@openneuro/search": "^4.21.0",
+    "@openneuro/search": "^4.21.2",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@sentry/node": "^4.5.3",
     "base64url": "^3.0.0",
@@ -30,7 +30,7 @@
     "date-fns": "^2.16.1",
     "draft-js": "^0.11.7",
     "draft-js-export-html": "^1.4.1",
-    "elastic-apm-node": "3.49.1",
+    "elastic-apm-node": "^4.3.0",
     "express": "4.18.2",
     "graphql": "16.8.1",
     "graphql-bigint": "^1.0.0",
@@ -85,5 +85,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "81b67df1ed76e8072f2b44897d004eb67ac6b1ae"
+  "gitHead": "5607a295f9130a0e51bffdaf8d3d6235c8eab8f8"
 }

package/src/config.ts

@@ -63,6 +63,8 @@ const config = {
   },
   elasticsearch: {
     connection: process.env.ELASTICSEARCH_CONNECTION,
+    apmServerUrl: process.env.ELASTIC_APM_SERVER_URL,
+    apmApiKey: process.env.ELASTIC_APM_API_KEY,
   },
   github: {
     token: process.env.DATALAD_GITHUB_TOKEN,

package/src/datalad/files.ts

@@ -1,4 +1,3 @@
-import request from "superagent"
 import { redis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDatasetWorker } from "../libs/datalad-service"
@@ -86,36 +85,38 @@ export const computeTotalSize = (files: [DatasetFile]): number =>
  * @param {string} datasetId - Dataset accession number
  * @param {string} treeish - Git treeish hexsha
  */
-export const getFiles = (datasetId, treeish): Promise<[DatasetFile]> => {
+export const getFiles = (datasetId, treeish): Promise<[DatasetFile?]> => {
   const cache = new CacheItem(redis, CacheType.commitFiles, [
     datasetId,
     treeish.substring(0, 7),
   ])
   return cache.get(
-    (doNotCache) =>
-      request
-        .get(
-          `${
-            getDatasetWorker(
-              datasetId,
-            )
-          }/datasets/${datasetId}/tree/${treeish}`,
-        )
-        .set("Accept", "application/json")
-        .then((response) => {
-          if (response.status === 200) {
-            const {
-              body: { files },
-            } = response
-            for (const f of files) {
-              // Skip caching this tree if it doesn't contain S3 URLs - likely still exporting
-              if (!f.directory && !f.urls[0].includes("s3.amazonaws.com")) {
-                doNotCache(true)
-                break
-              }
-            }
-            return files as [DatasetFile]
+    async (doNotCache): Promise<[DatasetFile?]> => {
+      const response = await fetch(
+        `http://${
+          getDatasetWorker(
+            datasetId,
+          )
+        }/datasets/${datasetId}/tree/${treeish}`,
+        {
+          signal: AbortSignal.timeout(10000),
+        },
+      )
+      const body = await response.json()
+      const files = body?.files
+      if (files) {
+        for (const f of files) {
+          // Skip caching this tree if it doesn't contain S3 URLs - likely still exporting
+          if (!f.directory && !f.urls[0].includes("s3.amazonaws.com")) {
+            doNotCache(true)
+            break
           }
-        }) as Promise<[DatasetFile]>,
+        }
+        return files
+      } else {
+        // Possible to have zero files here, return an empty array
+        return []
+      }
+    },
   )
 }

package/src/handlers/datalad.ts

@@ -24,11 +24,30 @@ export const getFile = async (req, res) => {
   let tree = snapshotId || "HEAD"
   let file
   for (const level of pathComponents) {
-    const files = await getFiles(datasetId, tree)
-    if (level == pathComponents.slice(-1)) {
-      file = files.find((f) => !f.directory && f.filename === level)
-    } else {
-      tree = files.find((f) => f.directory && f.filename === level).id
+    try {
+      const files = await getFiles(datasetId, tree)
+      if (level == pathComponents.slice(-1)) {
+        file = files.find((f) => !f.directory && f.filename === level)
+      } else {
+        // This tree may exist but have no children
+        if (files) {
+          tree = files.find((f) => f.directory && f.filename === level).id
+        }
+      }
+    } catch (err) {
+      // ConnectTimeoutError is Node/Undici and TimeoutError is the standard DOMException name
+      if (
+        err?.cause?.name === "ConnectTimeoutError" ||
+        err?.name === "TimeoutError"
+      ) {
+        // Unreachable backend, forward this error
+        // Usually this is the service restarting due to node migrations or upgrades
+        res.status(503).send("Worker could not be reached")
+        return
+      } else {
+        // Unknown error should bubble up
+        throw err
+      }
     }
   }
   // Get the file URL and redirect if external or serve if local
@@ -44,12 +63,13 @@ export const getFile = async (req, res) => {
       .then((r) => {
         // Set the content length (allow clients to catch HTTP issues better)
         res.setHeader("Content-Length", Number(r.headers.get("content-length")))
-        return r.body
+        if (r.status === 404) {
+          res.status(404).send("Requested dataset or file cannot be found")
+        } else {
+          // @ts-expect-error
+          Readable.fromWeb(r.body, { highWaterMark: 4194304 }).pipe(res)
+        }
       })
-      .then((stream) =>
-        // @ts-expect-error
-        Readable.fromWeb(stream, { highWaterMark: 4194304 }).pipe(res)
-      )
       .catch((err) => {
         console.error(err)
         res.status(500).send("Internal error transferring requested file")

package/src/server.ts

@@ -1,14 +1,17 @@
-/** Needs to run before the other imports in Node */
 import apm from "elastic-apm-node"
-apm.start({
-  serviceName: "openneuro-server",
-  cloudProvider: "none",
-})
-
+import config from "./config"
+/** Needs to run before the other imports in Node */
+if (config.elasticsearch.apmServerUrl) {
+  apm.start({
+    serverUrl: config.elasticsearch.apmServerUrl,
+    apiKey: config.elasticsearch.apmApiKey,
+    serviceName: "openneuro-server",
+    cloudProvider: "none",
+  })
+}
 import { createServer } from "http"
 import mongoose from "mongoose"
 import { connect as redisConnect } from "./libs/redis"
-import config from "./config"
 import { expressApolloSetup } from "./app"
 
 const redisConnectionSetup = async () => {