@openneuro/server 4.23.0 → 4.24.0-alpha.2

package/Dockerfile

@@ -4,7 +4,7 @@ FROM openneuro/node AS build
 WORKDIR /srv/packages/openneuro-server
 RUN yarn build
 
-FROM node:18.17.1-alpine
+FROM node:20.12.2-alpine
 
 WORKDIR /srv
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.23.0",
+  "version": "4.24.0-alpha.2",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -18,10 +18,10 @@
     "@apollo/client": "3.7.2",
     "@apollo/server": "4.9.3",
     "@apollo/utils.keyvadapter": "3.0.0",
-    "@elastic/elasticsearch": "7.15.0",
+    "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/schema": "^10.0.0",
     "@keyv/redis": "^2.7.0",
-    "@openneuro/search": "^4.23.0",
+    "@openneuro/search": "^4.24.0-alpha.2",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@sentry/node": "^4.5.3",
     "base64url": "^3.0.0",
@@ -30,8 +30,8 @@
     "date-fns": "^2.16.1",
     "draft-js": "^0.11.7",
     "draft-js-export-html": "^1.4.1",
-    "elastic-apm-node": "^4.3.0",
-    "express": "4.18.2",
+    "elastic-apm-node": "4.5.4",
+    "express": "4.19.2",
     "graphql": "16.8.1",
     "graphql-bigint": "^1.0.0",
     "graphql-compose": "9.0.10",
@@ -39,11 +39,12 @@
     "graphql-tools": "9.0.0",
     "immutable": "^3.8.2",
     "ioredis": "4.17.3",
-    "jsdom": "^11.6.2",
+    "jsdom": "24.0.0",
     "jsonwebtoken": "^9.0.0",
     "keyv": "^4.5.3",
     "mime-types": "^2.1.19",
-    "mongoose": "^6.11.3",
+    "mongodb-memory-server": "^9.2.0",
+    "mongoose": "6.12.8",
     "morgan": "^1.6.1",
     "node-mailjet": "^3.3.5",
     "object-hash": "2.1.1",
@@ -76,14 +77,14 @@
     "@types/semver": "^5",
     "core-js": "^3.10.1",
     "ioredis-mock": "^8.8.1",
-    "nodemon": "^2.0.7",
+    "nodemon": "3.1.0",
     "ts-node-dev": "1.1.6",
     "tsc-watch": "^4.2.9",
-    "vitest": "0.34.4",
+    "vitest": "1.5.0",
     "vitest-fetch-mock": "0.2.2"
   },
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "e86df78635d7ac6cfa03a2add5760b8539103583"
+  "gitHead": "aa9717595d41e5e0eb55328c62fa59c80847b95e"
 }

package/src/cache/types.ts

@@ -12,4 +12,5 @@ export enum CacheType {
   snapshotIndex = "snapshotIndex",
   participantCount = "participantCount",
   snapshotDownload = "download",
+  draftRevision = "revision",
 }

package/src/datalad/__tests__/dataset.spec.ts

@@ -3,6 +3,7 @@ import request from "superagent"
 import { createDataset, datasetsFilter, testBlacklist } from "../dataset"
 import { getDatasetWorker } from "../../libs/datalad-service"
 import { connect } from "mongoose"
+import { MongoMemoryServer } from "mongodb-memory-server"
 
 // Mock requests to Datalad service
 vi.mock("superagent")
@@ -13,9 +14,11 @@ vi.mock("../../libs/notifications")
 
 describe("dataset model operations", () => {
   describe("createDataset()", () => {
-    beforeAll(() => {
+    let mongod
+    beforeAll(async () => {
       // Setup MongoDB with mongodb-memory-server
-      connect(globalThis.__MONGO_URI__)
+      mongod = await MongoMemoryServer.create()
+      connect(mongod.getUri())
     })
     it("resolves to dataset id string", async () => {
       const user = { id: "1234" }

package/src/datalad/__tests__/pagination.spec.ts

@@ -1,5 +1,6 @@
 import { vi } from "vitest"
 vi.mock("ioredis")
+import { MongoMemoryServer } from "mongodb-memory-server"
 import * as pagination from "../pagination.js"
 import { connect, Types } from "mongoose"
 import Dataset from "../../models/dataset"
@@ -37,8 +38,11 @@ describe("pagination model operations", () => {
     })
   })
   describe("datasetsConnection()", () => {
+    let mongod
     beforeAll(async () => {
-      await connect(globalThis.__MONGO_URI__)
+      // Setup MongoDB with mongodb-memory-server
+      mongod = await MongoMemoryServer.create()
+      connect(mongod.getUri())
       const ds = new Dataset({
         _id: new ObjectID("5bef51a1ed211400c08e5524"),
         id: "ds001001",

package/src/datalad/__tests__/snapshots.spec.ts

@@ -1,5 +1,6 @@
 import { vi } from "vitest"
 vi.mock("ioredis")
+import { MongoMemoryServer } from "mongodb-memory-server"
 import request from "superagent"
 import { createDataset } from "../dataset"
 import { createSnapshot } from "../snapshots"
@@ -25,10 +26,15 @@ vi.mock("../../libs/notifications.ts")
 
 describe("snapshot model operations", () => {
   describe("createSnapshot()", () => {
+    let mongod
+    beforeAll(async () => {
+      // Setup MongoDB with mongodb-memory-server
+      mongod = await MongoMemoryServer.create()
+      connect(mongod.getUri())
+    })
     it("posts to the DataLad /datasets/{dsId}/snapshots/{snapshot} endpoint", async () => {
       const user = { id: "1234" }
       const tag = "snapshot"
-      await connect(globalThis.__MONGO_URI__)
       const { id: dsId } = await createDataset(user.id, user, {
         affirmedDefaced: true,
         affirmedConsent: true,

package/src/datalad/description.ts

@@ -88,6 +88,12 @@ export const repairDescriptionTypes = (description) => {
     newDescription.HowToAcknowledge =
       JSON.stringify(description.HowToAcknowledge) || ""
   }
+  if (
+    description.hasOwnProperty("DatasetType") &&
+    typeof description.DatasetType !== "string"
+  ) {
+    newDescription.DatasetType = "raw"
+  }
   return newDescription
 }
 

package/src/datalad/draft.ts

@@ -4,16 +4,21 @@
 import request from "superagent"
 import Dataset from "../models/dataset"
 import { getDatasetWorker } from "../libs/datalad-service"
+import CacheItem, { CacheType } from "../cache/item"
+import { redis } from "../libs/redis"
 
 export const getDraftRevision = async (datasetId) => {
-  const draftUrl = `http://${
-    getDatasetWorker(
-      datasetId,
-    )
-  }/datasets/${datasetId}/draft`
-  const response = await fetch(draftUrl)
-  const { hexsha } = await response.json()
-  return hexsha
+  const cache = new CacheItem(redis, CacheType.draftRevision, [datasetId], 10)
+  return cache.get(async (_doNotCache): Promise<string> => {
+    const draftUrl = `http://${
+      getDatasetWorker(
+        datasetId,
+      )
+    }/datasets/${datasetId}/draft`
+    const response = await fetch(draftUrl)
+    const { hexsha } = await response.json()
+    return hexsha
+  })
 }
 
 export const updateDatasetRevision = (datasetId, gitRef) => {

package/src/graphql/__tests__/__snapshots__/permissions.spec.ts.snap

@@ -1,5 +1,5 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`resolver permissions helpers > checkDatasetAdmin() > resolves to false for anonymous users 1`] = `"You do not have admin access to this dataset."`;
+exports[`resolver permissions helpers > checkDatasetAdmin() > resolves to false for anonymous users 1`] = `[Error: You do not have admin access to this dataset.]`;
 
-exports[`resolver permissions helpers > checkDatasetWrite() > resolves to false for anonymous users 1`] = `"You do not have access to modify this dataset."`;
+exports[`resolver permissions helpers > checkDatasetWrite() > resolves to false for anonymous users 1`] = `[Error: You do not have access to modify this dataset.]`;

package/src/graphql/__tests__/comment.spec.ts

@@ -1,6 +1,7 @@
 import { vi } from "vitest"
 import { connect } from "mongoose"
 import { deleteComment, flatten } from "../resolvers/comment"
+import { MongoMemoryServer } from "mongodb-memory-server"
 import Comment from "../../models/comment"
 
 vi.mock("ioredis")
@@ -16,8 +17,11 @@ describe("comment resolver helpers", () => {
       user: "5678",
       userInfo: { admin: false },
     }
+    let mongod
     beforeAll(async () => {
-      await connect(globalThis.__MONGO_URI__)
+      // Setup MongoDB with mongodb-memory-server
+      mongod = await MongoMemoryServer.create()
+      connect(mongod.getUri())
       const comment = new Comment({
         text: "a",
         createDate: new Date().toISOString(),

package/src/graphql/resolvers/__tests__/dataset-search.spec.ts

@@ -28,11 +28,9 @@ describe("dataset search resolvers", () => {
   describe("elasticRelayConnection()", () => {
     it("returns a relay cursor for empty ApiResponse", async () => {
       const emptyApiResponse = {
-        body: {
-          hits: {
-            hits: [],
-            total: { value: 0 },
-          },
+        hits: {
+          hits: [],
+          total: { value: 0 },
         },
       }
 
@@ -60,20 +58,18 @@ describe("dataset search resolvers", () => {
       }
 
       const expectedApiResponse = {
-        body: {
-          hits: {
-            hits: [
-              { _source: { id: "testdataset1" } },
-              { _source: { id: "testdataset2" } },
-              { _source: { id: "testdataset3" }, sort: [1] },
-            ],
-            total: { value: 10 },
-          },
+        hits: {
+          hits: [
+            { _source: { id: "testdataset1" } },
+            { _source: { id: "testdataset2" } },
+            { _source: { id: "testdataset3" }, sort: [1] },
+          ],
+          total: { value: 10 },
         },
       }
 
       const resultsRelayConnection = {
-        edges: expectedApiResponse.body.hits.hits.map((hit) => {
+        edges: expectedApiResponse.hits.hits.map((hit) => {
           // This skips the dataset resolver logic and passes this back
           mockResolvers.dataset.mockReturnValueOnce(hit._source)
           return {

package/src/graphql/resolvers/__tests__/dataset.spec.ts

@@ -1,4 +1,5 @@
 import { vi } from "vitest"
+import { MongoMemoryServer } from "mongodb-memory-server"
 import { connect } from "mongoose"
 import request from "superagent"
 import * as ds from "../dataset"
@@ -9,8 +10,11 @@ vi.mock("../../../config.ts")
 vi.mock("../../../libs/notifications.ts")
 
 describe("dataset resolvers", () => {
-  beforeAll(() => {
-    connect(globalThis.__MONGO_URI__)
+  let mongod
+  beforeAll(async () => {
+    // Setup MongoDB with mongodb-memory-server
+    mongod = await MongoMemoryServer.create()
+    connect(mongod.getUri())
   })
   describe("createDataset()", () => {
     it("createDataset mutation succeeds", async () => {

package/src/graphql/resolvers/dataset-search.ts

@@ -4,6 +4,7 @@ import Star from "../../models/stars"
 import Subscription from "../../models/subscription"
 import Permission from "../../models/permission"
 import { hashObject } from "../../libs/authentication/crypto"
+import util from "util"
 
 const elasticIndex = "datasets"
 
@@ -37,7 +38,7 @@ export const decodeCursor = (cursor) =>
  * @param {import ('@elastic/elasticsearch').ApiResponse} result
  */
 export const elasticRelayConnection = (
-  { body },
+  body,
   id,
   size,
   childResolvers = { dataset },
@@ -102,9 +103,9 @@ export const datasetSearchConnection = async (
     index: elasticIndex,
     size: first,
     q: `${q} AND public:true`,
-    body: requestBody,
+    ...requestBody,
   })
-  return elasticRelayConnection(result, searchId, first)
+  return elasticRelayConnection(requestBody, searchId, first)
 }
 
 export const datasetSearch = {
@@ -215,6 +216,7 @@ export const advancedDatasetSearchConnection = async (
   },
   { user, userInfo },
 ) => {
+  // Create an identity for this search (used to cache connections)
   const searchId = hashObject({
     query,
     datasetType,
@@ -223,26 +225,30 @@ export const advancedDatasetSearchConnection = async (
     user,
   })
   const sort = [{ _score: "desc" }, { id: "desc" }]
-  if (sortBy) sort.unshift(sortBy)
-  const requestBody = {
-    sort,
-    query: allDatasets
-      ? query
-      : await parseQuery(query, datasetType, datasetStatus, user),
-    search_after: undefined,
+  if (sortBy) {
+    sort.unshift(sortBy)
   }
+  // Parse out the decode token and add it to our query if successful
+  let search_after
   if (after) {
     try {
-      requestBody.search_after = decodeCursor(after)
-    } catch (err) {
+      search_after = decodeCursor(after)
+    } catch (_err) {
       // Don't include search_after if parsing fails
     }
   }
-  const result = await elasticClient.search({
+  const requestBody = {
     index: elasticIndex,
     size: first,
-    body: requestBody,
-  })
+    sort,
+    query: allDatasets
+      ? query
+      : await parseQuery(query, datasetType, datasetStatus, user),
+    search_after,
+  }
+  // Run the query
+  const result = await elasticClient.search(requestBody)
+  // Extend with relay connection pagination
   return elasticRelayConnection(
     result,
     searchId,

package/src/graphql/schema.ts

@@ -571,6 +571,8 @@ export const typeDefs = `
     ReferencesAndLinks: [String]
     # The Document Object Identifier of the dataset (not the corresponding paper).
     DatasetDOI: String
+    # The BIDS DatasetType field defined as "raw" or "derivative"
+    DatasetType: String
     # List of ethics committee approvals of the research protocols and/or protocol identifiers.
     EthicsApprovals: [String]
   }

package/src/handlers/datalad.ts

@@ -3,6 +3,7 @@ import { Readable } from "node:stream"
 import mime from "mime-types"
 import { getFiles } from "../datalad/files"
 import { getDatasetWorker } from "../libs/datalad-service"
+import { getDraftRevision } from "../datalad/draft"
 
 /**
  * Handlers for datalad dataset manipulation
@@ -21,7 +22,9 @@ export const getFile = async (req, res) => {
   const worker = getDatasetWorker(datasetId)
   // Find the right tree
   const pathComponents = filename.split(":")
-  let tree = snapshotId || "HEAD"
+  // Get the draft commit for cache busting
+  const draftCommit = await getDraftRevision(datasetId)
+  let tree = snapshotId || draftCommit
   let file
   for (const level of pathComponents) {
     try {

package/src/libs/__tests__/dataset.spec.ts

@@ -1,13 +1,17 @@
 import { vi } from "vitest"
-import { connect } from "mongoose"
+import { MongoMemoryServer } from "mongodb-memory-server"
+import { connect, disconnect } from "mongoose"
 import { getAccessionNumber } from "../dataset"
 
 vi.mock("ioredis")
 
 describe("libs/dataset", () => {
   describe("getAccessionNumber", () => {
-    beforeAll(() => {
-      connect(globalThis.__MONGO_URI__)
+    let mongod
+    beforeAll(async () => {
+      // Setup MongoDB with mongodb-memory-server
+      mongod = await MongoMemoryServer.create()
+      await connect(mongod.getUri())
     })
     it('returns strings starting with "ds"', async () => {
       const ds = await getAccessionNumber()

package/src/libs/doi/__tests__/__snapshots__/doi.spec.ts.snap

@@ -1,17 +1,17 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`DOI minting utils > template() > accepts expected arguments 1`] = `
-"<?xml version=\\"1.0\\" encoding=\\"UTF-8\\"?>
-<resource xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://datacite.org/schema/kernel-4\\" xsi:schemaLocation=\\"http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4/metadata.xsd\\">
-  <identifier identifierType=\\"DOI\\">12345</identifier>
+"<?xml version="1.0" encoding="UTF-8"?>
+<resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://datacite.org/schema/kernel-4" xsi:schemaLocation="http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4/metadata.xsd">
+  <identifier identifierType="DOI">12345</identifier>
   <creators>
   <creator><creatorName>A. User</creatorName></creator><creator><creatorName>B. User</creatorName></creator>
   </creators>
   <titles>
-    <title xml:lang=\\"en-us\\">Test Dataset</title>
+    <title xml:lang="en-us">Test Dataset</title>
   </titles>
   <publisher>Openneuro</publisher>
   <publicationYear>1999</publicationYear>
-  <resourceType resourceTypeGeneral=\\"Dataset\\">fMRI</resourceType>
+  <resourceType resourceTypeGeneral="Dataset">fMRI</resourceType>
 </resource>"
 `;

package/src/libs/email/templates/__tests__/__snapshots__/comment-created.spec.ts.snap

@@ -56,22 +56,22 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 </style>
 </head>
 <body>
-	<div class=\\"top-bar\\">
-		<img src=\\"https://openneuro.org/assets/email-header.1cb8bf76.png\\" />
+	<div class="top-bar">
+		<img src="https://openneuro.org/assets/email-header.1cb8bf76.png" />
 	</div>
-	<div class=\\"content\\">
+	<div class="content">
 		<h2>Hi, J. Doe</h2>
 
 		<p>
 			A new new has been posted on a dataset you follow, <b>Not Real Dataset</b>. 
 		</p>
-		<div class=\\"comment\\">
+		<div class="comment">
 			<p>By: <b>56789</b> on 2063-04-05</p>
 			
 			Test comment, please ignore		
 		</div>
-		<div class=\\"link-div\\">
-			<a class=\\"dataset-link\\" href=\\"https://openneuro.org/datasets/ds1245678#comment-12345\\">Click here to view this comment on OpenNeuro &raquo;</a>
+		<div class="link-div">
+			<a class="dataset-link" href="https://openneuro.org/datasets/ds1245678#comment-12345">Click here to view this comment on OpenNeuro &raquo;</a>
 		</div>
 
 		<p>
@@ -82,7 +82,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 	</div>
 </body>
 <footer>
-	If you would like to stop receiving notifications about this dataset, please <a class='link' href=\\"https://openneuro.org/datasets/ds1245678\\">visit the dataset page</a> and click the 'unfollow' icon.
+	If you would like to stop receiving notifications about this dataset, please <a class='link' href="https://openneuro.org/datasets/ds1245678">visit the dataset page</a> and click the 'unfollow' icon.
 </footer>
 <html>"
 `;

package/src/libs/email/templates/__tests__/__snapshots__/dataset-deleted.spec.ts.snap

@@ -38,10 +38,10 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 </style>
 </head>
 <body>
-	<div class=\\"top-bar\\">
-		<img src=\\"https://openneuro.org/assets/email-header.1cb8bf76.png\\" />
+	<div class="top-bar">
+		<img src="https://openneuro.org/assets/email-header.1cb8bf76.png" />
 	</div>
-	<div class=\\"content\\">
+	<div class="content">
 		<h2>Hi, J. Doe</h2>
 
 		<p>

package/src/libs/email/templates/__tests__/__snapshots__/owner-unsubscribed.spec.ts.snap

@@ -38,10 +38,10 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 </style>
 </head>
 <body>
-	<div class=\\"top-bar\\">
-		<img src=\\"https://openneuro.org/assets/email-header.1cb8bf76.png\\" />
+	<div class="top-bar">
+		<img src="https://openneuro.org/assets/email-header.1cb8bf76.png" />
 	</div>
-	<div class=\\"content\\">
+	<div class="content">
 		<h2>Hi, J. Doe</h2>
 
 		<p>

package/src/libs/email/templates/__tests__/__snapshots__/snapshot-created.spec.ts.snap

@@ -56,10 +56,10 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 </style>
 </head>
 <body>
-	<div class=\\"top-bar\\">
-		<img src=\\"https://openneuro.org/assets/email-header.1cb8bf76.png\\" />
+	<div class="top-bar">
+		<img src="https://openneuro.org/assets/email-header.1cb8bf76.png" />
 	</div>
-	<div class=\\"content\\">
+	<div class="content">
 		<h2>Hi, J. Doe</h2>
 
 		<p>
@@ -67,12 +67,12 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 		</p>
 
 		<div>
-			<a class='dataset-link' href=\\"https://openneuro.org/datasets/ds1245678/versions/1.2.4\\">Click here to view this snapshot on OpenNeuro &raquo;</a>
+			<a class='dataset-link' href="https://openneuro.org/datasets/ds1245678/versions/1.2.4">Click here to view this snapshot on OpenNeuro &raquo;</a>
 		</div>
 
 		<div>
 			<h4>Changelog</h4>
-			<p class=\\"changelog\\">New changes...</p>
+			<p class="changelog">New changes...</p>
 		</div>
 
 		<p>
@@ -82,7 +82,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 	</div>
 </body>
 <footer>
-	If you would like to stop receiving notifications about this dataset, please <a class='link' href=\\"https://openneuro.org/datasets/ds1245678\\">visit the dataset page</a> and click the 'unfollow' icon.
+	If you would like to stop receiving notifications about this dataset, please <a class='link' href="https://openneuro.org/datasets/ds1245678">visit the dataset page</a> and click the 'unfollow' icon.
 </footer>
 <html>"
 `;

package/src/libs/email/templates/__tests__/__snapshots__/snapshot-reminder.spec.ts.snap

@@ -38,10 +38,10 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 </style>
 </head>
 <body>
-	<div class=\\"top-bar\\">
-		<img src=\\"https://openneuro.org/assets/email-header.1cb8bf76.png\\" />
+	<div class="top-bar">
+		<img src="https://openneuro.org/assets/email-header.1cb8bf76.png" />
 	</div>
-	<div class=\\"content\\">
+	<div class="content">
 		<h2>Hi, J. Doe</h2>
 
 		<p>
@@ -53,7 +53,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 			The CRN Team
 		</p>
 
-		<a class=\\"dataset-link\\" href=\\"https://openneuro.org/datasets/ds12345678/snapshot\\">Create a snapshot. &raquo;</a>
+		<a class="dataset-link" href="https://openneuro.org/datasets/ds12345678/snapshot">Create a snapshot. &raquo;</a>
 	</div>
 </body>
 <html>"