@openneuro/server 5.0.0-alpha.0 → 5.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "5.0.0-alpha.0",
+  "version": "5.1.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -22,7 +22,10 @@
     "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/schema": "^10.0.0",
     "@keyv/redis": "^4.5.0",
-    "@openneuro/search": "^5.0.0-alpha.0",
+    "@openneuro/search": "^5.1.0",
+    "@pothos/core": "^4.12.0",
+    "@pothos/plugin-directives": "^4.3.0",
+    "@pothos/plugin-simple-objects": "^4.1.3",
     "@sentry/node": "^10.37.0",
     "@sentry/profiling-node": "^10.37.0",
     "base64url": "^3.0.0",
@@ -34,9 +37,9 @@
     "express": "5",
     "graphql": "16.8.1",
     "graphql-bigint": "^1.0.0",
-    "graphql-compose": "9.0.10",
     "graphql-iso-date": "^3.6.1",
     "graphql-tools": "9.0.0",
+    "graphql-type-json": "^0.3.2",
     "hash-wasm": "^4.12.0",
     "immutable": "^4.3.8",
     "ioredis": "^5.6.1",
@@ -69,7 +72,7 @@
     "ts-node": "10.9.2",
     "typescript": "5.6.3",
     "underscore": "^1.8.3",
-    "uuid": "10.0.0",
+    "uuid": "14.0.0",
     "xmldoc": "^1.1.0"
   },
   "devDependencies": {
@@ -92,5 +95,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "ee5b58a1623626ef79a439918e6c065a1c7d5e2e"
+  "gitHead": "e3444f63f43cb7e9d3ecfac51d2a42cdcc4f4e60"
 }

package/src/app.ts

@@ -22,13 +22,12 @@ import * as jwt from "./libs/authentication/jwt"
 import * as auth from "./libs/authentication/states"
 import { sitemapHandler } from "./handlers/sitemap"
 import { setupPassportAuth } from "./libs/authentication/passport"
-import { redis } from "./libs/redis"
+import { getRedis } from "./libs/redis"
 import { version } from "./lerna.json"
 export { Express } from "express-serve-static-core"
 
 interface OpenNeuroRequestContext {
   user: string
-  isSuperUser: boolean
   userInfo: {
     id: string
     exp: string
@@ -68,7 +67,7 @@ export async function expressApolloSetup() {
     // Always allow introspection - our schema is public
     introspection: true,
     // @ts-expect-error Type mismatch for keyv and ioredis recent releases
-    cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(redis) })),
+    cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(getRedis()) })),
     plugins: [
       ApolloServerPluginLandingPageLocalDefault(),
       ApolloServerPluginDrainHttpServer({
@@ -109,7 +108,6 @@ export async function expressApolloSetup() {
         if (req.isAuthenticated()) {
           return {
             user: req.user.id,
-            isSuperUser: req.user.admin,
             userInfo: req.user,
           }
         }

package/src/cache/__tests__/tree.spec.ts

@@ -22,6 +22,8 @@ function makeEntry(overrides: Partial<TreeEntry> = {}): TreeEntry {
     b: "",
     p: false,
     d: false,
+    a: false,
+    l: false,
     ...overrides,
   }
 }

package/src/cache/tree.ts

@@ -19,6 +19,10 @@ export interface TreeEntry {
   p: boolean
   /** is directory */
   d: boolean
+  /** is annexed */
+  a: boolean
+  /** is a symlink */
+  l: boolean
 }
 
 function treeKey(hash: string): string {

package/src/cache/types.ts

@@ -16,4 +16,5 @@ export enum CacheType {
   brainInitiative = "brainInitiative",
   validation = "validation",
   dataciteYml = "dataciteYml",
+  gitRef = "ref",
 }

package/src/datalad/__tests__/contributors.spec.ts

@@ -27,7 +27,7 @@ vi.mock("../../cache/item")
 vi.mock("../files")
 vi.mock("../../utils/datasetOrSnapshot")
 vi.mock("../libs/redis", () => ({
-  redis: vi.fn(),
+  getRedis: vi.fn(),
 }))
 
 const mockYamlLoad = vi.mocked(yaml.load)

package/src/datalad/__tests__/dataRetentionNotifications.spec.ts

@@ -89,7 +89,7 @@ describe("checkDataRetentionNotifications", () => {
     await Deletion.create({
       datasetId: TEST_DATASET,
       reason: "test deletion",
-      user: { _id: TEST_USER.id },
+      user: { id: TEST_USER.id },
     })
     mockDraft(daysAgo(15))
     mockSnapshots([{ hexsha: "other" }])
@@ -131,7 +131,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("no_snapshot_reminder"),
+        id: expect.stringContaining("no_snapshot_reminder"),
       }),
     )
   })
@@ -145,7 +145,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_14day"),
+        id: expect.stringContaining("retention_14day"),
       }),
     )
   })
@@ -158,7 +158,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_14day"),
+        id: expect.stringContaining("retention_14day"),
       }),
     )
   })
@@ -172,7 +172,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_14day"),
+        id: expect.stringContaining("retention_14day"),
       }),
     )
 
@@ -214,7 +214,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_7day"),
+        id: expect.stringContaining("retention_7day"),
       }),
     )
   })
@@ -235,7 +235,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_deletion"),
+        id: expect.stringContaining("retention_deletion"),
       }),
     )
   })
@@ -274,7 +274,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_14day"),
+        id: expect.stringContaining("retention_14day"),
       }),
     )
 
@@ -295,7 +295,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenLastCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_14day"),
+        id: expect.stringContaining("retention_14day"),
       }),
     )
 
@@ -312,7 +312,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenLastCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_7day"),
+        id: expect.stringContaining("retention_7day"),
       }),
     )
 
@@ -329,7 +329,7 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).toHaveBeenCalledTimes(1)
     expect(notifications.send).toHaveBeenLastCalledWith(
       expect.objectContaining({
-        _id: expect.stringContaining("retention_deletion"),
+        id: expect.stringContaining("retention_deletion"),
       }),
     )
 

package/src/datalad/__tests__/files.spec.ts

@@ -13,7 +13,7 @@ import type { TreeEntry } from "../../cache/tree"
 
 vi.mock("ioredis")
 vi.mock("../../config.ts")
-vi.mock("../../libs/redis", () => ({ redis: {} }))
+vi.mock("../../libs/redis", () => ({ getRedis: () => ({}) }))
 vi.mock("../../libs/presign", () => ({
   getPresignedUrl: vi.fn().mockResolvedValue(
     "https://s3.amazonaws.com/bucket/key?presigned=true",
@@ -74,6 +74,8 @@ describe("datalad files", () => {
         b: "",
         p: false,
         d: true,
+        a: false,
+        l: false,
       }
       const result = await entryToDatasetFile(entry, "ds000001")
       expect(result).toEqual({
@@ -82,6 +84,8 @@ describe("datalad files", () => {
         directory: true,
         size: 0,
         urls: [],
+        annexed: false,
+        symlink: false,
       })
     })
     it("returns a presigned URL for private files", async () => {
@@ -94,6 +98,8 @@ describe("datalad files", () => {
         b: "private-bucket",
         p: true,
         d: false,
+        a: false,
+        l: false,
       }
       const result = await entryToDatasetFile(entry, "ds000001")
       expect(result).toEqual({
@@ -102,6 +108,8 @@ describe("datalad files", () => {
         directory: false,
         size: 12345,
         urls: ["https://s3.amazonaws.com/bucket/key?presigned=true"],
+        annexed: false,
+        symlink: false,
       })
     })
     it("returns a public S3 URL for public files with S3 keys", async () => {
@@ -114,6 +122,8 @@ describe("datalad files", () => {
         b: "",
         p: false,
         d: false,
+        a: false,
+        l: false,
       }
       const result = await entryToDatasetFile(entry, "ds000001")
       expect(result).toEqual({
@@ -122,6 +132,8 @@ describe("datalad files", () => {
         directory: false,
         size: 500,
         urls: ["https://s3.amazonaws.com/bucket/key?versionId=abc123"],
+        annexed: false,
+        symlink: false,
       })
     })
     it("falls back to server object URL when no S3 key/version", async () => {
@@ -135,6 +147,8 @@ describe("datalad files", () => {
         b: "",
         p: false,
         d: false,
+        a: false,
+        l: false,
       }
       const result = await entryToDatasetFile(entry, "ds000001")
       expect(result).toEqual({
@@ -145,6 +159,8 @@ describe("datalad files", () => {
         urls: [
           "https://openneuro.org/crn/datasets/ds000001/objects/jkl012?filename=sub-01_T1w.nii.gz",
         ],
+        annexed: false,
+        symlink: false,
       })
     })
   })
@@ -192,6 +208,8 @@ describe("datalad files", () => {
         directory: true,
         size: 0,
         urls: [],
+        annexed: false,
+        symlink: false,
       }
       expect(workerFileToEntry(file, false)).toEqual({
         n: "sub-01",
@@ -202,6 +220,8 @@ describe("datalad files", () => {
         b: "",
         p: false,
         d: true,
+        a: false,
+        l: false,
       })
     })
     it("converts a public file with S3 URL", () => {
@@ -213,6 +233,8 @@ describe("datalad files", () => {
         urls: [
           "https://s3.amazonaws.com/openneuro.org/datasets/ds000001/README?versionId=ver2",
         ],
+        annexed: false,
+        symlink: false,
       }
       const result = workerFileToEntry(file, false)
       expect(result).toEqual({
@@ -224,6 +246,8 @@ describe("datalad files", () => {
         b: "openneuro.org",
         p: false,
         d: false,
+        a: false,
+        l: false,
       })
     })
     it("stores empty bucket when it matches the default bucket", () => {
@@ -237,6 +261,8 @@ describe("datalad files", () => {
         urls: [
           "https://s3.amazonaws.com/openneuro.org/datasets/ds000001/README?versionId=ver2",
         ],
+        symlink: false,
+        annexed: false,
       }
       const result = workerFileToEntry(file, false)
       expect(result.b).toBe("")
@@ -251,6 +277,8 @@ describe("datalad files", () => {
         urls: [
           "https://s3.amazonaws.com/private-bucket/data.nii.gz?versionId=v1",
         ],
+        symlink: false,
+        annexed: false,
       }
       expect(workerFileToEntry(file, true).p).toBe(true)
       expect(workerFileToEntry(file, false).p).toBe(false)
@@ -262,6 +290,8 @@ describe("datalad files", () => {
         directory: false,
         size: 1000,
         urls: [],
+        symlink: false,
+        annexed: false,
       }
       const result = workerFileToEntry(file, false)
       expect(result.k).toBe("")

package/src/datalad/__tests__/snapshots.spec.ts

@@ -10,12 +10,12 @@ import { connect } from "mongoose"
 // Mock requests to Datalad service
 vi.mock("superagent")
 vi.mock("../../libs/redis.js", () => ({
-  redis: {
+  getRedis: () => ({
     del: vi.fn(),
-  },
-  redlock: {
+  }),
+  getRedlock: () => ({
     lock: vi.fn().mockImplementation(() => ({ unlock: vi.fn() })),
-  },
+  }),
 }))
 // Mock draft files calls
 vi.mock("../draft.ts", () => ({

package/src/datalad/contributors.ts

@@ -1,6 +1,6 @@
 import * as Sentry from "@sentry/node"
 import CacheItem, { CacheType } from "../cache/item"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import {
   type DatasetOrSnapshot,
   datasetOrSnapshot,
@@ -26,7 +26,7 @@ export const contributors = async (
   if (!datasetId) return []
 
   const revisionShort = revision ? revision.substring(0, 7) : "HEAD"
-  const dataciteCache = new CacheItem(redis, CacheType.dataciteYml, [
+  const dataciteCache = new CacheItem(getRedis(), CacheType.dataciteYml, [
     datasetId,
     revisionShort,
   ])

package/src/datalad/dataRetentionNotifications.ts

@@ -18,7 +18,7 @@ const DAY = 24 * 60 * 60 * 1000
  */
 async function notifyWriteUsers(
   datasetId: string,
-  makeEmail: (user: { _id: string; email: string; name: string }) => object,
+  makeEmail: (user: { id: string; email: string; name: string }) => object,
 ) {
   const permissions = await Permission.find({
     datasetId,
@@ -86,7 +86,7 @@ export async function checkDataRetentionNotifications(
     !lastSnapshot && age >= DAY && age < 14 * DAY && !record.notifiedNoSnapshot
   ) {
     await notifyWriteUsers(datasetId, (user) => ({
-      _id: `${datasetId}_${user._id}_no_snapshot_reminder`,
+      id: `${datasetId}_${user.id}_no_snapshot_reminder`,
       type: "email",
       email: {
         to: user.email,
@@ -107,7 +107,7 @@ export async function checkDataRetentionNotifications(
   // Retention warnings sent in order from 14 days, 7 days, and 0 days.
   if (age >= 14 * DAY && !record.notifiedAt14Days) {
     await notifyWriteUsers(datasetId, (user) => ({
-      _id: `${datasetId}_${user._id}_retention_14day`,
+      id: `${datasetId}_${user.id}_retention_14day`,
       type: "email",
       email: {
         to: user.email,
@@ -129,7 +129,7 @@ export async function checkDataRetentionNotifications(
     now.getTime() - new Date(record.notifiedAt14Days).getTime() >= 7 * DAY
   ) {
     await notifyWriteUsers(datasetId, (user) => ({
-      _id: `${datasetId}_${user._id}_retention_7day`,
+      id: `${datasetId}_${user.id}_retention_7day`,
       type: "email",
       email: {
         to: user.email,
@@ -151,7 +151,7 @@ export async function checkDataRetentionNotifications(
     now.getTime() - new Date(record.notifiedAt7Days).getTime() >= 7 * DAY
   ) {
     await notifyWriteUsers(datasetId, (user) => ({
-      _id: `${datasetId}_${user._id}_retention_deletion`,
+      id: `${datasetId}_${user.id}_retention_deletion`,
       type: "email",
       email: {
         to: user.email,

package/src/datalad/dataset.ts

@@ -12,12 +12,12 @@ import type * as Mongoose from "mongoose"
 import config from "../config"
 import * as subscriptions from "../handlers/subscriptions"
 import { generateDataladCookie } from "../libs/authentication/jwt"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDraftRevision, updateDatasetRevision } from "./draft"
 import { encodeFilePath, filesUrl, fileUrl, getFileName } from "./files"
 import { getAccessionNumber } from "../libs/dataset"
-import Dataset from "../models/dataset"
+import Dataset, { type DatasetDocument } from "../models/dataset"
 import Metadata from "../models/metadata"
 import Permission from "../models/permission"
 import Star from "../models/stars"
@@ -80,12 +80,16 @@ export const createDataset = async (
   }
 }
 
+type DatasetWithRevision = Mongoose.FlattenMaps<DatasetDocument> & {
+  revision: Promise<string>
+}
+
 /**
  * Fetch dataset document and related fields
  */
-export const getDataset = async (id) => {
+export const getDataset = async (id): Promise<DatasetWithRevision | null> => {
   const dataset = await Dataset.findOne({ id }).lean()
-  return {
+  return dataset && {
     ...dataset,
     revision: getDraftRevision(id),
   }
@@ -114,7 +118,7 @@ export const deleteDataset = async (datasetId, user) => {
 export const cacheDatasetConnection = (options) => (connectionArguments) => {
   const connection = datasetsConnection(options)
   const cache = new CacheItem(
-    redis,
+    getRedis(),
     CacheType.datasetsConnection,
     [objectHash(options)],
     60,

package/src/datalad/description.ts

@@ -3,7 +3,7 @@
  */
 import config from "../config"
 import request from "superagent"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import { commitFiles } from "./dataset"
 import { fileUrl } from "./files"
 import { generateDataladCookie } from "../libs/authentication/jwt"
@@ -135,7 +135,7 @@ export const description = async (obj) => {
     Name: datasetId,
     BIDSVersion: "1.8.0",
   }
-  const cache = new CacheItem(redis, CacheType.datasetDescription, [
+  const cache = new CacheItem(getRedis(), CacheType.datasetDescription, [
     datasetId,
     revision.substring(0, 7),
   ])

package/src/datalad/draft.ts

@@ -5,7 +5,7 @@ import request from "superagent"
 import Dataset from "../models/dataset"
 import { getDatasetWorker } from "../libs/datalad-service"
 import CacheItem, { CacheType } from "../cache/item"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 
 // Draft info resolver
 type DraftInfo = {
@@ -13,7 +13,7 @@ type DraftInfo = {
   hexsha: string // Duplicate of ref for backwards compatibility
   tree: string
   message: string
-  modified: Date
+  modified: string
 }
 
 export const getDraftRevision = async (datasetId): Promise<string> => {
@@ -22,7 +22,12 @@ export const getDraftRevision = async (datasetId): Promise<string> => {
 }
 
 export const getDraftInfo = async (datasetId) => {
-  const cache = new CacheItem(redis, CacheType.draftRevision, [datasetId], 10)
+  const cache = new CacheItem(
+    getRedis(),
+    CacheType.draftRevision,
+    [datasetId],
+    10,
+  )
   return cache.get(async (_doNotCache): Promise<DraftInfo> => {
     const draftUrl = `http://${
       getDatasetWorker(