@openneuro/server 4.47.6 → 5.0.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.47.6",
+  "version": "5.0.0-alpha.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -16,12 +16,13 @@
   "author": "Squishymedia",
   "dependencies": {
     "@apollo/client": "3.13.8",
-    "@apollo/server": "~4.13.0",
+    "@apollo/server": "~5.5.0",
     "@apollo/utils.keyvadapter": "3.0.0",
+    "@as-integrations/express5": "^1.1.2",
     "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/schema": "^10.0.0",
     "@keyv/redis": "^4.5.0",
-    "@openneuro/search": "^4.47.6",
+    "@openneuro/search": "^5.0.0-alpha.0",
     "@sentry/node": "^10.37.0",
     "@sentry/profiling-node": "^10.37.0",
     "base64url": "^3.0.0",
@@ -30,12 +31,13 @@
     "date-fns": "^2.16.1",
     "draft-js": "^0.11.7",
     "draft-js-export-html": "^1.4.1",
-    "express": "4.20.0",
+    "express": "5",
     "graphql": "16.8.1",
     "graphql-bigint": "^1.0.0",
     "graphql-compose": "9.0.10",
     "graphql-iso-date": "^3.6.1",
     "graphql-tools": "9.0.0",
+    "hash-wasm": "^4.12.0",
     "immutable": "^4.3.8",
     "ioredis": "^5.6.1",
     "js-yaml": "^4.1.0",
@@ -46,6 +48,7 @@
     "mongodb-memory-server": "^9.2.0",
     "mongoose": "^8.9.5",
     "morgan": "^1.6.1",
+    "msgpackr": "^1.11.9",
     "node-mailjet": "^3.3.5",
     "object-hash": "2.1.1",
     "passport": "0.7.0",
@@ -72,8 +75,8 @@
   "devDependencies": {
     "@types/cors": "^2",
     "@types/draft-js": "^0.10.43",
-    "@types/express": "^4.17.17",
-    "@types/express-serve-static-core": "^4.17.35",
+    "@types/express": "^5.0.6",
+    "@types/express-serve-static-core": "^5.1.1",
     "@types/ioredis": "^4.17.1",
     "@types/ioredis-mock": "^8.2.2",
     "@types/js-yaml": "^4",
@@ -89,5 +92,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "e00e369a538900524544a396e7fd60e5e41d715e"
+  "gitHead": "ee5b58a1623626ef79a439918e6c065a1c7d5e2e"
 }

package/src/app.ts

@@ -12,7 +12,7 @@ import morgan from "morgan"
 import schema from "./graphql/schema"
 import { ApolloServer } from "@apollo/server"
 import { ApolloServerPluginLandingPageLocalDefault } from "@apollo/server/plugin/landingPage/default"
-import { expressMiddleware } from "@apollo/server/express4"
+import { expressMiddleware } from "@as-integrations/express5"
 import { ApolloServerPluginDrainHttpServer } from "@apollo/server/plugin/drainHttpServer"
 import { KeyvAdapter } from "@apollo/utils.keyvadapter"
 import Keyv from "keyv"

package/src/cache/__tests__/tree.spec.ts

@@ -0,0 +1,212 @@
+import type { Redis } from "ioredis"
+import {
+  addDatasetTree,
+  addDatasetTrees,
+  clearDatasetTrees,
+  getCommitTrees,
+  getDatasetTrees,
+  getTree,
+  getTreesBulk,
+  setCommitTrees,
+  setTree,
+  type TreeEntry,
+} from "../tree"
+
+function makeEntry(overrides: Partial<TreeEntry> = {}): TreeEntry {
+  return {
+    n: "README.md",
+    h: "abc123",
+    s: 42,
+    k: "datasets/ds000001/README.md",
+    v: "v1",
+    b: "",
+    p: false,
+    d: false,
+    ...overrides,
+  }
+}
+
+function createRedisMock() {
+  const store = new Map<string, Buffer>()
+  const sets = new Map<string, Set<string>>()
+
+  const pipelineOps: (() => [Error | null, unknown])[] = []
+  const pipelineMock = {
+    getBuffer(key: string) {
+      pipelineOps.push(() => {
+        const val = store.get(key)
+        return [null, val ?? null]
+      })
+      return pipelineMock
+    },
+    del(key: string) {
+      pipelineOps.push(() => {
+        store.delete(key)
+        sets.delete(key)
+        return [null, 1]
+      })
+      return pipelineMock
+    },
+    exec: vi.fn(async () => {
+      const results = pipelineOps.map((op) => op())
+      pipelineOps.length = 0
+      return results
+    }),
+  }
+
+  return {
+    getBuffer: vi.fn(async (key: string) => store.get(key) ?? null),
+    set: vi.fn(async (key: string, value: Buffer) => {
+      store.set(key, value)
+    }),
+    setex: vi.fn(async (key: string, _ttl: number, value: Buffer) => {
+      store.set(key, value)
+    }),
+    sadd: vi.fn(async (key: string, ...members: string[]) => {
+      if (!sets.has(key)) sets.set(key, new Set())
+      for (const m of members) sets.get(key)!.add(m)
+    }),
+    smembers: vi.fn(async (key: string) => [...(sets.get(key) ?? [])]),
+    pipeline: vi.fn(() => {
+      pipelineOps.length = 0
+      return pipelineMock
+    }),
+    // expose internals for assertions
+    _store: store,
+    _sets: sets,
+  } as unknown as Redis
+}
+
+describe("tree cache", () => {
+  describe("getTree / setTree", () => {
+    it("returns null on cache miss", async () => {
+      const redis = createRedisMock()
+      expect(await getTree(redis, "nonexistent")).toBeNull()
+    })
+
+    it("round-trips entries through msgpack", async () => {
+      const redis = createRedisMock()
+      const entries = [makeEntry(), makeEntry({ n: "sub-01", d: true, s: 0 })]
+      await setTree(redis, "hash1", entries)
+      const result = await getTree(redis, "hash1")
+      expect(result).toEqual(entries)
+    })
+
+    it("stores with TTL when provided", async () => {
+      const redis = createRedisMock()
+      await setTree(redis, "hash2", [makeEntry()], 3600)
+      expect(redis.setex).toHaveBeenCalledWith(
+        "tree:hash2",
+        3600,
+        expect.any(Buffer),
+      )
+      expect(redis.set).not.toHaveBeenCalled()
+    })
+
+    it("stores without TTL when omitted", async () => {
+      const redis = createRedisMock()
+      await setTree(redis, "hash3", [makeEntry()])
+      expect(redis.set).toHaveBeenCalledWith("tree:hash3", expect.any(Buffer))
+      expect(redis.setex).not.toHaveBeenCalled()
+    })
+  })
+
+  describe("getTreesBulk", () => {
+    it("returns empty map for empty input", async () => {
+      const redis = createRedisMock()
+      const result = await getTreesBulk(redis, [])
+      expect(result.size).toBe(0)
+      expect(redis.pipeline).not.toHaveBeenCalled()
+    })
+
+    it("fetches multiple trees via pipeline", async () => {
+      const redis = createRedisMock()
+      const e1 = [makeEntry({ n: "file1" })]
+      const e2 = [makeEntry({ n: "file2" })]
+      await setTree(redis, "a", e1)
+      await setTree(redis, "b", e2)
+
+      const result = await getTreesBulk(redis, ["a", "b", "missing"])
+      expect(result.size).toBe(2)
+      expect(result.get("a")).toEqual(e1)
+      expect(result.get("b")).toEqual(e2)
+      expect(result.has("missing")).toBe(false)
+    })
+  })
+
+  describe("setCommitTrees / getCommitTrees", () => {
+    it("returns null on cache miss", async () => {
+      const redis = createRedisMock()
+      expect(await getCommitTrees(redis, "nonexistent")).toBeNull()
+    })
+
+    it("round-trips tree hash lists", async () => {
+      const redis = createRedisMock()
+      const hashes = ["hash1", "hash2", "hash3"]
+      await setCommitTrees(redis, "commit1", hashes)
+      const result = await getCommitTrees(redis, "commit1")
+      expect(result).toEqual(hashes)
+    })
+
+    it("uses correct key prefix and TTL", async () => {
+      const redis = createRedisMock()
+      await setCommitTrees(redis, "abc", ["h1"])
+      expect(redis.set).toHaveBeenCalledWith(
+        "ct:abc",
+        expect.any(Buffer),
+      )
+    })
+  })
+
+  describe("dataset tree index", () => {
+    it("addDatasetTree adds a single hash", async () => {
+      const redis = createRedisMock()
+      await addDatasetTree(redis, "ds000001", "hash1")
+      expect(redis.sadd).toHaveBeenCalledWith("dt:ds000001", "hash1")
+      const members = await getDatasetTrees(redis, "ds000001")
+      expect(members).toEqual(["hash1"])
+    })
+
+    it("addDatasetTrees adds multiple hashes", async () => {
+      const redis = createRedisMock()
+      await addDatasetTrees(redis, "ds000001", ["h1", "h2", "h3"])
+      expect(redis.sadd).toHaveBeenCalledWith("dt:ds000001", "h1", "h2", "h3")
+    })
+
+    it("addDatasetTrees skips empty array", async () => {
+      const redis = createRedisMock()
+      await addDatasetTrees(redis, "ds000001", [])
+      expect(redis.sadd).not.toHaveBeenCalled()
+    })
+
+    it("getDatasetTrees returns empty array for unknown dataset", async () => {
+      const redis = createRedisMock()
+      const result = await getDatasetTrees(redis, "ds999999")
+      expect(result).toEqual([])
+    })
+  })
+
+  describe("clearDatasetTrees", () => {
+    it("deletes all cached trees and the index", async () => {
+      const redis = createRedisMock()
+      // Populate tree data and index
+      await setTree(redis, "t1", [makeEntry({ n: "a" })])
+      await setTree(redis, "t2", [makeEntry({ n: "b" })])
+      await addDatasetTrees(redis, "ds000001", ["t1", "t2"])
+
+      await clearDatasetTrees(redis, "ds000001")
+
+      // Individual trees should be gone
+      expect(await getTree(redis, "t1")).toBeNull()
+      expect(await getTree(redis, "t2")).toBeNull()
+      // Index should be gone
+      expect(await getDatasetTrees(redis, "ds000001")).toEqual([])
+    })
+
+    it("does nothing when dataset has no trees", async () => {
+      const redis = createRedisMock()
+      await clearDatasetTrees(redis, "ds000001")
+      expect(redis.pipeline).not.toHaveBeenCalled()
+    })
+  })
+})

package/src/cache/tree.ts

@@ -0,0 +1,148 @@
+import type { Redis } from "ioredis"
+import { pack, unpack } from "msgpackr"
+
+/** Compact tree entry stored in Redis */
+export interface TreeEntry {
+  /** filename */
+  n: string
+  /** child hash (tree hash for dirs, blob/annex key for files) */
+  h: string
+  /** size in bytes (0 for directories) */
+  s: number
+  /** S3 object key without bucket prefix (empty for directories) */
+  k: string
+  /** S3 versionId (empty for directories) */
+  v: string
+  /** S3 bucket override (empty string = default AWS_S3_PUBLIC_BUCKET) */
+  b: string
+  /** needs presigned URL */
+  p: boolean
+  /** is directory */
+  d: boolean
+}
+
+function treeKey(hash: string): string {
+  return `tree:${hash}`
+}
+
+function commitTreesKey(commitHash: string): string {
+  return `ct:${commitHash}`
+}
+
+function datasetTreesKey(datasetId: string): string {
+  return `dt:${datasetId}`
+}
+
+/** Read and decode a single tree from cache */
+export async function getTree(
+  redis: Redis,
+  treeHash: string,
+): Promise<TreeEntry[] | null> {
+  const data = await redis.getBuffer(treeKey(treeHash))
+  if (data) {
+    return unpack(data) as TreeEntry[]
+  }
+  return null
+}
+
+/** Encode and write a tree to cache. Optional TTL for unexported trees. */
+export async function setTree(
+  redis: Redis,
+  treeHash: string,
+  entries: TreeEntry[],
+  ttl?: number,
+): Promise<void> {
+  const packed = pack(entries)
+  if (ttl) {
+    await redis.setex(treeKey(treeHash), ttl, packed)
+  } else {
+    await redis.set(treeKey(treeHash), packed)
+  }
+}
+
+/** Pipeline-fetch multiple trees by hash */
+export async function getTreesBulk(
+  redis: Redis,
+  treeHashes: string[],
+): Promise<Map<string, TreeEntry[]>> {
+  if (treeHashes.length === 0) return new Map()
+  const pipeline = redis.pipeline()
+  for (const hash of treeHashes) {
+    pipeline.getBuffer(treeKey(hash))
+  }
+  const results = await pipeline.exec()
+  const trees = new Map<string, TreeEntry[]>()
+  for (let i = 0; i < treeHashes.length; i++) {
+    const [err, data] = results[i]
+    if (!err && data) {
+      trees.set(treeHashes[i], unpack(data as Buffer) as TreeEntry[])
+    }
+  }
+  return trees
+}
+
+/** Store the set of tree hashes belonging to a commit */
+export async function setCommitTrees(
+  redis: Redis,
+  commitHash: string,
+  treeHashes: string[],
+): Promise<void> {
+  const packed = pack(treeHashes)
+  await redis.set(commitTreesKey(commitHash), packed)
+}
+
+/** Read the set of tree hashes for a commit */
+export async function getCommitTrees(
+  redis: Redis,
+  commitHash: string,
+): Promise<string[] | null> {
+  const data = await redis.getBuffer(commitTreesKey(commitHash))
+  if (data) {
+    return unpack(data) as string[]
+  }
+  return null
+}
+
+/** Add a tree hash to the dataset's reverse index */
+export async function addDatasetTree(
+  redis: Redis,
+  datasetId: string,
+  treeHash: string,
+): Promise<void> {
+  await redis.sadd(datasetTreesKey(datasetId), treeHash)
+}
+
+/** Add multiple tree hashes to the dataset's reverse index */
+export async function addDatasetTrees(
+  redis: Redis,
+  datasetId: string,
+  treeHashes: string[],
+): Promise<void> {
+  if (treeHashes.length > 0) {
+    await redis.sadd(datasetTreesKey(datasetId), ...treeHashes)
+  }
+}
+
+/** Get all tree hashes for a dataset (for cache clearing) */
+export async function getDatasetTrees(
+  redis: Redis,
+  datasetId: string,
+): Promise<string[]> {
+  return redis.smembers(datasetTreesKey(datasetId))
+}
+
+/** Delete all cached trees for a dataset and clean up the index */
+export async function clearDatasetTrees(
+  redis: Redis,
+  datasetId: string,
+): Promise<void> {
+  const treeHashes = await getDatasetTrees(redis, datasetId)
+  if (treeHashes.length > 0) {
+    const pipeline = redis.pipeline()
+    for (const hash of treeHashes) {
+      pipeline.del(treeKey(hash))
+    }
+    pipeline.del(datasetTreesKey(datasetId))
+    await pipeline.exec()
+  }
+}

package/src/datalad/__tests__/dataRetentionNotifications.spec.ts

@@ -22,6 +22,7 @@ import notifications from "../../libs/notifications"
 import DataRetention from "../../models/dataRetention"
 import Permission from "../../models/permission"
 import User from "../../models/user"
+import Dataset from "../../models/dataset"
 import Deletion from "../../models/deletion"
 import { checkDataRetentionNotifications } from "../dataRetentionNotifications"
 import * as draftModule from "../draft"
@@ -55,6 +56,7 @@ describe("checkDataRetentionNotifications", () => {
   })
 
   beforeEach(async () => {
+    await Dataset.deleteMany({})
     await DataRetention.deleteMany({})
     await Deletion.deleteMany({})
     await Permission.deleteMany({})
@@ -96,6 +98,15 @@ describe("checkDataRetentionNotifications", () => {
     expect(notifications.send).not.toHaveBeenCalled()
   })
 
+  it("skips notifications for datasets with holdDeletion flag", async () => {
+    await Dataset.create({ id: TEST_DATASET, holdDeletion: true })
+    mockDraft(daysAgo(15))
+    mockSnapshots([{ hexsha: "other" }])
+
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).not.toHaveBeenCalled()
+  })
+
   it("does nothing when draft matches the latest snapshot", async () => {
     mockDraft(daysAgo(30), TEST_HEXSHA)
     mockSnapshots([{ hexsha: TEST_HEXSHA }])