@openneuro/server 5.0.0 → 5.1.0

package/src/datalad/files.ts

@@ -1,4 +1,4 @@
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import { getDatasetWorker } from "../libs/datalad-service"
 import {
   getPresignedUrl,
@@ -15,7 +15,9 @@ import {
   setTree,
   type TreeEntry,
 } from "../cache/tree"
+import CacheItem, { CacheType } from "../cache/item"
 import { join } from "node:path"
+import { captureException } from "@sentry/node"
 
 /**
  * Convert to URL compatible path
@@ -86,6 +88,8 @@ export type DatasetFile = {
   directory: boolean
   size: number
   urls: string[]
+  symlink: boolean
+  annexed: boolean
 }
 
 /**
@@ -140,6 +144,8 @@ export function workerFileToEntry(
       b: "",
       p: false,
       d: true,
+      a: false,
+      l: false,
     }
   }
   const parsed = file.urls[0] ? parseS3Url(file.urls[0]) : null
@@ -155,6 +161,8 @@ export function workerFileToEntry(
     b: bucket,
     p: needsPresign,
     d: false,
+    a: file.annexed,
+    l: file.symlink,
   }
 }
 
@@ -170,11 +178,13 @@ export async function entryToDatasetFile(
       directory: true,
       size: 0,
       urls: [],
+      symlink: false,
+      annexed: false,
     }
   }
   let url: string
   if (entry.p && entry.k && entry.v) {
-    url = await getPresignedUrl(redis, entry.b, entry.k, entry.v)
+    url = await getPresignedUrl(getRedis(), entry.b, entry.k, entry.v)
   } else if (entry.k && entry.v) {
     url = publicS3Url(entry.b, entry.k, entry.v)
   } else {
@@ -189,6 +199,8 @@ export async function entryToDatasetFile(
     directory: false,
     size: entry.s,
     urls: [url],
+    symlink: entry.l,
+    annexed: entry.a,
   }
 }
 
@@ -248,19 +260,53 @@ async function cacheWorkerTrees(
         (f) => f.directory || f.urls[0]?.includes("s3.amazonaws.com"),
       )
       if (allExported) {
-        void setTree(redis, hash, entries)
+        void setTree(getRedis(), hash, entries)
         permanentHashes.push(hash)
       } else {
-        void setTree(redis, hash, entries, 600)
+        void setTree(getRedis(), hash, entries, 600)
       }
     }
   }
   if (permanentHashes.length > 0) {
-    void addDatasetTrees(redis, datasetId, permanentHashes)
+    void addDatasetTrees(getRedis(), datasetId, permanentHashes)
   }
   return result
 }
 
+/**
+ * Resolve a git reference (branch, tag, or short commit hash) to a full commit hash.
+ * @param datasetId The dataset ID.
+ * @param treeish The git reference to resolve.
+ * @returns The full commit hash.
+ */
+export const resolveGitRef = async (
+  datasetId: string,
+  treeish: string,
+): Promise<string> => {
+  const cache = new CacheItem(getRedis(), CacheType.gitRef, [
+    datasetId,
+    treeish,
+  ])
+  return cache.get(async () => {
+    const url = `http://${
+      getDatasetWorker(datasetId)
+    }/datasets/${datasetId}/refs/${treeish}`
+    const response = await fetch(url)
+    if (!response.ok) {
+      throw new Error(
+        `Failed to resolve git reference ${treeish}: ${response.statusText}`,
+      )
+    }
+    const data = await response.json()
+    if (!data.hash) {
+      throw new Error(
+        `Invalid response from datalad worker for git reference ${treeish}`,
+      )
+    }
+    return data.hash
+  })
+}
+
 /**
  * Get files for a specific revision (tree hash or commit hash).
  * Uses content-addressed caching keyed by full git hash.
@@ -269,8 +315,19 @@ export const getFiles = async (
   datasetId: string,
   treeish: string,
 ): Promise<DatasetFile[]> => {
+  // Guard against requests without a full git hash (40 = SHA-1, 64 = SHA-256)
+  if (treeish.length !== 40 && treeish.length !== 64) {
+    try {
+      treeish = await resolveGitRef(datasetId, treeish)
+    } catch (error) {
+      captureException(error, {
+        tags: { datasetId, treeish, source: "resolveGitRef" },
+      })
+      throw new Error(`Invalid git reference: ${treeish}`, { cause: error })
+    }
+  }
   // Try cache first
-  const cached = await getTree(redis, treeish)
+  const cached = await getTree(getRedis(), treeish)
   if (cached) {
     return entriesToDatasetFiles(cached, datasetId)
   }
@@ -300,10 +357,10 @@ export async function getFilesRecursive(
 ): Promise<DatasetFile[]> {
   const needsPresign = await datasetNeedsPresign(datasetId)
   // Check for cached commit-to-trees mapping
-  const cachedTreeHashes = await getCommitTrees(redis, tree)
+  const cachedTreeHashes = await getCommitTrees(getRedis(), tree)
   if (cachedTreeHashes) {
     // Bulk-fetch all trees in one pipeline
-    const treesMap = await getTreesBulk(redis, cachedTreeHashes)
+    const treesMap = await getTreesBulk(getRedis(), cachedTreeHashes)
     if (treesMap.size < cachedTreeHashes.length) {
       // Batch-fetch all missing trees from the worker in one request
       const missingHashes = cachedTreeHashes.filter((h) => !treesMap.has(h))
@@ -327,7 +384,7 @@ export async function getFilesRecursive(
 
   while (pendingHashes.length > 0) {
     // Check cache for all pending hashes
-    const cached = await getTreesBulk(redis, pendingHashes)
+    const cached = await getTreesBulk(getRedis(), pendingHashes)
     const uncached = pendingHashes.filter((h) => !cached.has(h))
 
     // Fetch all uncached trees in one worker request
@@ -363,8 +420,8 @@ export async function getFilesRecursive(
   // Cache the commit-to-trees mapping for next time
   if (collectedHashes.size > 0) {
     const hashArray = [...collectedHashes]
-    void setCommitTrees(redis, tree, hashArray)
-    void addDatasetTrees(redis, datasetId, hashArray)
+    void setCommitTrees(getRedis(), tree, hashArray)
+    void addDatasetTrees(getRedis(), datasetId, hashArray)
   }
 
   return reconstructFromTrees(treesMap, tree, path, datasetId)
@@ -409,6 +466,8 @@ async function reconstructFromTrees(
       directory: false,
       size: entry.s,
       urls: [],
+      symlink: entry.l,
+      annexed: entry.a,
     }
     if (entry.p && entry.k && entry.v) {
       // To be presigned
@@ -429,7 +488,7 @@ async function reconstructFromTrees(
   // Bulk-resolve presigned URLs in minimal Redis requests
   if (presignIndices.length > 0) {
     const urls = await getPresignedUrlsBulk(
-      redis,
+      getRedis(),
       presignIndices.map((i) => ({
         bucket: fileEntries[i].entry.b,
         s3Key: fileEntries[i].entry.k,

package/src/datalad/readme.ts

@@ -1,5 +1,5 @@
 import { addFileString, commitFiles } from "./dataset"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDatasetWorker } from "../libs/datalad-service"
 import { datasetOrSnapshot } from "../utils/datasetOrSnapshot"
@@ -14,7 +14,7 @@ export const readmeUrl = (datasetId, revision) => {
 
 export const readme = (obj) => {
   const { datasetId, revision } = datasetOrSnapshot(obj)
-  const cache = new CacheItem(redis, CacheType.readme, [
+  const cache = new CacheItem(getRedis(), CacheType.readme, [
     datasetId,
     revision.substring(0, 7),
   ])

package/src/datalad/snapshots.ts

@@ -3,13 +3,10 @@
  */
 import * as Sentry from "@sentry/node"
 import request from "superagent"
-import { redis, redlock } from "../libs/redis"
+import { getRedis, getRedlock } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import config from "../config"
-import {
-  snapshotCreationComparison,
-  updateDatasetName,
-} from "../graphql/resolvers/dataset"
+import { snapshotCreationComparison } from "../utils/snapshots"
 import { createDraftDoi } from "../libs/doi/index"
 import { assembleMetadata } from "../libs/doi/metadata"
 import Doi from "../models/doi"
@@ -25,7 +22,7 @@ import { createEvent, updateEvent } from "../libs/events"
 import { queueIndexDataset } from "../queues/producer-methods"
 
 const lockSnapshot = (datasetId, tag) => {
-  return redlock.lock(
+  return getRedlock().lock(
     `openneuro:create-snapshot-lock:${datasetId}:${tag}`,
     1800000,
   )
@@ -76,7 +73,7 @@ const createIfNotExistsDoi = async (
     Sentry.captureException(err)
     // eslint-disable-next-line no-console
     console.error(err)
-    throw new Error(`DOI minting failed: ${err.message}`)
+    throw new Error(`DOI minting failed: ${err.message}`, { cause: err })
   }
 }
 
@@ -110,7 +107,12 @@ const postSnapshot = async (
 export const getSnapshots = async (datasetId): Promise<SnapshotDocument[]> => {
   const dataset = await Dataset.findOne({ id: datasetId })
   if (!dataset) return null
-  const cache = new CacheItem(redis, CacheType.snapshot, [datasetId], 432000)
+  const cache = new CacheItem(
+    getRedis(),
+    CacheType.snapshot,
+    [datasetId],
+    432000,
+  )
   return cache.get(() => {
     const url = `${getDatasetWorker(datasetId)}/datasets/${datasetId}/snapshots`
     return request
@@ -144,7 +146,7 @@ export const createSnapshot = async (
   descriptionFieldUpdates = {},
   snapshotChanges = [],
 ) => {
-  const snapshotCache = new CacheItem(redis, CacheType.snapshot, [
+  const snapshotCache = new CacheItem(getRedis(), CacheType.snapshot, [
     datasetId,
     tag,
   ])
@@ -182,10 +184,13 @@ export const createSnapshot = async (
       createSnapshotMetadata(datasetId, tag, snapshot.hexsha, snapshot.created),
 
       // Trigger an async update for the name field (cache for sorting)
-      updateDatasetName(datasetId),
+      // Dynamic import breaks circular dependency: datalad/snapshots → resolvers/dataset
+      import("../graphql/resolvers/dataset").then((m) =>
+        m.updateDatasetName(datasetId)
+      ),
     ])
 
-    const snapshotListCache = new CacheItem(redis, CacheType.snapshot, [
+    const snapshotListCache = new CacheItem(getRedis(), CacheType.snapshot, [
       datasetId,
     ])
     await snapshotListCache.drop()
@@ -215,12 +220,12 @@ export const deleteSnapshot = (datasetId, tag) => {
     )
   }/datasets/${datasetId}/snapshots/${tag}`
   return request.del(url).then(async ({ body }) => {
-    const snapshotCache = new CacheItem(redis, CacheType.snapshot, [
+    const snapshotCache = new CacheItem(getRedis(), CacheType.snapshot, [
       datasetId,
       tag,
     ])
     await snapshotCache.drop()
-    const snapshotListCache = new CacheItem(redis, CacheType.snapshot, [
+    const snapshotListCache = new CacheItem(getRedis(), CacheType.snapshot, [
       datasetId,
     ])
     await snapshotListCache.drop()
@@ -244,7 +249,7 @@ export const getSnapshot = (
     )
   }/datasets/${datasetId}/snapshots/${commitRef}`
   const cache = new CacheItem(
-    redis,
+    getRedis(),
     CacheType.snapshot,
     [datasetId, commitRef],
     432000,

package/src/elasticsearch/elastic-client.ts

@@ -1,11 +1,16 @@
 import config from "../config"
 import { Client } from "@elastic/elasticsearch"
 
-const elasticConfig = {
-  node: config.elasticsearch.connection || "http://mock-client",
-  maxRetries: 3,
-}
+let _client: Client | null = null
 
-export const elasticClient = new Client(elasticConfig)
+export function getElasticClient(): Client {
+  if (!_client) {
+    _client = new Client({
+      node: config.elasticsearch.connection || "http://mock-client",
+      maxRetries: 3,
+    })
+  }
+  return _client
+}
 
-export default elasticClient
+export default getElasticClient

package/src/elasticsearch/reindex-dataset.ts

@@ -1,6 +1,6 @@
 import config from "../config"
 import { indexDataset, indexingToken, queryForIndex } from "@openneuro/search"
-import { elasticClient } from "./elastic-client"
+import { getElasticClient } from "./elastic-client"
 import { ApolloClient, from, InMemoryCache } from "@apollo/client"
 import type { NormalizedCacheObject } from "@apollo/client"
 import { setContext } from "@apollo/client/link/context"
@@ -28,9 +28,13 @@ export const schemaLinkClient = (): ApolloClient<NormalizedCacheObject> => {
   })
 }
 
-const client = schemaLinkClient()
+let _client: ApolloClient<NormalizedCacheObject> | null = null
+function getClient(): ApolloClient<NormalizedCacheObject> {
+  if (!_client) _client = schemaLinkClient()
+  return _client
+}
 
 export const reindexDataset = async (datasetId: string): Promise<void> => {
-  const datasetIndexQueryResult = await queryForIndex(client, datasetId)
-  await indexDataset(elasticClient, datasetIndexQueryResult.data.dataset)
+  const datasetIndexQueryResult = await queryForIndex(getClient(), datasetId)
+  await indexDataset(getElasticClient(), datasetIndexQueryResult.data.dataset)
 }

package/src/graphql/__tests__/comment.spec.ts

@@ -3,6 +3,7 @@ import { connect } from "mongoose"
 import { deleteComment, flatten } from "../resolvers/comment"
 import { MongoMemoryServer } from "mongodb-memory-server"
 import Comment from "../../models/comment"
+import type { GraphQLContext } from "../builder"
 
 vi.mock("ioredis")
 
@@ -12,11 +13,11 @@ describe("comment resolver helpers", () => {
     const adminUser = {
       user: "1234",
       userInfo: { admin: true },
-    }
+    } as GraphQLContext
     const nonAdminUser = {
       user: "5678",
       userInfo: { admin: false },
-    }
+    } as GraphQLContext
     let mongod
     beforeAll(async () => {
       // Setup MongoDB with mongodb-memory-server

package/src/graphql/__tests__/schema.spec.ts

@@ -0,0 +1,28 @@
+import { vi } from "vitest"
+
+vi.mock("ioredis", () => {
+  const RedisMock = vi.fn()
+  RedisMock.prototype.on = vi.fn()
+  RedisMock.prototype.connect = vi.fn()
+  return { default: RedisMock }
+})
+vi.mock("../../elasticsearch/elastic-client", () => ({
+  elasticClient: {},
+}))
+vi.mock("../../config", () => ({
+  default: {
+    url: "http://localhost",
+    auth: { jwt: { secret: "test-secret-for-schema-smoke" } },
+    datalad: { uri: "http://localhost" },
+    mongo: { url: "mongodb://localhost", dbName: "test" },
+    redis: {},
+  },
+}))
+
+import schema from "../schema"
+
+it("builds the schema without error", () => {
+  expect(schema).toBeDefined()
+  expect(schema.getQueryType()).toBeDefined()
+  expect(schema.getMutationType()).toBeDefined()
+})

package/src/graphql/builder.ts

@@ -0,0 +1,42 @@
+import SchemaBuilder from "@pothos/core"
+import SimpleObjectsPlugin from "@pothos/plugin-simple-objects"
+import DirectivesPlugin from "@pothos/plugin-directives"
+
+export interface UserInfo {
+  id: string
+  userId: string
+  admin: boolean
+  username?: string
+  provider?: string
+  providerId?: string
+  blocked?: boolean
+  orcidConsent?: boolean | null
+  reviewer?: boolean
+  exp?: string
+  scopes?: string[]
+  indexer?: boolean
+}
+
+export interface GraphQLContext {
+  user: string
+  userInfo: UserInfo
+}
+
+export const builder = new SchemaBuilder<{
+  Context: GraphQLContext
+  Scalars: {
+    ID: { Input: string; Output: string }
+    Date: { Input: string; Output: Date }
+    DateTime: { Input: string; Output: Date }
+    BigInt: { Input: number; Output: number }
+    JSON: { Input: unknown; Output: unknown }
+  }
+  DefaultFieldNullability: true
+}>({
+  plugins: [SimpleObjectsPlugin, DirectivesPlugin],
+  notStrict:
+    "Pothos may not work correctly when strict mode is not enabled in tsconfig.json",
+  directives: {
+    useGraphQLToolsUnorderedDirectives: true,
+  },
+})

package/src/graphql/resolvers/__tests__/dataset.spec.ts

@@ -21,127 +21,12 @@ describe("dataset resolvers", () => {
       const { id: dsId } = await ds.createDataset(
         null,
         { affirmedDefaced: true, affirmedConsent: false },
-        { user: "123456", userInfo: {} },
-      )
-      expect(dsId).toEqual(expect.stringMatching(/^ds[0-9]{6}$/))
-    })
-  })
-  describe("snapshotCreationComparison()", () => {
-    it('sorts array of objects by the "created" and "tag" properties', () => {
-      const testArray = [
-        { id: 2, created: new Date("2018-11-20T00:05:43.473Z"), tag: "1.0.0" },
-        { id: 1, created: new Date("2018-11-19T00:05:43.473Z"), tag: "1.0.1" },
-        { id: 3, created: new Date("2018-11-23T00:05:43.473Z"), tag: "1.0.2" },
-        { id: 5, created: new Date("2018-11-23T00:05:43.473Z"), tag: "1.0.10" },
-        { id: 4, created: new Date("2018-11-23T00:05:43.473Z"), tag: "1.0.3" },
-      ]
-      const sorted = testArray.sort(ds.snapshotCreationComparison)
-      expect(sorted[0].id).toBe(2)
-      expect(sorted[1].id).toBe(1)
-      expect(sorted[2].id).toBe(3)
-      expect(sorted[3].id).toBe(4)
-      expect(sorted[4].id).toBe(5)
-    })
-    it('sorts array of objects by the "created" property as strings', () => {
-      const testArray = [
-        { id: 2, created: "2018-11-20T00:05:43.473Z", tag: "2.0.0" },
-        { id: 1, created: "2018-11-19T00:05:43.473Z", tag: "1.0.0" },
-        { id: 3, created: "2018-11-23T00:05:43.473Z", tag: "3.0.0" },
-      ]
-      const sorted = testArray.sort(ds.snapshotCreationComparison)
-      expect(sorted[0].id).toBe(1)
-      expect(sorted[1].id).toBe(2)
-      expect(sorted[2].id).toBe(3)
-    })
-    it("sorts non-semver tags mixed with semver tags", () => {
-      const testArray = [
-        { id: 2, created: new Date("2018-11-19T00:05:43.473Z"), tag: "1.0.2" },
-        {
-          id: 1,
-          created: new Date("2018-11-19T00:05:43.473Z"),
-          tag: "57fed018cce88d000ac1757f",
-        },
-        { id: 3, created: new Date("2018-11-19T00:05:43.473Z"), tag: "1.0.1" },
-      ]
-      const sorted = testArray.sort(ds.snapshotCreationComparison)
-      expect(sorted[0].id).toBe(2)
-      expect(sorted[1].id).toBe(1)
-      expect(sorted[2].id).toBe(3)
-    })
-    it("sorts snapshots with only non-semver tags", () => {
-      const testArray = [
-        {
-          id: 2,
-          created: new Date("2018-11-19T00:05:43.473Z"),
-          tag: "00001",
-        },
-        {
-          id: 1,
-          created: new Date("2018-11-19T00:05:43.473Z"),
-          tag: "57fed018cce88d000ac1757f",
-        },
-        {
-          id: 3,
-          created: new Date("2018-11-19T00:05:43.473Z"),
-          tag: "57fed018cce88d000ac1757f",
-        },
-      ]
-      const sorted = testArray.sort(ds.snapshotCreationComparison)
-      expect(sorted[0].id).toBe(2)
-      expect(sorted[1].id).toBe(1)
-      expect(sorted[2].id).toBe(3)
-    })
-    it("sorts very similar creation times by semver order", () => {
-      const testSnapshots = [
         {
-          id: "ds002680:1.0.0",
-          created: "2020-04-03T23:19:56.000Z",
-          tag: "1.0.0",
+          user: "123456",
+          userInfo: { id: "123456", userId: "123456", admin: false },
         },
-        {
-          id: "ds002680:1.2.0",
-          created: "2021-10-19T16:26:43.000Z",
-          tag: "1.2.0",
-        },
-        {
-          id: "ds002680:1.1.0",
-          created: "2021-10-19T16:26:44.000Z",
-          tag: "1.1.0",
-        },
-      ]
-      const sorted = testSnapshots.sort(ds.snapshotCreationComparison)
-      expect(sorted[0].id).toBe("ds002680:1.0.0")
-      expect(sorted[1].id).toBe("ds002680:1.1.0")
-      expect(sorted[2].id).toBe("ds002680:1.2.0")
-    })
-    it("sorts 000002 (legacy snapshots) before 1.0.1 (current format)", () => {
-      const testSnapshots = [
-        {
-          id: "ds000247:00002",
-          created: "2018-07-18T02:27:39.000Z",
-          tag: "00002",
-        },
-        {
-          id: "ds000247:00001",
-          created: "2018-07-18T02:35:37.000Z",
-          tag: "00001",
-        },
-        {
-          id: "ds000247:1.0.0",
-          created: "2021-07-05T15:58:18.000Z",
-          tag: "1.0.0",
-        },
-        {
-          id: "ds000247:1.0.1",
-          created: "2021-08-25T23:37:53.000Z",
-          tag: "1.0.1",
-        },
-      ]
-      const sorted = testSnapshots.sort(ds.snapshotCreationComparison)
-      expect(sorted[0].id).toBe("ds000247:00002")
-      expect(sorted[1].id).toBe("ds000247:00001")
-      expect(sorted[2].id).toBe("ds000247:1.0.0")
-      expect(sorted[3].id).toBe("ds000247:1.0.1")
+      )
+      expect(dsId).toEqual(expect.stringMatching(/^ds[0-9]{6}$/))
     })
   })
   describe("deleteFiles", () => {
@@ -170,6 +55,8 @@ describe("dataset resolvers", () => {
           user: "a_user_id",
           userInfo: {
             // bypass permission checks
+            id: "a_user_id",
+            userId: "a_user_id",
             admin: true,
           },
         },

package/src/graphql/resolvers/__tests__/importRemoteDataset.spec.ts

@@ -1,5 +1,6 @@
 import { vi } from "vitest"
 import { allowedImportUrl, importRemoteDataset } from "../importRemoteDataset"
+import type { GraphQLContext } from "../../builder"
 
 vi.mock("ioredis")
 vi.mock("../../../config")
@@ -10,7 +11,7 @@ describe("importRemoteDataset mutation", () => {
     await importRemoteDataset(
       {},
       { datasetId: "ds000000", url: "" },
-      { user: "1234", userInfo: { admin: true } },
+      { user: "1234", userInfo: { admin: true } } as GraphQLContext,
     )
   })
   describe("allowedImportUrl()", () => {

package/src/graphql/resolvers/__tests__/permssions.spec.ts

@@ -1,5 +1,6 @@
 import { vi } from "vitest"
 import { updatePermissions } from "../permissions"
+import type { GraphQLContext } from "../../builder"
 
 vi.mock("ioredis")
 vi.mock("../../permissions", () => ({
@@ -22,8 +23,8 @@ describe("permissions resolvers", () => {
       try {
         await updatePermissions(
           {},
-          { datasetId: "ds01234", userEmail: "fake@test.com" },
-          { user: "1234", userInfo: { id: "1234" } },
+          { datasetId: "ds01234", userEmail: "fake@test.com", level: "ro" },
+          { user: "1234", userInfo: { id: "1234" } } as GraphQLContext,
         )
       } catch (err) {
         error = err