@openneuro/server 4.47.7 → 5.0.0

package/src/libs/doi/__tests__/doi.spec.ts

@@ -1,25 +1,63 @@
 import { vi } from "vitest"
-import { formatBasicAuth, template } from "../index.js"
+import { buildPayload, createDOI, formatBasicAuth } from "../index.js"
 
 vi.mock("ioredis")
 
 describe("DOI minting utils", () => {
-  describe("auth()", () => {
+  describe("formatBasicAuth()", () => {
     it("returns a base64 basic auth string", () => {
       const doiConfig = { username: "test", password: "12345" }
       expect(formatBasicAuth(doiConfig)).toBe("Basic dGVzdDoxMjM0NQ==")
     })
   })
-  describe("template()", () => {
-    it("accepts expected arguments", () => {
-      const context = {
-        doi: "12345",
-        creators: ["A. User", "B. User"],
-        title: "Test Dataset",
-        year: "1999",
-        resourceType: "fMRI",
-      }
-      expect(template(context)).toMatchSnapshot()
+
+  describe("createDOI()", () => {
+    it("creates a DOI without snapshot", () => {
+      const doi = createDOI("ds000001")
+      expect(doi).toMatch(/\/openneuro\.ds000001$/)
+    })
+    it("creates a DOI with snapshot", () => {
+      const doi = createDOI("ds000001", "1.0.0")
+      expect(doi).toMatch(/\/openneuro\.ds000001\.v1\.0\.0$/)
+    })
+  })
+
+  describe("buildPayload()", () => {
+    const attributes = {
+      doi: "10.18112/openneuro.ds000001.v1.0.0",
+      url: "https://openneuro.org/datasets/ds000001/versions/1.0.0",
+      creators: [{ name: "A. User", nameType: "Personal" as const }],
+      titles: [{ title: "Test Dataset" }],
+      publisher: { name: "OpenNeuro" },
+      publicationYear: "2024",
+      types: { resourceTypeGeneral: "Dataset" as const },
+      schemaVersion: "http://datacite.org/schema/kernel-4" as const,
+    }
+
+    it("builds a valid Datacite JSON API payload", () => {
+      const payload = buildPayload(attributes)
+      expect(payload.data.type).toBe("dois")
+      expect(payload.data.attributes.doi).toBe(
+        "10.18112/openneuro.ds000001.v1.0.0",
+      )
+      expect(payload.data.attributes.event).toBeUndefined()
+      expect(payload.data.attributes.schemaVersion).toBe(
+        "http://datacite.org/schema/kernel-4",
+      )
+    })
+
+    it("omits event when not provided", () => {
+      const payload = buildPayload(attributes)
+      expect(payload.data.attributes.event).toBeUndefined()
+    })
+
+    it("preserves all metadata attributes", () => {
+      const payload = buildPayload(attributes, "publish")
+      expect(payload.data.attributes.creators).toHaveLength(1)
+      expect(payload.data.attributes.titles[0].title).toBe("Test Dataset")
+      expect(payload.data.attributes.publisher.name).toBe("OpenNeuro")
+      expect(payload.data.attributes.publicationYear).toBe("2024")
+      expect(payload.data.attributes.types.resourceTypeGeneral).toBe("Dataset")
     })
   })
 })

package/src/libs/doi/__tests__/validate.spec.ts

@@ -0,0 +1,110 @@
+import { vi } from "vitest"
+import { validateDataciteMetadata } from "../validate.js"
+import type { ResourceTypeGeneral } from "../../../types/datacite/datacite-v4.5.ts"
+
+vi.mock("ioredis")
+
+describe("validateDataciteMetadata", () => {
+  const validAttrs = {
+    doi: "10.18112/openneuro.ds000001.v1.0.0",
+    url: "https://openneuro.org/datasets/ds000001/versions/1.0.0",
+    creators: [{ name: "A. User", nameType: "Personal" as const }],
+    titles: [{ title: "Test Dataset" }],
+    publisher: { name: "OpenNeuro" },
+    publicationYear: "2024",
+    types: { resourceTypeGeneral: "Dataset" as const },
+    schemaVersion: "http://datacite.org/schema/kernel-4" as const,
+  }
+
+  it("returns no errors for valid metadata", () => {
+    expect(validateDataciteMetadata(validAttrs)).toEqual([])
+  })
+
+  it("requires at least one creator", () => {
+    const errors = validateDataciteMetadata({ ...validAttrs, creators: [] })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "creators" }),
+      ]),
+    )
+  })
+
+  it("requires each creator to have a name", () => {
+    const errors = validateDataciteMetadata({
+      ...validAttrs,
+      creators: [{ name: "", nameType: "Personal" }],
+    })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "creators" }),
+      ]),
+    )
+  })
+
+  it("requires at least one title", () => {
+    const errors = validateDataciteMetadata({ ...validAttrs, titles: [] })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "titles" }),
+      ]),
+    )
+  })
+
+  it("requires a non-empty title", () => {
+    const errors = validateDataciteMetadata({
+      ...validAttrs,
+      titles: [{ title: "" }],
+    })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "titles" }),
+      ]),
+    )
+  })
+
+  it("requires publisher name", () => {
+    const errors = validateDataciteMetadata({
+      ...validAttrs,
+      publisher: { name: "" },
+    })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "publisher" }),
+      ]),
+    )
+  })
+
+  it("requires a four-digit year string for publicationYear", () => {
+    const errors = validateDataciteMetadata({
+      ...validAttrs,
+      publicationYear: "0",
+    })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "publicationYear" }),
+      ]),
+    )
+  })
+
+  it("requires resourceTypeGeneral", () => {
+    const errors = validateDataciteMetadata({
+      ...validAttrs,
+      types: {
+        resourceTypeGeneral: "" as unknown as ResourceTypeGeneral,
+      },
+    })
+    expect(errors).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ field: "types" }),
+      ]),
+    )
+  })
+
+  it("returns multiple errors when multiple fields are invalid", () => {
+    const errors = validateDataciteMetadata({
+      doi: "10.18112/test",
+      url: "https://example.com",
+    })
+    expect(errors.length).toBeGreaterThanOrEqual(4)
+  })
+})

package/src/libs/doi/index.ts

@@ -1,32 +1,5 @@
-import request from "superagent"
 import config from "../../config"
-
-export const template = ({
-  doi,
-  creators,
-  title,
-  year,
-  resourceType,
-}) =>
-  `<?xml version="1.0" encoding="UTF-8"?>
-<resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://datacite.org/schema/kernel-4" xsi:schemaLocation="http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4/metadata.xsd">
-  <identifier identifierType="DOI">${doi}</identifier>
-  <creators>
-  ${
-    creators
-      .map((creator) =>
-        `<creator><creatorName>${creator}</creatorName></creator>`
-      )
-      .join("")
-  }
-  </creators>
-  <titles>
-    <title xml:lang="en-us">${title}</title>
-  </titles>
-  <publisher>Openneuro</publisher>
-  <publicationYear>${year}</publicationYear>
-  <resourceType resourceTypeGeneral="Dataset">${resourceType}</resourceType>
-</resource>`
+import type { DataCite, DataciteDoiRequest } from "../../types/datacite"
 
 /**
  * @param {Object} doiConfig
@@ -37,50 +10,114 @@ export const formatBasicAuth = (doiConfig) =>
   "Basic " +
   Buffer.from(doiConfig.username + ":" + doiConfig.password).toString("base64")
 
-export default {
-  auth: formatBasicAuth(config.doi),
-  createDOI(accNumber, snapshotId) {
-    let doi = config.doi.prefix + "/openneuro." + accNumber
-    if (snapshotId) {
-      doi = doi + ".v" + snapshotId
-    }
-    return doi
-  },
+/**
+ * Build a DOI string from dataset accession number and optional snapshot ID.
+ */
+export function createDOI(accNumber: string, snapshotId?: string): string {
+  let doi = config.doi.prefix + "/openneuro." + accNumber
+  if (snapshotId) {
+    doi = doi + ".v" + snapshotId
+  }
+  return doi
+}
+
+/**
+ * Build the Datacite JSON API request payload.
+ */
+export function buildPayload(
+  attributes: DataCite,
+  event?: DataCite["event"],
+): DataciteDoiRequest {
+  return {
+    data: {
+      type: "dois",
+      attributes: {
+        ...attributes,
+        ...(event ? { event } : {}),
+        schemaVersion: "http://datacite.org/schema/kernel-4",
+      },
+    },
+  }
+}
+
+/**
+ * Create or update a DOI via the Datacite JSON REST API.
+ * Uses PUT to {baseUrl}dois/{doi} which handles both create and update.
+ */
+export async function upsertDoi(
+  payload: DataciteDoiRequest,
+): Promise<Response> {
+  const doi = payload.data.attributes.doi
+  const url = `${config.doi.url}dois/${encodeURIComponent(doi)}`
+  const response = await fetch(url, {
+    method: "PUT",
+    headers: {
+      "Authorization": formatBasicAuth(config.doi),
+      "Content-Type": "application/vnd.api+json",
+    },
+    body: JSON.stringify(payload),
+  })
+  if (!response.ok) {
+    const body = await response.text()
+    throw new Error(
+      `Datacite API error ${response.status} for ${doi}: ${body}`,
+    )
+  }
+  return response
+}
+
+/**
+ * Transition a DOI's state without re-sending full metadata.
+ */
+export async function updateDoiState(
+  doi: string,
+  event: DataCite["event"],
+): Promise<void> {
+  const url = `${config.doi.url}dois/${encodeURIComponent(doi)}`
+  const payload = {
+    data: {
+      type: "dois",
+      attributes: { event },
+    },
+  }
+  const response = await fetch(url, {
+    method: "PUT",
+    headers: {
+      "Authorization": formatBasicAuth(config.doi),
+      "Content-Type": "application/vnd.api+json",
+    },
+    body: JSON.stringify(payload),
+  })
+  if (!response.ok) {
+    const body = await response.text()
+    throw new Error(
+      `Datacite API state transition error ${response.status} for ${doi}: ${body}`,
+    )
+  }
+}
 
-  async mintDOI(doi, url) {
-    return await request
-      .put(config.doi.url + "doi/" + doi)
-      .set("Authorization", this.auth)
-      .set("Content-Type", "text/plain;charset=UTF-8")
-      .send("doi=" + doi + "\nurl=" + url)
-  },
+/**
+ * Create a draft DOI for a dataset snapshot.
+ * Returns the DOI string.
+ */
+export async function createDraftDoi(
+  attributes: DataCite,
+): Promise<string> {
+  const payload = buildPayload(attributes)
+  await upsertDoi(payload)
+  return attributes.doi
+}
 
-  registerMetadata(context) {
-    const xml = template(context)
-    return request
-      .post(config.doi.url + "metadata/")
-      .set("Authorization", this.auth)
-      .set("Content-Type", "application/xml;charset=UTF-8")
-      .send(xml)
-  },
+/**
+ * Transition a DOI from draft to findable.
+ */
+export async function publishDoi(doi: string): Promise<void> {
+  await updateDoiState(doi, "publish")
+}
 
-  registerSnapshotDoi(datasetId, snapshotId, oldDesc) {
-    const baseDoi = this.createDOI(datasetId, snapshotId)
-    const url =
-      `https://openneuro.org/datasets/${datasetId}/versions/${snapshotId}`
-    const context = {
-      doi: baseDoi,
-      creators: oldDesc.Authors.filter((x) => x),
-      title: oldDesc.Name,
-      year: new Date().getFullYear(),
-      resourceType: "fMRI",
-    }
-    return this.registerMetadata(context)
-      .then(() => {
-        return this.mintDOI(baseDoi, url)
-      })
-      .then(() => {
-        return baseDoi
-      })
-  },
+/**
+ * Transition a DOI from findable to registered (hidden but reserved).
+ */
+export async function hideDoi(doi: string): Promise<void> {
+  await updateDoiState(doi, "hide")
 }

package/src/libs/doi/metadata.ts

@@ -0,0 +1,101 @@
+import config from "../../config"
+import { createDOI } from "./index"
+import { validateDataciteMetadata } from "./validate"
+import { getDataciteYml } from "../../utils/datacite-utils"
+import { description } from "../../datalad/description"
+import { getPrimaryModality } from "../../graphql/resolvers/summary"
+import type { Creator, DataCite } from "../../types/datacite"
+
+/**
+ * Assemble Datacite metadata for a DOI from datacite.yml or BIDS fallback.
+ *
+ * Priority:
+ * 1. If datacite.yml exists and has creators, use its attributes as the base.
+ * 2. Otherwise, build minimal metadata from dataset_description.json.
+ *
+ * Always ensures publisher, publicationYear, types.resourceTypeGeneral,
+ * doi, and url are set.
+ */
+export async function assembleMetadata(
+  datasetId: string,
+  snapshotId: string,
+  revision?: string,
+): Promise<DataCite> {
+  const doi = createDOI(datasetId, snapshotId)
+  const url = `${config.url}/datasets/${datasetId}/versions/${snapshotId}`
+
+  const dataciteYml = await getDataciteYml(datasetId, revision)
+  const ymlAttrs = dataciteYml?.data?.attributes
+
+  // Check if datacite.yml provided meaningful creator data
+  const hasDataciteCreators = Array.isArray(ymlAttrs?.creators) &&
+    ymlAttrs.creators.length > 0
+
+  let creators: Creator[]
+  let titles: DataCite["titles"]
+  let descriptions: DataCite["descriptions"]
+  let contributors: DataCite["contributors"]
+  let resourceType: string | undefined
+
+  if (hasDataciteCreators) {
+    // Use datacite.yml metadata
+    creators = ymlAttrs.creators
+    titles = ymlAttrs.descriptions?.length
+      ? [{ title: ymlAttrs.descriptions[0].description }]
+      : []
+    descriptions = ymlAttrs.descriptions
+    contributors = ymlAttrs.contributors
+    resourceType = ymlAttrs.types?.resourceType
+  } else {
+    // Fall back to BIDS dataset_description.json
+    const desc = await description({
+      id: datasetId,
+      revision: revision || "HEAD",
+    })
+    creators = (desc.Authors || [])
+      .filter((author: string) => author)
+      .map((author: string) => ({
+        name: author,
+        nameType: "Personal" as const,
+      }))
+    titles = [{ title: desc.Name || datasetId }]
+    descriptions = desc.Description
+      ? [{ description: desc.Description, descriptionType: "Abstract" }]
+      : undefined
+    contributors = undefined
+    resourceType = await getPrimaryModality(datasetId)
+  }
+
+  // If datacite.yml had titles via a different path, use them
+  if (hasDataciteCreators && titles.length === 0) {
+    const desc = await description({
+      id: datasetId,
+      revision: revision || "HEAD",
+    })
+    titles = [{ title: desc.Name || datasetId }]
+  }
+
+  const attributes: DataCite = {
+    doi,
+    url,
+    creators: creators as DataCite["creators"],
+    titles: titles as DataCite["titles"],
+    publisher: { name: "OpenNeuro" },
+    publicationYear: String(new Date().getFullYear()),
+    types: {
+      resourceTypeGeneral: "Dataset",
+      ...(resourceType ? { resourceType } : {}),
+    },
+    schemaVersion: "http://datacite.org/schema/kernel-4",
+    ...(descriptions ? { descriptions } : {}),
+    ...(contributors?.length ? { contributors } : {}),
+  }
+
+  const errors = validateDataciteMetadata(attributes)
+  if (errors.length > 0) {
+    const messages = errors.map((e) => `${e.field}: ${e.message}`).join("; ")
+    throw new Error(`DOI metadata validation failed: ${messages}`)
+  }
+
+  return attributes
+}

package/src/libs/doi/validate.ts

@@ -0,0 +1,59 @@
+import type { DataCite } from "../../types/datacite"
+
+export interface ValidationError {
+  field: string
+  message: string
+}
+
+/**
+ * Validate required Datacite metadata fields before submitting to the API.
+ * Returns an empty array if valid.
+ */
+export function validateDataciteMetadata(
+  attrs: Partial<DataCite>,
+): ValidationError[] {
+  const errors: ValidationError[] = []
+
+  if (!Array.isArray(attrs.creators) || attrs.creators.length === 0) {
+    errors.push({
+      field: "creators",
+      message: "At least one creator is required",
+    })
+  } else {
+    for (const creator of attrs.creators) {
+      if (!creator.name) {
+        errors.push({
+          field: "creators",
+          message: "Each creator must have a name",
+        })
+        break
+      }
+    }
+  }
+
+  if (!Array.isArray(attrs.titles) || attrs.titles.length === 0) {
+    errors.push({ field: "titles", message: "At least one title is required" })
+  } else if (!attrs.titles[0].title) {
+    errors.push({ field: "titles", message: "Title must not be empty" })
+  }
+
+  if (!attrs.publisher?.name) {
+    errors.push({ field: "publisher", message: "Publisher name is required" })
+  }
+
+  if (!attrs.publicationYear || !/^[0-9]{4}$/.test(attrs.publicationYear)) {
+    errors.push({
+      field: "publicationYear",
+      message: "Publication year must be a four-digit year string",
+    })
+  }
+
+  if (!attrs.types?.resourceTypeGeneral) {
+    errors.push({
+      field: "types",
+      message: "resourceTypeGeneral is required",
+    })
+  }
+
+  return errors
+}

package/src/libs/presign.ts

@@ -0,0 +1,137 @@
+import type { Redis } from "ioredis"
+import { createHMAC, createSHA1 } from "hash-wasm"
+
+const PRESIGN_TTL = 5 * 24 * 60 * 60 // 5 days in seconds
+const PRESIGN_EXPIRATION = 7 * 24 * 60 * 60 // 7 days for the presigned URL itself
+
+const defaultBucket = process.env.AWS_S3_PUBLIC_BUCKET
+const accessKeyId = process.env.AWS_ACCESS_KEY_ID
+const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY
+
+/** Resolve bucket name, falling back to the default configured bucket */
+function resolveBucket(bucket: string): string {
+  return bucket || defaultBucket
+}
+
+function presignKey(bucket: string, s3Key: string, versionId: string): string {
+  return `ps:${bucket}:${s3Key}:${versionId}`
+}
+
+/**
+ * Pre-initialized HMAC-SHA1 signer keyed with the AWS secret.
+ */
+let hmacPromise: ReturnType<typeof createHMAC> | null = null
+
+function getHMAC(): ReturnType<typeof createHMAC> {
+  if (!hmacPromise) {
+    if (!secretAccessKey) {
+      throw new Error("AWS_SECRET_ACCESS_KEY is required for presigned URLs")
+    }
+    hmacPromise = createHMAC(
+      createSHA1(),
+      new TextEncoder().encode(secretAccessKey),
+    )
+  }
+  return hmacPromise
+}
+
+/**
+ * Generate a V2 query-string presigned URL for an S3 GetObject request.
+ * Uses HMAC-SHA1 via hash-wasm (WASM)
+ * Replaced V4 signatures for performance reasons
+ */
+function presignV2(
+  hmac: Awaited<ReturnType<typeof createHMAC>>,
+  bucket: string,
+  s3Key: string,
+  versionId: string,
+  expires: number,
+): string {
+  // StringToSign = HTTP-Verb + "\n" + "\n" + "\n" + Expires + "\n" + CanonicalizedResource
+  const resource = `/${bucket}/${s3Key}?versionId=${versionId}`
+  const stringToSign = `GET\n\n\n${expires}\n${resource}`
+  hmac.init()
+  hmac.update(stringToSign)
+  const signature = Buffer.from(hmac.digest("binary")).toString("base64")
+  const encodedSig = encodeURIComponent(signature)
+  const encodedKey = encodeURIComponent(accessKeyId!)
+  return `https://s3.amazonaws.com/${bucket}/${s3Key}?versionId=${versionId}&AWSAccessKeyId=${encodedKey}&Expires=${expires}&Signature=${encodedSig}`
+}
+
+/**
+ * Get or generate a presigned URL, caching it in Redis.
+ * @param bucket - S3 bucket name, or empty string for the default bucket
+ */
+export async function getPresignedUrl(
+  redis: Redis,
+  bucket: string,
+  s3Key: string,
+  versionId: string,
+): Promise<string> {
+  const resolvedBucket = resolveBucket(bucket)
+  const key = presignKey(resolvedBucket, s3Key, versionId)
+  const cached = await redis.get(key)
+  if (cached) {
+    return cached
+  }
+  const hmac = await getHMAC()
+  const expires = Math.floor(Date.now() / 1000) + PRESIGN_EXPIRATION
+  const url = presignV2(hmac, resolvedBucket, s3Key, versionId, expires)
+  await redis.setex(key, PRESIGN_TTL, url)
+  return url
+}
+
+/**
+ * Bulk-resolve presigned URLs for many files in two pipelined Redis calls.
+ * Returns an array of resolved URLs matching the input order.
+ */
+export async function getPresignedUrlsBulk(
+  redis: Redis,
+  items: { bucket: string; s3Key: string; versionId: string }[],
+): Promise<string[]> {
+  if (items.length === 0) return []
+
+  const resolved = items.map((item) => ({
+    ...item,
+    bucket: resolveBucket(item.bucket),
+  }))
+  const keys = resolved.map((r) => presignKey(r.bucket, r.s3Key, r.versionId))
+  const cached = await redis.mget(...keys)
+
+  // Fill hits from cache, sign misses and queue them for write-back
+  const hmac = await getHMAC()
+  const expires = Math.floor(Date.now() / 1000) + PRESIGN_EXPIRATION
+  const writePipeline = redis.pipeline()
+  let misses = 0
+
+  const results = cached.map((val, i) => {
+    if (val) return val
+    misses++
+    const url = presignV2(
+      hmac,
+      resolved[i].bucket,
+      resolved[i].s3Key,
+      resolved[i].versionId,
+      expires,
+    )
+    writePipeline.setex(keys[i], PRESIGN_TTL, url)
+    return url
+  })
+
+  if (misses > 0) await writePipeline.exec()
+  return results
+}
+
+/**
+ * Build a public (non-presigned) S3 URL from key and versionId.
+ * @param bucket - S3 bucket name, or empty string for the default bucket
+ */
+export function publicS3Url(
+  bucket: string,
+  s3Key: string,
+  versionId: string,
+): string {
+  return `https://s3.amazonaws.com/${
+    resolveBucket(bucket)
+  }/${s3Key}?versionId=${versionId}`
+}

package/src/models/dataset.ts

@@ -29,6 +29,7 @@ export interface DatasetDocument extends Document {
   views: number
   related: [DatasetRelationDocument]
   schemaValidator: boolean
+  holdDeletion: boolean
   _conditions: object
 }
 
@@ -45,6 +46,7 @@ const datasetSchema = new Schema<DatasetDocument>(
     views: Number,
     related: [RelationSchema],
     schemaValidator: { type: Boolean, default: false },
+    holdDeletion: { type: Boolean, default: false },
   },
   { toJSON: { virtuals: true }, toObject: { virtuals: true } },
 )