@openneuro/server 4.47.6 → 5.0.0-alpha.0

package/src/libs/presign.ts

@@ -0,0 +1,137 @@
+import type { Redis } from "ioredis"
+import { createHMAC, createSHA1 } from "hash-wasm"
+
+const PRESIGN_TTL = 5 * 24 * 60 * 60 // 5 days in seconds
+const PRESIGN_EXPIRATION = 7 * 24 * 60 * 60 // 7 days for the presigned URL itself
+
+const defaultBucket = process.env.AWS_S3_PUBLIC_BUCKET
+const accessKeyId = process.env.AWS_ACCESS_KEY_ID
+const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY
+
+/** Resolve bucket name, falling back to the default configured bucket */
+function resolveBucket(bucket: string): string {
+  return bucket || defaultBucket
+}
+
+function presignKey(bucket: string, s3Key: string, versionId: string): string {
+  return `ps:${bucket}:${s3Key}:${versionId}`
+}
+
+/**
+ * Pre-initialized HMAC-SHA1 signer keyed with the AWS secret.
+ */
+let hmacPromise: ReturnType<typeof createHMAC> | null = null
+
+function getHMAC(): ReturnType<typeof createHMAC> {
+  if (!hmacPromise) {
+    if (!secretAccessKey) {
+      throw new Error("AWS_SECRET_ACCESS_KEY is required for presigned URLs")
+    }
+    hmacPromise = createHMAC(
+      createSHA1(),
+      new TextEncoder().encode(secretAccessKey),
+    )
+  }
+  return hmacPromise
+}
+
+/**
+ * Generate a V2 query-string presigned URL for an S3 GetObject request.
+ * Uses HMAC-SHA1 via hash-wasm (WASM)
+ * Replaced V4 signatures for performance reasons
+ */
+function presignV2(
+  hmac: Awaited<ReturnType<typeof createHMAC>>,
+  bucket: string,
+  s3Key: string,
+  versionId: string,
+  expires: number,
+): string {
+  // StringToSign = HTTP-Verb + "\n" + "\n" + "\n" + Expires + "\n" + CanonicalizedResource
+  const resource = `/${bucket}/${s3Key}?versionId=${versionId}`
+  const stringToSign = `GET\n\n\n${expires}\n${resource}`
+  hmac.init()
+  hmac.update(stringToSign)
+  const signature = Buffer.from(hmac.digest("binary")).toString("base64")
+  const encodedSig = encodeURIComponent(signature)
+  const encodedKey = encodeURIComponent(accessKeyId!)
+  return `https://s3.amazonaws.com/${bucket}/${s3Key}?versionId=${versionId}&AWSAccessKeyId=${encodedKey}&Expires=${expires}&Signature=${encodedSig}`
+}
+
+/**
+ * Get or generate a presigned URL, caching it in Redis.
+ * @param bucket - S3 bucket name, or empty string for the default bucket
+ */
+export async function getPresignedUrl(
+  redis: Redis,
+  bucket: string,
+  s3Key: string,
+  versionId: string,
+): Promise<string> {
+  const resolvedBucket = resolveBucket(bucket)
+  const key = presignKey(resolvedBucket, s3Key, versionId)
+  const cached = await redis.get(key)
+  if (cached) {
+    return cached
+  }
+  const hmac = await getHMAC()
+  const expires = Math.floor(Date.now() / 1000) + PRESIGN_EXPIRATION
+  const url = presignV2(hmac, resolvedBucket, s3Key, versionId, expires)
+  await redis.setex(key, PRESIGN_TTL, url)
+  return url
+}
+
+/**
+ * Bulk-resolve presigned URLs for many files in two pipelined Redis calls.
+ * Returns an array of resolved URLs matching the input order.
+ */
+export async function getPresignedUrlsBulk(
+  redis: Redis,
+  items: { bucket: string; s3Key: string; versionId: string }[],
+): Promise<string[]> {
+  if (items.length === 0) return []
+
+  const resolved = items.map((item) => ({
+    ...item,
+    bucket: resolveBucket(item.bucket),
+  }))
+  const keys = resolved.map((r) => presignKey(r.bucket, r.s3Key, r.versionId))
+  const cached = await redis.mget(...keys)
+
+  // Fill hits from cache, sign misses and queue them for write-back
+  const hmac = await getHMAC()
+  const expires = Math.floor(Date.now() / 1000) + PRESIGN_EXPIRATION
+  const writePipeline = redis.pipeline()
+  let misses = 0
+
+  const results = cached.map((val, i) => {
+    if (val) return val
+    misses++
+    const url = presignV2(
+      hmac,
+      resolved[i].bucket,
+      resolved[i].s3Key,
+      resolved[i].versionId,
+      expires,
+    )
+    writePipeline.setex(keys[i], PRESIGN_TTL, url)
+    return url
+  })
+
+  if (misses > 0) await writePipeline.exec()
+  return results
+}
+
+/**
+ * Build a public (non-presigned) S3 URL from key and versionId.
+ * @param bucket - S3 bucket name, or empty string for the default bucket
+ */
+export function publicS3Url(
+  bucket: string,
+  s3Key: string,
+  versionId: string,
+): string {
+  return `https://s3.amazonaws.com/${
+    resolveBucket(bucket)
+  }/${s3Key}?versionId=${versionId}`
+}

package/src/models/dataset.ts

@@ -29,6 +29,7 @@ export interface DatasetDocument extends Document {
   views: number
   related: [DatasetRelationDocument]
   schemaValidator: boolean
+  holdDeletion: boolean
   _conditions: object
 }
 
@@ -45,6 +46,7 @@ const datasetSchema = new Schema<DatasetDocument>(
     views: Number,
     related: [RelationSchema],
     schemaValidator: { type: Boolean, default: false },
+    holdDeletion: { type: Boolean, default: false },
   },
   { toJSON: { virtuals: true }, toObject: { virtuals: true } },
 )

package/src/models/doi.ts

@@ -1,17 +1,24 @@
 import mongoose from "mongoose"
 import type { Document } from "mongoose"
+import type { DoiState } from "../types/datacite"
 const { Schema, model } = mongoose
 
 export interface DoiDocument extends Document {
   datasetId: string
   snapshotId: string
   doi: string
+  state: DoiState
 }
 
 const doiSchema = new Schema({
   datasetId: String,
   snapshotId: String,
   doi: String,
+  state: {
+    type: String,
+    enum: ["draft", "registered", "findable"],
+    default: "draft",
+  },
 })
 
 const Doi = model<DoiDocument>("Doi", doiSchema)

package/src/queues/producer-methods.ts

@@ -25,14 +25,18 @@ export function queueIndexDataset(datasetId: string) {
  * Queue data retention check for a dataset
  * @param datasetId Dataset to check
  */
-export function queueDataRetentionCheck(datasetId: string) {
+export async function queueDataRetentionCheck(
+  datasetId: string,
+): Promise<void> {
   try {
     const msg = new ProducibleMessage()
     msg.setQueue(OpenNeuroQueues.DATARETENTION).setBody({ datasetId })
-    producer.produce(msg, (err) => {
-      if (err) {
-        Sentry.captureException(err)
-      }
+    msg.setTTL(64800000) // 18 hours in ms to survive the consumer rate limits
+    await new Promise<void>((resolve, reject) => {
+      producer.produce(msg, (err) => {
+        if (err) reject(err)
+        else resolve()
+      })
     })
   } catch (err) {
     Sentry.captureException(err)

package/src/queues/queue-schedule.ts

@@ -14,7 +14,7 @@ async function enqueueAllDatasetChecks(): Promise<void> {
   const cursor = Dataset.find({}, "id").cursor()
   for await (const dataset of cursor) {
     // Check data retention policy status and send notifications
-    queueDataRetentionCheck(dataset.id)
+    await queueDataRetentionCheck(dataset.id)
   }
 }
 

package/src/queues/queues.ts

@@ -55,6 +55,6 @@ export async function setupQueues(): Promise<void> {
   // Limit indexing queue to 8 runs per minute to avoid stacking indexing excessively
   await setRateLimit(OpenNeuroQueues.INDEXING, 8, 60000)
 
-  // Rate limit data retention queue to 16 runs per minute
-  await setRateLimit(OpenNeuroQueues.DATARETENTION, 16, 60000)
+  // Rate limit data retention queue to 60 runs per minute
+  await setRateLimit(OpenNeuroQueues.DATARETENTION, 60, 60000)
 }

package/src/routes.ts

@@ -190,8 +190,16 @@ const routes = [
   // git redirect routes
   { method: "get", url: "/git/:datasetId", handler: datalad.gitRepo },
   { method: "post", url: "/git/:datasetId", handler: datalad.gitRepo },
-  { method: "get", url: "/git/:datasetId/*", handler: datalad.gitRepo },
-  { method: "post", url: "/git/:datasetId/*", handler: datalad.gitRepo },
+  {
+    method: "get",
+    url: "/git/:datasetId/*arguments",
+    handler: datalad.gitRepo,
+  },
+  {
+    method: "post",
+    url: "/git/:datasetId/*arguments",
+    handler: datalad.gitRepo,
+  },
 ]
 
 // initialize routes -------------------------------

package/src/types/datacite/LICENSE

@@ -0,0 +1,37 @@
+DataCite is free software; you can redistribute it and/or modify
+it under the terms of the Revised BSD License quoted below.
+
+Copyright (C) 2015-2018 CERN.
+Copyright (C) 2018 Center for Open Science.
+Copyright (C) 2019-2024 Caltech.
+Copyright (C) 2024 Institute of Biotechnology of the Czech Academy of Sciences.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.

package/src/types/datacite/README.md

@@ -0,0 +1,3 @@
+Generate `datacite-v4.5.ts` with `npx json-schema-to-typescript datacite-v4.5.json`.
+
+`datacite-v4.5.json` sourced from https://github.com/inveniosoftware/datacite/blob/5506d1347a070952d2c2b96c213f44c5fa46d0dd/datacite/schemas/datacite-v4.5.json