@openneuro/server 5.0.0 → 5.1.0

package/src/handlers/subscriptions.ts

@@ -64,7 +64,7 @@ export const deleteAll = (req, res, next) => {
     .then((subscriptions) => {
       subscriptions.forEach((subscription) => {
         Subscription.deleteOne({
-          _id: new ObjectID(subscription._id),
+          _id: new ObjectID(subscription._id.toString()),
         })
       })
       return res.send()

package/src/libs/apikey.ts

@@ -20,7 +20,7 @@ export const generateApiKey = (user) => {
       id: userId,
       hash: token,
     },
-    { upsert: true, new: true },
+    { upsert: true },
   )
     .exec()
     .then(() => ({ key: token }))

package/src/libs/authentication/crypto.ts

@@ -1,17 +1,22 @@
 import crypto from "crypto"
 import config from "../../config"
 
-const secret = config.auth.jwt.secret
 const algorithm = "aes256"
-const key = crypto
-  .createHash("sha256")
-  .update(secret)
-  .digest("base64")
-  .substr(0, 32)
-
 const delimiter = "."
 const encoding = "base64"
 
+let _key: string | null = null
+function getKey(): string {
+  if (!_key) {
+    _key = crypto
+      .createHash("sha256")
+      .update(config.auth.jwt.secret)
+      .digest("base64")
+      .substr(0, 32)
+  }
+  return _key
+}
+
 const pack = (iv, encrypted) => iv.toString(encoding) + delimiter + encrypted
 
 const unpack = (encryptedPackage) => {
@@ -22,7 +27,7 @@ const unpack = (encryptedPackage) => {
 
 export const encrypt = (plainText) => {
   const iv = crypto.randomBytes(16)
-  const cipher = crypto.createCipheriv(algorithm, key, iv)
+  const cipher = crypto.createCipheriv(algorithm, getKey(), iv)
   const encryptedText = cipher.update(plainText, "utf8", "hex") +
     cipher.final("hex")
   const encryptedPackage = pack(iv, encryptedText)
@@ -31,7 +36,7 @@ export const encrypt = (plainText) => {
 
 export const decrypt = (encryptedPackage) => {
   const [iv, encryptedText] = unpack(encryptedPackage)
-  const decipher = crypto.createDecipheriv(algorithm, key, iv)
+  const decipher = crypto.createDecipheriv(algorithm, getKey(), iv)
   const decryptedText = decipher.update(encryptedText, "hex", "utf8") +
     decipher.final("utf8")
   return decryptedText

package/src/libs/notifications.ts

@@ -141,7 +141,7 @@ const notifications = {
                     from: "reply-" +
                       encodeURIComponent(comment._id) +
                       "-" +
-                      encodeURIComponent(user._id),
+                      encodeURIComponent(user._id.toString()),
                     subject: "Comment Created",
                     html: commentCreated({
                       name: user.name,

package/src/libs/presign.ts

@@ -82,7 +82,7 @@ export async function getPresignedUrl(
 }
 
 /**
- * Bulk-resolve presigned URLs for many files in two pipelined Redis calls.
+ * Bulk-resolve presigned URLs for many files in pipelined Redis calls.
  * Returns an array of resolved URLs matching the input order.
  */
 export async function getPresignedUrlsBulk(
@@ -96,29 +96,41 @@ export async function getPresignedUrlsBulk(
     bucket: resolveBucket(item.bucket),
   }))
   const keys = resolved.map((r) => presignKey(r.bucket, r.s3Key, r.versionId))
-  const cached = await redis.mget(...keys)
 
-  // Fill hits from cache, sign misses and queue them for write-back
   const hmac = await getHMAC()
   const expires = Math.floor(Date.now() / 1000) + PRESIGN_EXPIRATION
-  const writePipeline = redis.pipeline()
-  let misses = 0
-
-  const results = cached.map((val, i) => {
-    if (val) return val
-    misses++
-    const url = presignV2(
-      hmac,
-      resolved[i].bucket,
-      resolved[i].s3Key,
-      resolved[i].versionId,
-      expires,
-    )
-    writePipeline.setex(keys[i], PRESIGN_TTL, url)
-    return url
-  })
+  const results: string[] = []
+  const CHUNK_SIZE = 1000
+
+  // Process in chunks to avoid blocking Redis
+  for (let i = 0; i < keys.length; i += CHUNK_SIZE) {
+    const chunkKeys = keys.slice(i, i + CHUNK_SIZE)
+    const chunkResolved = resolved.slice(i, i + CHUNK_SIZE)
+    const cached = await redis.mget(chunkKeys)
+    const writePipeline = redis.pipeline()
+    let misses = 0
+
+    for (let j = 0; j < chunkKeys.length; j++) {
+      let val = cached[j]
+      if (!val) {
+        misses++
+        val = presignV2(
+          hmac,
+          chunkResolved[j].bucket,
+          chunkResolved[j].s3Key,
+          chunkResolved[j].versionId,
+          expires,
+        )
+        writePipeline.setex(chunkKeys[j], PRESIGN_TTL, val)
+      }
+      results.push(val)
+    }
+
+    if (misses > 0) {
+      await writePipeline.exec()
+    }
+  }
 
-  if (misses > 0) await writePipeline.exec()
   return results
 }
 

package/src/libs/redis.ts

@@ -5,5 +5,15 @@ import Redis from "ioredis"
 import Redlock from "redlock"
 import config from "../config"
 
-export const redis = new Redis(config.redis)
-export const redlock = new Redlock([redis])
+let _redis: Redis | null = null
+let _redlock: Redlock | null = null
+
+export function getRedis(): Redis {
+  if (!_redis) _redis = new Redis(config.redis)
+  return _redis
+}
+
+export function getRedlock(): Redlock {
+  if (!_redlock) _redlock = new Redlock([getRedis()])
+  return _redlock
+}

package/src/models/comment.ts

@@ -1,15 +1,13 @@
 import mongoose from "mongoose"
 import type { Document } from "mongoose"
+import type { UserDocument } from "./user"
 const { Schema, model } = mongoose
 
 export interface CommentDocument extends Document {
-  _id: string
   createDate: Date
   datasetId: string
   datasetLabel: string
-  user: {
-    _id: string
-  }
+  user: UserDocument
   parentId: string
   text: string
 }

package/src/models/counter.ts

@@ -2,12 +2,12 @@ import mongoose from "mongoose"
 import type { Document } from "mongoose"
 const { Schema, model } = mongoose
 
-export interface CounterDocument extends Document {
+export interface CounterDocument extends Omit<Document, "_id"> {
   _id: string
   sequence_value: number
 }
 const countersSchema = new Schema({
-  _id: { type: String },
+  _id: { type: String, required: true },
   sequence_value: { type: Number, default: 0 },
 })
 

package/src/models/ingestDataset.ts

@@ -2,12 +2,11 @@
  * Model for ingest of new datasets from a remote URL (zip/tarball)
  */
 import mongoose from "mongoose"
-import type { Document, ObjectId } from "mongoose"
+import type { Document } from "mongoose"
 const { Schema, model } = mongoose
 import { validateUrl } from "../utils/validateUrl"
 
 export interface IngestDatasetDocument extends Document {
-  _id: ObjectId
   datasetId: string
   userId: string
   url: string

package/src/models/notification.ts

@@ -3,7 +3,6 @@ import type { Document } from "mongoose"
 const { Schema, model } = mongoose
 
 export interface NotificationDocument extends Document {
-  _id: string
   type: string
   email: {
     to: string
@@ -38,7 +37,6 @@ export interface NotificationDocument extends Document {
 }
 
 const notificationSchema = new Schema({
-  _id: String,
   type: String,
   email: {
     to: String,

package/src/models/subscription.ts

@@ -3,7 +3,6 @@ import type { Document } from "mongoose"
 const { Schema, model } = mongoose
 
 export interface SubscriptionDocument extends Document {
-  _id: string
   datasetId: string
   userId: string
 }

package/src/models/user.ts

@@ -4,7 +4,6 @@ import type { Document } from "mongoose"
 const { Schema, model } = mongoose
 
 export interface UserDocument extends Document {
-  _id: string
   // OpenNeuro specific user uuid
   id: string
   // Best contact email for the user (notifications)
@@ -12,7 +11,7 @@ export interface UserDocument extends Document {
   // User's preferred name (visible)
   name: string
   // Login provider
-  provider: StaticRangeInit
+  provider: string
   // The id from the login provider
   providerId: string
   // ORCID iD associated with this OpenNeuro user

package/src/models/userMigration.ts

@@ -4,7 +4,7 @@ import type { Document } from "mongoose"
 const { Schema, model } = mongoose
 
 export interface UserMigrationDocument extends Document {
-  _id: string
+  id: string
   orcid: string
   google: string
   users: object[]

package/src/models/userNotificationStatus.ts

@@ -5,7 +5,6 @@ const { Schema, model } = mongoose
 export type NotificationStatusType = "UNREAD" | "SAVED" | "ARCHIVED"
 
 export interface UserNotificationStatusDocument extends Document {
-  _id: string
   userId: string
   datasetEventId: string
   status: NotificationStatusType

package/src/utils/__tests__/snapshots.spec.ts

@@ -0,0 +1,120 @@
+import { snapshotCreationComparison } from "../snapshots"
+
+describe("snapshotCreationComparison()", () => {
+  it('sorts array of objects by the "created" and "tag" properties', () => {
+    const testArray = [
+      { id: 2, created: new Date("2018-11-20T00:05:43.473Z"), tag: "1.0.0" },
+      { id: 1, created: new Date("2018-11-19T00:05:43.473Z"), tag: "1.0.1" },
+      { id: 3, created: new Date("2018-11-23T00:05:43.473Z"), tag: "1.0.2" },
+      { id: 5, created: new Date("2018-11-23T00:05:43.473Z"), tag: "1.0.10" },
+      { id: 4, created: new Date("2018-11-23T00:05:43.473Z"), tag: "1.0.3" },
+    ]
+    const sorted = testArray.sort(snapshotCreationComparison)
+    expect(sorted[0].id).toBe(2)
+    expect(sorted[1].id).toBe(1)
+    expect(sorted[2].id).toBe(3)
+    expect(sorted[3].id).toBe(4)
+    expect(sorted[4].id).toBe(5)
+  })
+  it('sorts array of objects by the "created" property as strings', () => {
+    const testArray = [
+      { id: 2, created: "2018-11-20T00:05:43.473Z", tag: "2.0.0" },
+      { id: 1, created: "2018-11-19T00:05:43.473Z", tag: "1.0.0" },
+      { id: 3, created: "2018-11-23T00:05:43.473Z", tag: "3.0.0" },
+    ]
+    const sorted = testArray.sort(snapshotCreationComparison)
+    expect(sorted[0].id).toBe(1)
+    expect(sorted[1].id).toBe(2)
+    expect(sorted[2].id).toBe(3)
+  })
+  it("sorts non-semver tags mixed with semver tags", () => {
+    const testArray = [
+      { id: 2, created: new Date("2018-11-19T00:05:43.473Z"), tag: "1.0.2" },
+      {
+        id: 1,
+        created: new Date("2018-11-19T00:05:43.473Z"),
+        tag: "57fed018cce88d000ac1757f",
+      },
+      { id: 3, created: new Date("2018-11-19T00:05:43.473Z"), tag: "1.0.1" },
+    ]
+    const sorted = testArray.sort(snapshotCreationComparison)
+    expect(sorted[0].id).toBe(2)
+    expect(sorted[1].id).toBe(1)
+    expect(sorted[2].id).toBe(3)
+  })
+  it("sorts snapshots with only non-semver tags", () => {
+    const testArray = [
+      {
+        id: 2,
+        created: new Date("2018-11-19T00:05:43.473Z"),
+        tag: "00001",
+      },
+      {
+        id: 1,
+        created: new Date("2018-11-19T00:05:43.473Z"),
+        tag: "57fed018cce88d000ac1757f",
+      },
+      {
+        id: 3,
+        created: new Date("2018-11-19T00:05:43.473Z"),
+        tag: "57fed018cce88d000ac1757f",
+      },
+    ]
+    const sorted = testArray.sort(snapshotCreationComparison)
+    expect(sorted[0].id).toBe(2)
+    expect(sorted[1].id).toBe(1)
+    expect(sorted[2].id).toBe(3)
+  })
+  it("sorts very similar creation times by semver order", () => {
+    const testSnapshots = [
+      {
+        id: "ds002680:1.0.0",
+        created: "2020-04-03T23:19:56.000Z",
+        tag: "1.0.0",
+      },
+      {
+        id: "ds002680:1.2.0",
+        created: "2021-10-19T16:26:43.000Z",
+        tag: "1.2.0",
+      },
+      {
+        id: "ds002680:1.1.0",
+        created: "2021-10-19T16:26:44.000Z",
+        tag: "1.1.0",
+      },
+    ]
+    const sorted = testSnapshots.sort(snapshotCreationComparison)
+    expect(sorted[0].id).toBe("ds002680:1.0.0")
+    expect(sorted[1].id).toBe("ds002680:1.1.0")
+    expect(sorted[2].id).toBe("ds002680:1.2.0")
+  })
+  it("sorts 000002 (legacy snapshots) before 1.0.1 (current format)", () => {
+    const testSnapshots = [
+      {
+        id: "ds000247:00002",
+        created: "2018-07-18T02:27:39.000Z",
+        tag: "00002",
+      },
+      {
+        id: "ds000247:00001",
+        created: "2018-07-18T02:35:37.000Z",
+        tag: "00001",
+      },
+      {
+        id: "ds000247:1.0.0",
+        created: "2021-07-05T15:58:18.000Z",
+        tag: "1.0.0",
+      },
+      {
+        id: "ds000247:1.0.1",
+        created: "2021-08-25T23:37:53.000Z",
+        tag: "1.0.1",
+      },
+    ]
+    const sorted = testSnapshots.sort(snapshotCreationComparison)
+    expect(sorted[0].id).toBe("ds000247:00002")
+    expect(sorted[1].id).toBe("ds000247:00001")
+    expect(sorted[2].id).toBe("ds000247:1.0.0")
+    expect(sorted[3].id).toBe("ds000247:1.0.1")
+  })
+})

package/src/utils/snapshots.ts

@@ -0,0 +1,12 @@
+import semver from "semver"
+
+export const snapshotCreationComparison = (
+  { created: a, tag: a_tag }: { created: Date | string; tag: string },
+  { created: b, tag: b_tag }: { created: Date | string; tag: string },
+) => {
+  if (semver.valid(a_tag) && semver.valid(b_tag)) {
+    return semver.compare(a_tag, b_tag)
+  } else {
+    return new Date(a).getTime() - new Date(b).getTime()
+  }
+}