@openneuro/server 4.4.9 → 4.5.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.4.9",
+  "version": "4.5.1",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -47,7 +47,7 @@
     "jsonwebtoken": "^8.3.0",
     "mime-types": "^2.1.19",
     "moment": "^2.14.1",
-    "mongoose": "6.2.0",
+    "mongoose": "^6.2.3",
     "morgan": "^1.6.1",
     "node-fetch": "^2.6.0",
     "node-mailjet": "^3.3.5",
@@ -104,5 +104,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "af313c7d80f6aade5786075853d00023c52db60e"
+  "gitHead": "92a2879d0f00400c240bfe3319f26c38db8ef866"
 }

package/src/datalad/__tests__/files.spec.js

@@ -3,6 +3,7 @@ import {
   decodeFilePath,
   fileUrl,
   filterFiles,
+  computeTotalSize,
 } from '../files.js'
 
 jest.mock('../../config.js')
@@ -104,4 +105,16 @@ describe('datalad files', () => {
       expect(filterFiles('sub-01/func')(mockFiles)).toEqual([mockSub01[1]])
     })
   })
+  describe('computeTotalSize()', () => {
+    const mockFileSizes = [
+      { filename: 'README', size: 234 },
+      { filename: 'dataset_description.json', size: 432 },
+      { filename: 'sub-01/anat/sub-01_T1w.nii.gz', size: 10858 },
+      {
+        filename: 'sub-01/func/sub-01_task-onebacktask_run-01_bold.nii.gz',
+        size: 1945682,
+      },
+    ]
+    expect(computeTotalSize(mockFileSizes)).toBe(1957206)
+  })
 })

package/src/datalad/description.js

@@ -127,6 +127,7 @@ export const description = obj => {
   return cache
     .get(() => {
       return getFiles(datasetId, revision)
+        .then(response => response.files)
         .then(getDescriptionObject(datasetId))
         .then(uncachedDescription => ({ id: revision, ...uncachedDescription }))
     })

package/src/datalad/files.js

@@ -51,6 +51,12 @@ export const fileUrl = (datasetId, path, filename) => {
 export const filesUrl = datasetId =>
   `http://${getDatasetWorker(datasetId)}/datasets/${datasetId}/files`
 
+/**
+ * Sum all file sizes for total dataset size
+ */
+export const computeTotalSize = files =>
+  files.reduce((size, f) => size + f.size, 0)
+
 /**
  * Get files for a specific revision
  * Similar to getDraftFiles but different cache key and fixed revisions
@@ -75,7 +81,11 @@ export const getFiles = (datasetId, hexsha) => {
           const {
             body: { files },
           } = response
-          return files.map(addFileUrl(datasetId, hexsha))
+          const size = computeTotalSize(files)
+          return {
+            files: files.map(addFileUrl(datasetId, hexsha)),
+            size,
+          }
         }
       }),
   )

package/src/datalad/snapshots.js

@@ -154,7 +154,9 @@ export const createSnapshot = async (
       snapshotChanges,
     )
     snapshot.created = new Date()
-    snapshot.files = await getFiles(datasetId, tag)
+    const { files, size } = await getFiles(datasetId, tag)
+    snapshot.files = files
+    snapshot.size = size
 
     await Promise.all([
       // Update the draft status in datasets collection in case any changes were made (DOI, License)

package/src/graphql/resolvers/brainlife.ts

@@ -1,3 +1,4 @@
+import fetch from 'node-fetch'
 import {
   DatasetOrSnapshot,
   getDatasetFromSnapshotId,

package/src/graphql/resolvers/dataset.js

@@ -16,9 +16,7 @@ import { history } from './history.js'
 import * as dataladAnalytics from '../../datalad/analytics.js'
 import DatasetModel from '../../models/dataset'
 import Deletion from '../../models/deletion'
-import fetch from 'node-fetch'
 import { reviewers } from './reviewer'
-import { UpdatedFile } from '../utils/file.js'
 import { getDatasetWorker } from '../../libs/datalad-service.js'
 import { getDraftHead } from '../../datalad/dataset.js'
 import { getFileName } from '../../datalad/files.js'
@@ -152,9 +150,7 @@ export const deleteFiles = async (
 ) => {
   try {
     await checkDatasetWrite(datasetId, user, userInfo)
-    const deletedFiles = await datalad
-      .deleteFiles(datasetId, files, userInfo)
-      .then(filenames => filenames.map(filename => new UpdatedFile(filename)))
+    const deletedFiles = await datalad.deleteFiles(datasetId, files, userInfo)
     pubsub.publish('filesUpdated', {
       datasetId,
       filesUpdated: {

package/src/graphql/resolvers/draft.js

@@ -10,11 +10,17 @@ import { filterRemovedAnnexObjects } from '../utils/file.js'
 // A draft must have a dataset parent
 const draftFiles = async (dataset, args, { userInfo }) => {
   const hexsha = await getDraftRevision(dataset.id)
-  const files = await getFiles(dataset.id, hexsha)
+  const { files } = await getFiles(dataset.id, hexsha)
   const prefixFiltered = filterFiles('prefix' in args && args.prefix)(files)
   return filterRemovedAnnexObjects(dataset.id, userInfo)(prefixFiltered)
 }
 
+const draftSize = async (dataset, args, { userInfo }) => {
+  const hexsha = await getDraftRevision(dataset.id)
+  const { size } = await getFiles(dataset.id, hexsha)
+  return size
+}
+
 /**
  * Deprecated mutation to move the draft HEAD reference forward or backward
  *
@@ -39,6 +45,7 @@ export const revalidate = async (obj, { datasetId }, { user, userInfo }) => {
 const draft = {
   id: obj => obj.id,
   files: draftFiles,
+  size: draftSize,
   summary,
   issues,
   modified: obj => obj.modified,

package/src/graphql/resolvers/permissions.js

@@ -44,7 +44,9 @@ const publishPermissions = async datasetId => {
 export const updatePermissions = async (obj, args, { user, userInfo }) => {
   await checkDatasetAdmin(args.datasetId, user, userInfo)
   // get all users the the email specified by permissions arg
-  const users = await User.find({ email: args.userEmail }).exec()
+  const users = await User.find({ email: args.userEmail })
+    .collation({ locale: 'en', strength: 2 })
+    .exec()
 
   if (!users.length) {
     throw new Error('A user with that email address does not exist')

package/src/graphql/resolvers/snapshots.js

@@ -29,8 +29,11 @@ export const snapshot = (obj, { datasetId, tag }, context) => {
         summary: () => summary({ id: datasetId, revision: snapshot.hexsha }),
         files: ({ prefix }) =>
           getFiles(datasetId, snapshot.hexsha)
+            .then(response => response.files)
             .then(filterFiles(prefix))
             .then(filterRemovedAnnexObjects(datasetId, context.userInfo)),
+        size: () =>
+          getFiles(datasetId, snapshot.hexsha).then(response => response.size),
         deprecated: () => deprecated({ datasetId, tag }),
         related: () => related(datasetId),
         onBrainlife: () => onBrainlife(snapshot),

package/src/graphql/schema.js

@@ -430,6 +430,8 @@ export const typeDefs = `
     uploads: [UploadMetadata]
     # Git commit hash
     head: String
+    # Total size in bytes of this draft
+    size: Int
   }
 
   # Tagged snapshot of a draft
@@ -461,6 +463,8 @@ export const typeDefs = `
     related: [RelatedObject]
     # Is the snapshot available for analysis on Brainlife?
     onBrainlife: Boolean @cacheControl(maxAge: 10080, scope: PUBLIC)
+    # Total size in bytes of this snapshot
+    size: Int
   }
 
   # RelatedObject nature of relationship

package/src/graphql/utils/file.js

@@ -1,34 +1,5 @@
 import BadAnnexObject from '../../models/badAnnexObject'
 
-/**
- * Generates unique id for untracked files.
- * @param {string} filepath - filepath ('/' delimiters)
- * @param {number|string} [size] - of file
- */
-export const generateFileId = (filepath, size) => `${filepath}:${size}`
-
-/**
- * Creates a file object with an ApolloGQL cache-safe id.
- * @class
- * @param {string} filepath ':' delimited
- * @param {string|number} [size]
- */
-export function UpdatedFile(filepath, size) {
-  /**
-   * unique id
-   * @id UpdatedFile#id
-   * @type {string}
-   */
-  this.id = generateFileId(filepath, size)
-  /**
-   * filename with '/' delimiters
-   * @filename UpdatedFile#filename
-   * @type {string}
-   */
-  this.filename = filepath
-  if (size) this.size = size
-}
-
 export const filterRemovedAnnexObjects =
   (datasetId, userInfo) => async files => {
     const removedAnnexObjectKeys = (

package/src/models/user.ts

@@ -29,6 +29,16 @@ const userSchema = new Schema({
 })
 
 userSchema.index({ id: 1, provider: 1 }, { unique: true })
+// Allow case insensitive email queries
+userSchema.index(
+  { email: 1 },
+  {
+    collation: {
+      locale: 'en',
+      strength: 2,
+    },
+  },
+)
 
 const User = model<UserDocument>('User', userSchema)