@openneuro/server 4.4.10 → 4.6.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.4.10",
+  "version": "4.6.0-alpha.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -18,6 +18,7 @@
   "dependencies": {
     "@apollo/client": "3.4.17",
     "@elastic/elasticsearch": "7.15.0",
+    "@openneuro/search": "^4.6.0-alpha.0",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@sentry/node": "^4.5.3",
     "apollo-server": "2.25.3",
@@ -104,5 +105,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "d3e7a6459132b92d678665a081efd666e3088461"
+  "gitHead": "ce2db27f750c2614e9cf2f0461add04ca8c3cb48"
 }

package/src/datalad/__tests__/files.spec.js

@@ -3,6 +3,7 @@ import {
   decodeFilePath,
   fileUrl,
   filterFiles,
+  computeTotalSize,
 } from '../files.js'
 
 jest.mock('../../config.js')
@@ -104,4 +105,16 @@ describe('datalad files', () => {
       expect(filterFiles('sub-01/func')(mockFiles)).toEqual([mockSub01[1]])
     })
   })
+  describe('computeTotalSize()', () => {
+    const mockFileSizes = [
+      { filename: 'README', size: 234 },
+      { filename: 'dataset_description.json', size: 432 },
+      { filename: 'sub-01/anat/sub-01_T1w.nii.gz', size: 10858 },
+      {
+        filename: 'sub-01/func/sub-01_task-onebacktask_run-01_bold.nii.gz',
+        size: 1945682,
+      },
+    ]
+    expect(computeTotalSize(mockFileSizes)).toBe(1957206)
+  })
 })

package/src/datalad/description.js

@@ -127,6 +127,7 @@ export const description = obj => {
   return cache
     .get(() => {
       return getFiles(datasetId, revision)
+        .then(response => response.files)
         .then(getDescriptionObject(datasetId))
         .then(uncachedDescription => ({ id: revision, ...uncachedDescription }))
     })

package/src/datalad/files.js

@@ -51,6 +51,12 @@ export const fileUrl = (datasetId, path, filename) => {
 export const filesUrl = datasetId =>
   `http://${getDatasetWorker(datasetId)}/datasets/${datasetId}/files`
 
+/**
+ * Sum all file sizes for total dataset size
+ */
+export const computeTotalSize = files =>
+  files.reduce((size, f) => size + f.size, 0)
+
 /**
  * Get files for a specific revision
  * Similar to getDraftFiles but different cache key and fixed revisions
@@ -75,7 +81,11 @@ export const getFiles = (datasetId, hexsha) => {
           const {
             body: { files },
           } = response
-          return files.map(addFileUrl(datasetId, hexsha))
+          const size = computeTotalSize(files)
+          return {
+            files: files.map(addFileUrl(datasetId, hexsha)),
+            size,
+          }
         }
       }),
   )

package/src/datalad/snapshots.js

@@ -2,6 +2,7 @@
  * Get snapshots from datalad-service tags
  */
 import request from 'superagent'
+import { reindexDataset } from '../elasticsearch/reindex-dataset'
 import { redis, redlock } from '../libs/redis'
 import CacheItem, { CacheType } from '../cache/item'
 import config from '../config.js'
@@ -154,7 +155,9 @@ export const createSnapshot = async (
       snapshotChanges,
     )
     snapshot.created = new Date()
-    snapshot.files = await getFiles(datasetId, tag)
+    const { files, size } = await getFiles(datasetId, tag)
+    snapshot.files = files
+    snapshot.size = size
 
     await Promise.all([
       // Update the draft status in datasets collection in case any changes were made (DOI, License)
@@ -167,15 +170,17 @@ export const createSnapshot = async (
       updateDatasetName(datasetId),
     ])
 
-    snapshotLock.unlock()
+    await reindexDataset(datasetId)
+
     announceNewSnapshot(snapshot, datasetId, user)
     return snapshot
   } catch (err) {
     // delete the keys if any step fails
     // this avoids inconsistent cache state after failures
     snapshotCache.drop()
-    snapshotLock.unlock()
     return err
+  } finally {
+    snapshotLock.unlock()
   }
 }
 

package/src/elasticsearch/elastic-client.js

@@ -6,6 +6,6 @@ const elasticConfig = {
   maxRetries: 3,
 }
 
-const elasticClient = new Client(elasticConfig)
+export const elasticClient = new Client(elasticConfig)
 
 export default elasticClient

package/src/elasticsearch/reindex-dataset.ts

@@ -0,0 +1,41 @@
+import { indexDataset, queryForIndex, indexingToken } from '@openneuro/search'
+import { elasticClient } from './elastic-client'
+import {
+  from,
+  ApolloClient,
+  InMemoryCache,
+  NormalizedCacheObject,
+} from '@apollo/client'
+import { setContext } from '@apollo/client/link/context'
+import { HttpLink } from '@apollo/client/link/http'
+import fetch from 'node-fetch'
+
+/**
+ * Setup SchemaLink based client for querying
+ */
+export const schemaLinkClient = (): ApolloClient<NormalizedCacheObject> => {
+  const accessToken = indexingToken()
+  const authLink = setContext((_, { headers }) => {
+    return {
+      headers: {
+        ...headers,
+        Cookie: `accessToken=${accessToken}`,
+      },
+    }
+  })
+  const httpLink = new HttpLink({
+    uri: process.env.GRAPHQL_URI,
+    fetch,
+  })
+  return new ApolloClient({
+    link: from([authLink, httpLink]),
+    cache: new InMemoryCache(),
+  })
+}
+
+const client = schemaLinkClient()
+
+export const reindexDataset = async (datasetId: string): Promise<void> => {
+  const datasetIndexQueryResult = await queryForIndex(client, datasetId)
+  await indexDataset(elasticClient, datasetIndexQueryResult.data.dataset)
+}

package/src/graphql/resolvers/dataset.js

@@ -16,9 +16,7 @@ import { history } from './history.js'
 import * as dataladAnalytics from '../../datalad/analytics.js'
 import DatasetModel from '../../models/dataset'
 import Deletion from '../../models/deletion'
-import fetch from 'node-fetch'
 import { reviewers } from './reviewer'
-import { UpdatedFile } from '../utils/file.js'
 import { getDatasetWorker } from '../../libs/datalad-service.js'
 import { getDraftHead } from '../../datalad/dataset.js'
 import { getFileName } from '../../datalad/files.js'
@@ -152,9 +150,7 @@ export const deleteFiles = async (
 ) => {
   try {
     await checkDatasetWrite(datasetId, user, userInfo)
-    const deletedFiles = await datalad
-      .deleteFiles(datasetId, files, userInfo)
-      .then(filenames => filenames.map(filename => new UpdatedFile(filename)))
+    const deletedFiles = await datalad.deleteFiles(datasetId, files, userInfo)
     pubsub.publish('filesUpdated', {
       datasetId,
       filesUpdated: {

package/src/graphql/resolvers/draft.js

@@ -10,11 +10,17 @@ import { filterRemovedAnnexObjects } from '../utils/file.js'
 // A draft must have a dataset parent
 const draftFiles = async (dataset, args, { userInfo }) => {
   const hexsha = await getDraftRevision(dataset.id)
-  const files = await getFiles(dataset.id, hexsha)
+  const { files } = await getFiles(dataset.id, hexsha)
   const prefixFiltered = filterFiles('prefix' in args && args.prefix)(files)
   return filterRemovedAnnexObjects(dataset.id, userInfo)(prefixFiltered)
 }
 
+const draftSize = async (dataset, args, { userInfo }) => {
+  const hexsha = await getDraftRevision(dataset.id)
+  const { size } = await getFiles(dataset.id, hexsha)
+  return size
+}
+
 /**
  * Deprecated mutation to move the draft HEAD reference forward or backward
  *
@@ -39,6 +45,7 @@ export const revalidate = async (obj, { datasetId }, { user, userInfo }) => {
 const draft = {
   id: obj => obj.id,
   files: draftFiles,
+  size: draftSize,
   summary,
   issues,
   modified: obj => obj.modified,

package/src/graphql/resolvers/permissions.js

@@ -44,7 +44,9 @@ const publishPermissions = async datasetId => {
 export const updatePermissions = async (obj, args, { user, userInfo }) => {
   await checkDatasetAdmin(args.datasetId, user, userInfo)
   // get all users the the email specified by permissions arg
-  const users = await User.find({ email: args.userEmail }).exec()
+  const users = await User.find({ email: args.userEmail })
+    .collation({ locale: 'en', strength: 2 })
+    .exec()
 
   if (!users.length) {
     throw new Error('A user with that email address does not exist')

package/src/graphql/resolvers/snapshots.js

@@ -29,8 +29,11 @@ export const snapshot = (obj, { datasetId, tag }, context) => {
         summary: () => summary({ id: datasetId, revision: snapshot.hexsha }),
         files: ({ prefix }) =>
           getFiles(datasetId, snapshot.hexsha)
+            .then(response => response.files)
             .then(filterFiles(prefix))
             .then(filterRemovedAnnexObjects(datasetId, context.userInfo)),
+        size: () =>
+          getFiles(datasetId, snapshot.hexsha).then(response => response.size),
         deprecated: () => deprecated({ datasetId, tag }),
         related: () => related(datasetId),
         onBrainlife: () => onBrainlife(snapshot),

package/src/graphql/schema.js

@@ -430,6 +430,8 @@ export const typeDefs = `
     uploads: [UploadMetadata]
     # Git commit hash
     head: String
+    # Total size in bytes of this draft
+    size: Int
   }
 
   # Tagged snapshot of a draft
@@ -461,6 +463,8 @@ export const typeDefs = `
     related: [RelatedObject]
     # Is the snapshot available for analysis on Brainlife?
     onBrainlife: Boolean @cacheControl(maxAge: 10080, scope: PUBLIC)
+    # Total size in bytes of this snapshot
+    size: Int
   }
 
   # RelatedObject nature of relationship

package/src/graphql/utils/file.js

@@ -1,34 +1,5 @@
 import BadAnnexObject from '../../models/badAnnexObject'
 
-/**
- * Generates unique id for untracked files.
- * @param {string} filepath - filepath ('/' delimiters)
- * @param {number|string} [size] - of file
- */
-export const generateFileId = (filepath, size) => `${filepath}:${size}`
-
-/**
- * Creates a file object with an ApolloGQL cache-safe id.
- * @class
- * @param {string} filepath ':' delimited
- * @param {string|number} [size]
- */
-export function UpdatedFile(filepath, size) {
-  /**
-   * unique id
-   * @id UpdatedFile#id
-   * @type {string}
-   */
-  this.id = generateFileId(filepath, size)
-  /**
-   * filename with '/' delimiters
-   * @filename UpdatedFile#filename
-   * @type {string}
-   */
-  this.filename = filepath
-  if (size) this.size = size
-}
-
 export const filterRemovedAnnexObjects =
   (datasetId, userInfo) => async files => {
     const removedAnnexObjectKeys = (

package/src/models/user.ts

@@ -29,6 +29,16 @@ const userSchema = new Schema({
 })
 
 userSchema.index({ id: 1, provider: 1 }, { unique: true })
+// Allow case insensitive email queries
+userSchema.index(
+  { email: 1 },
+  {
+    collation: {
+      locale: 'en',
+      strength: 2,
+    },
+  },
+)
 
 const User = model<UserDocument>('User', userSchema)
 

package/tsconfig.json

@@ -7,5 +7,8 @@
   },
   "include": ["./src"],
   "files": ["./src/lerna.json"],
-  "references": [{ "path": "../openneuro-client" }]
+  "references": [
+    { "path": "../openneuro-client" },
+    { "path": "../openneuro-search" }
+  ]
 }