@openneuro/server 4.11.0 → 4.12.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.11.0",
+  "version": "4.12.0-alpha.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -18,7 +18,7 @@
   "dependencies": {
     "@apollo/client": "3.4.17",
     "@elastic/elasticsearch": "7.15.0",
-    "@openneuro/search": "^4.11.0",
+    "@openneuro/search": "^4.12.0-alpha.0",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@sentry/node": "^4.5.3",
     "apollo-server": "2.25.3",
@@ -105,5 +105,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "4795359e0756b2f6d83f385172a80d9b26b5fdc8"
+  "gitHead": "3d8f7524b2233605f2da66f05de90b2e96e7a127"
 }

package/src/datalad/dataset.js

@@ -12,7 +12,7 @@ import * as subscriptions from '../handlers/subscriptions.js'
 import { generateDataladCookie } from '../libs/authentication/jwt'
 import { redis } from '../libs/redis'
 import CacheItem, { CacheType } from '../cache/item'
-import { updateDatasetRevision, expireDraftFiles } from './draft.js'
+import { updateDatasetRevision } from './draft.js'
 import { fileUrl, getFileName, encodeFilePath, filesUrl } from './files'
 import { getAccessionNumber } from '../libs/dataset.js'
 import Dataset from '../models/dataset'
@@ -309,7 +309,6 @@ export const testBlacklist = (path, filename) =>
 export const addFile = async (datasetId, path, file) => {
   try {
     const { filename, mimetype, createReadStream, capacitor } = await file
-    await expireDraftFiles(datasetId)
 
     // Apply blacklist to uploaded files
     if (testBlacklist(path, filename)) {

package/src/datalad/description.js

@@ -127,7 +127,6 @@ export const description = obj => {
   return cache
     .get(() => {
       return getFiles(datasetId, revision)
-        .then(response => response.files)
         .then(getDescriptionObject(datasetId))
         .then(uncachedDescription => ({ id: revision, ...uncachedDescription }))
     })

package/src/datalad/draft.js

@@ -4,16 +4,9 @@
 import fetch from 'node-fetch'
 import request from 'superagent'
 import Dataset from '../models/dataset'
-import { redis } from '../libs/redis'
-import CacheItem, { CacheType } from '../cache/item'
 import publishDraftUpdate from '../graphql/utils/publish-draft-update.js'
 import { getDatasetWorker } from '../libs/datalad-service'
 
-export const expireDraftFiles = datasetId => {
-  const cache = new CacheItem(redis, CacheType.commitFiles, [datasetId])
-  return cache.drop()
-}
-
 export const getDraftRevision = async datasetId => {
   const draftUrl = `http://${getDatasetWorker(
     datasetId,
@@ -29,10 +22,6 @@ export const updateDatasetRevision = (datasetId, gitRef) => {
    */
   return Dataset.updateOne({ id: datasetId }, { modified: new Date() })
     .exec()
-    .then(() => {
-      // Remove the now invalid draft files cache
-      return expireDraftFiles(datasetId)
-    })
     .then(() => publishDraftUpdate(datasetId, gitRef))
 }
 

package/src/datalad/files.js

@@ -61,19 +61,17 @@ export const computeTotalSize = files =>
  * Get files for a specific revision
  * Similar to getDraftFiles but different cache key and fixed revisions
  * @param {string} datasetId - Dataset accession number
- * @param {string} hexsha - Git treeish hexsha
+ * @param {string} treeish - Git treeish hexsha
  */
-export const getFiles = (datasetId, hexsha) => {
+export const getFiles = (datasetId, treeish) => {
   const cache = new CacheItem(redis, CacheType.commitFiles, [
     datasetId,
-    hexsha.substring(0, 7),
+    treeish.substring(0, 7),
   ])
   return cache.get(() =>
     request
       .get(
-        `${getDatasetWorker(
-          datasetId,
-        )}/datasets/${datasetId}/snapshots/${hexsha}/files`,
+        `${getDatasetWorker(datasetId)}/datasets/${datasetId}/tree/${treeish}`,
       )
       .set('Accept', 'application/json')
       .then(response => {
@@ -81,11 +79,7 @@ export const getFiles = (datasetId, hexsha) => {
           const {
             body: { files },
           } = response
-          const size = computeTotalSize(files)
-          return {
-            files: files.map(addFileUrl(datasetId, hexsha)),
-            size,
-          }
+          return files.map(addFileUrl(datasetId, treeish))
         }
       }),
   )

package/src/datalad/snapshots.js

@@ -155,9 +155,7 @@ export const createSnapshot = async (
       snapshotChanges,
     )
     snapshot.created = new Date()
-    const { files, size } = await getFiles(datasetId, tag)
-    snapshot.files = files
-    snapshot.size = size
+    snapshot.files = await getFiles(datasetId, tag)
 
     await Promise.all([
       // Update the draft status in datasets collection in case any changes were made (DOI, License)

package/src/graphql/resolvers/draft.js

@@ -1,24 +1,25 @@
+import Summary from '../../models/summary'
 import { summary } from './summary.js'
 import { issues } from './issues.js'
 import { description } from './description.js'
 import { readme } from './readme.js'
 import { getDraftRevision, updateDatasetRevision } from '../../datalad/draft.js'
 import { checkDatasetWrite } from '../permissions.js'
-import { getFiles, filterFiles } from '../../datalad/files.js'
+import { getFiles } from '../../datalad/files.js'
 import { filterRemovedAnnexObjects } from '../utils/file.js'
 
 // A draft must have a dataset parent
 const draftFiles = async (dataset, args, { userInfo }) => {
   const hexsha = await getDraftRevision(dataset.id)
-  const { files } = await getFiles(dataset.id, hexsha)
-  const prefixFiltered = filterFiles('prefix' in args && args.prefix)(files)
-  return filterRemovedAnnexObjects(dataset.id, userInfo)(prefixFiltered)
+  const files = await getFiles(dataset.id, args.tree || hexsha)
+  return filterRemovedAnnexObjects(dataset.id, userInfo)(files)
 }
 
 const draftSize = async (dataset, args, { userInfo }) => {
   const hexsha = await getDraftRevision(dataset.id)
-  const { size } = await getFiles(dataset.id, hexsha)
-  return size
+  return Summary.findOne({ datasetId: dataset.id, id: hexsha })
+    .exec()
+    .then(res => res?.toObject()?.size)
 }
 
 /**

package/src/graphql/resolvers/snapshots.js

@@ -6,7 +6,8 @@ import { readme } from './readme.js'
 import { description } from './description.js'
 import { summary } from './summary.js'
 import { snapshotIssues } from './issues.js'
-import { getFiles, filterFiles } from '../../datalad/files.js'
+import { getFiles } from '../../datalad/files.js'
+import Summary from '../../models/summary'
 import DatasetModel from '../../models/dataset'
 import { filterRemovedAnnexObjects } from '../utils/file.js'
 import DeprecatedSnapshot from '../../models/deprecatedSnapshot'
@@ -28,13 +29,14 @@ export const snapshot = (obj, { datasetId, tag }, context) => {
         description: () => description(snapshot),
         readme: () => readme(snapshot),
         summary: () => summary({ id: datasetId, revision: snapshot.hexsha }),
-        files: ({ prefix }) =>
-          getFiles(datasetId, snapshot.hexsha)
-            .then(response => response.files)
-            .then(filterFiles(prefix))
-            .then(filterRemovedAnnexObjects(datasetId, context.userInfo)),
+        files: ({ tree }) =>
+          getFiles(datasetId, tree || snapshot.hexsha).then(
+            filterRemovedAnnexObjects(datasetId, context.userInfo),
+          ),
         size: () =>
-          getFiles(datasetId, snapshot.hexsha).then(response => response.size),
+          Summary.findOne({ datasetId: datasetId, id: snapshot.hexsha })
+            .exec()
+            .then(res => res?.toObject()?.size),
         deprecated: () => deprecated({ datasetId, tag }),
         related: () => related(datasetId),
         onBrainlife: () => onBrainlife(snapshot),

package/src/graphql/schema.js

@@ -434,7 +434,7 @@ export const typeDefs = `
     # Validator issues
     issues: [ValidationIssue]
     # Committed files in the working tree
-    files(prefix: String = ""): [DatasetFile]
+    files(tree: String): [DatasetFile]
     # dataset_description.json fields
     description: Description
     # Dataset README
@@ -461,7 +461,7 @@ export const typeDefs = `
     # bids-validator issues for this snapshot
     issues: [ValidationIssue]
     # Snapshot files
-    files(prefix: String = ""): [DatasetFile]
+    files(tree: String): [DatasetFile]
     # dataset_description.json fields
     description: Description
     # Snapshot usage and download statistics
@@ -689,7 +689,6 @@ export const typeDefs = `
     size: BigInt
     annexed: Boolean
     urls: [String]
-    objectpath: String
     # Return a flag if this is a directory which contains more files
     directory: Boolean
   }