@openneuro/server 4.20.5 → 4.20.6-alpha.3

package/src/datalad/{dataset.js → dataset.ts}

@@ -3,30 +3,31 @@
  *
  * See resolvers for interaction with other data sources.
  */
-import request from 'superagent'
-import requestNode from 'request'
-import objectHash from 'object-hash'
-import { Readable } from 'stream'
-import config from '../config'
-import * as subscriptions from '../handlers/subscriptions.js'
-import { generateDataladCookie } from '../libs/authentication/jwt'
-import { redis } from '../libs/redis'
-import CacheItem, { CacheType } from '../cache/item'
-import { updateDatasetRevision } from './draft.js'
-import { fileUrl, getFileName, encodeFilePath, filesUrl } from './files'
-import { getAccessionNumber } from '../libs/dataset.js'
-import Dataset from '../models/dataset'
-import Metadata from '../models/metadata'
-import Permission from '../models/permission'
-import Star from '../models/stars'
-import Subscription from '../models/subscription'
-import BadAnnexObject from '../models/badAnnexObject'
-import { datasetsConnection } from './pagination'
-import { getDatasetWorker } from '../libs/datalad-service'
-import notifications from '../libs/notifications'
+import request from "superagent"
+import requestNode from "request"
+import objectHash from "object-hash"
+import { Readable } from "stream"
+import * as Mongoose from "mongoose"
+import config from "../config"
+import * as subscriptions from "../handlers/subscriptions"
+import { generateDataladCookie } from "../libs/authentication/jwt"
+import { redis } from "../libs/redis"
+import CacheItem, { CacheType } from "../cache/item"
+import { updateDatasetRevision } from "./draft"
+import { encodeFilePath, filesUrl, fileUrl, getFileName } from "./files"
+import { getAccessionNumber } from "../libs/dataset"
+import Dataset from "../models/dataset"
+import Metadata from "../models/metadata"
+import Permission from "../models/permission"
+import Star from "../models/stars"
+import Subscription from "../models/subscription"
+import BadAnnexObject from "../models/badAnnexObject"
+import { datasetsConnection } from "./pagination"
+import { getDatasetWorker } from "../libs/datalad-service"
+import notifications from "../libs/notifications"
 
 export const giveUploaderPermission = (datasetId, userId) => {
-  const permission = new Permission({ datasetId, userId, level: 'admin' })
+  const permission = new Permission({ datasetId, userId, level: "admin" })
   return permission.save()
 }
 
@@ -51,8 +52,8 @@ export const createDataset = async (
     const ds = new Dataset({ id: datasetId, uploader })
     await request
       .post(`${getDatasetWorker(datasetId)}/datasets/${datasetId}`)
-      .set('Accept', 'application/json')
-      .set('Cookie', generateDataladCookie(config)(userInfo))
+      .set("Accept", "application/json")
+      .set("Cookie", generateDataladCookie(config)(userInfo))
     // Write the new dataset to mongo after creation
     await ds.save()
     const md = new Metadata({ datasetId, affirmedDefaced, affirmedConsent })
@@ -72,17 +73,17 @@ export const createDataset = async (
  * Return the latest commit
  * @param {string} id Dataset accession number
  */
-export const getDraftHead = async id => {
+export const getDraftHead = async (id) => {
   const draftRes = await request
     .get(`${getDatasetWorker(id)}/datasets/${id}/draft`)
-    .set('Accept', 'application/json')
+    .set("Accept", "application/json")
   return draftRes.body.hexsha
 }
 
 /**
  * Fetch dataset document and related fields
  */
-export const getDataset = async id => {
+export const getDataset = async (id) => {
   const dataset = await Dataset.findOne({ id }).lean()
   return {
     ...dataset,
@@ -93,7 +94,7 @@ export const getDataset = async id => {
 /**
  * Delete dataset and associated documents
  */
-export const deleteDataset = id =>
+export const deleteDataset = (id) =>
   request
     .del(`${getDatasetWorker(id)}/datasets/${id}`)
     .then(() => Dataset.deleteOne({ id }).exec())
@@ -103,7 +104,7 @@ export const deleteDataset = id =>
  * For public datasets, cache combinations of sorts/limits/cursors to speed responses
  * @param {object} options getDatasets options object
  */
-export const cacheDatasetConnection = options => connectionArguments => {
+export const cacheDatasetConnection = (options) => (connectionArguments) => {
   const connection = datasetsConnection(options)
   const cache = new CacheItem(
     redis,
@@ -119,134 +120,134 @@ export const cacheDatasetConnection = options => connectionArguments => {
  * @param {object} match MongoDB $match aggregate
  * @returns {Array<object>} Array of MongoDB aggregate pipelines
  */
-const aggregateArraySetup = match => [{ $match: match }]
+const aggregateArraySetup = (match): Mongoose.Expression => [{ $match: match }]
 
 /**
  * Add any filter steps based on the filterBy options provided
  * @param {object} options GraphQL query parameters
  * @returns {(match: object) => Array<any>} Array of aggregate stages
  */
-export const datasetsFilter = options => match => {
+export const datasetsFilter = (options) => (match) => {
   const aggregates = aggregateArraySetup(match)
   if (options.modality) {
     aggregates.push(
       ...[
         {
           $lookup: {
-            from: 'snapshots',
-            localField: 'id',
-            foreignField: 'datasetId',
-            as: 'snapshots',
+            from: "snapshots",
+            localField: "id",
+            foreignField: "datasetId",
+            as: "snapshots",
           },
         },
-        { $addFields: { snapshots: { $slice: ['$snapshots', -1] } } },
+        { $addFields: { snapshots: { $slice: ["$snapshots", -1] } } },
         {
           $lookup: {
-            from: 'summaries',
-            localField: 'snapshots.0.hexsha',
-            foreignField: 'id',
-            as: 'summaries',
+            from: "summaries",
+            localField: "snapshots.0.hexsha",
+            foreignField: "id",
+            as: "summaries",
           },
         },
         {
           $match: {
-            'summaries.0.modalities': options.modality,
+            "summaries.0.modalities": options.modality,
           },
         },
       ],
     )
     return aggregates
   }
-  const filterMatch = {}
-  if ('filterBy' in options) {
+  const filterMatch: Mongoose.Expression = {}
+  if ("filterBy" in options) {
     const filters = options.filterBy
     if (
-      'admin' in options &&
+      "admin" in options &&
       options.admin &&
-      'all' in filters &&
+      "all" in filters &&
       filters.all
     ) {
       // For admins and {filterBy: all}, ignore any passed in matches
       aggregates.length = 0
     }
     // Apply any filters as needed
-    if ('public' in filters && filters.public) {
+    if ("public" in filters && filters.public) {
       filterMatch.public = true
     }
-    if ('saved' in filters && filters.saved) {
+    if ("saved" in filters && filters.saved) {
       aggregates.push({
         $lookup: {
-          from: 'stars',
-          let: { datasetId: '$id' },
+          from: "stars",
+          let: { datasetId: "$id" },
           pipeline: [
             {
               $match: {
                 $expr: {
                   $and: [
-                    { $eq: ['$userId', options.userId] },
-                    { $eq: ['$datasetId', '$$datasetId'] },
+                    { $eq: ["$userId", options.userId] },
+                    { $eq: ["$datasetId", "$$datasetId"] },
                   ],
                 },
               },
             },
           ],
-          as: 'saved',
+          as: "saved",
         },
       })
       filterMatch.saved = { $exists: true, $ne: [] } // arr datasetIds
     }
 
-    if ('userId' in options && 'shared' in filters && filters.shared) {
+    if ("userId" in options && "shared" in filters && filters.shared) {
       filterMatch.uploader = { $ne: options.userId }
     }
-    if ('userId' in options && 'starred' in filters && filters.starred) {
+    if ("userId" in options && "starred" in filters && filters.starred) {
       aggregates.push({
         $lookup: {
-          from: 'stars',
-          let: { datasetId: '$id' },
+          from: "stars",
+          let: { datasetId: "$id" },
           pipeline: [
             {
               $match: {
                 $expr: {
                   $and: [
-                    { $eq: ['$datasetId', '$$datasetId'] },
-                    { $eq: ['$userId', options.userId] },
+                    { $eq: ["$datasetId", "$$datasetId"] },
+                    { $eq: ["$userId", options.userId] },
                   ],
                 },
               },
             },
           ],
-          as: 'starred',
+          as: "starred",
         },
       })
       filterMatch.starred = { $exists: true, $ne: [] }
     }
-    if ('invalid' in filters && filters.invalid) {
+    if ("invalid" in filters && filters.invalid) {
       // SELECT * FROM datasets JOIN issues ON datasets.revision = issues.id WHERE ...
       aggregates.push({
         $lookup: {
-          from: 'issues', //look at issues collection
-          let: { revision: '$revision' }, // find issue match revision datasets.revision
+          from: "issues", //look at issues collection
+          let: { revision: "$revision" }, // find issue match revision datasets.revision
           pipeline: [
-            { $unwind: '$issues' },
+            { $unwind: "$issues" },
             {
               $match: {
                 $expr: {
                   $and: [
-                    { $eq: ['$id', '$$revision'] }, // JOIN CONSTRAINT  issues.id = datasets.revision
-                    { $eq: ['$issues.severity', 'error'] }, // WHERE severity = 'error'  issues.severity db
+                    { $eq: ["$id", "$$revision"] }, // JOIN CONSTRAINT  issues.id = datasets.revision
+                    { $eq: ["$issues.severity", "error"] }, // WHERE severity = 'error'  issues.severity db
                   ],
                 },
               },
             },
           ],
-          as: 'issues',
+          as: "issues",
         },
       })
       // Count how many error fields matched in previous step
       aggregates.push({
         $addFields: {
-          errorCount: { $size: '$issues' },
+          errorCount: { $size: "$issues" },
         },
       })
       // Filter any datasets with no errors
@@ -260,19 +261,19 @@ export const datasetsFilter = options => match => {
  * Fetch all datasets
  * @param {object} options {orderBy: {created: 'ascending'}, filterBy: {public: true}}
  */
-export const getDatasets = options => {
+export const getDatasets = (options) => {
   const filter = datasetsFilter(options)
   const connection = datasetsConnection(options)
-  if (options && 'userId' in options) {
+  if (options && "userId" in options) {
     // Authenticated request
     return Permission.find({ userId: options.userId })
       .exec()
-      .then(datasetsAllowed => {
+      .then((datasetsAllowed) => {
         const datasetIds = datasetsAllowed.map(
-          permission => permission.datasetId,
+          (permission) => permission.datasetId,
         )
         // Match allowed datasets
-        if ('myDatasets' in options && options.myDatasets) {
+        if ("myDatasets" in options && options.myDatasets) {
           // Exclude other users public datasets even though we have access to those
           return connection(filter({ id: { $in: datasetIds } }))
         } else {
@@ -330,18 +331,18 @@ export const addFile = async (datasetId, path, file) => {
       const downstreamRequest = requestNode(
         {
           url: fileUrl(datasetId, path, filename),
-          method: 'post',
-          headers: { 'Content-Type': mimetype },
+          method: "post",
+          headers: { "Content-Type": mimetype },
         },
-        err => (err ? reject(err) : resolve(responseFile)),
+        (err) => (err ? reject(err) : resolve(responseFile)),
       )
       // Attach error handler for incoming request and start feeding downstream
       stream
-        .on('data', chunk => {
+        .on("data", (chunk) => {
           responseFile.size += chunk.length
         })
-        .on('error', err => {
-          if (err.constructor.name === 'FileStreamDisconnectUploadError') {
+        .on("error", (err) => {
+          if (err.constructor.name === "FileStreamDisconnectUploadError") {
             // Catch client disconnects.
             // eslint-disable-next-line no-console
             console.warn(
@@ -356,7 +357,7 @@ export const addFile = async (datasetId, path, file) => {
         .pipe(downstreamRequest)
     })
   } catch (err) {
-    if (err.constructor.name === 'UploadPromiseDisconnectUploadError') {
+    if (err.constructor.name === "UploadPromiseDisconnectUploadError") {
       // Catch client aborts silently
     } else {
       // Raise any unknown errors
@@ -371,7 +372,7 @@ export const addFile = async (datasetId, path, file) => {
  * Used to mock the stream interface in addFile
  */
 export const addFileString = (datasetId, filename, mimetype, content) =>
-  addFile(datasetId, '', {
+  addFile(datasetId, "", {
     filename,
     mimetype,
     // Mock a stream so we can reuse addFile
@@ -401,9 +402,9 @@ export const commitFiles = (datasetId, user) => {
   const url = `${getDatasetWorker(datasetId)}/datasets/${datasetId}/draft`
   return request
     .post(url)
-    .set('Cookie', generateDataladCookie(config)(user))
-    .set('Accept', 'application/json')
-    .then(res => {
+    .set("Cookie", generateDataladCookie(config)(user))
+    .set("Accept", "application/json")
+    .then((res) => {
       gitRef = res.body.ref
       return updateDatasetRevision(datasetId, gitRef).then(() => gitRef)
     })
@@ -414,12 +415,12 @@ export const commitFiles = (datasetId, user) => {
  */
 export const deleteFiles = (datasetId, files, user) => {
   const filenames = files.map(({ filename, path }) =>
-    filename ? getFileName(path, filename) : encodeFilePath(path),
+    filename ? getFileName(path, filename) : encodeFilePath(path)
   )
   return request
     .del(filesUrl(datasetId))
-    .set('Cookie', generateDataladCookie(config)(user))
-    .set('Accept', 'application/json')
+    .set("Cookie", generateDataladCookie(config)(user))
+    .set("Accept", "application/json")
     .send({ filenames })
     .then(() => filenames)
 }
@@ -435,15 +436,16 @@ export const removeAnnexObject = (
   user,
 ) => {
   const worker = getDatasetWorker(datasetId)
-  const url = `http://${worker}/datasets/${datasetId}/snapshots/${snapshot}/annex-key/${annexKey}`
+  const url =
+    `http://${worker}/datasets/${datasetId}/snapshots/${snapshot}/annex-key/${annexKey}`
   return request
     .del(url)
-    .set('Cookie', generateDataladCookie(config)(user))
-    .set('Accept', 'application/json')
+    .set("Cookie", generateDataladCookie(config)(user))
+    .set("Accept", "application/json")
     .then(async () => {
       const existingBAO = await BadAnnexObject.find({ annexKey }).exec()
       if (existingBAO) {
-        existingBAO.forEach(bAO => {
+        existingBAO.forEach((bAO) => {
           bAO.remover = user._id
           bAO.removed = true
           bAO.save()
@@ -494,19 +496,19 @@ export const updatePublic = (datasetId, publicFlag) =>
   ).exec()
 
 export const getDatasetAnalytics = (datasetId, tag) => {
-  return Dataset.findOne({ id: datasetId }).then(ds => ({
+  return Dataset.findOne({ id: datasetId }).then((ds) => ({
     datasetId,
     views: ds.views || 0,
     downloads: ds.downloads || 0,
   }))
 }
 
-export const getStars = datasetId => Star.find({ datasetId })
+export const getStars = (datasetId) => Star.find({ datasetId })
 
 export const getUserStarred = (datasetId, userId) =>
   Star.countDocuments({ datasetId, userId }).exec()
 
-export const getFollowers = datasetId => {
+export const getFollowers = (datasetId) => {
   return Subscription.find({
     datasetId: datasetId,
   }).exec()

package/src/datalad/{description.js → description.ts}

@@ -1,15 +1,15 @@
 /**
  * Get description data from backend
  */
-import config from '../config'
-import request from 'superagent'
-import { redis } from '../libs/redis.js'
-import { commitFiles } from './dataset.js'
-import { fileUrl } from './files'
-import { generateDataladCookie } from '../libs/authentication/jwt'
-import { getDatasetWorker } from '../libs/datalad-service'
-import CacheItem, { CacheType } from '../cache/item'
-import { datasetOrSnapshot } from '../utils/datasetOrSnapshot'
+import config from "../config"
+import request from "superagent"
+import { redis } from "../libs/redis"
+import { commitFiles } from "./dataset"
+import { fileUrl } from "./files"
+import { generateDataladCookie } from "../libs/authentication/jwt"
+import { getDatasetWorker } from "../libs/datalad-service"
+import CacheItem, { CacheType } from "../cache/item"
+import { datasetOrSnapshot } from "../utils/datasetOrSnapshot"
 
 /**
  * Find dataset_description.json id and fetch description object
@@ -18,10 +18,10 @@ import { datasetOrSnapshot } from '../utils/datasetOrSnapshot'
  */
 export const getDescriptionObject = async (datasetId, revision) => {
   const res = await fetch(
-    fileUrl(datasetId, '', 'dataset_description.json', revision),
+    fileUrl(datasetId, "", "dataset_description.json", revision),
   )
-  const contentType = res.headers.get('content-type')
-  if (res.status === 200 && contentType.includes('application/json')) {
+  const contentType = res.headers.get("content-type")
+  if (res.status === 200 && contentType.includes("application/json")) {
     return await res.json()
   } else {
     throw new Error(
@@ -34,59 +34,59 @@ export const descriptionCacheKey = (datasetId, revision) => {
   return `openneuro:dataset_description.json:${datasetId}:${revision}`
 }
 
-export const repairDescriptionTypes = description => {
+export const repairDescriptionTypes = (description) => {
   const newDescription = { ...description }
   // Array types
   if (
-    description.hasOwnProperty('Authors') &&
+    description.hasOwnProperty("Authors") &&
     !Array.isArray(description.Authors)
   ) {
     newDescription.Authors = [description.Authors]
   }
   if (
-    description.hasOwnProperty('ReferencesAndLinks') &&
+    description.hasOwnProperty("ReferencesAndLinks") &&
     !Array.isArray(description.ReferencesAndLinks)
   ) {
     newDescription.ReferencesAndLinks = [description.ReferencesAndLinks]
   }
   if (
-    description.hasOwnProperty('Funding') &&
+    description.hasOwnProperty("Funding") &&
     !Array.isArray(description.Funding)
   ) {
     newDescription.Funding = [description.Funding]
   }
   if (
-    description.hasOwnProperty('EthicsApprovals') &&
+    description.hasOwnProperty("EthicsApprovals") &&
     !Array.isArray(description.EthicsApprovals)
   ) {
     newDescription.EthicsApprovals = [description.EthicsApprovals]
   }
   // String types
   if (
-    description.hasOwnProperty('Name') &&
-    typeof description.Name !== 'string'
+    description.hasOwnProperty("Name") &&
+    typeof description.Name !== "string"
   ) {
-    newDescription.Name = JSON.stringify(description.Name) || ''
+    newDescription.Name = JSON.stringify(description.Name) || ""
   }
   if (
-    description.hasOwnProperty('DatasetDOI') &&
-    typeof description.DatasetDOI !== 'string'
+    description.hasOwnProperty("DatasetDOI") &&
+    typeof description.DatasetDOI !== "string"
   ) {
-    newDescription.DatasetDOI = JSON.stringify(description.DatasetDOI) || ''
+    newDescription.DatasetDOI = JSON.stringify(description.DatasetDOI) || ""
   }
   if (
-    description.hasOwnProperty('Acknowledgements') &&
-    typeof description.Acknowledgements !== 'string'
+    description.hasOwnProperty("Acknowledgements") &&
+    typeof description.Acknowledgements !== "string"
   ) {
     newDescription.Acknowledgements =
-      JSON.stringify(description.Acknowledgements) || ''
+      JSON.stringify(description.Acknowledgements) || ""
   }
   if (
-    description.hasOwnProperty('HowToAcknowledge') &&
-    typeof description.HowToAcknowledge !== 'string'
+    description.hasOwnProperty("HowToAcknowledge") &&
+    typeof description.HowToAcknowledge !== "string"
   ) {
     newDescription.HowToAcknowledge =
-      JSON.stringify(description.HowToAcknowledge) || ''
+      JSON.stringify(description.HowToAcknowledge) || ""
   }
   return newDescription
 }
@@ -94,7 +94,7 @@ export const repairDescriptionTypes = description => {
 /**
  * Return the last author in dataset_description as the senior author if available
  */
-export const appendSeniorAuthor = description => {
+export const appendSeniorAuthor = (description) => {
   try {
     const SeniorAuthor = description?.Authors[description.Authors.length - 1]
     return { ...description, SeniorAuthor }
@@ -107,13 +107,13 @@ export const appendSeniorAuthor = description => {
  * Get a parsed dataset_description.json
  * @param {object} obj dataset or snapshot object
  */
-export const description = async obj => {
+export const description = async (obj) => {
   // Obtain datasetId from Dataset or Snapshot objects
   const { datasetId, revision } = datasetOrSnapshot(obj)
   // Default fallback if dataset_description.json is not valid or missing
   const defaultDescription = {
     Name: datasetId,
-    BIDSVersion: '1.8.0',
+    BIDSVersion: "1.8.0",
   }
   const cache = new CacheItem(redis, CacheType.datasetDescription, [
     datasetId,
@@ -122,7 +122,7 @@ export const description = async obj => {
   try {
     const datasetDescription = await cache.get(() => {
       return getDescriptionObject(datasetId, revision).then(
-        uncachedDescription => ({ id: revision, ...uncachedDescription }),
+        (uncachedDescription) => ({ id: revision, ...uncachedDescription }),
       )
     })
     return appendSeniorAuthor(repairDescriptionTypes(datasetDescription))
@@ -136,11 +136,11 @@ export const setDescription = (datasetId, user, descriptionFieldUpdates) => {
   return request
     .post(url)
     .send({ description_fields: descriptionFieldUpdates })
-    .set('Accept', 'application/json')
-    .set('Cookie', generateDataladCookie(config)(user))
-    .then(res => {
+    .set("Accept", "application/json")
+    .set("Cookie", generateDataladCookie(config)(user))
+    .then((res) => {
       const description = res.body
-      return commitFiles(datasetId, user).then(gitRef => ({
+      return commitFiles(datasetId, user).then((gitRef) => ({
         id: gitRef,
         ...description,
       }))

package/src/datalad/draft.ts

@@ -0,0 +1,38 @@
+/**
+ * Manage serving a Draft object based on DataLad working trees
+ */
+import request from "superagent"
+import Dataset from "../models/dataset"
+import { getDatasetWorker } from "../libs/datalad-service"
+
+export const getDraftRevision = async (datasetId) => {
+  const draftUrl = `http://${
+    getDatasetWorker(
+      datasetId,
+    )
+  }/datasets/${datasetId}/draft`
+  const response = await fetch(draftUrl)
+  const { hexsha } = await response.json()
+  return hexsha
+}
+
+export const updateDatasetRevision = (datasetId, gitRef) => {
+  /**
+   * Update the revision modified time in a draft on changes
+   */
+  return Dataset.updateOne({ id: datasetId }, { modified: new Date() }).exec()
+}
+
+/**
+ * Run a git reset on the draft
+ * @param {string} datasetId Accession number
+ * @param {string} ref Git hexsha
+ */
+export const resetDraft = (datasetId, ref) => {
+  const resetUrl = `${
+    getDatasetWorker(
+      datasetId,
+    )
+  }/datasets/${datasetId}/reset/${ref}`
+  return request.post(resetUrl).set("Accept", "application/json")
+}