@openneuro/server 4.3.0 → 4.4.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.3.0",
+  "version": "4.4.0-alpha.2",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -42,6 +42,7 @@
     "graphql-tools": "4.0.6",
     "immutable": "^3.8.2",
     "ioredis": "4.17.3",
+    "jest-fetch-mock": "^3.0.3",
     "jsdom": "^11.6.2",
     "jsonwebtoken": "^8.3.0",
     "mime-types": "^2.1.19",
@@ -49,7 +50,8 @@
     "mongoose": "5.12.3",
     "morgan": "^1.6.1",
     "node-fetch": "^2.6.0",
-    "nodemailer": "6.4.16",
+    "nodemailer": "^6.7.2",
+    "nodemailer-mailjet-transport": "^1.0.4",
     "object-hash": "2.1.1",
     "passport": "^0.4.0",
     "passport-globus": "^0.0.1",
@@ -103,5 +105,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "2e777ea3f979e05dab85d612ea2b0788e699b96b"
+  "gitHead": "69aec829ed581c71c46bb4bdb3561f669e619f4e"
 }

package/src/datalad/__tests__/dataset.spec.js

@@ -95,10 +95,16 @@ describe('dataset model operations', () => {
         expect(testBlacklist('.git', 'HEAD')).toBe(true)
       })
       it('returns true for root level .DS_Store files', () => {
-        expect(testBlacklist('', '.DS_Store'))
+        expect(testBlacklist('', '.DS_Store')).toBe(true)
       })
       it('returns true for nested .DS_Store files', () => {
-        expect(testBlacklist('sub-01/anat/', '.DS_Store'))
+        expect(testBlacklist('sub-01/anat/', '.DS_Store')).toBe(true)
+      })
+      // https://github.com/OpenNeuroOrg/openneuro/issues/2519
+      it('skips ._ prefixed files created by macOS', () => {
+        expect(testBlacklist('', '._.DS_Store')).toBe(true)
+        expect(testBlacklist('stimuli/', '._1002.png')).toBe(true)
+        expect(testBlacklist('stimuli/', 'test._1002.png')).toBe(false)
       })
     })
   })

package/src/datalad/dataset.js

@@ -298,7 +298,7 @@ export const getDatasets = options => {
 }
 
 // Files to skip in uploads
-const filenameBlacklist = new RegExp(/.DS_Store|Icon\r/)
+const filenameBlacklist = new RegExp(/.DS_Store|Icon\r|^\._/)
 const pathBlacklist = new RegExp(/^.git|^.gitattributes|^.datalad|^.heudiconv/)
 export const testBlacklist = (path, filename) =>
   filenameBlacklist.test(filename) || pathBlacklist.test(path)
@@ -505,7 +505,7 @@ export const getDatasetAnalytics = (datasetId, tag) => {
 export const getStars = datasetId => Star.find({ datasetId })
 
 export const getUserStarred = (datasetId, userId) =>
-  Star.count({ datasetId, userId }).exec()
+  Star.countDocuments({ datasetId, userId }).exec()
 
 export const getFollowers = datasetId => {
   return Subscription.find({

package/src/datalad/snapshots.js

@@ -28,7 +28,7 @@ const lockSnapshot = (datasetId, tag) => {
 }
 
 const createSnapshotMetadata = (datasetId, tag, hexsha, created) => {
-  return Snapshot.update(
+  return Snapshot.updateOne(
     { datasetId: datasetId, tag: tag },
     {
       $set: {

package/src/graphql/permissions.js

@@ -50,7 +50,7 @@ export const checkPermissionLevel = (permission, state) => {
 }
 
 export const checkDatasetExists = async datasetId => {
-  const found = await Dataset.count({ id: datasetId }).exec()
+  const found = await Dataset.countDocuments({ id: datasetId }).exec()
   if (!found) throw new Error(`Dataset ${datasetId} does not exist.`)
 }
 

package/src/graphql/resolvers/__tests__/importRemoteDataset.spec.ts

@@ -0,0 +1,41 @@
+import fetchMock from 'jest-fetch-mock'
+import { importRemoteDataset, allowedImportUrl } from '../importRemoteDataset'
+import { checkDatasetWrite } from '../../permissions'
+
+jest.mock('../../../config')
+jest.mock('../../permissions')
+
+describe('importRemoteDataset mutation', () => {
+  it('given a user with access, it creates an import record for later processing', () => {
+    fetchMock.mockOnce(JSON.stringify(true))
+    importRemoteDataset(
+      {},
+      { datasetId: 'ds000000', url: '' },
+      { user: '1234', userInfo: { admin: true } },
+    )
+  })
+  describe('allowedImportUrl()', () => {
+    it('allows brainlife.io', () => {
+      expect(
+        allowedImportUrl('https://brainlife.io/ezbids/dataset-to-import.zip'),
+      ).toBe(true)
+    })
+    it('allows a test bucket for OpenNeuro use', () => {
+      expect(
+        allowedImportUrl(
+          'https://openneuro-test-import-bucket.s3.us-west-2.amazonaws.com/ds000003.zip',
+        ),
+      ).toBe(true)
+    })
+    it('does not allow other URLs', () => {
+      expect(allowedImportUrl('https://openneuro.org')).toBe(false)
+      expect(allowedImportUrl('iiajsdfoijawe')).toBe(false)
+      expect(allowedImportUrl('http://google.com/some-zip-file.zip')).toBe(
+        false,
+      )
+      expect(
+        allowedImportUrl('http://github.com/brainlife.io/somewhere-else.zip'),
+      ).toBe(false)
+    })
+  })
+})

package/src/graphql/resolvers/dataset.js

@@ -83,7 +83,7 @@ export const datasetName = obj => {
  */
 export const updateDatasetName = datasetId =>
   datasetName({ id: datasetId }).then(name =>
-    DatasetModel.update({ id: datasetId }, { $set: { name } }).exec(),
+    DatasetModel.updateOne({ id: datasetId }, { $set: { name } }).exec(),
   )
 
 /**

package/src/graphql/resolvers/importRemoteDataset.ts

@@ -0,0 +1,81 @@
+import fetch from 'node-fetch'
+import IngestDataset from '../../models/ingestDataset'
+import { checkDatasetWrite } from '../permissions.js'
+import { getDatasetWorker } from '../../libs/datalad-service'
+import { generateDataladCookie } from '../../libs/authentication/jwt'
+import notifications from '../../libs/notifications'
+import config from '../../config'
+
+/**
+ * Test if a URL is allowed to be imported
+ * @param raw String URL
+ * @returns {boolean}
+ */
+export function allowedImportUrl(raw: string): boolean {
+  let url
+  try {
+    url = new URL(raw)
+  } catch (_) {
+    return false
+  }
+  if (url.hostname === 'brainlife.io') {
+    return true
+  } else if (
+    url.hostname === 'openneuro-test-import-bucket.s3.us-west-2.amazonaws.com'
+  ) {
+    return true
+  } else {
+    return false
+  }
+}
+
+/**
+ * Queue a bundle of files for import into an existing dataset
+ */
+export async function importRemoteDataset(
+  _: Record<string, unknown>,
+  { datasetId, url }: { datasetId: string; url: string },
+  { user, userInfo }: { user: string; userInfo: Record<string, unknown> },
+): Promise<string | null> {
+  await checkDatasetWrite(datasetId, user, userInfo)
+  if (!allowedImportUrl(url)) {
+    return
+  }
+  const ingest = new IngestDataset({ datasetId, url, userId: user })
+  // undefined validateSync() means no errors
+  if (ingest.validateSync() == undefined) {
+    await ingest.save()
+    const importId = ingest._id.toString()
+    const worker = getDatasetWorker(datasetId)
+    const importUrl = `http://${worker}/datasets/${datasetId}/import/${importId}`
+    await fetch(importUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Cookie: generateDataladCookie(config)(userInfo),
+      },
+      body: JSON.stringify({ url }),
+    })
+    return ingest._id.toString()
+  } else {
+    return
+  }
+}
+
+export async function finishImportRemoteDataset(
+  _: Record<string, unknown>,
+  { id, success, message }: { id: string; success: boolean; message: string },
+  { user, userInfo }: { user: string; userInfo: Record<string, unknown> },
+): Promise<boolean> {
+  const ingest = await IngestDataset.findById(id)
+  ingest.imported = success
+  await ingest.save()
+  await notifications.datasetImported(
+    ingest.datasetId,
+    ingest.userId,
+    success,
+    message,
+    ingest.url,
+  )
+  return true
+}

package/src/graphql/resolvers/mutation.js

@@ -36,6 +36,10 @@ import { reexportRemotes } from './reexporter'
 import { resetDraft } from './reset'
 import { createReviewer, deleteReviewer } from './reviewer'
 import { createRelation, deleteRelation } from './relation'
+import {
+  importRemoteDataset,
+  finishImportRemoteDataset,
+} from './importRemoteDataset'
 
 const Mutation = {
   createDataset,
@@ -79,6 +83,8 @@ const Mutation = {
   deleteReviewer,
   createRelation,
   deleteRelation,
+  importRemoteDataset,
+  finishImportRemoteDataset,
 }
 
 export default Mutation

package/src/graphql/schema.js

@@ -187,6 +187,10 @@ export const typeDefs = `
     createRelation(datasetId: ID!, doi: String!, relation: RelatedObjectRelation!, kind: RelatedObjectKind!, description: String): Dataset
     # Remove a relationship to an external DOI
     deleteRelation(datasetId: ID!, doi: String!): Dataset
+    # Submit an import for a remote dataset, returns id if the URL is accepted for import
+    importRemoteDataset(datasetId: ID!, url: String!): ID
+    # Finish and notify import is done, returns true if successful
+    finishImportRemoteDataset(id: ID!, success: Boolean!, message: String): Boolean
   }
 
   # Anonymous dataset reviewer

package/src/libs/counter.js

@@ -15,7 +15,7 @@ export default {
   getNext(type, callback) {
     Counter.findOne({ _id: type }).then(found => {
       if (found) {
-        Counter.update({ _id: type }, { $inc: { sequence_value: 1 } }).then(
+        Counter.updateOne({ _id: type }, { $inc: { sequence_value: 1 } }).then(
           callback(found.sequence_value + 1),
         )
       } else {

package/src/libs/email/templates/dataset-import-failed.ts

@@ -0,0 +1,93 @@
+export const datasetImportFailed = ({
+  siteUrl,
+  name,
+  datasetId,
+  message,
+  retryUrl,
+}: {
+  siteUrl: string
+  name: string
+  datasetId: string
+  message: string
+  retryUrl: string
+}): string => `<html>
+<head>
+<style>
+	body {
+		font-family: 'Open Sans', sans-serif;
+		font-weight: lighter;
+		background: #F5F5F5;
+	}
+	footer {
+		border-top: 1px solid #333;
+		padding-top: 15px;
+		background: #F5F5F5;
+	}
+	.link {
+		color: #00505c
+	}
+	.link:hover {
+		color: #0093a9
+	}
+	.top-bar {
+		width: 100%;
+		background: #333;
+		padding: 8px 0px 8px 15px;
+	}
+	.content {
+		padding: 15px;
+	}
+	p {
+		font-size: 16px;
+		font-weight: lighter;
+	}
+	b {
+		font-weight: bold;
+	}
+	.dataset-link {
+		display: inline-block;
+		background: #008599;
+		color: #FFF;
+		font-size: 20px;
+		padding: 8px 15px;
+		text-decoration: none;
+		cursor: pointer;
+	}
+	.comment {
+		border: 1px solid #ccc;
+		padding: 15px;
+	}
+	.log {
+		white-space: pre-wrap;
+	}
+	.FAILED {color: #d9534f;}
+	.FINISHED {color: #5cb85c;}
+</style>
+</head>
+<body>
+	<div class="top-bar">
+		<img src="${siteUrl}/assets/CRN-Logo-Placeholder.png" />
+	</div>
+	<div class="content">
+		<h2>Hi, ${name}</h2>
+
+		<p>
+			A dataset imported you requested failed. It was imported as <b>${datasetId}</b>. 
+		</p>
+
+		<div>
+			<a class='dataset-link' href="${siteUrl}/import?url=${retryUrl}&datasetId=${datasetId}">Click here to retry the import &raquo;</a>
+		</div>
+
+		<div>
+			<h4>Import Log</h4>
+			<p class="log">${message}</p>
+		</div>
+
+		<p>
+			Sincerely,
+			The CRN Team
+		</p>
+	</div>
+</body>
+<html>`

package/src/libs/email/templates/dataset-imported.ts

@@ -0,0 +1,84 @@
+export const datasetImportEmail = ({
+  siteUrl,
+  name,
+  datasetId,
+}: {
+  siteUrl: string
+  name: string
+  datasetId: string
+}): string => `<html>
+<head>
+<style>
+	body {
+		font-family: 'Open Sans', sans-serif;
+		font-weight: lighter;
+		background: #F5F5F5;
+	}
+	footer {
+		border-top: 1px solid #333;
+		padding-top: 15px;
+		background: #F5F5F5;
+	}
+	.link {
+		color: #00505c
+	}
+	.link:hover {
+		color: #0093a9
+	}
+	.top-bar {
+		width: 100%;
+		background: #333;
+		padding: 8px 0px 8px 15px;
+	}
+	.content {
+		padding: 15px;
+	}
+	p {
+		font-size: 16px;
+		font-weight: lighter;
+	}
+	b {
+		font-weight: bold;
+	}
+	.dataset-link {
+		display: inline-block;
+		background: #008599;
+		color: #FFF;
+		font-size: 20px;
+		padding: 8px 15px;
+		text-decoration: none;
+		cursor: pointer;
+	}
+	.comment {
+		border: 1px solid #ccc;
+		padding: 15px;
+	}
+	.log {
+		white-space: pre-wrap;
+	}
+	.FAILED {color: #d9534f;}
+	.FINISHED {color: #5cb85c;}
+</style>
+</head>
+<body>
+	<div class="top-bar">
+		<img src="${siteUrl}/assets/CRN-Logo-Placeholder.png" />
+	</div>
+	<div class="content">
+		<h2>Hi, ${name}</h2>
+
+		<p>
+			A dataset imported you requested has finished. It was imported as <b>${datasetId}</b>. 
+		</p>
+
+		<div>
+			<a class='dataset-link' href="${siteUrl}/datasets/${datasetId}">Click here to view the dataset on OpenNeuro &raquo;</a>
+		</div>
+
+		<p>
+			Sincerely,
+			The CRN Team
+		</p>
+	</div>
+</body>
+<html>`

package/src/libs/notifications.js

@@ -19,6 +19,8 @@ import { datasetDeleted } from '../libs/email/templates/dataset-deleted'
 import { ownerUnsubscribed } from '../libs/email/templates/owner-unsubscribed'
 import { snapshotCreated } from '../libs/email/templates/snapshot-created'
 import { snapshotReminder } from '../libs/email/templates/snapshot-reminder'
+import { datasetImportEmail } from '../libs/email/templates/dataset-imported'
+import { datasetImportFailed } from '../libs/email/templates/dataset-import-failed'
 
 function noop() {
   // No callback helper
@@ -60,32 +62,23 @@ const notifications = {
    */
   async snapshotCreated(datasetId, body, uploader) {
     const tag = body.tag
-    // if we still have a promise for the body files, await it
-    const files = await body.files
     const uploaderId = uploader ? uploader.id : null
-    const datasetDescription = files.find(
-      file => file.filename == 'dataset_description.json',
-    )
-    const datasetDescriptionId = datasetDescription
-      ? datasetDescription.id
-      : null
     const URI = getDatasetWorker(datasetId)
-    const datasetDescriptionUrl = `${URI}/datasets/${datasetId}/objects/${datasetDescriptionId}`
-
-    const changesFile = files.find(file => file.filename == 'CHANGES')
-    const changesId = changesFile ? changesFile.id : null
-    const changesUrl = `${URI}/datasets/${datasetId}/objects/${changesId}`
+    const datasetDescriptionUrl = `${URI}/datasets/${datasetId}/snapshots/${tag}/files/dataset_description.json`
+    const changesUrl = `${URI}/datasets/${datasetId}/snapshots/${tag}/files/CHANGES`
 
     // get the dataset description
-    let response = await request.get(datasetDescriptionUrl)
-    const description = response.body
+    const descriptionResponse = await request.get(datasetDescriptionUrl)
+    const description = descriptionResponse.body
     const datasetLabel = description.Name ? description.Name : 'Unnamed Dataset'
 
     // get the snapshot changelog
-    response = await request
+    const changesResponse = await request
       .get(changesUrl)
       .responseType('application/octet-stream')
-    const changelog = response.body ? response.body.toString() : null
+    const changelog = changesResponse.body
+      ? changesResponse.body.toString()
+      : null
     // get all users that are subscribed to the dataset
     const subscriptions = await Subscription.find({
       datasetId: datasetId,
@@ -342,6 +335,50 @@ const notifications = {
     )
   },
 
+  /**
+   * Import of a remote resource finished
+   * @param {string} datasetId
+   * @param {string} userId
+   * @param {boolean} success
+   * @param {string} message
+   */
+  async datasetImported(datasetId, userId, success, message, retryUrl) {
+    const user = await User.findOne({ id: userId }).exec()
+    let html
+    if (success) {
+      html = datasetImportEmail({
+        name: user.name,
+        datasetId: datasetId,
+        siteUrl:
+          url.parse(config.url).protocol +
+          '//' +
+          url.parse(config.url).hostname,
+      })
+    } else {
+      html = datasetImportFailed({
+        name: user.name,
+        datasetId: datasetId,
+        message: success ? '' : message,
+        siteUrl:
+          url.parse(config.url).protocol +
+          '//' +
+          url.parse(config.url).hostname,
+        retryUrl: retryUrl,
+      })
+    }
+    const emailContent = {
+      _id: datasetId + '_' + user._id + '_' + 'dataset_imported',
+      type: 'email',
+      email: {
+        to: user.email,
+        subject: `Dataset Import ${success ? 'Success' : 'Failed'}`,
+        html: html,
+      },
+    }
+    // send the email to the notifications database for distribution
+    notifications.add(emailContent, noop)
+  },
+
   initCron() {
     setInterval(() => {
       // After one hour, retry a notification even if we have a lock

package/src/models/__tests__/ingestDataset.spec.ts

@@ -0,0 +1,37 @@
+import IngestDataset from '../ingestDataset'
+
+describe('IngestDataset model', () => {
+  it('IngestDataset model fails if required fields are missing', done => {
+    const model = new IngestDataset()
+    model.validate(result => {
+      expect(result.name).toEqual('ValidationError')
+      done()
+    })
+  })
+  it('IngestDataset model URL validation fails with a bad URL', done => {
+    const badUrlModel = new IngestDataset({
+      datasetId: 'ds00000',
+      userId: 'b3df6399-d1be-4e07-b997-9f7aa3ed1f8e',
+      url: 'this is not a valid URL',
+      imported: false,
+      notified: false,
+    })
+    badUrlModel.validate(result => {
+      expect(result.name).toEqual('ValidationError')
+      done()
+    })
+  })
+  it('IngestDataset model URL validation succeeds with a good URL', done => {
+    const goodUrlModel = new IngestDataset({
+      datasetId: 'ds00000',
+      userId: 'b3df6399-d1be-4e07-b997-9f7aa3ed1f8e',
+      url: 'https://example.com',
+      imported: false,
+      notified: false,
+    })
+    goodUrlModel.validate(result => {
+      expect(result).toBe(null)
+      done()
+    })
+  })
+})

package/src/models/ingestDataset.ts

@@ -0,0 +1,35 @@
+/**
+ * Model for ingest of new datasets from a remote URL (zip/tarball)
+ */
+import mongoose, { Document, ObjectId } from 'mongoose'
+const { Schema, model } = mongoose
+import { validateUrl } from '../utils/validateUrl'
+
+export interface IngestDatasetDocument extends Document {
+  _id: ObjectId
+  datasetId: string
+  userId: string
+  url: string
+  imported: boolean
+}
+
+const ingestDatasetSchema = new Schema({
+  datasetId: { type: String, required: true },
+  userId: { type: String, required: true },
+  url: {
+    type: String,
+    required: true,
+    validate: {
+      validator: validateUrl,
+      message: 'Must be a valid HTTPS URL',
+    },
+  },
+  imported: { type: Boolean, required: true, default: false },
+})
+
+const IngestDataset = model<IngestDatasetDocument>(
+  'IngestDataset',
+  ingestDatasetSchema,
+)
+
+export default IngestDataset

package/src/utils/__tests__/validateUrl.spec.ts

@@ -0,0 +1,13 @@
+import { validateUrl } from '../validateUrl'
+
+describe('validateUrl', () => {
+  it('returns true for a regular HTTPS url', () => {
+    expect(validateUrl('https://openneuro.org')).toBe(true)
+  })
+  it('returns false for a regular HTTP url', () => {
+    expect(validateUrl('http://openneuro.org')).toBe(false)
+  })
+  it('returns false for something that is not really a URL', () => {
+    expect(validateUrl('openneuro.org/robots.txt')).toBe(false)
+  })
+})

package/src/utils/validateUrl.ts

@@ -0,0 +1,13 @@
+/**
+ * Test if a string is an HTTPS URL
+ * @param value Text string to test for a URL
+ * @returns {boolean} True if valid
+ */
+export function validateUrl(value: string): boolean {
+  try {
+    const url = new URL(value)
+    return url.protocol === 'https:'
+  } catch (err) {
+    return false
+  }
+}

package/src/migrations/05-nameField.js

@@ -1,20 +0,0 @@
-/* eslint-disable no-console */
-/**
- * Prepopulate Dataset.name field on upgrade
- */
-import path from 'path'
-import Dataset from '../models/dataset'
-import { datasetName } from '../graphql/resolvers/dataset.js'
-
-export default {
-  id: path.basename(module.filename),
-  update: async () => {
-    const datasets = await Dataset.find().exec()
-    for (const dataset of datasets) {
-      await datasetName(dataset).then(name => {
-        console.log(`Updating ${dataset.id} with name "${name}"`)
-        return Dataset.update({ id: dataset.id }, { $set: { name } }).exec()
-      })
-    }
-  },
-}

package/src/migrations/index.js

@@ -1,4 +0,0 @@
-import nameField from './05-nameField.js'
-
-// Ordered list should match a sort of the filenames
-export default [nameField]

package/src/migrations/upgrade.js

@@ -1,50 +0,0 @@
-/* eslint-disable no-console */
-// Run all pending migrations
-import config from '../config.js'
-import { connect as redisConnect } from '../libs/redis.js'
-import mongoose from 'mongoose'
-import Migration from '../models/migration'
-import migrations from './index.js'
-
-// Setup Mongoose
-mongoose.connect(`${config.mongo.url}crn`)
-
-/**
- * This is a basic migration system, runs any unapplied updates in order
- * from the index provided in index.js
- *
- * Will yell at you if there are errors.
- *
- * Runs manually for now but could run at startup.
- */
-const upgradeAll = async () => {
-  await redisConnect(config.redis)
-  // Connect to old database(s)
-  for (const migrationDefinition of migrations) {
-    const key = migrationDefinition.id
-    const migrate = await Migration.findOneAndUpdate(
-      { id: key },
-      {},
-      { upsert: true, new: true, setDefaultsOnInsert: true },
-    )
-    try {
-      if (migrate.complete) {
-        console.log(`${key} has already run - continuing`)
-      } else {
-        await migrationDefinition.update()
-        console.log(`${key} migration complete`)
-        migrate.complete = true
-        await migrate.save()
-      }
-    } catch (e) {
-      console.log(`${key} failed to execute - exiting`)
-      throw e
-    }
-  }
-}
-
-// Entrypoint
-upgradeAll().then(() => {
-  mongoose.connection.close()
-  process.exit(0)
-})

package/src/models/migration.ts

@@ -1,16 +0,0 @@
-import mongoose, { Document } from 'mongoose'
-const { Schema, model } = mongoose
-
-export interface MigrationDocument extends Document {
-  id: string
-  complete: boolean
-}
-
-const migrationSchema = new Schema({
-  id: String,
-  complete: { type: Boolean, default: false },
-})
-
-const Migration = model<MigrationDocument>('Migration', migrationSchema)
-
-export default Migration