@openneuro/server 4.17.2 → 4.18.1

package/Dockerfile

@@ -4,7 +4,7 @@ FROM openneuro/node AS build
 WORKDIR /srv/packages/openneuro-server
 RUN yarn build
 
-FROM node:18.12.0-alpine
+FROM node:18.15.0-alpine
 
 WORKDIR /srv
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.17.2",
+  "version": "4.18.1",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -17,7 +17,7 @@
   "dependencies": {
     "@apollo/client": "3.7.2",
     "@elastic/elasticsearch": "7.15.0",
-    "@openneuro/search": "^4.17.2",
+    "@openneuro/search": "^4.18.1",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@sentry/node": "^4.5.3",
     "apollo-server": "2.25.4",
@@ -92,5 +92,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "7e0baf02f32539a1ca3c2bf146e9e5f7ba437854"
+  "gitHead": "fbd8afe9033abf11e39e17968b3e9f60261a9c01"
 }

package/src/cache/item.ts

@@ -52,26 +52,34 @@ class CacheItem {
     const deserialized: T = JSON.parse(decompressed.toString())
     return deserialized
   }
-  public async get<T>(miss: () => Promise<T>): Promise<T> {
+  public async get<T>(
+    miss: (setDoNotCache?: (doNotCache) => void) => Promise<T>,
+  ): Promise<T> {
     try {
       const data = await this.redis.getBuffer(this.key)
       if (data) {
         return this.deserialize(data)
       } else {
+        let skipCaching = false
+        const doNotCache = (doNotCache: boolean): void => {
+          skipCaching = doNotCache
+        }
         // Call the cache miss function if we didn't get anything
-        const data = await miss()
+        const data = await miss(doNotCache)
         const serialized = await this.serialize(data)
-        // Allow for the simple case of aging out keys
-        if (this.expiration > 0) {
-          void this.redis.setex(this.key, this.expiration, serialized)
-        } else {
-          void this.redis.set(this.key, serialized)
+        // Allow the cache miss function to return a value that isn't cached
+        if (!skipCaching) {
+          // Allow for the simple case of aging out keys
+          if (this.expiration > 0) {
+            void this.redis.setex(this.key, this.expiration, serialized)
+          } else {
+            void this.redis.set(this.key, serialized)
+          }
         }
         return data
       }
     } catch {
       // Keep going as though we had a cache miss if there is a problem but don't cache it
-      // TODO: Sentry reporting doesn't work here but should be fixed
       return miss()
     }
   }

package/src/cache/types.ts

@@ -11,4 +11,5 @@ export enum CacheType {
   snapshot = 'snapshot',
   snapshotIndex = 'snapshotIndex',
   participantCount = 'participantCount',
+  snapshotDownload = 'download',
 }

package/src/datalad/__tests__/files.spec.js

@@ -2,9 +2,8 @@ import {
   encodeFilePath,
   decodeFilePath,
   fileUrl,
-  filterFiles,
   computeTotalSize,
-} from '../files.js'
+} from '../files'
 
 vi.mock('ioredis')
 vi.mock('../../config.js')
@@ -59,53 +58,6 @@ describe('datalad files', () => {
       )
     })
   })
-  describe('filterFiles()', () => {
-    it('disables the filter when set to null', () => {
-      expect(filterFiles(null)(mockFiles)).toBe(mockFiles)
-    })
-    it("returns only root level files with '' filter", () => {
-      const mockDirs = [
-        {
-          filename: 'sub-01',
-          id: 'directory:sub-01',
-          urls: [],
-          size: 2,
-          directory: true,
-        },
-        {
-          filename: 'sub-02',
-          id: 'directory:sub-02',
-          urls: [],
-          size: 2,
-          directory: true,
-        },
-        {
-          filename: 'sub-03',
-          id: 'directory:sub-03',
-          urls: [],
-          size: 2,
-          directory: true,
-        },
-        {
-          filename: 'derivatives',
-          id: 'directory:derivatives',
-          urls: [],
-          size: 1,
-          directory: true,
-        },
-      ]
-      expect(filterFiles('')(mockFiles)).toEqual([
-        ...mockRootFiles,
-        ...mockDirs,
-      ])
-    })
-    it('returns only matching prefixed files with a directory name filter', () => {
-      expect(filterFiles('sub-01')(mockFiles)).toEqual(mockSub01)
-    })
-    it('works correctly for deeply nested files', () => {
-      expect(filterFiles('sub-01/func')(mockFiles)).toEqual([mockSub01[1]])
-    })
-  })
   describe('computeTotalSize()', () => {
     it('computes the size correctly', () => {
       const mockFileSizes = [

package/src/datalad/description.js

@@ -5,7 +5,7 @@ import config from '../config'
 import request from 'superagent'
 import { redis } from '../libs/redis.js'
 import { commitFiles } from './dataset.js'
-import { fileUrl } from './files.js'
+import { fileUrl } from './files'
 import { generateDataladCookie } from '../libs/authentication/jwt'
 import { getDatasetWorker } from '../libs/datalad-service'
 import CacheItem, { CacheType } from '../cache/item'

package/src/datalad/files.ts

@@ -0,0 +1,115 @@
+import request from 'superagent'
+import { redis } from '../libs/redis'
+import CacheItem, { CacheType } from '../cache/item'
+import { getDatasetWorker } from '../libs/datalad-service'
+
+/**
+ * Convert to URL compatible path
+ * @param {String} path
+ */
+export const encodeFilePath = (path: string): string => {
+  return path.replace(new RegExp('/', 'g'), ':')
+}
+
+/**
+ * Convert to from URL compatible path fo filepath
+ * @param {String} path
+ */
+export const decodeFilePath = (path: string): string => {
+  return path.replace(new RegExp(':', 'g'), '/')
+}
+
+/**
+ * If path is provided, this is a subdirectory, otherwise a root level file.
+ * @param {String} path
+ * @param {String} filename
+ */
+export const getFileName = (path: string, filename: string): string => {
+  const filePath = path ? [path, filename].join('/') : filename
+  return filename ? encodeFilePath(filePath) : encodeFilePath(path)
+}
+
+/**
+ * Generate file URL for DataLad service
+ * @param {string} datasetId
+ * @param {string} path - Relative path for the file
+ * @param {string} filename
+ * @param {string} [revision] - Git hash of commit or tree owning this file
+ */
+export const fileUrl = (
+  datasetId: string,
+  path: string,
+  filename: string,
+  revision?: string,
+): string => {
+  const fileName = getFileName(path, filename)
+  if (revision) {
+    return `http://${getDatasetWorker(
+      datasetId,
+    )}/datasets/${datasetId}/snapshots/${revision}/files/${fileName}`
+  } else {
+    return `http://${getDatasetWorker(
+      datasetId,
+    )}/datasets/${datasetId}/files/${fileName}`
+  }
+}
+
+/**
+ * Generate path URL (such a directory or virtual path) for DataLad service
+ * @param {String} datasetId
+ */
+export const filesUrl = (datasetId: string): string =>
+  `http://${getDatasetWorker(datasetId)}/datasets/${datasetId}/files`
+
+/** Minimal variant of DatasetFile type from GraphQL API */
+type DatasetFile = {
+  id: string
+  filename: string
+  directory: boolean
+  size: number
+  urls: string[]
+}
+
+/**
+ * Sum all file sizes for total dataset size
+ */
+export const computeTotalSize = (files: [DatasetFile]): number =>
+  files.reduce((size, f) => size + f.size, 0)
+
+/**
+ * Get files for a specific revision
+ * Similar to getDraftFiles but different cache key and fixed revisions
+ * @param {string} datasetId - Dataset accession number
+ * @param {string} treeish - Git treeish hexsha
+ */
+export const getFiles = (datasetId, treeish): Promise<[DatasetFile]> => {
+  const cache = new CacheItem(redis, CacheType.commitFiles, [
+    datasetId,
+    treeish.substring(0, 7),
+  ])
+  return cache.get(
+    doNotCache =>
+      request
+        .get(
+          `${getDatasetWorker(
+            datasetId,
+          )}/datasets/${datasetId}/tree/${treeish}`,
+        )
+        .set('Accept', 'application/json')
+        .then(response => {
+          if (response.status === 200) {
+            const {
+              body: { files },
+            } = response
+            for (const f of files) {
+              // Skip caching this tree if it doesn't contain S3 URLs - likely still exporting
+              if (!f.directory && !f.urls[0].includes('s3.amazonaws.com')) {
+                doNotCache(true)
+                break
+              }
+            }
+            return files as [DatasetFile]
+          }
+        }) as Promise<[DatasetFile]>,
+  )
+}

package/src/datalad/snapshots.js

@@ -20,6 +20,7 @@ import Dataset from '../models/dataset'
 import Snapshot from '../models/snapshot'
 import { updateDatasetRevision } from './draft.js'
 import { getDatasetWorker } from '../libs/datalad-service'
+import { join } from 'path'
 
 const lockSnapshot = (datasetId, tag) => {
   return redlock.lock(
@@ -264,3 +265,35 @@ export const getPublicSnapshots = () => {
       ]).exec()
     })
 }
+
+/**
+ * For snapshots, precache all trees for downloads
+ */
+export const downloadFiles = (datasetId, tag) => {
+  const downloadCache = new CacheItem(redis, CacheType.snapshotDownload, [
+    datasetId,
+    tag,
+  ])
+  // Return an existing cache object if we have one
+  return downloadCache.get(async () => {
+    // If not, fetch all trees sequentially and cache the result (hopefully some or all trees are cached)
+    const files = await getFilesRecursive(datasetId, tag, '')
+    files.sort()
+    return files
+  })
+}
+
+export async function getFilesRecursive(datasetId, tree, path = '') {
+  const files = []
+  // Fetch files
+  const fileTree = await getFiles(datasetId, tree)
+  for (const file of fileTree) {
+    const absPath = join(path, file.filename)
+    if (file.directory) {
+      files.push(...(await getFilesRecursive(datasetId, file.id, absPath)))
+    } else {
+      files.push({ ...file, filename: absPath })
+    }
+  }
+  return files
+}

package/src/graphql/resolvers/cache.ts

@@ -1,21 +1,25 @@
 import { redis } from '../../libs/redis.js'
+import CacheItem from '../../cache/item'
+import { CacheType } from '../../cache/types'
 
+/**
+ * Clear the snapshotDownload cache after exports
+ */
 export async function cacheClear(
   obj: Record<string, unknown>,
-  { datasetId }: { datasetId: string },
+  { datasetId, tag }: { datasetId: string; tag: string },
   { userInfo }: { userInfo: { admin: boolean } },
 ): Promise<boolean> {
   // Check for admin and validate datasetId argument
   if (userInfo?.admin && datasetId.length == 8 && datasetId.startsWith('ds')) {
-    const keys = await redis.keys(`*${datasetId}*`)
-    if (keys.length) {
-      const transaction = redis.pipeline()
-      keys.forEach(key => {
-        transaction.unlink(key)
-      })
-      transaction.exec()
+    const downloadCache = new CacheItem(redis, CacheType.snapshotDownload, [
+      datasetId,
+      tag,
+    ])
+    try {
+      await downloadCache.drop()
       return true
-    } else {
+    } catch (err) {
       return false
     }
   } else {

package/src/graphql/resolvers/dataset.js

@@ -19,7 +19,7 @@ import Deletion from '../../models/deletion'
 import { reviewers } from './reviewer'
 import { getDatasetWorker } from '../../libs/datalad-service.js'
 import { getDraftHead } from '../../datalad/dataset.js'
-import { getFileName } from '../../datalad/files.js'
+import { getFileName } from '../../datalad/files'
 import { onBrainlife } from './brainlife'
 import { derivatives } from './derivatives'
 import semver from 'semver'

package/src/graphql/resolvers/draft.js

@@ -5,7 +5,7 @@ import { description } from './description.js'
 import { readme } from './readme.js'
 import { getDraftRevision, updateDatasetRevision } from '../../datalad/draft.js'
 import { checkDatasetWrite } from '../permissions.js'
-import { getFiles } from '../../datalad/files.js'
+import { getFiles } from '../../datalad/files'
 import { filterRemovedAnnexObjects } from '../utils/file.js'
 
 // A draft must have a dataset parent

package/src/graphql/resolvers/query.js

@@ -4,7 +4,6 @@
 import { dataset, datasets } from './dataset.js'
 import { snapshot, participantCount } from './snapshots.js'
 import { user, users } from './user.js'
-import { datasetChanges } from './dataset-change.js'
 import { flaggedFiles } from './flaggedFiles'
 
 const Query = {
@@ -14,7 +13,6 @@ const Query = {
   users,
   snapshot,
   participantCount,
-  datasetChanges,
   flaggedFiles,
 }
 

package/src/graphql/resolvers/snapshots.js

@@ -6,7 +6,7 @@ import { readme } from './readme.js'
 import { description } from './description.js'
 import { summary } from './summary.js'
 import { snapshotIssues } from './issues.js'
-import { getFiles } from '../../datalad/files.js'
+import { getFiles } from '../../datalad/files'
 import Summary from '../../models/summary'
 import DatasetModel from '../../models/dataset'
 import { filterRemovedAnnexObjects } from '../utils/file.js'
@@ -15,6 +15,7 @@ import { redis } from '../../libs/redis'
 import CacheItem, { CacheType } from '../../cache/item'
 import { normalizeDOI } from '../../libs/doi/normalize'
 import { getDraftHead } from '../../datalad/dataset'
+import { downloadFiles } from '../../datalad/snapshots'
 
 export const snapshots = obj => {
   return datalad.getSnapshots(obj.id)
@@ -40,6 +41,7 @@ export const snapshot = (obj, { datasetId, tag }, context) => {
         deprecated: () => deprecated({ datasetId, tag }),
         related: () => related(datasetId),
         onBrainlife: () => onBrainlife(snapshot),
+        downloadFiles: () => downloadFiles(datasetId, tag),
       }))
     },
   )

package/src/graphql/resolvers/subscriptions.js

@@ -70,19 +70,12 @@ export const filesUpdated = {
   },
 }
 
-export const datasetChanged = {
-  type: 'DatasetChange',
-  subscribe: () => pubsub.asyncIterator('datasetChanged'),
-  args: {},
-}
-
 const Subscription = {
   datasetDeleted,
   snapshotsUpdated,
   permissionsUpdated,
   draftUpdated,
   filesUpdated,
-  datasetChanged,
 }
 
 export default Subscription

package/src/graphql/schema.js

@@ -90,11 +90,6 @@ export const typeDefs = `
     participantCount(modality: String): Int @cacheControl(maxAge: 3600, scope: PUBLIC)
     # Request one snapshot
     snapshot(datasetId: ID!, tag: String!): Snapshot
-    # Get recent dataset changes (newest first)
-    datasetChanges(
-      "Limit results, default 100, max 1000"
-      limit: Int = 100
-    ): [DatasetChange]
     # Get annexed files that have been flagged or removed.
     flaggedFiles(
       "Get files that have been flagged, default true."
@@ -165,8 +160,8 @@ export const typeDefs = `
     prepareUpload(datasetId: ID!, uploadId: ID!): UploadMetadata
     # Add files from a completed upload to the dataset draft
     finishUpload(uploadId: ID!): Boolean
-    # Drop caches for a given dataset - requires site admin access
-    cacheClear(datasetId: ID!): Boolean
+    # Drop download cache for a snapshot - requires site admin access
+    cacheClear(datasetId: ID!, tag: String!): Boolean
     # Rerun the latest validator on a given commit
     revalidate(datasetId: ID!, ref: String!): Boolean
     # Request a temporary token for git access
@@ -478,6 +473,8 @@ export const typeDefs = `
     onBrainlife: Boolean @cacheControl(maxAge: 10080, scope: PUBLIC)
     # Total size in bytes of this snapshot
     size: BigInt
+    # Single list of files to download this snapshot (only available on snapshots)
+    downloadFiles: [DatasetFile]
   }
 
   # RelatedObject nature of relationship
@@ -700,15 +697,6 @@ export const typeDefs = `
     payload: [DatasetFile]
   }
 
-  # Recent changes to datasets
-  type DatasetChange {
-    datasetId: String!
-    created: Boolean
-    modified: Boolean
-    deleted: Boolean
-    timestamp: DateTime
-  }
-
   # Analytics for a dataset
   type Analytic @cacheControl(maxAge: 300, scope: PUBLIC) {
     datasetId: ID!

package/src/handlers/datalad.js

@@ -1,5 +1,7 @@
 import request from 'superagent'
+import { Readable } from 'node:stream'
 import mime from 'mime-types'
+import { getFiles } from '../datalad/files'
 import { getDatasetWorker } from '../libs/datalad-service'
 
 /**
@@ -14,14 +16,48 @@ import { getDatasetWorker } from '../libs/datalad-service'
 /**
  * Get a file from a dataset
  */
-export const getFile = (req, res) => {
+export const getFile = async (req, res) => {
   const { datasetId, snapshotId, filename } = req.params
   const worker = getDatasetWorker(datasetId)
-  res.set('Content-Type', mime.lookup(filename) || 'application/octet-stream')
-  const uri = snapshotId
-    ? `${worker}/datasets/${datasetId}/snapshots/${snapshotId}/files/${filename}`
-    : `${worker}/datasets/${datasetId}/files/${filename}`
-  return request.get(uri).pipe(res)
+  // Find the right tree
+  const pathComponents = filename.split(':')
+  let tree = snapshotId || 'HEAD'
+  let file
+  for (const level of pathComponents) {
+    const files = await getFiles(datasetId, tree)
+    if (level == pathComponents.slice(-1)) {
+      file = files.find(f => !f.directory && f.filename === level)
+    } else {
+      tree = files.find(f => f.directory && f.filename === level).id
+    }
+  }
+  // Get the file URL and redirect if external or serve if local
+  if (file && file.urls[0].startsWith('https://s3.amazonaws.com/')) {
+    res.redirect(file.urls[0])
+  } else {
+    // Serve the file directly
+    res.set('Content-Type', mime.lookup(filename) || 'application/octet-stream')
+    const uri = snapshotId
+      ? `http://${worker}/datasets/${datasetId}/snapshots/${snapshotId}/files/${filename}`
+      : `http://${worker}/datasets/${datasetId}/files/${filename}`
+    return (
+      fetch(uri)
+        .then(r => {
+          // Set the content length (allow clients to catch HTTP issues better)
+          res.setHeader(
+            'Content-Length',
+            Number(r.headers.get('content-length')),
+          )
+          return r.body
+        })
+        // @ts-expect-error
+        .then(stream => Readable.fromWeb(stream).pipe(res))
+        .catch(err => {
+          console.error(err)
+          res.status(500).send('Internal error transferring requested file')
+        })
+    )
+  }
 }
 
 /**

package/src/models/dataset.ts

@@ -1,6 +1,5 @@
 import mongoose, { Document } from 'mongoose'
 const { Schema, model } = mongoose
-import DatasetChange from './datasetChange'
 
 // External relations annotating the whole dataset
 export interface DatasetRelationDocument extends Document {
@@ -74,22 +73,6 @@ datasetSchema.virtual('subscriptions', {
   justOne: true,
 })
 
-datasetSchema.post('updateOne', function () {
-  const datasetId = this.getQuery()?.['id']
-  return new DatasetChange({
-    datasetId,
-    modified: true,
-  }).save()
-})
-
-datasetSchema.post('deleteOne', function () {
-  const datasetId = this.getQuery()?.['id']
-  return new DatasetChange({
-    datasetId,
-    deleted: true,
-  }).save()
-})
-
 const Dataset = model<DatasetDocument>('Dataset', datasetSchema)
 
 export default Dataset

package/src/datalad/files.js

@@ -1,132 +0,0 @@
-import request from 'superagent'
-import { redis } from '../libs/redis'
-import CacheItem, { CacheType } from '../cache/item'
-import { getDatasetWorker } from '../libs/datalad-service'
-
-/**
- * Convert to URL compatible path
- * @param {String} path
- */
-export const encodeFilePath = path => {
-  return path.replace(new RegExp('/', 'g'), ':')
-}
-
-/**
- * Convert to from URL compatible path fo filepath
- * @param {String} path
- */
-export const decodeFilePath = path => {
-  return path.replace(new RegExp(':', 'g'), '/')
-}
-
-/**
- * If path is provided, this is a subdirectory, otherwise a root level file.
- * @param {String} path
- * @param {String} filename
- */
-export const getFileName = (path, filename) => {
-  const filePath = path ? [path, filename].join('/') : filename
-  return filename ? encodeFilePath(filePath) : encodeFilePath(path)
-}
-
-/**
- * Generate file URL for DataLad service
- * @param {String} datasetId
- * @param {String} path - Relative path for the file
- * @param {String} filename
- * @param {String} [revision] - Git hash of commit or tree owning this file
- */
-export const fileUrl = (datasetId, path, filename, revision) => {
-  const fileName = getFileName(path, filename)
-  if (revision) {
-    return `http://${getDatasetWorker(
-      datasetId,
-    )}/datasets/${datasetId}/snapshots/${revision}/files/${fileName}`
-  } else {
-    return `http://${getDatasetWorker(
-      datasetId,
-    )}/datasets/${datasetId}/files/${fileName}`
-  }
-}
-
-/**
- * Generate path URL (such a directory or virtual path) for DataLad service
- * @param {String} datasetId
- */
-export const filesUrl = datasetId =>
-  `http://${getDatasetWorker(datasetId)}/datasets/${datasetId}/files`
-
-/**
- * Sum all file sizes for total dataset size
- */
-export const computeTotalSize = files =>
-  files.reduce((size, f) => size + f.size, 0)
-
-/**
- * Get files for a specific revision
- * Similar to getDraftFiles but different cache key and fixed revisions
- * @param {string} datasetId - Dataset accession number
- * @param {string} treeish - Git treeish hexsha
- */
-export const getFiles = (datasetId, treeish) => {
-  const cache = new CacheItem(redis, CacheType.commitFiles, [
-    datasetId,
-    treeish.substring(0, 7),
-  ])
-  return cache.get(() =>
-    request
-      .get(
-        `${getDatasetWorker(datasetId)}/datasets/${datasetId}/tree/${treeish}`,
-      )
-      .set('Accept', 'application/json')
-      .then(response => {
-        if (response.status === 200) {
-          const {
-            body: { files },
-          } = response
-          return files
-        }
-      }),
-  )
-}
-
-/**
- * Given a list of files (from getFiles), return a subset matching the prefix
- * @param {string} prefix The prefix to filter on
- * @returns {(files: Object[]) => Object[]}
- */
-export const filterFiles =
-  (prefix = '') =>
-  files => {
-    // Disable on null
-    if (prefix === null) {
-      return files
-    }
-    // Track potential directories and include those as "files"
-    const directoryFacades = {}
-    // Return only root level files if prefix is set
-    const matchingFiles = files.filter(f => {
-      if (prefix === '') {
-        if (f.filename.includes('/')) {
-          const dirName = f.filename.split('/').slice(0, 1)[0]
-          if (directoryFacades[dirName] !== undefined) {
-            directoryFacades[dirName].size += 1
-          } else {
-            directoryFacades[dirName] = {
-              id: `directory:${dirName}`,
-              urls: [],
-              filename: dirName,
-              size: 1,
-              directory: true,
-            }
-          }
-          return false
-        } else {
-          return true
-        }
-      } else {
-        return f.filename.startsWith(prefix)
-      }
-    })
-    return [...matchingFiles, ...Object.values(directoryFacades)]
-  }

package/src/graphql/resolvers/dataset-change.js

@@ -1,5 +0,0 @@
-import DatasetChange from '../../models/datasetChange'
-
-export const datasetChanges = (_, { limit = 100 }) => {
-  return DatasetChange.find().sort({ $natural: -1 }).limit(limit)
-}

package/src/models/datasetChange.ts

@@ -1,39 +0,0 @@
-import mongoose, { Document } from 'mongoose'
-const { Schema, model } = mongoose
-import pubsub from '../graphql/pubsub'
-
-export interface DatasetChangeDocument extends Document {
-  datasetId: string
-  created: boolean
-  modified: boolean
-  deleted: boolean
-  timestamp: Date
-}
-
-const datasetChangeSchema = new Schema(
-  {
-    datasetId: { type: String, required: true },
-    created: { type: Boolean, default: false },
-    modified: { type: Boolean, default: false },
-    deleted: { type: Boolean, default: false },
-    timestamp: { type: Date, default: Date.now },
-  },
-  {
-    // limits the collection size to 1000 documents
-    // works like a circular buffer
-    capped: 1000,
-  },
-)
-
-datasetChangeSchema.post('save', doc => {
-  pubsub.publish('datasetChanged', {
-    datasetChanged: doc,
-  })
-})
-
-const DatasetChange = model<DatasetChangeDocument>(
-  'DatasetChange',
-  datasetChangeSchema,
-)
-
-export default DatasetChange