@sanity/export 3.37.4 → 3.38.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/export",
-  "version": "3.37.4",
+  "version": "3.38.0",
   "description": "Export Sanity documents and assets",
   "keywords": [
     "sanity",
@@ -27,9 +27,10 @@
   ],
   "scripts": {
     "lint": "eslint .",
-    "test": "jest"
+    "test": "jest --verbose"
   },
   "dependencies": {
+    "@sanity/client": "^6.15.20",
     "@sanity/util": "3.37.2",
     "archiver": "^7.0.0",
     "debug": "^4.3.4",
@@ -38,7 +39,9 @@
     "mississippi": "^4.0.0",
     "p-queue": "^2.3.0",
     "rimraf": "^3.0.2",
-    "split2": "^4.2.0"
+    "split2": "^4.2.0",
+    "tar": "^7.0.1",
+    "yaml": "^2.4.2"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
@@ -48,9 +51,11 @@
     "eslint-config-sanity": "^7.1.2",
     "eslint-plugin-prettier": "^5.1.3",
     "jest": "^29.7.0",
+    "nock": "^13.5.4",
     "prettier": "^3.2.5",
     "prettier-plugin-packagejson": "^2.5.0",
-    "string-to-stream": "^1.1.0"
+    "string-to-stream": "^1.1.0",
+    "tar": "^7.0.1"
   },
   "engines": {
     "node": ">=18"

package/src/AssetHandler.js

@@ -9,29 +9,11 @@ const pkg = require('../package.json')
 const debug = require('./debug')
 const requestStream = require('./requestStream')
 const rimraf = require('./util/rimraf')
+const {ASSET_DOWNLOAD_MAX_RETRIES, ASSET_DOWNLOAD_CONCURRENCY} = require('./constants')
 
 const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url']
 const ACTION_REMOVE = 'remove'
 const ACTION_REWRITE = 'rewrite'
-const ASSET_DOWNLOAD_CONCURRENCY = 8
-
-const retryHelper = (times, fn, onError) => {
-  let attempt = 0
-  const caller = (...args) => {
-    return fn(...args).catch((err) => {
-      if (onError) {
-        onError(err, attempt)
-      }
-      if (attempt < times) {
-        attempt++
-        return caller(...args)
-      }
-
-      throw err
-    })
-  }
-  return caller
-}
 
 class AssetHandler {
   constructor(options) {
@@ -47,6 +29,7 @@ class AssetHandler {
     this.assetMap = {}
     this.filesWritten = 0
     this.queueSize = 0
+    this.maxRetries = options.maxRetries || ASSET_DOWNLOAD_MAX_RETRIES
     this.queue = options.queue || new PQueue({concurrency})
 
     this.rejectedError = null
@@ -123,26 +106,42 @@ class AssetHandler {
     this.queueSize++
     this.downloading.push(assetDoc.url)
 
-    const doDownload = retryHelper(
-      10, // try 10 times
-      () => this.downloadAsset(assetDoc, dstPath),
-      (err, attempt) => {
-        debug(
-          `Error downloading asset %s (destination: %s), attempt %d`,
-          assetDoc._id,
-          dstPath,
-          attempt,
-          err,
-        )
-      },
-    )
-    this.queue.add(() =>
-      doDownload().catch((err) => {
-        debug('Error downloading asset', err)
-        this.queue.clear()
-        this.reject(err)
-      }),
-    )
+    const doDownload = async () => {
+      let dlError
+      for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+        try {
+          return await this.downloadAsset(assetDoc, dstPath)
+        } catch (err) {
+          debug(
+            `Error downloading asset %s (destination: %s), attempt %d`,
+            assetDoc._id,
+            dstPath,
+            attempt,
+            err,
+          )
+
+          dlError = err
+
+          if ('statusCode' in err && err.statusCode >= 400 && err.statusCode < 500) {
+            // Don't retry on client errors
+            break
+          }
+        }
+      }
+      throw dlError
+    }
+
+    this.queue
+      .add(() =>
+        doDownload().catch((err) => {
+          debug('Failed to download the asset, aborting download', err)
+          this.queue.clear()
+          this.reject(err)
+        }),
+      )
+      .catch((error) => {
+        debug('Queued task failed', error)
+      })
   }
 
   maybeCreateAssetDirs() {
@@ -163,7 +162,13 @@ class AssetHandler {
     const isImage = assetDoc._type === 'sanity.imageAsset'
 
     const url = parseUrl(assetDoc.url, true)
-    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
+    if (
+      isImage &&
+      token &&
+      (['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) ||
+        // used in tests. use a very specific port to avoid conflicts
+        url.host === 'localhost:43216')
+    ) {
       headers.Authorization = `Bearer ${token}`
       url.query = {...(url.query || {}), dlRaw: 'true'}
     }
@@ -191,15 +196,13 @@ class AssetHandler {
     }
 
     if (stream.statusCode !== 200) {
-      this.queue.clear()
+      let errMsg
       try {
         const err = await tryGetErrorFromStream(stream)
-        let errMsg = `Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`
+        errMsg = `Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`
         if (err) {
-          errMsg = `${errMsg}:\n\n${err}`
+          errMsg = `${errMsg}: ${err}`
         }
-
-        throw new Error(errMsg)
       } catch (err) {
         const message = 'Failed to parse error response from asset stream'
         if (typeof err.message === 'string') {
@@ -210,6 +213,10 @@ class AssetHandler {
 
         throw new Error(message, {cause: err})
       }
+
+      const streamError = new Error(errMsg)
+      streamError.statusCode = stream.statusCode
+      throw streamError
     }
 
     this.maybeCreateAssetDirs()
@@ -385,9 +392,20 @@ function writeHashedStream(filePath, stream) {
 
 function tryGetErrorFromStream(stream) {
   return new Promise((resolve, reject) => {
-    miss.pipe(stream, miss.concat(parse), (err) => (err ? reject(err) : noop))
+    let receivedData = false
+
+    miss.pipe(stream, miss.concat(parse), (err) => {
+      if (err) {
+        reject(err)
+      } else if (!receivedData) {
+        // Resolve with null if no data was received, to let the caller
+        // know we couldn't parse the error.
+        resolve(null)
+      }
+    })
 
     function parse(body) {
+      receivedData = true
       try {
         const parsed = JSON.parse(body.toString('utf8'))
         resolve(parsed.message || parsed.error || null)

package/src/constants.js

@@ -0,0 +1,22 @@
+/**
+ * How many retries to attempt when retrieving the document stream.
+ * User overridable as `options.maxRetries`.
+ *
+ * Note: Only for initial connection - if download fails while streaming, we cannot easily resume.
+ * @internal
+ */
+exports.DOCUMENT_STREAM_MAX_RETRIES = 5
+
+/**
+ * How many retries to attempt when downloading an asset.
+ * User overridable as `options.maxAssetRetries`.
+ * @internal
+ */
+exports.ASSET_DOWNLOAD_MAX_RETRIES = 10
+
+/**
+ * How many concurrent asset downloads to allow.
+ * User overridable as `options.assetConcurrency`.
+ * @internal
+ */
+exports.ASSET_DOWNLOAD_CONCURRENCY = 8

package/src/export.js

@@ -20,12 +20,16 @@ const validateOptions = require('./validateOptions')
 
 const noop = () => null
 
-function exportDataset(opts) {
+async function exportDataset(opts) {
   const options = validateOptions(opts)
   const onProgress = options.onProgress || noop
   const archive = archiver('tar', {
     gzip: true,
-    gzipOptions: {level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION},
+    gzipOptions: {
+      level: options.compress
+        ? zlib.constants.Z_DEFAULT_COMPRESSION
+        : zlib.constants.Z_NO_COMPRESSION,
+    },
   })
 
   const slugDate = new Date()
@@ -45,10 +49,11 @@ function exportDataset(opts) {
     tmpDir,
     prefix,
     concurrency: options.assetConcurrency,
+    maxRetries: options.maxAssetRetries,
   })
 
-  debug('Outputting assets (temporarily) to %s', tmpDir)
-  debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath)
+  debug('Downloading assets (temporarily) to %s', tmpDir)
+  debug('Downloading to %s', options.outputPath === '-' ? 'stdout' : options.outputPath)
 
   let outputStream
   if (isWritableStream(options.outputPath)) {
@@ -63,144 +68,157 @@ function exportDataset(opts) {
     assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets
   }
 
-  // eslint-disable-next-line no-async-promise-executor
-  return new Promise(async (resolve, reject) => {
-    miss.finished(archive, async (archiveErr) => {
-      if (archiveErr) {
-        debug('Archiving errored! %s', archiveErr.stack)
-        await cleanup()
-        reject(archiveErr)
-        return
-      }
-
-      debug('Archive finished!')
-    })
-
-    debug('Getting dataset export stream')
-    onProgress({step: 'Exporting documents...'})
-
-    let documentCount = 0
-    let lastReported = Date.now()
-    const reportDocumentCount = (chunk, enc, cb) => {
-      ++documentCount
+  let resolve
+  let reject
+  const result = new Promise((res, rej) => {
+    resolve = res
+    reject = rej
+  })
 
-      const now = Date.now()
-      if (now - lastReported > 50) {
-        onProgress({
-          step: 'Exporting documents...',
-          current: documentCount,
-          total: '?',
-          update: true,
-        })
+  miss.finished(archive, async (archiveErr) => {
+    if (archiveErr) {
+      debug('Archiving errored: %s', archiveErr.stack)
+      await cleanup()
+      reject(archiveErr)
+      return
+    }
 
-        lastReported = now
-      }
+    debug('Archive finished')
+  })
 
-      cb(null, chunk)
-    }
+  debug('Getting dataset export stream')
+  onProgress({step: 'Exporting documents...'})
 
-    const inputStream = await getDocumentsStream(options.client, options.dataset)
-    debug('Got HTTP %d', inputStream.statusCode)
-    debug('Response headers: %o', inputStream.headers)
-
-    const jsonStream = miss.pipeline(
-      inputStream,
-      logFirstChunk(),
-      split(tryParseJson),
-      rejectOnApiError(),
-      filterSystemDocuments(),
-      assetStreamHandler,
-      filterDocumentTypes(options.types),
-      options.drafts ? miss.through.obj() : filterDrafts(),
-      stringifyStream(),
-      miss.through(reportDocumentCount),
-    )
-
-    miss.finished(jsonStream, async (err) => {
-      if (err) {
-        return
-      }
+  let documentCount = 0
+  let lastReported = Date.now()
+  const reportDocumentCount = (chunk, enc, cb) => {
+    ++documentCount
 
+    const now = Date.now()
+    if (now - lastReported > 50) {
       onProgress({
         step: 'Exporting documents...',
         current: documentCount,
-        total: documentCount,
+        total: '?',
         update: true,
       })
 
-      if (!options.raw && options.assets) {
-        onProgress({step: 'Downloading assets...'})
-      }
+      lastReported = now
+    }
 
-      let prevCompleted = 0
-      const progressInterval = setInterval(() => {
-        const completed =
-          assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending
-
-        if (prevCompleted === completed) {
-          return
-        }
-
-        prevCompleted = completed
-        onProgress({
-          step: 'Downloading assets...',
-          current: completed,
-          total: assetHandler.queueSize,
-          update: true,
-        })
-      }, 500)
-
-      debug('Waiting for asset handler to complete downloads')
-      try {
-        const assetMap = await assetHandler.finish()
-
-        // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
-        onProgress({
-          step: 'Downloading assets...',
-          current: assetHandler.queueSize,
-          total: assetHandler.queueSize,
-          update: true,
-        })
-
-        archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix})
-        clearInterval(progressInterval)
-      } catch (assetErr) {
-        clearInterval(progressInterval)
-        await cleanup()
-        reject(assetErr)
-        return
-      }
+    cb(null, chunk)
+  }
 
-      // Add all downloaded assets to archive
-      archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {store: true})
-      archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {store: true})
+  const inputStream = await getDocumentsStream(options)
+  debug('Got HTTP %d', inputStream.statusCode)
+  debug('Response headers: %o', inputStream.headers)
+
+  const jsonStream = miss.pipeline(
+    inputStream,
+    logFirstChunk(),
+    split(tryParseJson),
+    rejectOnApiError(),
+    filterSystemDocuments(),
+    assetStreamHandler,
+    filterDocumentTypes(options.types),
+    options.drafts ? miss.through.obj() : filterDrafts(),
+    stringifyStream(),
+    miss.through(reportDocumentCount),
+  )
 
-      debug('Finalizing archive, flushing streams')
-      onProgress({step: 'Adding assets to archive...'})
-      archive.finalize()
-    })
+  miss.finished(jsonStream, async (err) => {
+    if (err) {
+      debug('Export stream error: ', err)
+      reject(err)
+      return
+    }
 
-    archive.on('warning', (err) => {
-      debug('Archive warning: %s', err.message)
+    debug('Export stream completed')
+    onProgress({
+      step: 'Exporting documents...',
+      current: documentCount,
+      total: documentCount,
+      update: true,
     })
 
-    archive.append(jsonStream, {name: 'data.ndjson', prefix})
-    miss.pipe(archive, outputStream, onComplete)
+    if (!options.raw && options.assets) {
+      onProgress({step: 'Downloading assets...'})
+    }
 
-    async function onComplete(err) {
-      onProgress({step: 'Clearing temporary files...'})
-      await cleanup()
+    let prevCompleted = 0
+    const progressInterval = setInterval(() => {
+      const completed =
+        assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending
 
-      if (!err) {
-        resolve()
+      if (prevCompleted === completed) {
         return
       }
 
-      debug('Error during streaming: %s', err.stack)
-      assetHandler.clear()
-      reject(err)
+      prevCompleted = completed
+      onProgress({
+        step: 'Downloading assets...',
+        current: completed,
+        total: assetHandler.queueSize,
+        update: true,
+      })
+    }, 500)
+
+    debug('Waiting for asset handler to complete downloads')
+    try {
+      const assetMap = await assetHandler.finish()
+
+      // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
+      onProgress({
+        step: 'Downloading assets...',
+        current: assetHandler.queueSize,
+        total: assetHandler.queueSize,
+        update: true,
+      })
+
+      archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix})
+      clearInterval(progressInterval)
+    } catch (assetErr) {
+      clearInterval(progressInterval)
+      await cleanup()
+      reject(assetErr)
+      return
     }
+
+    // Add all downloaded assets to archive
+    archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {store: true})
+    archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {store: true})
+
+    debug('Finalizing archive, flushing streams')
+    onProgress({step: 'Adding assets to archive...'})
+    await archive.finalize()
+  })
+
+  archive.on('warning', (err) => {
+    debug('Archive warning: %s', err.message)
   })
+
+  archive.append(jsonStream, {name: 'data.ndjson', prefix})
+  miss.pipe(archive, outputStream, onComplete)
+
+  async function onComplete(err) {
+    onProgress({step: 'Clearing temporary files...'})
+    await cleanup()
+
+    if (!err) {
+      resolve({
+        outputPath: options.outputPath,
+        documentCount,
+        assetCount: assetHandler.filesWritten,
+      })
+      return
+    }
+
+    debug('Error during streaming: %s', err.stack)
+    assetHandler.clear()
+    reject(err)
+  }
+
+  return result
 }
 
 function isWritableStream(val) {

package/src/getDocumentsStream.js

@@ -1,15 +1,15 @@
 const pkg = require('../package.json')
 const requestStream = require('./requestStream')
 
-module.exports = (client, dataset) => {
+module.exports = (options) => {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
-  const url = client.getUrl(`/data/export/${dataset}`)
-  const token = client.config().token
+  const url = options.client.getUrl(`/data/export/${options.dataset}`)
+  const token = options.client.config().token
   const headers = {
     'User-Agent': `${pkg.name}@${pkg.version}`,
     ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
 
-  return requestStream({url, headers})
+  return requestStream({url, headers, maxRetries: options.maxRetries})
 }

package/src/rejectOnApiError.js

@@ -3,7 +3,13 @@ const miss = require('mississippi')
 module.exports = () =>
   miss.through.obj((doc, enc, callback) => {
     if (doc.error && doc.statusCode) {
-      callback(new Error([doc.statusCode, doc.error].join(': ')))
+      callback(
+        new Error(
+          ['Export', `HTTP ${doc.statusCode}`, doc.error, doc.message]
+            .filter((part) => typeof part === 'string')
+            .join(': '),
+        ),
+      )
       return
     }
 

package/src/requestStream.js

@@ -1,13 +1,15 @@
 const {getIt} = require('get-it')
 const {keepAlive, promise} = require('get-it/middleware')
 const debug = require('./debug')
+const {extractFirstError} = require('./util/extractFirstError')
+const {DOCUMENT_STREAM_MAX_RETRIES} = require('./constants')
 
 const request = getIt([keepAlive(), promise({onlyBody: true})])
 const socketsWithTimeout = new WeakSet()
 
 const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds
 const READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes
-const MAX_RETRIES = 5
+const RETRY_DELAY_MS = 1500 // 1.5 seconds
 
 function delay(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms))
@@ -15,8 +17,11 @@ function delay(ms) {
 
 /* eslint-disable no-await-in-loop, max-depth */
 module.exports = async (options) => {
+  const maxRetries =
+    typeof options.maxRetries === 'number' ? options.maxRetries : DOCUMENT_STREAM_MAX_RETRIES
+
   let error
-  for (let i = 0; i < MAX_RETRIES; i++) {
+  for (let i = 0; i < maxRetries; i++) {
     try {
       const response = await request({
         ...options,
@@ -33,23 +38,24 @@ module.exports = async (options) => {
         socketsWithTimeout.add(response.connection)
         response.connection.setTimeout(READ_TIMEOUT, () => {
           response.destroy(
-            new Error(`Read timeout: No data received on socket for ${READ_TIMEOUT} ms`),
+            new Error(`Export: Read timeout: No data received on socket for ${READ_TIMEOUT} ms`),
           )
         })
       }
 
       return response
     } catch (err) {
-      error = err
+      error = extractFirstError(err)
 
       if (err.response && err.response.statusCode && err.response.statusCode < 500) {
         break
       }
 
-      debug('Error, retrying after 1500ms: %s', err.message)
-      await delay(1500)
+      debug('Error, retrying after %d ms: %s', RETRY_DELAY_MS, error.message)
+      await delay(RETRY_DELAY_MS)
     }
   }
 
+  error.message = `Export: Failed to fetch ${options.url}: ${error.message}`
   throw error
 }

package/src/util/extractFirstError.js

@@ -0,0 +1,14 @@
+exports.extractFirstError = function extractFirstError(err) {
+  if (
+    // eslint-disable-next-line no-undef
+    ((typeof AggregateError !== 'undefined' && err instanceof AggregateError) ||
+      ('name' in err && err.name === 'AggregateError')) &&
+    Array.isArray(err.errors) &&
+    err.errors.length > 0 &&
+    'message' in err.errors[0]
+  ) {
+    return err.errors[0]
+  }
+
+  return err
+}

package/src/validateOptions.js

@@ -1,12 +1,16 @@
 const defaults = require('lodash/defaults')
+const {DOCUMENT_STREAM_MAX_RETRIES, ASSET_DOWNLOAD_MAX_RETRIES} = require('./constants')
 
 const clientMethods = ['getUrl', 'config']
 const booleanFlags = ['assets', 'raw', 'compress', 'drafts']
+const numberFlags = ['maxAssetRetries', 'maxRetries', 'assetConcurrency']
 const exportDefaults = {
   compress: true,
   drafts: true,
   assets: true,
   raw: false,
+  maxRetries: DOCUMENT_STREAM_MAX_RETRIES,
+  maxAssetRetries: ASSET_DOWNLOAD_MAX_RETRIES,
 }
 
 function validateOptions(opts) {
@@ -42,6 +46,12 @@ function validateOptions(opts) {
     }
   })
 
+  numberFlags.forEach((flag) => {
+    if (typeof options[flag] !== 'undefined' && typeof options[flag] !== 'number') {
+      throw new Error(`Flag ${flag} must be a number if specified`)
+    }
+  })
+
   if (!options.outputPath) {
     throw new Error('outputPath must be specified (- for stdout)')
   }