@sanity/export 0.136.3-gql-rtb.375 → 0.136.3-purple-unicorn-patch.5629

package/src/AssetHandler.js

@@ -1,9 +1,10 @@
 const path = require('path')
 const crypto = require('crypto')
+const {parse: parseUrl, format: formatUrl} = require('url')
 const fse = require('fs-extra')
 const miss = require('mississippi')
 const PQueue = require('p-queue')
-const {omit} = require('lodash')
+const {omit, noop} = require('lodash')
 const pkg = require('../package.json')
 const requestStream = require('./requestStream')
 const debug = require('./debug')
@@ -11,20 +12,27 @@ const debug = require('./debug')
 const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url']
 const ACTION_REMOVE = 'remove'
 const ACTION_REWRITE = 'rewrite'
+const ASSET_DOWNLOAD_CONCURRENCY = 8
 
 class AssetHandler {
   constructor(options) {
+    const concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY
+    debug('Using asset download concurrency of %d', concurrency)
+
     this.client = options.client
     this.tmpDir = options.tmpDir
     this.assetDirsCreated = false
 
+    this.downloading = []
     this.assetsSeen = new Map()
     this.assetMap = {}
     this.filesWritten = 0
     this.queueSize = 0
-    this.queue = options.queue || new PQueue({concurrency: 3})
-    this.reject = () => {
-      throw new Error('Asset handler errored before `finish()` was called')
+    this.queue = options.queue || new PQueue({concurrency})
+
+    this.rejectedError = null
+    this.reject = (err) => {
+      this.rejectedError = err
     }
   }
 
@@ -36,6 +44,11 @@ class AssetHandler {
 
   finish() {
     return new Promise((resolve, reject) => {
+      if (this.rejectedError) {
+        reject(this.rejectedError)
+        return
+      }
+
       this.reject = reject
       this.queue.onIdle().then(() => resolve(this.assetMap))
     })
@@ -53,7 +66,7 @@ class AssetHandler {
       return
     }
 
-    callback(null, await this.findAndModify(doc, ACTION_REWRITE))
+    callback(null, this.findAndModify(doc, ACTION_REWRITE))
   })
 
   // Called in the case where we don't _want_ assets, so basically just remove all asset documents
@@ -64,7 +77,7 @@ class AssetHandler {
       return
     }
 
-    callback(null, await this.findAndModify(doc, ACTION_REMOVE))
+    callback(null, this.findAndModify(doc, ACTION_REMOVE))
   })
 
   // Called when we are using raw export mode along with `assets: false`, where we simply
@@ -89,45 +102,131 @@ class AssetHandler {
 
     debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath)
     this.queueSize++
+    this.downloading.push(assetDoc.url)
     this.queue.add(() => this.downloadAsset(assetDoc, dstPath))
   }
 
-  async downloadAsset(assetDoc, dstPath) {
-    const {url} = assetDoc
+  maybeCreateAssetDirs() {
+    if (this.assetDirsCreated) {
+      return
+    }
+
+    /* eslint-disable no-sync */
+    fse.ensureDirSync(path.join(this.tmpDir, 'files'))
+    fse.ensureDirSync(path.join(this.tmpDir, 'images'))
+    /* eslint-enable no-sync */
+    this.assetDirsCreated = true
+  }
+
+  getAssetRequestOptions(assetDoc) {
+    const token = this.client.config().token
     const headers = {'User-Agent': `${pkg.name}@${pkg.version}`}
-    const stream = await requestStream({url, headers})
+    const isImage = assetDoc._type === 'sanity.imageAsset'
+
+    const url = parseUrl(assetDoc.url, true)
+    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname)) {
+      headers.Authorization = `Bearer ${token}`
+      url.query = {...(url.query || {}), dlRaw: 'true'}
+    }
+
+    return {url: formatUrl(url), headers}
+  }
+
+  async downloadAsset(assetDoc, dstPath, attemptNum = 0) {
+    const {url} = assetDoc
+    const options = this.getAssetRequestOptions(assetDoc)
+
+    let stream
+    try {
+      stream = await requestStream(options)
+    } catch (err) {
+      this.reject(err)
+      return false
+    }
 
     if (stream.statusCode !== 200) {
       this.queue.clear()
-      this.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`))
-      return
-    }
+      const err = await tryGetErrorFromStream(stream)
+      let errMsg = `Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`
+      if (err) {
+        errMsg = `${errMsg}:\n\n${err}`
+      }
 
-    if (!this.assetDirsCreated) {
-      /* eslint-disable no-sync */
-      fse.ensureDirSync(path.join(this.tmpDir, 'files'))
-      fse.ensureDirSync(path.join(this.tmpDir, 'images'))
-      /* eslint-enable no-sync */
-      this.assetDirsCreated = true
+      this.reject(new Error(errMsg))
+      return false
     }
 
+    this.maybeCreateAssetDirs()
+
     debug('Asset stream ready, writing to filesystem at %s', dstPath)
-    const hash = await writeHashedStream(path.join(this.tmpDir, dstPath), stream)
-    const type = assetDoc._type === 'sanity.imageAsset' ? 'image' : 'file'
-    const id = `${type}-${hash}`
+    const tmpPath = path.join(this.tmpDir, dstPath)
+    const {sha1, md5, size} = await writeHashedStream(tmpPath, stream)
+
+    // Verify it against our downloaded stream to make sure we have the same copy
+    const contentLength = stream.headers['content-length']
+    const remoteSha1 = stream.headers['x-sanity-sha1']
+    const remoteMd5 = stream.headers['x-sanity-md5']
+    const hasHash = Boolean(remoteSha1 || remoteMd5)
+    const method = md5 ? 'md5' : 'sha1'
+
+    let differs = false
+    if (remoteMd5 && md5) {
+      differs = remoteMd5 !== md5
+    } else if (remoteSha1 && sha1) {
+      differs = remoteSha1 !== sha1
+    }
+
+    if (differs && attemptNum < 3) {
+      debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url)
+      return this.downloadAsset(assetDoc, dstPath, attemptNum + 1)
+    } else if (differs) {
+      const details = [
+        hasHash &&
+          (method === 'md5'
+            ? `md5 should be ${remoteMd5}, got ${md5}`
+            : `sha1 should be ${remoteSha1}, got ${sha1}`),
+
+        contentLength &&
+          parseInt(contentLength, 10) !== size &&
+          `Asset should be ${contentLength} bytes, got ${size}`,
+
+        `Did not succeed after ${attemptNum} attempts.`,
+      ]
+
+      const detailsString = `Details:\n - ${details.filter(Boolean).join('\n - ')}`
+
+      await fse.unlink(tmpPath)
+      this.queue.clear()
+
+      const error = new Error(
+        `Failed to download asset at ${assetDoc.url}, giving up. ${detailsString}`
+      )
+
+      this.reject(error)
+      return false
+    }
+
+    const isImage = assetDoc._type === 'sanity.imageAsset'
+    const type = isImage ? 'image' : 'file'
+    const id = `${type}-${sha1}`
 
     const metaProps = omit(assetDoc, EXCLUDE_PROPS)
     if (Object.keys(metaProps).length > 0) {
       this.assetMap[id] = metaProps
     }
 
+    this.downloading.splice(
+      this.downloading.findIndex((datUrl) => datUrl === url),
+      1
+    )
+
     this.filesWritten++
+    return true
   }
 
-  // eslint-disable-next-line complexity
-  findAndModify = async (item, action) => {
+  findAndModify = (item, action) => {
     if (Array.isArray(item)) {
-      const children = await Promise.all(item.map(child => this.findAndModify(child, action)))
+      const children = item.map((child) => this.findAndModify(child, action))
       return children.filter(Boolean)
     }
 
@@ -143,16 +242,12 @@ class AssetHandler {
     if (isAsset && action === ACTION_REWRITE) {
       const {asset, ...other} = item
       const assetId = asset._ref
-      if (isModernAsset(assetId)) {
-        const assetType = getAssetType(item)
-        const filePath = `${assetType}s/${generateFilename(assetId)}`
-        return {_sanityAsset: `${assetType}@file://./${filePath}`, ...other}
+      const assetType = getAssetType(item)
+      const filePath = `${assetType}s/${generateFilename(assetId)}`
+      return {
+        _sanityAsset: `${assetType}@file://./${filePath}`,
+        ...this.findAndModify(other, action),
       }
-
-      // Legacy asset
-      const type = this.assetsSeen.get(assetId) || (await this.lookupAssetType(assetId))
-      const filePath = `${type}s/${generateFilename(assetId)}`
-      return {_sanityAsset: `${type}@file://./${filePath}`, ...other}
     }
 
     const newItem = {}
@@ -161,8 +256,7 @@ class AssetHandler {
       const key = keys[i]
       const value = item[key]
 
-      // eslint-disable-next-line no-await-in-loop
-      newItem[key] = await this.findAndModify(value, action)
+      newItem[key] = this.findAndModify(value, action)
 
       if (typeof newItem[key] === 'undefined') {
         delete newItem[key]
@@ -171,15 +265,10 @@ class AssetHandler {
 
     return newItem
   }
-
-  lookupAssetType = async assetId => {
-    const docType = await this.client.fetch('*[_id == $id][0]._type', {id: assetId})
-    return docType === 'sanity.imageAsset' ? 'image' : 'file'
-  }
 }
 
 function isAssetField(item) {
-  return item.asset && item.asset._ref
+  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref)
 }
 
 function getAssetType(item) {
@@ -191,8 +280,11 @@ function getAssetType(item) {
   return type || null
 }
 
-function isModernAsset(assetId) {
-  return /^(image|file)/.test(assetId)
+function isSanityAsset(assetId) {
+  return (
+    /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) ||
+    /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId)
+  )
 }
 
 function generateFilename(assetId) {
@@ -202,22 +294,46 @@ function generateFilename(assetId) {
 }
 
 function writeHashedStream(filePath, stream) {
-  const hash = crypto.createHash('sha1')
+  let size = 0
+  const md5 = crypto.createHash('md5')
+  const sha1 = crypto.createHash('sha1')
+
   const hasher = miss.through((chunk, enc, cb) => {
-    hash.update(chunk)
+    size += chunk.length
+    md5.update(chunk)
+    sha1.update(chunk)
     cb(null, chunk)
   })
 
   return new Promise((resolve, reject) =>
-    miss.pipe(
-      stream,
-      hasher,
-      fse.createWriteStream(filePath),
-      err => {
-        return err ? reject(err) : resolve(hash.digest('hex'))
+    miss.pipe(stream, hasher, fse.createWriteStream(filePath), (err) => {
+      if (err) {
+        reject(err)
+        return
       }
-    )
+
+      resolve({
+        size,
+        sha1: sha1.digest('hex'),
+        md5: md5.digest('hex'),
+      })
+    })
   )
 }
 
+function tryGetErrorFromStream(stream) {
+  return new Promise((resolve, reject) => {
+    miss.pipe(stream, miss.concat(parse), (err) => (err ? reject(err) : noop))
+
+    function parse(body) {
+      try {
+        const parsed = JSON.parse(body.toString('utf8'))
+        resolve(parsed.message || parsed.error || null)
+      } catch (err) {
+        resolve(body.toString('utf8').slice(0, 16000))
+      }
+    }
+  })
+}
+
 module.exports = AssetHandler

package/src/export.js

@@ -14,6 +14,8 @@ const getDocumentsStream = require('./getDocumentsStream')
 const filterSystemDocuments = require('./filterSystemDocuments')
 const filterDocumentTypes = require('./filterDocumentTypes')
 const filterDrafts = require('./filterDrafts')
+const logFirstChunk = require('./logFirstChunk')
+const tryParseJson = require('./tryParseJson')
 
 const noop = () => null
 
@@ -22,7 +24,7 @@ function exportDataset(opts) {
   const onProgress = options.onProgress || noop
   const archive = archiver('tar', {
     gzip: true,
-    gzipOptions: {level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION}
+    gzipOptions: {level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION},
   })
 
   const slugDate = new Date()
@@ -32,16 +34,28 @@ function exportDataset(opts) {
 
   const prefix = `${opts.dataset}-export-${slugDate}`
   const tmpDir = path.join(os.tmpdir(), prefix)
+  const cleanup = () =>
+    fse.remove(tmpDir).catch((err) => {
+      debug(`Error while cleaning up temporary files: ${err.message}`)
+    })
+
   const assetHandler = new AssetHandler({
     client: options.client,
     tmpDir,
-    prefix
+    prefix,
+    concurrency: options.assetConcurrency,
   })
 
   debug('Outputting assets (temporarily) to %s', tmpDir)
   debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath)
-  const outputStream =
-    options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath)
+
+  let outputStream
+  if (isWritableStream(options.outputPath)) {
+    outputStream = options.outputPath
+  } else {
+    outputStream =
+      options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath)
+  }
 
   let assetStreamHandler = assetHandler.noop
   if (!options.raw) {
@@ -49,9 +63,10 @@ function exportDataset(opts) {
   }
 
   return new Promise(async (resolve, reject) => {
-    miss.finished(archive, archiveErr => {
+    miss.finished(archive, async (archiveErr) => {
       if (archiveErr) {
         debug('Archiving errored! %s', archiveErr.stack)
+        await cleanup()
         reject(archiveErr)
         return
       }
@@ -73,7 +88,7 @@ function exportDataset(opts) {
           step: 'Exporting documents...',
           current: documentCount,
           total: '?',
-          update: true
+          update: true,
         })
 
         lastReported = now
@@ -83,19 +98,23 @@ function exportDataset(opts) {
     }
 
     const inputStream = await getDocumentsStream(options.client, options.dataset)
+    debug('Got HTTP %d', inputStream.statusCode)
+    debug('Response headers: %o', inputStream.headers)
+
     const jsonStream = miss.pipeline(
       inputStream,
-      split(JSON.parse),
-      rejectOnApiError,
-      filterSystemDocuments,
+      logFirstChunk(),
+      split(tryParseJson),
+      rejectOnApiError(),
+      filterSystemDocuments(),
       assetStreamHandler,
       filterDocumentTypes(options.types),
-      options.drafts ? miss.through.obj() : filterDrafts,
-      stringifyStream,
+      options.drafts ? miss.through.obj() : filterDrafts(),
+      stringifyStream(),
       miss.through(reportDocumentCount)
     )
 
-    miss.finished(jsonStream, async err => {
+    miss.finished(jsonStream, async (err) => {
       if (err) {
         return
       }
@@ -104,7 +123,7 @@ function exportDataset(opts) {
         step: 'Exporting documents...',
         current: documentCount,
         total: documentCount,
-        update: true
+        update: true,
       })
 
       if (!options.raw && options.assets) {
@@ -113,7 +132,9 @@ function exportDataset(opts) {
 
       let prevCompleted = 0
       const progressInterval = setInterval(() => {
-        const completed = assetHandler.queueSize - assetHandler.queue.size
+        const completed =
+          assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending
+
         if (prevCompleted === completed) {
           return
         }
@@ -123,17 +144,27 @@ function exportDataset(opts) {
           step: 'Downloading assets...',
           current: completed,
           total: assetHandler.queueSize,
-          update: true
+          update: true,
         })
       }, 500)
 
       debug('Waiting for asset handler to complete downloads')
       try {
         const assetMap = await assetHandler.finish()
+
+        // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
+        onProgress({
+          step: 'Downloading assets...',
+          current: assetHandler.queueSize,
+          total: assetHandler.queueSize,
+          update: true,
+        })
+
         archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix})
         clearInterval(progressInterval)
       } catch (assetErr) {
         clearInterval(progressInterval)
+        await cleanup()
         reject(assetErr)
         return
       }
@@ -147,20 +178,16 @@ function exportDataset(opts) {
       archive.finalize()
     })
 
-    archive.on('warning', err => {
+    archive.on('warning', (err) => {
       debug('Archive warning: %s', err.message)
     })
 
     archive.append(jsonStream, {name: 'data.ndjson', prefix})
-    miss.pipe(
-      archive,
-      outputStream,
-      onComplete
-    )
+    miss.pipe(archive, outputStream, onComplete)
 
     async function onComplete(err) {
       onProgress({step: 'Clearing temporary files...'})
-      await fse.remove(tmpDir)
+      await cleanup()
 
       if (!err) {
         resolve()
@@ -174,4 +201,14 @@ function exportDataset(opts) {
   })
 }
 
+function isWritableStream(val) {
+  return (
+    val !== null &&
+    typeof val === 'object' &&
+    typeof val.pipe === 'function' &&
+    typeof val._write === 'function' &&
+    typeof val._writableState === 'object'
+  )
+}
+
 module.exports = exportDataset

package/src/filterDocumentTypes.js

@@ -1,6 +1,6 @@
 const miss = require('mississippi')
 
-module.exports = allowedTypes =>
+module.exports = (allowedTypes) =>
   allowedTypes
     ? miss.through.obj((doc, enc, callback) => {
         const type = doc && doc._type

package/src/filterDrafts.js

@@ -1,11 +1,12 @@
 const miss = require('mississippi')
 
-const isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0
+const isDraft = (doc) => doc && doc._id && doc._id.indexOf('drafts.') === 0
 
-module.exports = miss.through.obj((doc, enc, callback) => {
-  if (isDraft(doc)) {
-    return callback()
-  }
+module.exports = () =>
+  miss.through.obj((doc, enc, callback) => {
+    if (isDraft(doc)) {
+      return callback()
+    }
 
-  return callback(null, doc)
-})
+    return callback(null, doc)
+  })

package/src/filterSystemDocuments.js

@@ -1,13 +1,14 @@
 const miss = require('mississippi')
 const debug = require('./debug')
 
-const isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0
+const isSystemDocument = (doc) => doc && doc._id && doc._id.indexOf('_.') === 0
 
-module.exports = miss.through.obj((doc, enc, callback) => {
-  if (isSystemDocument(doc)) {
-    debug('%s is a system document, skipping', doc && doc._id)
-    return callback()
-  }
+module.exports = () =>
+  miss.through.obj((doc, enc, callback) => {
+    if (isSystemDocument(doc)) {
+      debug('%s is a system document, skipping', doc && doc._id)
+      return callback()
+    }
 
-  return callback(null, doc)
-})
+    return callback(null, doc)
+  })

package/src/getDocumentsStream.js

@@ -5,9 +5,10 @@ module.exports = (client, dataset) => {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
   const url = client.getUrl(`/data/export/${dataset}`)
+  const token = client.config().token
   const headers = {
-    Authorization: `Bearer ${client.config().token}`,
-    'User-Agent': `${pkg.name}@${pkg.version}`
+    'User-Agent': `${pkg.name}@${pkg.version}`,
+    ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
 
   return requestStream({url, headers})

package/src/logFirstChunk.js

@@ -0,0 +1,15 @@
+const miss = require('mississippi')
+const debug = require('./debug')
+
+module.exports = () => {
+  let firstChunk = true
+  return miss.through((chunk, enc, callback) => {
+    if (firstChunk) {
+      const string = chunk.toString('utf8').split('\n')[0]
+      debug('First chunk received: %s', string.slice(0, 300))
+      firstChunk = false
+    }
+
+    callback(null, chunk)
+  })
+}

package/src/rejectOnApiError.js

@@ -1,10 +1,16 @@
 const miss = require('mississippi')
 
-module.exports = miss.through.obj((doc, enc, callback) => {
-  if (doc.error && doc.statusCode) {
-    callback(new Error([doc.statusCode, doc.error].join(': ')))
-    return
-  }
+module.exports = () =>
+  miss.through.obj((doc, enc, callback) => {
+    if (doc.error && doc.statusCode) {
+      callback(new Error([doc.statusCode, doc.error].join(': ')))
+      return
+    }
 
-  callback(null, doc)
-})
+    if (!doc._id && doc.error) {
+      callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)))
+      return
+    }
+
+    callback(null, doc)
+  })