@sanity/export 4.0.1 → 5.0.0

package/README.md

@@ -11,7 +11,7 @@ npm install --save @sanity/export
 ## Usage
 
 ```js
-const exportDataset = require('@sanity/export')
+import {exportDataset} from '@sanity/export'
 
 exportDataset({
   // Instance of @sanity/client configured to correct project ID and dataset

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/export",
-  "version": "4.0.1",
+  "version": "5.0.0",
   "description": "Export Sanity documents and assets",
   "keywords": [
     "sanity",
@@ -21,7 +21,7 @@
   },
   "license": "MIT",
   "author": "Sanity.io <hello@sanity.io>",
-  "type": "commonjs",
+  "type": "module",
   "exports": "./src/export.js",
   "main": "./src/export.js",
   "files": [
@@ -29,36 +29,29 @@
   ],
   "scripts": {
     "lint": "eslint .",
-    "test": "jest --verbose"
+    "test": "vitest"
   },
   "dependencies": {
-    "@sanity/client": "^7.8.2",
-    "@sanity/util": "^4.3.0",
-    "archiver": "^7.0.0",
+    "archiver": "^7.0.1",
     "debug": "^4.3.4",
     "get-it": "^8.6.10",
-    "json-stream-stringify": "^2.0.2",
-    "lodash": "^4.17.21",
-    "mississippi": "^4.0.0",
-    "p-queue": "^2.3.0",
-    "rimraf": "^6.0.1",
-    "split2": "^4.2.0",
-    "tar": "^7.0.1",
-    "yaml": "^2.4.2"
+    "json-stream-stringify": "^3.1.6",
+    "p-queue": "^9.0.1",
+    "rimraf": "^6.1.2"
   },
   "devDependencies": {
-    "@jest/globals": "^29.7.0",
-    "@sanity/semantic-release-preset": "^4.1.7",
+    "@sanity/client": "^7.13.1",
+    "@sanity/semantic-release-preset": "^5.0.0",
+    "@vitest/coverage-v8": "^4.0.15",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-sanity": "^7.1.2",
-    "eslint-plugin-prettier": "^5.1.3",
-    "jest": "^29.7.0",
-    "nock": "^13.5.4",
-    "prettier": "^3.6.2",
-    "prettier-plugin-packagejson": "^2.5.19",
-    "string-to-stream": "^1.1.0",
-    "tar": "^7.0.1"
+    "eslint-config-sanity": "^7.1.4",
+    "eslint-plugin-prettier": "^5.5.4",
+    "nock": "^14.0.10",
+    "prettier": "^3.7.4",
+    "prettier-plugin-packagejson": "^2.5.20",
+    "tar": "^7.5.2",
+    "vitest": "^4.0.15"
   },
   "engines": {
     "node": ">=20.19 <22 || >=22.12"

package/src/AssetHandler.js

@@ -1,21 +1,27 @@
-const crypto = require('crypto')
-const {mkdirSync, createWriteStream} = require('fs')
-const path = require('path')
-const {parse: parseUrl, format: formatUrl} = require('url')
-const {omit} = require('lodash')
-const miss = require('mississippi')
-const PQueue = require('p-queue')
-const pkg = require('../package.json')
-const debug = require('./debug')
-const requestStream = require('./requestStream')
-const rimraf = require('./util/rimraf')
-const {ASSET_DOWNLOAD_MAX_RETRIES, ASSET_DOWNLOAD_CONCURRENCY} = require('./constants')
+import {createHash} from 'node:crypto'
+import {createWriteStream, mkdirSync} from 'node:fs'
+import {join as joinPath} from 'node:path'
+import {pipeline} from 'node:stream/promises'
+
+import PQueue from 'p-queue'
+import {rimraf} from 'rimraf'
+
+import {
+  ASSET_DOWNLOAD_CONCURRENCY,
+  ASSET_DOWNLOAD_MAX_RETRIES,
+  DEFAULT_RETRY_DELAY,
+} from './constants.js'
+import {debug} from './debug.js'
+import {getUserAgent} from './getUserAgent.js'
+import {requestStream} from './requestStream.js'
+import {delay} from './util/delay.js'
+import {through, throughObj} from './util/streamHelpers.js'
 
 const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url']
 const ACTION_REMOVE = 'remove'
 const ACTION_REWRITE = 'rewrite'
 
-class AssetHandler {
+export class AssetHandler {
   constructor(options) {
     const concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY
     debug('Using asset download concurrency of %d', concurrency)
@@ -30,6 +36,7 @@ class AssetHandler {
     this.filesWritten = 0
     this.queueSize = 0
     this.maxRetries = options.maxRetries || ASSET_DOWNLOAD_MAX_RETRIES
+    this.retryDelayMs = options.retryDelayMs
     this.queue = options.queue || new PQueue({concurrency})
 
     this.rejectedError = null
@@ -58,7 +65,7 @@ class AssetHandler {
 
   // Called when we want to download all assets to local filesystem and rewrite documents to hold
   // placeholder asset references (_sanityAsset: 'image@file:///local/path')
-  rewriteAssets = miss.through.obj(async (doc, enc, callback) => {
+  rewriteAssets = throughObj(async (doc, enc, callback) => {
     if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
       const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file'
       const filePath = `${type}s/${generateFilename(doc._id)}`
@@ -73,7 +80,7 @@ class AssetHandler {
 
   // Called in the case where we don't _want_ assets, so basically just remove all asset documents
   // as well as references to assets (*.asset._ref ^= (image|file)-)
-  stripAssets = miss.through.obj(async (doc, enc, callback) => {
+  stripAssets = throughObj(async (doc, enc, callback) => {
     if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
       callback()
       return
@@ -84,7 +91,7 @@ class AssetHandler {
 
   // Called when we are using raw export mode along with `assets: false`, where we simply
   // want to skip asset documents but retain asset references (useful for data mangling)
-  skipAssets = miss.through.obj((doc, enc, callback) => {
+  skipAssets = throughObj((doc, enc, callback) => {
     const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)
     if (isAsset) {
       callback()
@@ -94,9 +101,9 @@ class AssetHandler {
     callback(null, doc)
   })
 
-  noop = miss.through.obj((doc, enc, callback) => callback(null, doc))
+  noop = throughObj((doc, enc, callback) => callback(null, doc))
 
-  queueAssetDownload(assetDoc, dstPath, type) {
+  queueAssetDownload(assetDoc, dstPath) {
     if (!assetDoc.url) {
       debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id)
       return
@@ -140,6 +147,8 @@ class AssetHandler {
             // Don't retry on client errors
             break
           }
+
+          await delay(this.retryDelayMs || DEFAULT_RETRY_DELAY)
         }
       }
       throw dlError
@@ -164,18 +173,23 @@ class AssetHandler {
     }
 
     /* eslint-disable no-sync */
-    mkdirSync(path.join(this.tmpDir, 'files'), {recursive: true})
-    mkdirSync(path.join(this.tmpDir, 'images'), {recursive: true})
+    mkdirSync(joinPath(this.tmpDir, 'files'), {recursive: true})
+    mkdirSync(joinPath(this.tmpDir, 'images'), {recursive: true})
     /* eslint-enable no-sync */
     this.assetDirsCreated = true
   }
 
   getAssetRequestOptions(assetDoc) {
     const token = this.client.config().token
-    const headers = {'User-Agent': `${pkg.name}@${pkg.version}`}
+    const headers = {'User-Agent': getUserAgent()}
     const isImage = assetDoc._type === 'sanity.imageAsset'
 
-    const url = parseUrl(assetDoc.url, true)
+    const url = URL.parse(assetDoc.url)
+    // If we can't parse it, return as-is
+    if (!url) {
+      return {url: assetDoc.url, headers}
+    }
+
     if (
       isImage &&
       token &&
@@ -184,10 +198,10 @@ class AssetHandler {
         url.host === 'localhost:43216')
     ) {
       headers.Authorization = `Bearer ${token}`
-      url.query = {...(url.query || {}), dlRaw: 'true'}
+      url.searchParams.set('dlRaw', 'true')
     }
 
-    return {url: formatUrl(url), headers}
+    return {url: url.toString(), headers}
   }
 
   // eslint-disable-next-line max-statements
@@ -200,7 +214,10 @@ class AssetHandler {
 
     let stream
     try {
-      stream = await requestStream(options)
+      stream = await requestStream({
+        maxRetries: 0, // We handle retries ourselves in queueAssetDownload
+        ...options,
+      })
     } catch (err) {
       const message = 'Failed to create asset stream'
       if (typeof err.message === 'string') {
@@ -239,7 +256,7 @@ class AssetHandler {
     this.maybeCreateAssetDirs()
 
     debug('Asset stream ready, writing to filesystem at %s', dstPath)
-    const tmpPath = path.join(this.tmpDir, dstPath)
+    const tmpPath = joinPath(this.tmpDir, dstPath)
     let sha1 = ''
     let md5 = ''
     let size = 0
@@ -381,58 +398,61 @@ function generateFilename(assetId) {
   return asset ? `${asset}.${extension}` : `${assetId}.bin`
 }
 
-function writeHashedStream(filePath, stream) {
+async function writeHashedStream(filePath, stream) {
   let size = 0
-  const md5 = crypto.createHash('md5')
-  const sha1 = crypto.createHash('sha1')
+  const md5 = createHash('md5')
+  const sha1 = createHash('sha1')
 
-  const hasher = miss.through((chunk, enc, cb) => {
+  const hasher = through((chunk, enc, cb) => {
     size += chunk.length
     md5.update(chunk)
     sha1.update(chunk)
     cb(null, chunk)
   })
 
-  return new Promise((resolve, reject) =>
-    miss.pipe(stream, hasher, createWriteStream(filePath), (err) => {
-      if (err) {
-        reject(err)
-        return
-      }
-
-      resolve({
-        size,
-        sha1: sha1.digest('hex'),
-        md5: md5.digest('hex'),
-      })
-    }),
-  )
+  await pipeline(stream, hasher, createWriteStream(filePath))
+  return {
+    size,
+    sha1: sha1.digest('hex'),
+    md5: md5.digest('hex'),
+  }
 }
 
 function tryGetErrorFromStream(stream) {
   return new Promise((resolve, reject) => {
+    const chunks = []
     let receivedData = false
 
-    miss.pipe(stream, miss.concat(parse), (err) => {
-      if (err) {
-        reject(err)
-      } else if (!receivedData) {
-        // Resolve with null if no data was received, to let the caller
-        // know we couldn't parse the error.
+    stream.on('data', (chunk) => {
+      receivedData = true
+      chunks.push(chunk)
+    })
+
+    stream.on('end', () => {
+      if (!receivedData) {
         resolve(null)
+        return
       }
-    })
 
-    function parse(body) {
-      receivedData = true
+      const body = Buffer.concat(chunks)
       try {
         const parsed = JSON.parse(body.toString('utf8'))
         resolve(parsed.message || parsed.error || null)
-      } catch (err) {
+      } catch {
         resolve(body.toString('utf8').slice(0, 16000))
       }
-    }
+    })
+
+    stream.on('error', reject)
   })
 }
 
-module.exports = AssetHandler
+function omit(obj, keys) {
+  const copy = {}
+  Object.entries(obj).forEach(([key, value]) => {
+    if (!keys.includes(key)) {
+      copy[key] = value
+    }
+  })
+  return copy
+}

package/src/constants.js

@@ -5,39 +5,46 @@
  * Note: Only for initial connection - if download fails while streaming, we cannot easily resume.
  * @internal
  */
-exports.DOCUMENT_STREAM_MAX_RETRIES = 5
+export const DOCUMENT_STREAM_MAX_RETRIES = 5
 
 /**
  * How many retries to attempt when downloading an asset.
  * User overridable as `options.maxAssetRetries`.
  * @internal
  */
-exports.ASSET_DOWNLOAD_MAX_RETRIES = 10
+export const ASSET_DOWNLOAD_MAX_RETRIES = 10
+
+/**
+ * Default delay between retries when retrieving assets or document stream.
+ * User overridable as `options.retryDelayMs`.
+ * @internal
+ */
+export const DEFAULT_RETRY_DELAY = 1500
 
 /**
  * How many concurrent asset downloads to allow.
  * User overridable as `options.assetConcurrency`.
  * @internal
  */
-exports.ASSET_DOWNLOAD_CONCURRENCY = 8
+export const ASSET_DOWNLOAD_CONCURRENCY = 8
 
 /**
  * How frequently we will `debug` log while streaming the documents.
  * @internal
  */
-exports.DOCUMENT_STREAM_DEBUG_INTERVAL = 10000
+export const DOCUMENT_STREAM_DEBUG_INTERVAL = 10000
 
 /**
  * How long to wait before timing out the read of a request due to inactivity.
  * User overridable as `options.readTimeout`.
  * @internal
  */
-exports.REQUEST_READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes
+export const REQUEST_READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes
 
 /**
   What mode to use when exporting documents.
   stream: Export all documents in the dataset in one request, this will be consistent but might be slow on large datasets.
   cursor: Export documents using a cursor, this might lead to inconsistent results if a mutation is performed while exporting.
 */
-exports.MODE_STREAM = 'stream'
-exports.MODE_CURSOR = 'cursor'
+export const MODE_STREAM = 'stream'
+export const MODE_CURSOR = 'cursor'

package/src/debug.js

@@ -1 +1,3 @@
-module.exports = require('debug')('sanity:export')
+import debugIt from 'debug'
+
+export const debug = debugIt('sanity:export')

package/src/export.js

@@ -1,37 +1,38 @@
-const fs = require('fs')
-const os = require('os')
-const path = require('path')
-const zlib = require('zlib')
-const archiver = require('archiver')
-const miss = require('mississippi')
-const split = require('split2')
-const JsonStreamStringify = require('json-stream-stringify')
-const AssetHandler = require('./AssetHandler')
-const debug = require('./debug')
-const pipeAsync = require('./util/pipeAsync')
-const filterDocuments = require('./filterDocuments')
-const filterDocumentTypes = require('./filterDocumentTypes')
-const getDocumentsStream = require('./getDocumentsStream')
-const getDocumentCursorStream = require('./getDocumentCursorStream')
-const logFirstChunk = require('./logFirstChunk')
-const rejectOnApiError = require('./rejectOnApiError')
-const stringifyStream = require('./stringifyStream')
-const tryParseJson = require('./tryParseJson')
-const rimraf = require('./util/rimraf')
-const validateOptions = require('./validateOptions')
-const {DOCUMENT_STREAM_DEBUG_INTERVAL, MODE_CURSOR, MODE_STREAM} = require('./constants')
+import {createWriteStream} from 'node:fs'
+import {mkdir} from 'node:fs/promises'
+import {tmpdir} from 'node:os'
+import {join as joinPath} from 'node:path'
+import {PassThrough} from 'node:stream'
+import {finished, pipeline} from 'node:stream/promises'
+import {constants as zlib} from 'node:zlib'
+
+import archiver from 'archiver'
+import {JsonStreamStringify} from 'json-stream-stringify'
+import {rimraf} from 'rimraf'
+
+import {AssetHandler} from './AssetHandler.js'
+import {DOCUMENT_STREAM_DEBUG_INTERVAL, MODE_CURSOR, MODE_STREAM} from './constants.js'
+import {debug} from './debug.js'
+import {filterDocuments} from './filterDocuments.js'
+import {filterDocumentTypes} from './filterDocumentTypes.js'
+import {getDocumentCursorStream} from './getDocumentCursorStream.js'
+import {getDocumentsStream} from './getDocumentsStream.js'
+import {logFirstChunk} from './logFirstChunk.js'
+import {rejectOnApiError} from './rejectOnApiError.js'
+import {stringifyStream} from './stringifyStream.js'
+import {tryParseJson} from './tryParseJson.js'
+import {isWritableStream, split, throughObj} from './util/streamHelpers.js'
+import {validateOptions} from './validateOptions.js'
 
 const noop = () => null
 
-async function exportDataset(opts) {
+export async function exportDataset(opts) {
   const options = validateOptions(opts)
   const onProgress = options.onProgress || noop
   const archive = archiver('tar', {
     gzip: true,
     gzipOptions: {
-      level: options.compress
-        ? zlib.constants.Z_DEFAULT_COMPRESSION
-        : zlib.constants.Z_NO_COMPRESSION,
+      level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION,
     },
   })
   archive.on('warning', (err) => {
@@ -47,10 +48,10 @@ async function exportDataset(opts) {
     .toLowerCase()
 
   const prefix = `${opts.dataset ?? opts.mediaLibraryId}-export-${slugDate}`
-  const tmpDir = path.join(os.tmpdir(), prefix)
-  fs.mkdirSync(tmpDir, {recursive: true})
-  const dataPath = path.join(tmpDir, 'data.ndjson')
-  const assetsPath = path.join(tmpDir, 'assets.json')
+  const tmpDir = joinPath(tmpdir(), prefix)
+  await mkdir(tmpDir, {recursive: true})
+  const dataPath = joinPath(tmpDir, 'data.ndjson')
+  const assetsPath = joinPath(tmpDir, 'assets.json')
 
   const cleanup = () =>
     rimraf(tmpDir).catch((err) => {
@@ -63,6 +64,7 @@ async function exportDataset(opts) {
     prefix,
     concurrency: options.assetConcurrency,
     maxRetries: options.maxAssetRetries,
+    retryDelayMs: options.retryDelayMs,
   })
 
   debug('Downloading assets (temporarily) to %s', tmpDir)
@@ -73,7 +75,7 @@ async function exportDataset(opts) {
     outputStream = options.outputPath
   } else {
     outputStream =
-      options.outputPath === '-' ? process.stdout : fs.createWriteStream(options.outputPath)
+      options.outputPath === '-' ? process.stdout : createWriteStream(options.outputPath)
   }
 
   let assetStreamHandler = assetHandler.noop
@@ -88,16 +90,15 @@ async function exportDataset(opts) {
     reject = rej
   })
 
-  miss.finished(archive, async (archiveErr) => {
-    if (archiveErr) {
+  finished(archive)
+    .then(async () => {
+      debug('Archive finished')
+    })
+    .catch(async (archiveErr) => {
       debug('Archiving errored: %s', archiveErr.stack)
       await cleanup()
       reject(archiveErr)
-      return
-    }
-
-    debug('Archive finished')
-  })
+    })
 
   debug('Getting dataset export stream, mode: "%s"', options.mode)
   onProgress({step: 'Exporting documents...'})
@@ -150,7 +151,40 @@ async function exportDataset(opts) {
 
   scheduleDebugTimer()
 
-  const jsonStream = miss.pipeline(
+  const filterTransform = throughObj((doc, _enc, callback) => {
+    if (!options.filterDocument) {
+      return callback(null, doc)
+    }
+
+    try {
+      const include = options.filterDocument(doc)
+      return include ? callback(null, doc) : callback()
+    } catch (err) {
+      return callback(err)
+    }
+  })
+
+  const transformTransform = throughObj((doc, _enc, callback) => {
+    if (!options.transformDocument) {
+      return callback(null, doc)
+    }
+
+    try {
+      return callback(null, options.transformDocument(doc))
+    } catch (err) {
+      return callback(err)
+    }
+  })
+
+  const reportTransform = throughObj(reportDocumentCount)
+
+  // Use pipeline to chain streams with proper error handling
+  const jsonStream = new PassThrough()
+  finished(jsonStream)
+    .then(() => debug('JSON stream finished'))
+    .catch((err) => reject(err))
+
+  pipeline(
     inputStream,
     logFirstChunk(),
     split(tryParseJson),
@@ -158,98 +192,98 @@ async function exportDataset(opts) {
     filterDocuments(options.drafts),
     filterDocumentTypes(options.types),
     assetStreamHandler,
-    miss.through.obj((doc, _enc, callback) => {
-      if (options.filterDocument(doc)) {
-        return callback(null, doc)
-      }
-      return callback()
-    }),
-    miss.through.obj((doc, _enc, callback) => {
-      callback(null, options.transformDocument(doc))
-    }),
-    miss.through.obj(reportDocumentCount),
+    filterTransform,
+    transformTransform,
+    reportTransform,
     stringifyStream(),
-  )
-
-  miss.pipe(jsonStream, fs.createWriteStream(dataPath), async (err) => {
+    jsonStream,
+  ).catch((err) => {
     if (debugTimer !== null) clearTimeout(debugTimer)
+    debug(`Export stream error @ ${lastDocumentID}/${documentCount}: `, err)
+    reject(err)
+  })
 
-    if (err) {
-      debug(`Export stream error @ ${lastDocumentID}/${documentCount}: `, err)
-      reject(err)
-      return
-    }
-
-    debug('Export stream completed')
-    onProgress({
-      step: 'Exporting documents...',
-      current: documentCount,
-      total: documentCount,
-      update: true,
-    })
-
-    debug('Adding data.ndjson to archive')
-    archive.file(dataPath, {name: 'data.ndjson', prefix})
-
-    if (!options.raw && options.assets) {
-      onProgress({step: 'Downloading assets...'})
-    }
-
-    let prevCompleted = 0
-    const progressInterval = setInterval(() => {
-      const completed =
-        assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending
-
-      if (prevCompleted === completed) {
-        return
-      }
-
-      prevCompleted = completed
-      onProgress({
-        step: 'Downloading assets...',
-        current: completed,
-        total: assetHandler.queueSize,
-        update: true,
-      })
-    }, 500)
-
-    debug('Waiting for asset handler to complete downloads')
-    try {
-      const assetMap = await assetHandler.finish()
+  pipeline(jsonStream, createWriteStream(dataPath))
+    .then(async () => {
+      if (debugTimer !== null) clearTimeout(debugTimer)
 
-      // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
+      debug('Export stream completed')
       onProgress({
-        step: 'Downloading assets...',
-        current: assetHandler.queueSize,
-        total: assetHandler.queueSize,
+        step: 'Exporting documents...',
+        current: documentCount,
+        total: documentCount,
         update: true,
       })
 
-      const assetsStream = fs.createWriteStream(assetsPath)
-      await pipeAsync(new JsonStreamStringify(assetMap), assetsStream)
+      debug('Adding data.ndjson to archive')
+      archive.file(dataPath, {name: 'data.ndjson', prefix})
 
-      if (options.assetsMap) {
-        archive.file(assetsPath, {name: 'assets.json', prefix})
+      if (!options.raw && options.assets) {
+        onProgress({step: 'Downloading assets...'})
       }
 
-      clearInterval(progressInterval)
-    } catch (assetErr) {
-      clearInterval(progressInterval)
-      await cleanup()
-      reject(assetErr)
-      return
-    }
+      let prevCompleted = 0
+      const progressInterval = setInterval(() => {
+        const completed =
+          assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending
+
+        if (prevCompleted === completed) {
+          return
+        }
+
+        prevCompleted = completed
+        onProgress({
+          step: 'Downloading assets...',
+          current: completed,
+          total: assetHandler.queueSize,
+          update: true,
+        })
+      }, 500)
+
+      debug('Waiting for asset handler to complete downloads')
+      try {
+        const assetMap = await assetHandler.finish()
+
+        // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
+        onProgress({
+          step: 'Downloading assets...',
+          current: assetHandler.queueSize,
+          total: assetHandler.queueSize,
+          update: true,
+        })
+
+        const assetsStream = createWriteStream(assetsPath)
+        await pipeline(new JsonStreamStringify(assetMap), assetsStream)
+
+        if (options.assetsMap) {
+          archive.file(assetsPath, {name: 'assets.json', prefix})
+        }
+
+        clearInterval(progressInterval)
+      } catch (assetErr) {
+        clearInterval(progressInterval)
+        await cleanup()
+        reject(assetErr)
+        return
+      }
 
-    // Add all downloaded assets to archive
-    archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {store: true})
-    archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {store: true})
+      // Add all downloaded assets to archive
+      archive.directory(joinPath(tmpDir, 'files'), `${prefix}/files`, {store: true})
+      archive.directory(joinPath(tmpDir, 'images'), `${prefix}/images`, {store: true})
 
-    debug('Finalizing archive, flushing streams')
-    onProgress({step: 'Adding assets to archive...'})
-    await archive.finalize()
-  })
+      debug('Finalizing archive, flushing streams')
+      onProgress({step: 'Adding assets to archive...'})
+      await archive.finalize()
+    })
+    .catch(async (err) => {
+      if (debugTimer !== null) clearTimeout(debugTimer)
+      debug(`Export stream error @ ${lastDocumentID}/${documentCount}: `, err)
+      reject(err)
+    })
 
-  miss.pipe(archive, outputStream, onComplete)
+  pipeline(archive, outputStream)
+    .then(() => onComplete())
+    .catch(onComplete)
 
   async function onComplete(err) {
     onProgress({step: 'Clearing temporary files...'})
@@ -283,15 +317,3 @@ function getDocumentInputStream(options) {
 
   throw new Error(`Invalid mode: ${options.mode}`)
 }
-
-function isWritableStream(val) {
-  return (
-    val !== null &&
-    typeof val === 'object' &&
-    typeof val.pipe === 'function' &&
-    typeof val._write === 'function' &&
-    typeof val._writableState === 'object'
-  )
-}
-
-module.exports = exportDataset

package/src/filterDocumentTypes.js

@@ -1,14 +1,18 @@
-const miss = require('mississippi')
+import {throughObj} from './util/streamHelpers.js'
 
-module.exports = (allowedTypes) =>
-  allowedTypes && allowedTypes.length > 0
-    ? miss.through.obj((doc, enc, callback) => {
-        const type = doc && doc._type
-        if (allowedTypes.includes(type)) {
-          callback(null, doc)
-          return
-        }
+export function filterDocumentTypes(allowedTypes) {
+  if (!allowedTypes || allowedTypes.length === 0) {
+    // Pass-through
+    return throughObj((doc, enc, callback) => callback(null, doc))
+  }
 
-        callback()
-      })
-    : miss.through.obj()
+  return throughObj(function docTypesFilter(doc, enc, callback) {
+    const type = doc && doc._type
+    if (allowedTypes.includes(type)) {
+      callback(null, doc)
+      return
+    }
+
+    callback()
+  })
+}

package/src/filterDocuments.js

@@ -1,5 +1,5 @@
-const miss = require('mississippi')
-const debug = require('./debug')
+import {debug} from './debug.js'
+import {throughObj} from './util/streamHelpers.js'
 
 const isDraftOrVersion = (doc) =>
   doc && doc._id && (doc._id.indexOf('drafts.') === 0 || doc._id.indexOf('versions.') === 0)
@@ -8,8 +8,8 @@ const isSystemDocument = (doc) => doc && doc._id && doc._id.indexOf('_.') === 0
 const isReleaseDocument = (doc) => doc && doc._id && doc._id.indexOf('_.releases.') === 0
 const isCursor = (doc) => doc && !doc._id && doc.nextCursor !== undefined
 
-module.exports = (drafts) =>
-  miss.through.obj((doc, enc, callback) => {
+export function filterDocuments(drafts) {
+  return throughObj(function filterDocs(doc, enc, callback) {
     if (isCursor(doc)) {
       debug('%o is a cursor, skipping', doc)
       return callback()
@@ -21,7 +21,7 @@ module.exports = (drafts) =>
     }
 
     if (isSystemDocument(doc)) {
-      if (!drafts && isReleaseDocument(doc)) {
+      if (drafts && isReleaseDocument(doc)) {
         return callback(null, doc)
       }
       debug('%s is a system document, skipping', doc && doc._id)
@@ -30,3 +30,4 @@ module.exports = (drafts) =>
 
     return callback(null, doc)
   })
+}

package/src/getDocumentCursorStream.js

@@ -1,13 +1,13 @@
-const {Transform} = require('node:stream')
+import {Transform} from 'node:stream'
 
-const pkg = require('../package.json')
-const debug = require('./debug')
-const requestStream = require('./requestStream')
+import {debug} from './debug.js'
+import {getUserAgent} from './getUserAgent.js'
+import {requestStream} from './requestStream.js'
 
 // same regex as split2 is using by default: https://github.com/mcollina/split2/blob/53432f54bd5bf422bd55d91d38f898b6c9496fc1/index.js#L86
 const splitRegex = /\r?\n/
 
-module.exports = async (options) => {
+export async function getDocumentCursorStream(options) {
   let streamsInflight = 0
   function decrementInflight(stream) {
     streamsInflight--
@@ -80,17 +80,21 @@ function startStream(options, nextCursor) {
   }
   const token = options.client.config().token
   const headers = {
-    'User-Agent': `${pkg.name}@${pkg.version}`,
+    'User-Agent': getUserAgent(),
     ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
 
   debug('Starting stream with cursor "%s"', nextCursor)
 
-  return requestStream({url: url.toString(), headers, maxRetries: options.maxRetries}).then(
-    (res) => {
-      debug('Got stream with HTTP %d', res.statusCode)
+  return requestStream({
+    url: url.toString(),
+    headers,
+    maxRetries: options.maxRetries,
+    retryDelayMs: options.retryDelayMs,
+    readTimeout: options.readTimeout,
+  }).then((res) => {
+    debug('Got stream with HTTP %d', res.statusCode)
 
-      return res
-    },
-  )
+    return res
+  })
 }

package/src/getDocumentsStream.js

@@ -1,7 +1,7 @@
-const pkg = require('../package.json')
-const requestStream = require('./requestStream')
+import {getUserAgent} from './getUserAgent.js'
+import {requestStream} from './requestStream.js'
 
-module.exports = (options) => {
+export function getDocumentsStream(options) {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
   const baseUrl = options.client.getUrl(
@@ -17,7 +17,7 @@ module.exports = (options) => {
 
   const token = options.client.config().token
   const headers = {
-    'User-Agent': `${pkg.name}@${pkg.version}`,
+    'User-Agent': getUserAgent(),
     ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
 
@@ -25,6 +25,7 @@ module.exports = (options) => {
     url: url.toString(),
     headers,
     maxRetries: options.maxRetries,
+    retryDelayMs: options.retryDelayMs,
     readTimeout: options.readTimeout,
   })
 }

package/src/getUserAgent.js

@@ -0,0 +1,14 @@
+import {readFileSync} from 'node:fs'
+import {join as joinPath} from 'node:path'
+
+let ua = null
+
+export function getUserAgent() {
+  if (!ua) {
+    const data = readFileSync(joinPath(import.meta.dirname, '..', 'package.json'), 'utf-8')
+    const pkg = JSON.parse(data)
+    ua = `${pkg.name}@${pkg.version}`
+  }
+
+  return ua
+}

package/src/logFirstChunk.js

@@ -1,9 +1,9 @@
-const miss = require('mississippi')
-const debug = require('./debug')
+import {debug} from './debug.js'
+import {through} from './util/streamHelpers.js'
 
-module.exports = () => {
+export function logFirstChunk() {
   let firstChunk = true
-  return miss.through((chunk, enc, callback) => {
+  return through((chunk, enc, callback) => {
     if (firstChunk) {
       const string = chunk.toString('utf8').split('\n')[0]
       debug('First chunk received: %s', string.slice(0, 300))

package/src/rejectOnApiError.js

@@ -1,7 +1,7 @@
-const miss = require('mississippi')
+import {throughObj} from './util/streamHelpers.js'
 
-module.exports = () =>
-  miss.through.obj((doc, enc, callback) => {
+export function rejectOnApiError() {
+  return throughObj((doc, enc, callback) => {
     // check if the document passed contains a document attribtue first, and return early.
     if (doc._id) {
       callback(null, doc)
@@ -28,3 +28,4 @@ module.exports = () =>
 
     callback(null, doc)
   })
+}

package/src/requestStream.js

@@ -1,28 +1,37 @@
-const {getIt} = require('get-it')
-const {keepAlive, promise} = require('get-it/middleware')
-const debug = require('./debug')
-const {extractFirstError} = require('./util/extractFirstError')
-const {DOCUMENT_STREAM_MAX_RETRIES, REQUEST_READ_TIMEOUT} = require('./constants')
+import {getIt} from 'get-it'
+import {keepAlive, promise} from 'get-it/middleware'
+
+import {
+  DEFAULT_RETRY_DELAY,
+  DOCUMENT_STREAM_MAX_RETRIES,
+  REQUEST_READ_TIMEOUT,
+} from './constants.js'
+import {debug} from './debug.js'
+import {delay} from './util/delay.js'
+import {extractFirstError} from './util/extractFirstError.js'
+import {tryThrowFriendlyError} from './util/friendlyError.js'
 
 const request = getIt([keepAlive(), promise({onlyBody: true})])
 
 const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds
-const RETRY_DELAY_MS = 1500 // 1.5 seconds
-
-function delay(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms))
-}
 
 /* eslint-disable no-await-in-loop, max-depth */
-module.exports = async (options) => {
+export async function requestStream(options) {
   const maxRetries =
     typeof options.maxRetries === 'number' ? options.maxRetries : DOCUMENT_STREAM_MAX_RETRIES
 
   const readTimeout =
     typeof options.readTimeout === 'number' ? options.readTimeout : REQUEST_READ_TIMEOUT
 
+  const retryDelayMs =
+    typeof options.retryDelayMs === 'number' ? options.retryDelayMs : DEFAULT_RETRY_DELAY
+
   let error
-  for (let i = 0; i < maxRetries; i++) {
+
+  let i = 0
+  do {
+    i++
+
     try {
       return await request({
         ...options,
@@ -31,16 +40,24 @@ module.exports = async (options) => {
         timeout: {connect: CONNECTION_TIMEOUT, socket: readTimeout},
       })
     } catch (err) {
-      error = extractFirstError(err)
+      error = extractFirstError(err) || err
+
+      if (maxRetries === 0) {
+        throw error
+      }
 
       if (err.response && err.response.statusCode && err.response.statusCode < 500) {
         break
       }
 
-      debug('Error, retrying after %d ms: %s', RETRY_DELAY_MS, error.message)
-      await delay(RETRY_DELAY_MS)
+      if (i < maxRetries) {
+        debug('Error, retrying after %d ms: %s', retryDelayMs, error.message)
+        await delay(retryDelayMs)
+      }
     }
-  }
+  } while (i < maxRetries)
+
+  await tryThrowFriendlyError(error)
 
   error.message = `Export: Failed to fetch ${options.url}: ${error.message}`
   throw error

package/src/stringifyStream.js

@@ -1,4 +1,5 @@
-const miss = require('mississippi')
+import {throughObj} from './util/streamHelpers.js'
 
-module.exports = () =>
-  miss.through.obj((doc, enc, callback) => callback(null, `${JSON.stringify(doc)}\n`))
+export function stringifyStream() {
+  return throughObj((doc, enc, callback) => callback(null, `${JSON.stringify(doc)}\n`))
+}

package/src/tryParseJson.js

@@ -1,5 +1,3 @@
-const {createSafeJsonParser} = require('@sanity/util/createSafeJsonParser')
-
 /**
  * Safe JSON parser that is able to handle lines interrupted by an error object.
  *
@@ -8,6 +6,30 @@ const {createSafeJsonParser} = require('@sanity/util/createSafeJsonParser')
  * @internal
  * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request}
  */
-module.exports = createSafeJsonParser({
+export const tryParseJson = createSafeJsonParser({
   errorLabel: 'Error streaming dataset',
 })
+
+function createSafeJsonParser({errorLabel}) {
+  return function safeJsonParser(line) {
+    try {
+      return JSON.parse(line)
+    } catch (err) {
+      // Catch half-done lines with an error at the end
+      const errorPosition = line.lastIndexOf('{"error":')
+      if (errorPosition === -1) {
+        err.message = `${err.message} (${line})`
+        throw err
+      }
+
+      const errorJson = line.slice(errorPosition)
+      const errorLine = JSON.parse(errorJson)
+      const error = errorLine && errorLine.error
+      if (error && error.description) {
+        throw new Error(`${errorLabel}: ${error.description}\n\n${errorJson}\n`, {cause: err})
+      }
+
+      throw err
+    }
+  }
+}

package/src/util/delay.js

@@ -0,0 +1,3 @@
+export function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}

package/src/util/extractFirstError.js

@@ -1,4 +1,4 @@
-exports.extractFirstError = function extractFirstError(err) {
+export function extractFirstError(err) {
   if (
     // eslint-disable-next-line no-undef
     ((typeof AggregateError !== 'undefined' && err instanceof AggregateError) ||

package/src/util/friendlyError.js

@@ -0,0 +1,58 @@
+export async function tryThrowFriendlyError(err) {
+  if (!isRecord(err)) {
+    return null
+  }
+
+  if (!('response' in err) || !isRecord(err.response)) {
+    return null
+  }
+
+  if (
+    !('body' in err.response) ||
+    !('pipe' in err.response.body) ||
+    !('headers' in err.response) ||
+    !isRecord(err.response.headers)
+  ) {
+    return null
+  }
+
+  if (
+    typeof err.response.headers['content-type'] !== 'string' ||
+    !err.response.headers['content-type'].includes('application/json')
+  ) {
+    return null
+  }
+
+  const body = await readBodyJson(err.response.body)
+
+  if (!isRecord(body)) {
+    return null
+  }
+
+  // Look for Sanity API(ish) standard error shape
+  const status =
+    typeof err.response.statusCode === 'number' ? `HTTP ${err.response.statusCode}` : undefined
+  const error = typeof body.error === 'string' ? body.error : undefined
+  const message = typeof body.message === 'string' ? body.message : undefined
+  if (!error && !message) {
+    return null
+  }
+
+  throw new Error(['Export', status, error, message].filter(Boolean).join(': '))
+}
+
+function isRecord(thing) {
+  return typeof thing === 'object' && thing !== null && !Array.isArray(thing)
+}
+
+async function readBody(req) {
+  const chunks = []
+  for await (const chunk of req) {
+    chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk)
+  }
+  return Buffer.concat(chunks)
+}
+
+async function readBodyJson(req) {
+  return JSON.parse((await readBody(req)).toString('utf8'))
+}

package/src/util/streamHelpers.js

@@ -0,0 +1,103 @@
+import {Transform} from 'node:stream'
+
+export function through(transformFn) {
+  return new Transform({
+    transform(chunk, encoding, callback) {
+      transformFn(chunk, encoding, callback)
+    },
+  })
+}
+
+export function throughObj(transformFn) {
+  return new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback) {
+      transformFn(chunk, encoding, callback)
+    },
+  })
+}
+
+export function isWritableStream(val) {
+  return (
+    val !== null &&
+    typeof val === 'object' &&
+    typeof val.pipe === 'function' &&
+    typeof val._write === 'function' &&
+    typeof val._writableState === 'object'
+  )
+}
+
+export function concat(onData) {
+  const chunks = []
+  return new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback) {
+      chunks.push(chunk)
+      callback()
+    },
+    flush(callback) {
+      try {
+        onData(chunks)
+        callback()
+      } catch (err) {
+        callback(err)
+      }
+    },
+  })
+}
+
+export const split = (transformFn) => {
+  let buffer = ''
+  const splitRegex = /\r?\n/
+
+  return new Transform({
+    objectMode: !!transformFn,
+    transform(chunk, encoding, callback) {
+      buffer += chunk.toString()
+      const lines = buffer.split(splitRegex)
+
+      // Keep the last line in buffer as it might be incomplete
+      buffer = lines.pop() || ''
+
+      for (const line of lines) {
+        if (line.length === 0) continue
+
+        if (transformFn) {
+          try {
+            const result = transformFn(line)
+            if (result !== undefined) {
+              this.push(result)
+            }
+          } catch (err) {
+            callback(err)
+            return
+          }
+        } else {
+          this.push(line)
+        }
+      }
+      callback()
+    },
+    flush(callback) {
+      if (buffer.length === 0) {
+        callback()
+        return
+      }
+
+      if (!transformFn) {
+        callback(null, buffer)
+        return
+      }
+
+      try {
+        const result = transformFn(buffer)
+        if (result !== undefined) {
+          this.push(result)
+        }
+        callback()
+      } catch (err) {
+        callback(err)
+      }
+    },
+  })
+}

package/src/validateOptions.js

@@ -1,11 +1,10 @@
-const defaults = require('lodash/defaults')
-const {
-  DOCUMENT_STREAM_MAX_RETRIES,
+import {
   ASSET_DOWNLOAD_MAX_RETRIES,
-  REQUEST_READ_TIMEOUT,
-  MODE_STREAM,
+  DOCUMENT_STREAM_MAX_RETRIES,
   MODE_CURSOR,
-} = require('./constants')
+  MODE_STREAM,
+  REQUEST_READ_TIMEOUT,
+} from './constants.js'
 
 const clientMethods = ['getUrl', 'config']
 const booleanFlags = ['assets', 'raw', 'compress', 'drafts']
@@ -24,8 +23,8 @@ const exportDefaults = {
   transformDocument: (doc) => doc,
 }
 
-function validateOptions(opts) {
-  const options = defaults({}, opts, exportDefaults)
+export function validateOptions(opts) {
+  const options = {...exportDefaults, ...opts}
 
   const resources = [options.dataset, options.mediaLibraryId].filter(
     (resource) => typeof resource === 'string' && resource.length !== 0,
@@ -112,5 +111,3 @@ function validateOptions(opts) {
 
   return options
 }
-
-module.exports = validateOptions

package/src/util/pipeAsync.js

@@ -1,17 +0,0 @@
-const miss = require('mississippi')
-
-module.exports = async (readable, writable) => {
-  return new Promise((resolve, reject) => {
-    try {
-      miss.pipe(readable, writable, (jsonErr) => {
-        if (jsonErr) {
-          reject(jsonErr)
-        } else {
-          resolve()
-        }
-      })
-    } catch (assetErr) {
-      reject(assetErr)
-    }
-  })
-}

package/src/util/rimraf.js

@@ -1 +0,0 @@
-module.exports = require('rimraf').rimraf