@sanity/export 3.40.0 → 3.41.1

package/README.md

@@ -42,6 +42,11 @@ exportDataset({
 
   // Run 12 concurrent asset downloads
   assetConcurrency: 12,
+
+  // What mode to use when exporting documents, can be eiter `stream`(default) or `cursor`.
+  // Cursor mode might help when dealing with large datasets, but might yield inconsistent results if the dataset is mutated during export.
+  // Default: 'stream'
+  mode: 'stream',
 })
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/export",
-  "version": "3.40.0",
+  "version": "3.41.1",
   "description": "Export Sanity documents and assets",
   "keywords": [
     "sanity",
@@ -38,7 +38,7 @@
     "lodash": "^4.17.21",
     "mississippi": "^4.0.0",
     "p-queue": "^2.3.0",
-    "rimraf": "^3.0.2",
+    "rimraf": "^6.0.1",
     "split2": "^4.2.0",
     "tar": "^7.0.1",
     "yaml": "^2.4.2"

package/src/AssetHandler.js

@@ -2,7 +2,7 @@ const crypto = require('crypto')
 const {mkdirSync, createWriteStream} = require('fs')
 const path = require('path')
 const {parse: parseUrl, format: formatUrl} = require('url')
-const {omit, noop} = require('lodash')
+const {omit} = require('lodash')
 const miss = require('mississippi')
 const PQueue = require('p-queue')
 const pkg = require('../package.json')

package/src/export.js

@@ -91,7 +91,7 @@ async function exportDataset(opts) {
     debug('Archive finished')
   })
 
-  debug('Getting dataset export stream')
+  debug('Getting dataset export stream, mode: "%s"', options.mode)
   onProgress({step: 'Exporting documents...'})
 
   let documentCount = 0
@@ -120,8 +120,12 @@ async function exportDataset(opts) {
   }
 
   const inputStream = await getDocumentInputStream(options)
-  debug('Got HTTP %d', inputStream.statusCode)
-  debug('Response headers: %o', inputStream.headers)
+  if (inputStream.statusCode) {
+    debug('Got HTTP %d', inputStream.statusCode)
+  }
+  if (inputStream.headers) {
+    debug('Response headers: %o', inputStream.headers)
+  }
 
   let debugTimer = null
   function scheduleDebugTimer() {

package/src/filterSystemDocuments.js

@@ -2,6 +2,7 @@ const miss = require('mississippi')
 const debug = require('./debug')
 
 const isSystemDocument = (doc) => doc && doc._id && doc._id.indexOf('_.') === 0
+const isCursor = (doc) => doc && !doc._id && doc.nextCursor !== undefined
 
 module.exports = () =>
   miss.through.obj((doc, enc, callback) => {
@@ -9,6 +10,10 @@ module.exports = () =>
       debug('%s is a system document, skipping', doc && doc._id)
       return callback()
     }
+    if (isCursor(doc)) {
+      debug('%o is a cursor, skipping', doc)
+      return callback()
+    }
 
     return callback(null, doc)
   })

package/src/getDocumentCursorStream.js

@@ -1,60 +1,66 @@
 const {Transform} = require('node:stream')
 
 const pkg = require('../package.json')
+const debug = require('./debug')
 const requestStream = require('./requestStream')
 
+// same regex as split2 is using by default: https://github.com/mcollina/split2/blob/53432f54bd5bf422bd55d91d38f898b6c9496fc1/index.js#L86
+const splitRegex = /\r?\n/
+
 module.exports = async (options) => {
   let streamsInflight = 0
+  function decrementInflight(stream) {
+    streamsInflight--
+    if (streamsInflight === 0) {
+      stream.end()
+    }
+  }
+
   const stream = new Transform({
     async transform(chunk, encoding, callback) {
       if (encoding !== 'buffer' && encoding !== 'string') {
         callback(null, chunk)
         return
       }
+      this.push(chunk, encoding)
 
       let parsedChunk = null
-      try {
-        parsedChunk = JSON.parse(chunk.toString())
-      } catch (err) {
-        // Ignore JSON parse errors
-        // this can happen if the chunk is not a JSON object. We just pass it through and let the caller handle it.
-      }
+      for (const chunkStr of chunk.toString().split(splitRegex)) {
+        if (chunkStr.trim() === '') {
+          continue
+        }
 
-      if (
-        parsedChunk !== null &&
-        typeof parsedChunk === 'object' &&
-        'nextCursor' in parsedChunk &&
-        typeof parsedChunk.nextCursor === 'string' &&
-        !('_id' in parsedChunk)
-      ) {
-        streamsInflight++
+        try {
+          parsedChunk = JSON.parse(chunkStr)
+        } catch (err) {
+          // Ignore JSON parse errors
+          // this can happen if the chunk is not a JSON object. We just pass it through and let the caller handle it.
+          debug('Failed to parse JSON chunk, ignoring', err, chunkStr)
+        }
 
-        const reqStream = await startStream(options, parsedChunk.nextCursor)
-        reqStream.on('end', () => {
-          streamsInflight--
-          if (streamsInflight === 0) {
-            stream.end()
-          }
-        })
-        reqStream.pipe(this, {end: false})
+        if (
+          parsedChunk !== null &&
+          typeof parsedChunk === 'object' &&
+          'nextCursor' in parsedChunk &&
+          typeof parsedChunk.nextCursor === 'string' &&
+          !('_id' in parsedChunk)
+        ) {
+          debug('Got next cursor "%s", fetching next stream', parsedChunk.nextCursor)
+          streamsInflight++
 
-        callback()
-        return
+          const reqStream = await startStream(options, parsedChunk.nextCursor)
+          reqStream.on('end', () => decrementInflight(this))
+          reqStream.pipe(this, {end: false})
+        }
       }
 
-      callback(null, chunk)
+      callback()
     },
   })
 
   streamsInflight++
   const reqStream = await startStream(options, '')
-  reqStream.on('end', () => {
-    streamsInflight--
-    if (streamsInflight === 0) {
-      stream.end()
-    }
-  })
-
+  reqStream.on('end', () => decrementInflight(stream))
   reqStream.pipe(stream, {end: false})
   return stream
 }
@@ -69,5 +75,11 @@ function startStream(options, nextCursor) {
     ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
 
-  return requestStream({url, headers, maxRetries: options.maxRetries})
+  debug('Starting stream with cursor "%s"', nextCursor)
+
+  return requestStream({url, headers, maxRetries: options.maxRetries}).then((res) => {
+    debug('Got stream with HTTP %d', res.statusCode)
+
+    return res
+  })
 }

package/src/util/rimraf.js

@@ -1,4 +1 @@
-const {promisify} = require('util')
-const rimrafCb = require('rimraf')
-
-module.exports = promisify(rimrafCb)
+module.exports = require('rimraf').rimraf