@sanity/export 3.38.0 → 3.38.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/export",
-  "version": "3.38.0",
+  "version": "3.38.2",
   "description": "Export Sanity documents and assets",
   "keywords": [
     "sanity",

package/src/constants.js

@@ -20,3 +20,9 @@ exports.ASSET_DOWNLOAD_MAX_RETRIES = 10
  * @internal
  */
 exports.ASSET_DOWNLOAD_CONCURRENCY = 8
+
+/**
+ * How frequently we will `debug` log while streaming the documents.
+ * @internal
+ */
+exports.DOCUMENT_STREAM_DEBUG_INTERVAL = 10000

package/src/export.js

@@ -17,6 +17,7 @@ const stringifyStream = require('./stringifyStream')
 const tryParseJson = require('./tryParseJson')
 const rimraf = require('./util/rimraf')
 const validateOptions = require('./validateOptions')
+const {DOCUMENT_STREAM_DEBUG_INTERVAL} = require('./constants')
 
 const noop = () => null
 
@@ -39,6 +40,9 @@ async function exportDataset(opts) {
 
   const prefix = `${opts.dataset}-export-${slugDate}`
   const tmpDir = path.join(os.tmpdir(), prefix)
+  fs.mkdirSync(tmpDir, {recursive: true})
+  const dataPath = path.join(tmpDir, 'data.ndjson')
+
   const cleanup = () =>
     rimraf(tmpDir).catch((err) => {
       debug(`Error while cleaning up temporary files: ${err.message}`)
@@ -90,11 +94,14 @@ async function exportDataset(opts) {
   onProgress({step: 'Exporting documents...'})
 
   let documentCount = 0
+  let lastDocumentID = null
   let lastReported = Date.now()
-  const reportDocumentCount = (chunk, enc, cb) => {
+  const reportDocumentCount = (doc, enc, cb) => {
     ++documentCount
 
     const now = Date.now()
+    // We report to the `onProgress` handler every 50 ms.
+    // It's up to the caller to not do too much expensive work.
     if (now - lastReported > 50) {
       onProgress({
         step: 'Exporting documents...',
@@ -106,13 +113,30 @@ async function exportDataset(opts) {
       lastReported = now
     }
 
-    cb(null, chunk)
+    lastDocumentID = doc._id
+
+    cb(null, doc)
   }
 
   const inputStream = await getDocumentsStream(options)
   debug('Got HTTP %d', inputStream.statusCode)
   debug('Response headers: %o', inputStream.headers)
 
+  let debugTimer = null
+  function scheduleDebugTimer() {
+    debugTimer = setTimeout(() => {
+      debug('Still streaming documents', {
+        documentCount,
+        lastDocumentID,
+      })
+
+      // Schedule another tick:
+      scheduleDebugTimer()
+    }, DOCUMENT_STREAM_DEBUG_INTERVAL)
+  }
+
+  scheduleDebugTimer()
+
   const jsonStream = miss.pipeline(
     inputStream,
     logFirstChunk(),
@@ -122,13 +146,15 @@ async function exportDataset(opts) {
     assetStreamHandler,
     filterDocumentTypes(options.types),
     options.drafts ? miss.through.obj() : filterDrafts(),
+    miss.through.obj(reportDocumentCount),
     stringifyStream(),
-    miss.through(reportDocumentCount),
   )
 
-  miss.finished(jsonStream, async (err) => {
+  miss.pipe(jsonStream, fs.createWriteStream(dataPath), async (err) => {
+    if (debugTimer !== null) clearTimeout(debugTimer)
+
     if (err) {
-      debug('Export stream error: ', err)
+      debug(`Export stream error @ ${lastDocumentID}/${documentCount}: `, err)
       reject(err)
       return
     }
@@ -141,6 +167,9 @@ async function exportDataset(opts) {
       update: true,
     })
 
+    debug('Adding data.ndjson to archive')
+    archive.file(dataPath, {name: 'data.ndjson', prefix})
+
     if (!options.raw && options.assets) {
       onProgress({step: 'Downloading assets...'})
     }
@@ -197,7 +226,6 @@ async function exportDataset(opts) {
     debug('Archive warning: %s', err.message)
   })
 
-  archive.append(jsonStream, {name: 'data.ndjson', prefix})
   miss.pipe(archive, outputStream, onComplete)
 
   async function onComplete(err) {
@@ -205,6 +233,7 @@ async function exportDataset(opts) {
     await cleanup()
 
     if (!err) {
+      debug('Export completed')
       resolve({
         outputPath: options.outputPath,
         documentCount,

package/src/rejectOnApiError.js

@@ -2,18 +2,26 @@ const miss = require('mississippi')
 
 module.exports = () =>
   miss.through.obj((doc, enc, callback) => {
-    if (doc.error && doc.statusCode) {
-      callback(
-        new Error(
-          ['Export', `HTTP ${doc.statusCode}`, doc.error, doc.message]
-            .filter((part) => typeof part === 'string')
-            .join(': '),
-        ),
-      )
+    // check if the document passed contains a document attribtue first, and return early.
+    if (doc._id) {
+      callback(null, doc)
       return
     }
 
-    if (!doc._id && doc.error) {
+    if (doc.error) {
+      // if we got a statusCode we can decorate the error with it
+      if (doc.statusCode) {
+        callback(
+          new Error(
+            ['Export', `HTTP ${doc.statusCode}`, doc.error, doc.message]
+              .filter((part) => typeof part === 'string')
+              .join(': '),
+          ),
+        )
+        return
+      }
+
+      // no statusCode, just serialize and return the error
       callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)))
       return
     }