@sanity/export 3.41.1 → 3.42.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/export",
-  "version": "3.41.1",
+  "version": "3.42.0",
   "description": "Export Sanity documents and assets",
   "keywords": [
     "sanity",
@@ -41,7 +41,8 @@
     "rimraf": "^6.0.1",
     "split2": "^4.2.0",
     "tar": "^7.0.1",
-    "yaml": "^2.4.2"
+    "yaml": "^2.4.2",
+    "json-stream-stringify": "^2.0.2"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

package/src/AssetHandler.js

@@ -112,6 +112,20 @@ class AssetHandler {
         try {
           return await this.downloadAsset(assetDoc, dstPath)
         } catch (err) {
+          // Ignore inaccessible assets
+          switch (err.statusCode) {
+            case 401:
+            case 403:
+            case 404:
+              console.warn(
+                `⚠ Asset failed with HTTP %d (ignoring): %s`,
+                err.statusCode,
+                assetDoc._id,
+              )
+              return true
+            default:
+          }
+
           debug(
             `Error downloading asset %s (destination: %s), attempt %d`,
             assetDoc._id,
@@ -179,6 +193,9 @@ class AssetHandler {
   // eslint-disable-next-line max-statements
   async downloadAsset(assetDoc, dstPath) {
     const {url} = assetDoc
+
+    debug('Downloading asset %s', url)
+
     const options = this.getAssetRequestOptions(assetDoc)
 
     let stream

package/src/export.js

@@ -5,8 +5,10 @@ const zlib = require('zlib')
 const archiver = require('archiver')
 const miss = require('mississippi')
 const split = require('split2')
+const JsonStreamStringify = require('json-stream-stringify')
 const AssetHandler = require('./AssetHandler')
 const debug = require('./debug')
+const pipeAsync = require('./util/pipeAsync')
 const filterDocumentTypes = require('./filterDocumentTypes')
 const filterDrafts = require('./filterDrafts')
 const filterSystemDocuments = require('./filterSystemDocuments')
@@ -33,6 +35,12 @@ async function exportDataset(opts) {
         : zlib.constants.Z_NO_COMPRESSION,
     },
   })
+  archive.on('warning', (err) => {
+    debug('Archive warning: %s', err.message)
+  })
+  archive.on('entry', (entry) => {
+    debug('Adding archive entry: %s', entry.name)
+  })
 
   const slugDate = new Date()
     .toISOString()
@@ -43,6 +51,7 @@ async function exportDataset(opts) {
   const tmpDir = path.join(os.tmpdir(), prefix)
   fs.mkdirSync(tmpDir, {recursive: true})
   const dataPath = path.join(tmpDir, 'data.ndjson')
+  const assetsPath = path.join(tmpDir, 'assets.json')
 
   const cleanup = () =>
     rimraf(tmpDir).catch((err) => {
@@ -209,7 +218,9 @@ async function exportDataset(opts) {
         update: true,
       })
 
-      archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix})
+      const assetsStream = fs.createWriteStream(assetsPath)
+      await pipeAsync(new JsonStreamStringify(assetMap), assetsStream)
+      archive.file(assetsPath, {name: 'assets.json', prefix})
       clearInterval(progressInterval)
     } catch (assetErr) {
       clearInterval(progressInterval)
@@ -227,10 +238,6 @@ async function exportDataset(opts) {
     await archive.finalize()
   })
 
-  archive.on('warning', (err) => {
-    debug('Archive warning: %s', err.message)
-  })
-
   miss.pipe(archive, outputStream, onComplete)
 
   async function onComplete(err) {

package/src/util/pipeAsync.js

@@ -0,0 +1,17 @@
+const miss = require('mississippi')
+
+module.exports = async (readable, writable) => {
+  return new Promise((resolve, reject) => {
+    try {
+      miss.pipe(readable, writable, (jsonErr) => {
+        if (jsonErr) {
+          reject(jsonErr)
+        } else {
+          resolve()
+        }
+      })
+    } catch (assetErr) {
+      reject(assetErr)
+    }
+  })
+}