@sanity/export 0.136.3-gql-rtb.372 → 0.136.3-purple-unicorn-patch.5627

package/lib/export.js

@@ -1,9 +1,5 @@
 "use strict";
 
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
-
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
-
 const os = require('os');
 
 const path = require('path');
@@ -36,6 +32,10 @@ const filterDocumentTypes = require('./filterDocumentTypes');
 
 const filterDrafts = require('./filterDrafts');
 
+const logFirstChunk = require('./logFirstChunk');
+
+const tryParseJson = require('./tryParseJson');
+
 const noop = () => null;
 
 function exportDataset(opts) {
@@ -48,168 +48,173 @@ function exportDataset(opts) {
     }
   });
   const slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
-  const prefix = `${opts.dataset}-export-${slugDate}`;
+  const prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
   const tmpDir = path.join(os.tmpdir(), prefix);
+
+  const cleanup = () => fse.remove(tmpDir).catch(err => {
+    debug("Error while cleaning up temporary files: ".concat(err.message));
+  });
+
   const assetHandler = new AssetHandler({
     client: options.client,
     tmpDir,
-    prefix
+    prefix,
+    concurrency: options.assetConcurrency
   });
   debug('Outputting assets (temporarily) to %s', tmpDir);
   debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
-  const outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
+  let outputStream;
+
+  if (isWritableStream(options.outputPath)) {
+    outputStream = options.outputPath;
+  } else {
+    outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
+  }
+
   let assetStreamHandler = assetHandler.noop;
 
   if (!options.raw) {
     assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
   }
 
-  return new Promise(
-  /*#__PURE__*/
-  function () {
-    var _ref = _asyncToGenerator(function* (resolve, reject) {
-      miss.finished(archive, archiveErr => {
-        if (archiveErr) {
-          debug('Archiving errored! %s', archiveErr.stack);
-          reject(archiveErr);
-          return;
-        }
+  return new Promise(async (resolve, reject) => {
+    miss.finished(archive, async archiveErr => {
+      if (archiveErr) {
+        debug('Archiving errored! %s', archiveErr.stack);
+        await cleanup();
+        reject(archiveErr);
+        return;
+      }
+
+      debug('Archive finished!');
+    });
+    debug('Getting dataset export stream');
+    onProgress({
+      step: 'Exporting documents...'
+    });
+    let documentCount = 0;
+    let lastReported = Date.now();
+
+    const reportDocumentCount = (chunk, enc, cb) => {
+      ++documentCount;
+      const now = Date.now();
+
+      if (now - lastReported > 50) {
+        onProgress({
+          step: 'Exporting documents...',
+          current: documentCount,
+          total: '?',
+          update: true
+        });
+        lastReported = now;
+      }
+
+      cb(null, chunk);
+    };
+
+    const inputStream = await getDocumentsStream(options.client, options.dataset);
+    debug('Got HTTP %d', inputStream.statusCode);
+    debug('Response headers: %o', inputStream.headers);
+    const jsonStream = miss.pipeline(inputStream, logFirstChunk(), split(tryParseJson), rejectOnApiError(), filterSystemDocuments(), assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts(), stringifyStream(), miss.through(reportDocumentCount));
+    miss.finished(jsonStream, async err => {
+      if (err) {
+        return;
+      }
 
-        debug('Archive finished!');
-      });
-      debug('Getting dataset export stream');
       onProgress({
-        step: 'Exporting documents...'
+        step: 'Exporting documents...',
+        current: documentCount,
+        total: documentCount,
+        update: true
       });
-      let documentCount = 0;
-      let lastReported = Date.now();
-
-      const reportDocumentCount = (chunk, enc, cb) => {
-        ++documentCount;
-        const now = Date.now();
-
-        if (now - lastReported > 50) {
-          onProgress({
-            step: 'Exporting documents...',
-            current: documentCount,
-            total: '?',
-            update: true
-          });
-          lastReported = now;
+
+      if (!options.raw && options.assets) {
+        onProgress({
+          step: 'Downloading assets...'
+        });
+      }
+
+      let prevCompleted = 0;
+      const progressInterval = setInterval(() => {
+        const completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
+
+        if (prevCompleted === completed) {
+          return;
         }
 
-        cb(null, chunk);
-      };
-
-      const inputStream = yield getDocumentsStream(options.client, options.dataset);
-      const jsonStream = miss.pipeline(inputStream, split(JSON.parse), rejectOnApiError, filterSystemDocuments, assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts, stringifyStream, miss.through(reportDocumentCount));
-      miss.finished(jsonStream,
-      /*#__PURE__*/
-      function () {
-        var _ref2 = _asyncToGenerator(function* (err) {
-          if (err) {
-            return;
-          }
-
-          onProgress({
-            step: 'Exporting documents...',
-            current: documentCount,
-            total: documentCount,
-            update: true
-          });
-
-          if (!options.raw && options.assets) {
-            onProgress({
-              step: 'Downloading assets...'
-            });
-          }
-
-          let prevCompleted = 0;
-          const progressInterval = setInterval(() => {
-            const completed = assetHandler.queueSize - assetHandler.queue.size;
-
-            if (prevCompleted === completed) {
-              return;
-            }
-
-            prevCompleted = completed;
-            onProgress({
-              step: 'Downloading assets...',
-              current: completed,
-              total: assetHandler.queueSize,
-              update: true
-            });
-          }, 500);
-          debug('Waiting for asset handler to complete downloads');
-
-          try {
-            const assetMap = yield assetHandler.finish();
-            archive.append(JSON.stringify(assetMap), {
-              name: 'assets.json',
-              prefix
-            });
-            clearInterval(progressInterval);
-          } catch (assetErr) {
-            clearInterval(progressInterval);
-            reject(assetErr);
-            return;
-          } // Add all downloaded assets to archive
-
-
-          archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {
-            store: true
-          });
-          archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {
-            store: true
-          });
-          debug('Finalizing archive, flushing streams');
-          onProgress({
-            step: 'Adding assets to archive...'
-          });
-          archive.finalize();
+        prevCompleted = completed;
+        onProgress({
+          step: 'Downloading assets...',
+          current: completed,
+          total: assetHandler.queueSize,
+          update: true
+        });
+      }, 500);
+      debug('Waiting for asset handler to complete downloads');
+
+      try {
+        const assetMap = await assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
+
+        onProgress({
+          step: 'Downloading assets...',
+          current: assetHandler.queueSize,
+          total: assetHandler.queueSize,
+          update: true
+        });
+        archive.append(JSON.stringify(assetMap), {
+          name: 'assets.json',
+          prefix
         });
+        clearInterval(progressInterval);
+      } catch (assetErr) {
+        clearInterval(progressInterval);
+        await cleanup();
+        reject(assetErr);
+        return;
+      } // Add all downloaded assets to archive
+
 
-        return function (_x3) {
-          return _ref2.apply(this, arguments);
-        };
-      }());
-      archive.on('warning', err => {
-        debug('Archive warning: %s', err.message);
+      archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
+        store: true
+      });
+      archive.directory(path.join(tmpDir, 'images'), "".concat(prefix, "/images"), {
+        store: true
+      });
+      debug('Finalizing archive, flushing streams');
+      onProgress({
+        step: 'Adding assets to archive...'
       });
-      archive.append(jsonStream, {
-        name: 'data.ndjson',
-        prefix
+      archive.finalize();
+    });
+    archive.on('warning', err => {
+      debug('Archive warning: %s', err.message);
+    });
+    archive.append(jsonStream, {
+      name: 'data.ndjson',
+      prefix
+    });
+    miss.pipe(archive, outputStream, onComplete);
+
+    async function onComplete(err) {
+      onProgress({
+        step: 'Clearing temporary files...'
       });
-      miss.pipe(archive, outputStream, onComplete);
+      await cleanup();
 
-      function onComplete(_x4) {
-        return _onComplete.apply(this, arguments);
+      if (!err) {
+        resolve();
+        return;
       }
 
-      function _onComplete() {
-        _onComplete = _asyncToGenerator(function* (err) {
-          onProgress({
-            step: 'Clearing temporary files...'
-          });
-          yield fse.remove(tmpDir);
-
-          if (!err) {
-            resolve();
-            return;
-          }
-
-          debug('Error during streaming: %s', err.stack);
-          assetHandler.clear();
-          reject(err);
-        });
-        return _onComplete.apply(this, arguments);
-      }
-    });
+      debug('Error during streaming: %s', err.stack);
+      assetHandler.clear();
+      reject(err);
+    }
+  });
+}
 
-    return function (_x, _x2) {
-      return _ref.apply(this, arguments);
-    };
-  }());
+function isWritableStream(val) {
+  return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
 }
 
 module.exports = exportDataset;

package/lib/filterDrafts.js

@@ -4,7 +4,7 @@ const miss = require('mississippi');
 
 const isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
 
-module.exports = miss.through.obj((doc, enc, callback) => {
+module.exports = () => miss.through.obj((doc, enc, callback) => {
   if (isDraft(doc)) {
     return callback();
   }

package/lib/filterSystemDocuments.js

@@ -6,7 +6,7 @@ const debug = require('./debug');
 
 const isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
 
-module.exports = miss.through.obj((doc, enc, callback) => {
+module.exports = () => miss.through.obj((doc, enc, callback) => {
   if (isSystemDocument(doc)) {
     debug('%s is a system document, skipping', doc && doc._id);
     return callback();

package/lib/getDocumentsStream.js

@@ -7,10 +7,13 @@ const requestStream = require('./requestStream');
 module.exports = (client, dataset) => {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
-  const url = client.getUrl(`/data/export/${dataset}`);
+  const url = client.getUrl("/data/export/".concat(dataset));
+  const token = client.config().token;
   const headers = {
-    Authorization: `Bearer ${client.config().token}`,
-    'User-Agent': `${pkg.name}@${pkg.version}`
+    'User-Agent': "".concat(pkg.name, "@").concat(pkg.version),
+    ...(token ? {
+      Authorization: "Bearer ".concat(token)
+    } : {})
   };
   return requestStream({
     url,

package/lib/logFirstChunk.js

@@ -0,0 +1,18 @@
+"use strict";
+
+const miss = require('mississippi');
+
+const debug = require('./debug');
+
+module.exports = () => {
+  let firstChunk = true;
+  return miss.through((chunk, enc, callback) => {
+    if (firstChunk) {
+      const string = chunk.toString('utf8').split('\n')[0];
+      debug('First chunk received: %s', string.slice(0, 300));
+      firstChunk = false;
+    }
+
+    callback(null, chunk);
+  });
+};

package/lib/rejectOnApiError.js

@@ -2,11 +2,16 @@
 
 const miss = require('mississippi');
 
-module.exports = miss.through.obj((doc, enc, callback) => {
+module.exports = () => miss.through.obj((doc, enc, callback) => {
   if (doc.error && doc.statusCode) {
     callback(new Error([doc.statusCode, doc.error].join(': ')));
     return;
   }
 
+  if (!doc._id && doc.error) {
+    callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
+    return;
+  }
+
   callback(null, doc);
 });

package/lib/requestStream.js

@@ -1,7 +1,63 @@
 "use strict";
 
-const simpleGet = require('simple-get');
+const getIt = require('get-it');
 
-module.exports = options => new Promise((resolve, reject) => {
-  simpleGet(options, (err, res) => err ? reject(err) : resolve(res));
-});
+const {
+  keepAlive,
+  promise
+} = require('get-it/middleware');
+
+const debug = require('./debug');
+
+const request = getIt([keepAlive(), promise({
+  onlyBody: true
+})]);
+const socketsWithTimeout = new WeakSet();
+const CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
+
+const READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
+
+const MAX_RETRIES = 5;
+
+function delay(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+/* eslint-disable no-await-in-loop, max-depth */
+
+
+module.exports = async options => {
+  let error;
+
+  for (let i = 0; i < MAX_RETRIES; i++) {
+    try {
+      const response = await request({ ...options,
+        stream: true,
+        maxRedirects: 0,
+        timeout: {
+          connect: CONNECTION_TIMEOUT,
+          socket: READ_TIMEOUT
+        }
+      });
+
+      if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
+        socketsWithTimeout.add(response.connection);
+        response.connection.setTimeout(READ_TIMEOUT, () => {
+          response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
+        });
+      }
+
+      return response;
+    } catch (err) {
+      error = err;
+
+      if (err.response && err.response.statusCode && err.response.statusCode < 500) {
+        break;
+      }
+
+      debug('Error, retrying after 1500ms: %s', err.message);
+      await delay(1500);
+    }
+  }
+
+  throw error;
+};

package/lib/stringifyStream.js

@@ -2,4 +2,4 @@
 
 const miss = require('mississippi');
 
-module.exports = miss.through.obj((doc, enc, callback) => callback(null, `${JSON.stringify(doc)}\n`));
+module.exports = () => miss.through.obj((doc, enc, callback) => callback(null, "".concat(JSON.stringify(doc), "\n")));

package/lib/tryParseJson.js

@@ -0,0 +1,25 @@
+"use strict";
+
+module.exports = line => {
+  try {
+    return JSON.parse(line);
+  } catch (err) {
+    // Catch half-done lines with an error at the end
+    const errorPosition = line.lastIndexOf('{"error":');
+
+    if (errorPosition === -1) {
+      err.message = "".concat(err.message, " (").concat(line, ")");
+      throw err;
+    }
+
+    const errorJson = line.slice(errorPosition);
+    const errorLine = JSON.parse(errorJson);
+    const error = errorLine && errorLine.error;
+
+    if (error && error.description) {
+      throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
+    }
+
+    throw err;
+  }
+};

package/lib/validateOptions.js

@@ -15,11 +15,11 @@ function validateOptions(opts) {
   const options = defaults({}, opts, exportDefaults);
 
   if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
-    throw new Error(`options.dataset must be a valid dataset name`);
+    throw new Error("options.dataset must be a valid dataset name");
   }
 
   if (options.onProgress && typeof options.onProgress !== 'function') {
-    throw new Error(`options.onProgress must be a function`);
+    throw new Error("options.onProgress must be a function");
   }
 
   if (!options.client) {
@@ -29,7 +29,7 @@ function validateOptions(opts) {
   const missing = clientMethods.find(key => typeof options.client[key] !== 'function');
 
   if (missing) {
-    throw new Error(`\`options.client\` is not a valid @sanity/client instance - no "${missing}" method found`);
+    throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
   }
 
   const clientConfig = options.client.config();
@@ -40,7 +40,7 @@ function validateOptions(opts) {
 
   booleanFlags.forEach(flag => {
     if (typeof options[flag] !== 'boolean') {
-      throw new Error(`Flag ${flag} must be a boolean (true/false)`);
+      throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
     }
   });
 
@@ -48,6 +48,10 @@ function validateOptions(opts) {
     throw new Error('outputPath must be specified (- for stdout)');
   }
 
+  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
+    throw new Error('`assetConcurrency` must be between 1 and 24');
+  }
+
   return options;
 }
 

package/package.json

@@ -1,16 +1,19 @@
 {
   "name": "@sanity/export",
-  "version": "0.136.3-gql-rtb.372+c3bd9d4b7",
+  "version": "0.136.3-purple-unicorn-patch.5627+c8b4b71ac",
   "description": "Export Sanity documents and assets",
-  "main": "lib/export.js",
+  "main": "./lib/export.js",
+  "types": "./lib/dts/src/export.d.ts",
   "engines": {
-    "node": ">=6.0.0"
+    "node": ">=12.0.0"
   },
   "author": "Sanity.io <hello@sanity.io>",
   "license": "MIT",
   "scripts": {
+    "build": "../../../bin/pkg-utils transpile --target node",
     "clean": "rimraf lib",
-    "test": "jest"
+    "test": "jest",
+    "watch": "../../../bin/pkg-utils transpile --target node --watch"
   },
   "keywords": [
     "sanity",
@@ -22,20 +25,17 @@
     "ndjson"
   ],
   "dependencies": {
-    "archiver": "^2.1.1",
-    "debug": "^3.1.0",
-    "fs-extra": "^6.0.1",
-    "lodash": "^4.17.4",
-    "mississippi": "^2.0.0",
+    "archiver": "^5.0.0",
+    "debug": "^3.2.7",
+    "fs-extra": "^7.0.0",
+    "get-it": "^5.2.1",
+    "lodash": "^4.17.21",
+    "mississippi": "^4.0.0",
     "p-queue": "^2.3.0",
-    "simple-get": "^3.0.2",
-    "split2": "^2.1.1"
+    "split2": "^3.2.2"
   },
   "devDependencies": {
-    "@sanity/client": "0.136.3-gql-rtb.372+c3bd9d4b7",
-    "decompress": "^4.2.0",
-    "jest": "^23.6.0",
-    "rimraf": "^2.6.2",
+    "rimraf": "^3.0.2",
     "string-to-stream": "^1.1.0"
   },
   "publishConfig": {
@@ -43,11 +43,12 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/sanity-io/sanity.git"
+    "url": "git+https://github.com/sanity-io/sanity.git",
+    "directory": "packages/@sanity/export"
   },
   "bugs": {
     "url": "https://github.com/sanity-io/sanity/issues"
   },
   "homepage": "https://www.sanity.io/",
-  "gitHead": "c3bd9d4b7b47cdfb67b42607c28e8775ce6efefe"
+  "gitHead": "c8b4b71acd7cf035f6f0a2efd19c39ced849811f"
 }