@sanity/export 0.136.3-gql-rtb.375 → 0.136.3-purple-unicorn-patch.5629

package/src/requestStream.js

@@ -1,6 +1,55 @@
-const simpleGet = require('simple-get')
+const getIt = require('get-it')
+const {keepAlive, promise} = require('get-it/middleware')
+const debug = require('./debug')
 
-module.exports = options =>
-  new Promise((resolve, reject) => {
-    simpleGet(options, (err, res) => (err ? reject(err) : resolve(res)))
-  })
+const request = getIt([keepAlive(), promise({onlyBody: true})])
+const socketsWithTimeout = new WeakSet()
+
+const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds
+const READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes
+const MAX_RETRIES = 5
+
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/* eslint-disable no-await-in-loop, max-depth */
+module.exports = async (options) => {
+  let error
+  for (let i = 0; i < MAX_RETRIES; i++) {
+    try {
+      const response = await request({
+        ...options,
+        stream: true,
+        maxRedirects: 0,
+        timeout: {connect: CONNECTION_TIMEOUT, socket: READ_TIMEOUT},
+      })
+
+      if (
+        response.connection &&
+        typeof response.connection.setTimeout === 'function' &&
+        !socketsWithTimeout.has(response.connection)
+      ) {
+        socketsWithTimeout.add(response.connection)
+        response.connection.setTimeout(READ_TIMEOUT, () => {
+          response.destroy(
+            new Error(`Read timeout: No data received on socket for ${READ_TIMEOUT} ms`)
+          )
+        })
+      }
+
+      return response
+    } catch (err) {
+      error = err
+
+      if (err.response && err.response.statusCode && err.response.statusCode < 500) {
+        break
+      }
+
+      debug('Error, retrying after 1500ms: %s', err.message)
+      await delay(1500)
+    }
+  }
+
+  throw error
+}

package/src/stringifyStream.js

@@ -1,5 +1,4 @@
 const miss = require('mississippi')
 
-module.exports = miss.through.obj((doc, enc, callback) =>
-  callback(null, `${JSON.stringify(doc)}\n`)
-)
+module.exports = () =>
+  miss.through.obj((doc, enc, callback) => callback(null, `${JSON.stringify(doc)}\n`))

package/src/tryParseJson.js

@@ -0,0 +1,21 @@
+module.exports = (line) => {
+  try {
+    return JSON.parse(line)
+  } catch (err) {
+    // Catch half-done lines with an error at the end
+    const errorPosition = line.lastIndexOf('{"error":')
+    if (errorPosition === -1) {
+      err.message = `${err.message} (${line})`
+      throw err
+    }
+
+    const errorJson = line.slice(errorPosition)
+    const errorLine = JSON.parse(errorJson)
+    const error = errorLine && errorLine.error
+    if (error && error.description) {
+      throw new Error(`Error streaming dataset: ${error.description}\n\n${errorJson}\n`)
+    }
+
+    throw err
+  }
+}

package/src/validateOptions.js

@@ -6,7 +6,7 @@ const exportDefaults = {
   compress: true,
   drafts: true,
   assets: true,
-  raw: false
+  raw: false,
 }
 
 function validateOptions(opts) {
@@ -24,7 +24,7 @@ function validateOptions(opts) {
     throw new Error('`options.client` must be set to an instance of @sanity/client')
   }
 
-  const missing = clientMethods.find(key => typeof options.client[key] !== 'function')
+  const missing = clientMethods.find((key) => typeof options.client[key] !== 'function')
   if (missing) {
     throw new Error(
       `\`options.client\` is not a valid @sanity/client instance - no "${missing}" method found`
@@ -36,7 +36,7 @@ function validateOptions(opts) {
     throw new Error('Client is not instantiated with a `token`')
   }
 
-  booleanFlags.forEach(flag => {
+  booleanFlags.forEach((flag) => {
     if (typeof options[flag] !== 'boolean') {
       throw new Error(`Flag ${flag} must be a boolean (true/false)`)
     }
@@ -46,6 +46,10 @@ function validateOptions(opts) {
     throw new Error('outputPath must be specified (- for stdout)')
   }
 
+  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
+    throw new Error('`assetConcurrency` must be between 1 and 24')
+  }
+
   return options
 }