@sanity/export 5.0.0 → 6.0.0

package/src/{AssetHandler.js → AssetHandler.ts}

@@ -4,8 +4,9 @@ import {join as joinPath} from 'node:path'
 import {pipeline} from 'node:stream/promises'
 
 import PQueue from 'p-queue'
-import {rimraf} from 'rimraf'
 
+import {delay} from './util/delay.js'
+import {through, throughObj} from './util/streamHelpers.js'
 import {
   ASSET_DOWNLOAD_CONCURRENCY,
   ASSET_DOWNLOAD_MAX_RETRIES,
@@ -14,16 +15,70 @@ import {
 import {debug} from './debug.js'
 import {getUserAgent} from './getUserAgent.js'
 import {requestStream} from './requestStream.js'
-import {delay} from './util/delay.js'
-import {through, throughObj} from './util/streamHelpers.js'
+import type {
+  AssetDocument,
+  AssetMap,
+  AssetMetadata,
+  ResponseStream,
+  SanityClientLike,
+  SanityDocument,
+} from './types.js'
+import {rm} from 'node:fs/promises'
 
 const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url']
-const ACTION_REMOVE = 'remove'
-const ACTION_REWRITE = 'rewrite'
+const ACTION_REMOVE = 'remove' as const
+const ACTION_REWRITE = 'rewrite' as const
+
+type AssetAction = typeof ACTION_REMOVE | typeof ACTION_REWRITE
+
+interface AssetHandlerOptions {
+  client: SanityClientLike
+  tmpDir: string
+  prefix?: string
+  concurrency?: number
+  maxRetries?: number
+  retryDelayMs?: number
+  queue?: PQueue
+}
+
+interface AssetRequestOptions {
+  url: string
+  headers: Record<string, string>
+}
+
+interface AssetField {
+  asset: {
+    _ref: string
+  }
+  [key: string]: unknown
+}
+
+interface RewrittenAssetField {
+  _sanityAsset: string
+  [key: string]: unknown
+}
+
+interface DownloadError extends Error {
+  statusCode?: number
+}
 
 export class AssetHandler {
-  constructor(options) {
-    const concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY
+  client: SanityClientLike
+  tmpDir: string
+  assetDirsCreated: boolean
+  downloading: string[]
+  assetsSeen: Map<string, string>
+  assetMap: AssetMap
+  filesWritten: number
+  queueSize: number
+  maxRetries: number
+  retryDelayMs: number | undefined
+  queue: PQueue
+  rejectedError: Error | null
+  reject: (err: Error) => void
+
+  constructor(options: AssetHandlerOptions) {
+    const concurrency = options.concurrency ?? ASSET_DOWNLOAD_CONCURRENCY
     debug('Using asset download concurrency of %d', concurrency)
 
     this.client = options.client
@@ -35,23 +90,23 @@ export class AssetHandler {
     this.assetMap = {}
     this.filesWritten = 0
     this.queueSize = 0
-    this.maxRetries = options.maxRetries || ASSET_DOWNLOAD_MAX_RETRIES
+    this.maxRetries = options.maxRetries ?? ASSET_DOWNLOAD_MAX_RETRIES
     this.retryDelayMs = options.retryDelayMs
-    this.queue = options.queue || new PQueue({concurrency})
+    this.queue = options.queue ?? new PQueue({concurrency})
 
     this.rejectedError = null
-    this.reject = (err) => {
+    this.reject = (err: Error): void => {
       this.rejectedError = err
     }
   }
 
-  clear() {
+  clear(): void {
     this.assetsSeen.clear()
     this.queue.clear()
     this.queueSize = 0
   }
 
-  finish() {
+  finish(): Promise<AssetMap> {
     return new Promise((resolve, reject) => {
       if (this.rejectedError) {
         reject(this.rejectedError)
@@ -59,28 +114,31 @@ export class AssetHandler {
       }
 
       this.reject = reject
-      this.queue.onIdle().then(() => resolve(this.assetMap))
+      void this.queue.onIdle().then(() => resolve(this.assetMap))
     })
   }
 
   // Called when we want to download all assets to local filesystem and rewrite documents to hold
   // placeholder asset references (_sanityAsset: 'image@file:///local/path')
-  rewriteAssets = throughObj(async (doc, enc, callback) => {
-    if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-      const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file'
-      const filePath = `${type}s/${generateFilename(doc._id)}`
-      this.assetsSeen.set(doc._id, type)
-      this.queueAssetDownload(doc, filePath, type)
-      callback()
-      return
-    }
+  rewriteAssets = throughObj(
+    (doc: SanityDocument | AssetDocument, _enc: BufferEncoding, callback) => {
+      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
+        const assetDoc = doc as AssetDocument
+        const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file'
+        const filePath = `${type}s/${generateFilename(doc._id)}`
+        this.assetsSeen.set(doc._id, type)
+        this.queueAssetDownload(assetDoc, filePath)
+        callback()
+        return
+      }
 
-    callback(null, this.findAndModify(doc, ACTION_REWRITE))
-  })
+      callback(null, this.findAndModify(doc, ACTION_REWRITE))
+    },
+  )
 
   // Called in the case where we don't _want_ assets, so basically just remove all asset documents
   // as well as references to assets (*.asset._ref ^= (image|file)-)
-  stripAssets = throughObj(async (doc, enc, callback) => {
+  stripAssets = throughObj((doc: SanityDocument, _enc: BufferEncoding, callback) => {
     if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
       callback()
       return
@@ -91,7 +149,7 @@ export class AssetHandler {
 
   // Called when we are using raw export mode along with `assets: false`, where we simply
   // want to skip asset documents but retain asset references (useful for data mangling)
-  skipAssets = throughObj((doc, enc, callback) => {
+  skipAssets = throughObj((doc: SanityDocument, _enc: BufferEncoding, callback) => {
     const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)
     if (isAsset) {
       callback()
@@ -101,9 +159,9 @@ export class AssetHandler {
     callback(null, doc)
   })
 
-  noop = throughObj((doc, enc, callback) => callback(null, doc))
+  noop = throughObj((doc: SanityDocument, _enc: BufferEncoding, callback) => callback(null, doc))
 
-  queueAssetDownload(assetDoc, dstPath) {
+  queueAssetDownload(assetDoc: AssetDocument, dstPath: string): void {
     if (!assetDoc.url) {
       debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id)
       return
@@ -113,20 +171,21 @@ export class AssetHandler {
     this.queueSize++
     this.downloading.push(assetDoc.url)
 
-    const doDownload = async () => {
-      let dlError
+    const doDownload = async (): Promise<boolean> => {
+      let dlError: DownloadError | undefined
       for (let attempt = 0; attempt < this.maxRetries; attempt++) {
         try {
           return await this.downloadAsset(assetDoc, dstPath)
         } catch (err) {
+          const downloadError = err as DownloadError
           // Ignore inaccessible assets
-          switch (err.statusCode) {
+          switch (downloadError.statusCode) {
             case 401:
             case 403:
             case 404:
               console.warn(
                 `⚠ Asset failed with HTTP %d (ignoring): %s`,
-                err.statusCode,
+                downloadError.statusCode,
                 assetDoc._id,
               )
               return true
@@ -141,59 +200,62 @@ export class AssetHandler {
             err,
           )
 
-          dlError = err
+          dlError = downloadError
 
-          if ('statusCode' in err && err.statusCode >= 400 && err.statusCode < 500) {
+          if (
+            downloadError.statusCode &&
+            downloadError.statusCode >= 400 &&
+            downloadError.statusCode < 500
+          ) {
             // Don't retry on client errors
             break
           }
 
-          await delay(this.retryDelayMs || DEFAULT_RETRY_DELAY)
+          await delay(this.retryDelayMs ?? DEFAULT_RETRY_DELAY)
         }
       }
-      throw dlError
+      throw new Error(dlError?.message ?? 'Unknown error downloading asset')
     }
 
     this.queue
       .add(() =>
-        doDownload().catch((err) => {
+        doDownload().catch((err: unknown) => {
           debug('Failed to download the asset, aborting download', err)
           this.queue.clear()
-          this.reject(err)
+          this.reject(err instanceof Error ? err : new Error(String(err)))
         }),
       )
-      .catch((error) => {
+      .catch((error: unknown) => {
         debug('Queued task failed', error)
       })
   }
 
-  maybeCreateAssetDirs() {
+  maybeCreateAssetDirs(): void {
     if (this.assetDirsCreated) {
       return
     }
 
-    /* eslint-disable no-sync */
     mkdirSync(joinPath(this.tmpDir, 'files'), {recursive: true})
     mkdirSync(joinPath(this.tmpDir, 'images'), {recursive: true})
-    /* eslint-enable no-sync */
     this.assetDirsCreated = true
   }
 
-  getAssetRequestOptions(assetDoc) {
+  getAssetRequestOptions(assetDoc: AssetDocument): AssetRequestOptions {
     const token = this.client.config().token
-    const headers = {'User-Agent': getUserAgent()}
+    const headers: Record<string, string> = {'User-Agent': getUserAgent()}
     const isImage = assetDoc._type === 'sanity.imageAsset'
 
-    const url = URL.parse(assetDoc.url)
+    const url = URL.parse(assetDoc.url ?? '')
     // If we can't parse it, return as-is
     if (!url) {
-      return {url: assetDoc.url, headers}
+      return {url: assetDoc.url ?? '', headers}
     }
 
     if (
       isImage &&
       token &&
-      (['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) ||
+      (url.hostname === 'cdn.sanity.io' ||
+        url.hostname === 'cdn.sanity.work' ||
         // used in tests. use a very specific port to avoid conflicts
         url.host === 'localhost:43216')
     ) {
@@ -204,15 +266,14 @@ export class AssetHandler {
     return {url: url.toString(), headers}
   }
 
-  // eslint-disable-next-line max-statements
-  async downloadAsset(assetDoc, dstPath) {
+  async downloadAsset(assetDoc: AssetDocument, dstPath: string): Promise<boolean> {
     const {url} = assetDoc
 
     debug('Downloading asset %s', url)
 
     const options = this.getAssetRequestOptions(assetDoc)
 
-    let stream
+    let stream: ResponseStream
     try {
       stream = await requestStream({
         maxRetries: 0, // We handle retries ourselves in queueAssetDownload
@@ -220,8 +281,7 @@ export class AssetHandler {
       })
     } catch (err) {
       const message = 'Failed to create asset stream'
-      if (typeof err.message === 'string') {
-        // try to re-assign the error message so the stack trace is more visible
+      if (err instanceof Error) {
         err.message = `${message}: ${err.message}`
         throw err
       }
@@ -230,7 +290,7 @@ export class AssetHandler {
     }
 
     if (stream.statusCode !== 200) {
-      let errMsg
+      let errMsg: string
       try {
         const err = await tryGetErrorFromStream(stream)
         errMsg = `Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`
@@ -239,8 +299,7 @@ export class AssetHandler {
         }
       } catch (err) {
         const message = 'Failed to parse error response from asset stream'
-        if (typeof err.message === 'string') {
-          // try to re-assign the error message so the stack trace is more visible
+        if (err instanceof Error) {
           err.message = `${message}: ${err.message}`
           throw err
         }
@@ -248,8 +307,10 @@ export class AssetHandler {
         throw new Error(message, {cause: err})
       }
 
-      const streamError = new Error(errMsg)
-      streamError.statusCode = stream.statusCode
+      const streamError: DownloadError = new Error(errMsg)
+      if (stream.statusCode !== undefined) {
+        streamError.statusCode = stream.statusCode
+      }
       throw streamError
     }
 
@@ -268,7 +329,7 @@ export class AssetHandler {
     } catch (err) {
       const message = 'Failed to write asset stream to filesystem'
 
-      if (typeof err.message === 'string') {
+      if (err instanceof Error) {
         err.message = `${message}: ${err.message}`
         throw err
       }
@@ -277,9 +338,9 @@ export class AssetHandler {
     }
 
     // Verify it against our downloaded stream to make sure we have the same copy
-    const contentLength = stream.headers['content-length']
-    const remoteSha1 = stream.headers['x-sanity-sha1']
-    const remoteMd5 = stream.headers['x-sanity-md5']
+    const contentLength = stream.headers?.['content-length']
+    const remoteSha1 = stream.headers?.['x-sanity-sha1']
+    const remoteMd5 = stream.headers?.['x-sanity-md5']
     const hasHash = Boolean(remoteSha1 || remoteMd5)
     const method = sha1 ? 'sha1' : 'md5'
 
@@ -298,13 +359,13 @@ export class AssetHandler {
             : `sha1 should be ${remoteSha1}, got ${sha1}`),
 
         contentLength &&
-          parseInt(contentLength, 10) !== size &&
+          parseInt(String(contentLength), 10) !== size &&
           `Asset should be ${contentLength} bytes, got ${size}`,
       ]
 
       const detailsString = `Details:\n - ${details.filter(Boolean).join('\n - ')}`
 
-      await rimraf(tmpPath)
+      await rm(tmpPath, {recursive: true, force: true})
 
       throw new Error(`Failed to download asset at ${assetDoc.url}. ${detailsString}`)
     }
@@ -327,43 +388,44 @@ export class AssetHandler {
     return true
   }
 
-  findAndModify = (item, action) => {
+  findAndModify = (item: unknown, action: AssetAction): unknown => {
     if (Array.isArray(item)) {
-      const children = item.map((child) => this.findAndModify(child, action))
-      return children.filter(function (child) {
-        return child !== null && child !== undefined
-      })
+      const children = item.map((child: unknown) => this.findAndModify(child, action))
+      return children.filter((child): child is NonNullable<typeof child> => child != null)
     }
 
     if (!item || typeof item !== 'object') {
       return item
     }
 
-    const isAsset = isAssetField(item)
+    const record = item as Record<string, unknown>
+
+    const isAsset = isAssetField(record)
     if (isAsset && action === ACTION_REMOVE) {
       return undefined
     }
 
     if (isAsset && action === ACTION_REWRITE) {
-      const {asset, ...other} = item
+      const {asset, ...other} = record
       const assetId = asset._ref
-      const assetType = getAssetType(item)
+      const assetType = getAssetType(record)
       const filePath = `${assetType}s/${generateFilename(assetId)}`
+      const modified = this.findAndModify(other, action)
       return {
         _sanityAsset: `${assetType}@file://./${filePath}`,
-        ...this.findAndModify(other, action),
-      }
+        ...(typeof modified === 'object' && modified !== null ? modified : {}),
+      } as RewrittenAssetField
     }
 
-    const newItem = {}
-    const keys = Object.keys(item)
-    for (let i = 0; i < keys.length; i++) {
-      const key = keys[i]
-      const value = item[key]
+    const newItem: Record<string, unknown> = {}
+    const keys = Object.keys(record)
+    for (const key of keys) {
+      const value = record[key]
 
       newItem[key] = this.findAndModify(value, action)
 
       if (typeof newItem[key] === 'undefined') {
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
         delete newItem[key]
       }
     }
@@ -372,38 +434,51 @@ export class AssetHandler {
   }
 }
 
-function isAssetField(item) {
-  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref)
+function isAssetField(item: Record<string, unknown>): item is AssetField {
+  const asset = item.asset as {_ref?: unknown} | undefined
+  return Boolean(asset?._ref && typeof asset._ref === 'string' && isSanityAsset(asset._ref))
 }
 
-function getAssetType(item) {
-  if (!item.asset || typeof item.asset._ref !== 'string') {
+function getAssetType(item: Record<string, unknown>): string | null {
+  const asset = item.asset as {_ref?: unknown} | undefined
+  if (!asset || typeof asset._ref !== 'string') {
     return null
   }
 
-  const [, type] = item.asset._ref.match(/^(image|file)-/) || []
-  return type || null
+  const match = asset._ref.match(/^(image|file)-/)
+  return match?.[1] ?? null
 }
 
-function isSanityAsset(assetId) {
+function isSanityAsset(assetId: string): boolean {
   return (
     /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) ||
     /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId)
   )
 }
 
-function generateFilename(assetId) {
-  const [, , asset, ext] = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || []
-  const extension = (ext || 'bin').replace(/^-/, '')
+function generateFilename(assetId: string): string {
+  const match = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/)
+  const asset = match?.[2]
+  const ext = match?.[3]
+  const extension = (ext ?? 'bin').replace(/^-/, '')
   return asset ? `${asset}.${extension}` : `${assetId}.bin`
 }
 
-async function writeHashedStream(filePath, stream) {
+interface HashResult {
+  size: number
+  sha1: string
+  md5: string
+}
+
+async function writeHashedStream(
+  filePath: string,
+  stream: NodeJS.ReadableStream,
+): Promise<HashResult> {
   let size = 0
   const md5 = createHash('md5')
   const sha1 = createHash('sha1')
 
-  const hasher = through((chunk, enc, cb) => {
+  const hasher = through((chunk, _enc, cb) => {
     size += chunk.length
     md5.update(chunk)
     sha1.update(chunk)
@@ -418,12 +493,12 @@ async function writeHashedStream(filePath, stream) {
   }
 }
 
-function tryGetErrorFromStream(stream) {
+function tryGetErrorFromStream(stream: NodeJS.ReadableStream): Promise<string | null> {
   return new Promise((resolve, reject) => {
-    const chunks = []
+    const chunks: Buffer[] = []
     let receivedData = false
 
-    stream.on('data', (chunk) => {
+    stream.on('data', (chunk: Buffer) => {
       receivedData = true
       chunks.push(chunk)
     })
@@ -436,8 +511,8 @@ function tryGetErrorFromStream(stream) {
 
       const body = Buffer.concat(chunks)
       try {
-        const parsed = JSON.parse(body.toString('utf8'))
-        resolve(parsed.message || parsed.error || null)
+        const parsed = JSON.parse(body.toString('utf8')) as {message?: string; error?: string}
+        resolve(parsed.message ?? parsed.error ?? null)
       } catch {
         resolve(body.toString('utf8').slice(0, 16000))
       }
@@ -447,12 +522,12 @@ function tryGetErrorFromStream(stream) {
   })
 }
 
-function omit(obj, keys) {
-  const copy = {}
-  Object.entries(obj).forEach(([key, value]) => {
+function omit(obj: Record<string, unknown>, keys: string[]): AssetMetadata {
+  const copy: AssetMetadata = {}
+  for (const [key, value] of Object.entries(obj)) {
     if (!keys.includes(key)) {
       copy[key] = value
     }
-  })
+  }
   return copy
 }

package/src/constants.ts

@@ -0,0 +1,50 @@
+/**
+ * How many retries to attempt when retrieving the document stream.
+ * User overridable as `options.maxRetries`.
+ *
+ * Note: Only for initial connection - if download fails while streaming, we cannot easily resume.
+ * @internal
+ */
+export const DOCUMENT_STREAM_MAX_RETRIES: number = 5
+
+/**
+ * How many retries to attempt when downloading an asset.
+ * User overridable as `options.maxAssetRetries`.
+ * @internal
+ */
+export const ASSET_DOWNLOAD_MAX_RETRIES: number = 10
+
+/**
+ * Default delay between retries when retrieving assets or document stream.
+ * User overridable as `options.retryDelayMs`.
+ * @internal
+ */
+export const DEFAULT_RETRY_DELAY: number = 1500
+
+/**
+ * How many concurrent asset downloads to allow.
+ * User overridable as `options.assetConcurrency`.
+ * @internal
+ */
+export const ASSET_DOWNLOAD_CONCURRENCY: number = 8
+
+/**
+ * How frequently we will `debug` log while streaming the documents.
+ * @internal
+ */
+export const DOCUMENT_STREAM_DEBUG_INTERVAL: number = 10000
+
+/**
+ * How long to wait before timing out the read of a request due to inactivity.
+ * User overridable as `options.readTimeout`.
+ * @internal
+ */
+export const REQUEST_READ_TIMEOUT: number = 3 * 60 * 1000 // 3 minutes
+
+/**
+ * What mode to use when exporting documents.
+ * stream: Export all documents in the dataset in one request, this will be consistent but might be slow on large datasets.
+ * cursor: Export documents using a cursor, this might lead to inconsistent results if a mutation is performed while exporting.
+ */
+export const MODE_STREAM = 'stream' as const
+export const MODE_CURSOR = 'cursor' as const

package/src/debug.ts

@@ -0,0 +1,3 @@
+import createDebug, {type Debugger} from 'debug'
+
+export const debug: Debugger = createDebug('sanity:export')