@sanity/export 5.0.0 → 6.0.0

package/src/options.ts

@@ -0,0 +1,138 @@
+import {
+  ASSET_DOWNLOAD_MAX_RETRIES,
+  DOCUMENT_STREAM_MAX_RETRIES,
+  MODE_CURSOR,
+  MODE_STREAM,
+  REQUEST_READ_TIMEOUT,
+} from './constants.js'
+import type {ExportOptions, ExportSource, NormalizedExportOptions, SanityDocument} from './types.js'
+
+const booleanFlags = ['assets', 'raw', 'compress', 'drafts'] as const
+const numberFlags = ['maxAssetRetries', 'maxRetries', 'assetConcurrency', 'readTimeout'] as const
+
+const exportDefaults = {
+  compress: true,
+  drafts: true,
+  assets: true,
+  assetsMap: true,
+  raw: false,
+  mode: MODE_STREAM,
+  maxRetries: DOCUMENT_STREAM_MAX_RETRIES,
+  maxAssetRetries: ASSET_DOWNLOAD_MAX_RETRIES,
+  readTimeout: REQUEST_READ_TIMEOUT,
+  filterDocument: (): boolean => true,
+  transformDocument: (doc: SanityDocument): SanityDocument => doc,
+} as const
+
+export function validateOptions(opts: ExportOptions): NormalizedExportOptions {
+  const options = {...exportDefaults, ...opts}
+
+  const dataset =
+    'dataset' in options && typeof options.dataset === 'string' ? options.dataset : undefined
+  const mediaLibraryId =
+    'mediaLibraryId' in options && typeof options.mediaLibraryId === 'string'
+      ? options.mediaLibraryId
+      : undefined
+
+  if (dataset && mediaLibraryId) {
+    throw new Error(
+      'either `options.dataset` or `options.mediaLibraryId` must be specified, got both',
+    )
+  }
+
+  if (!dataset && !mediaLibraryId) {
+    throw new Error(
+      'either `options.dataset` or `options.mediaLibraryId` must be specified, got neither',
+    )
+  }
+
+  const source = getSource(options)
+  if (!source.id.trim()) {
+    throw new Error(`Source (${source.type}) specified but was empty`)
+  }
+
+  // Type narrowing for mode validation
+  const mode = options.mode as string
+  if (typeof mode !== 'string' || (mode !== MODE_STREAM && mode !== MODE_CURSOR)) {
+    throw new Error(
+      `options.mode must be either "${MODE_STREAM}" or "${MODE_CURSOR}", got "${mode}"`,
+    )
+  }
+
+  if (typeof options.onProgress !== 'undefined' && typeof options.onProgress !== 'function') {
+    throw new Error(`options.onProgress must be a function`)
+  }
+
+  if (
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    !options.client ||
+    !('config' in options.client) ||
+    typeof options.client.getUrl !== 'function'
+  ) {
+    throw new Error('`options.client` must be set to an instance of @sanity/client')
+  }
+
+  const clientConfig = options.client.config()
+  if (!clientConfig.token) {
+    throw new Error('Client is not instantiated with a `token`')
+  }
+
+  for (const flag of booleanFlags) {
+    if (typeof options[flag] !== 'boolean') {
+      throw new Error(`Flag ${flag} must be a boolean (true/false)`)
+    }
+  }
+
+  for (const flag of numberFlags) {
+    if (typeof options[flag] !== 'undefined' && typeof options[flag] !== 'number') {
+      throw new Error(`Flag ${flag} must be a number if specified`)
+    }
+  }
+
+  if (!options.outputPath) {
+    throw new Error('outputPath must be specified (- for stdout)')
+  }
+
+  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
+    throw new Error('`assetConcurrency` must be between 1 and 24')
+  }
+
+  if (
+    typeof options.filterDocument !== 'undefined' &&
+    typeof options.filterDocument !== 'function'
+  ) {
+    throw new Error('`filterDocument` must be a function')
+  }
+
+  if (
+    typeof options.transformDocument !== 'undefined' &&
+    typeof options.transformDocument !== 'function'
+  ) {
+    throw new Error('`transformDocument` must be a function')
+  }
+
+  if (typeof options.assetsMap !== 'undefined' && typeof options.assetsMap !== 'boolean') {
+    throw new Error('`assetsMap` must be a boolean')
+  }
+
+  return options
+}
+
+/**
+ * Determines the source type and ID from the provided options.
+ *
+ * @param options - The export options containing either dataset or mediaLibraryId.
+ * @returns An object with the source type and its corresponding ID.
+ * @internal
+ */
+export function getSource(options: ExportSource): {
+  type: 'dataset' | 'media-library'
+  id: string
+} {
+  if ('dataset' in options && typeof options.dataset === 'string') {
+    return {type: 'dataset', id: options.dataset}
+  } else if ('mediaLibraryId' in options && typeof options.mediaLibraryId === 'string') {
+    return {type: 'media-library', id: options.mediaLibraryId}
+  }
+  throw new Error('Either dataset or mediaLibraryId must be specified')
+}

package/src/rejectOnApiError.ts

@@ -0,0 +1,62 @@
+import type {Transform} from 'node:stream'
+
+import {throughObj} from './util/streamHelpers.js'
+import type {SanityDocument} from './types.js'
+
+interface ErrorDocument {
+  _id?: undefined
+  error:
+    | string
+    | {
+        description?: string
+        message?: string
+      }
+  message?: string
+  statusCode?: number
+}
+
+type DocumentOrError = SanityDocument | ErrorDocument
+
+function isErrorDocument(doc: DocumentOrError): doc is ErrorDocument {
+  return !('_id' in doc && doc._id) && 'error' in doc
+}
+
+export function rejectOnApiError(): Transform {
+  return throughObj((doc: DocumentOrError, _enc, callback) => {
+    // check if the document passed contains a document attribute first, and return early.
+    if ('_id' in doc && doc._id) {
+      callback(null, doc)
+      return
+    }
+
+    if (isErrorDocument(doc)) {
+      // if we got a statusCode we can decorate the error with it
+      if (doc.statusCode) {
+        const err = doc.error
+        const errorMessage =
+          typeof err === 'string'
+            ? err
+            : typeof err === 'object'
+              ? (err.description ?? err.message)
+              : undefined
+        callback(
+          new Error(
+            ['Export', `HTTP ${doc.statusCode}`, errorMessage, doc.message]
+              .filter((part): part is string => typeof part === 'string')
+              .join(': '),
+          ),
+        )
+        return
+      }
+
+      // no statusCode, just serialize and return the error
+      const error = doc.error
+      const errorMessage =
+        typeof error === 'object' ? (error.description ?? error.message) : undefined
+      callback(new Error(errorMessage ?? JSON.stringify(doc)))
+      return
+    }
+
+    callback(null, doc)
+  })
+}

package/src/requestStream.ts

@@ -0,0 +1,81 @@
+import {getIt} from 'get-it'
+import {keepAlive, promise} from 'get-it/middleware'
+
+import {delay} from './util/delay.js'
+import {extractFirstError} from './util/extractFirstError.js'
+import {tryThrowFriendlyError} from './util/friendlyError.js'
+import {
+  DEFAULT_RETRY_DELAY,
+  DOCUMENT_STREAM_MAX_RETRIES,
+  REQUEST_READ_TIMEOUT,
+} from './constants.js'
+import {debug} from './debug.js'
+import type {RequestStreamOptions, ResponseStream} from './types.js'
+
+interface GetItResponse extends ResponseStream {
+  statusCode: number
+  headers: Record<string, string | string[] | undefined>
+}
+
+interface ErrorWithResponse extends Error {
+  response?: {
+    statusCode?: number
+  }
+}
+
+const request = getIt([keepAlive(), promise({onlyBody: true})]) as unknown as (
+  options: Record<string, unknown>,
+) => Promise<GetItResponse>
+
+const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds
+
+export async function requestStream(options: RequestStreamOptions): Promise<ResponseStream> {
+  const maxRetries =
+    options.maxRetries !== undefined ? options.maxRetries : DOCUMENT_STREAM_MAX_RETRIES
+
+  const readTimeout = options.readTimeout !== undefined ? options.readTimeout : REQUEST_READ_TIMEOUT
+
+  const retryDelayMs =
+    options.retryDelayMs !== undefined ? options.retryDelayMs : DEFAULT_RETRY_DELAY
+
+  let error: ErrorWithResponse | undefined
+
+  let i = 0
+  do {
+    i++
+
+    try {
+      return await request({
+        ...options,
+        stream: true,
+        maxRedirects: 0,
+        timeout: {connect: CONNECTION_TIMEOUT, socket: readTimeout},
+      })
+    } catch (err) {
+      const firstError = extractFirstError(err) as ErrorWithResponse | null
+      error = firstError !== null ? firstError : (err as ErrorWithResponse)
+
+      if (maxRetries === 0) {
+        throw error
+      }
+
+      if (error.response?.statusCode && error.response.statusCode < 500) {
+        break
+      }
+
+      if (i < maxRetries) {
+        debug('Error, retrying after %d ms: %s', retryDelayMs, error.message)
+        await delay(retryDelayMs)
+      }
+    }
+  } while (i < maxRetries)
+
+  await tryThrowFriendlyError(error)
+
+  // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+  if (!error) {
+    throw new Error(`Export: Failed to fetch ${options.url}: Unknown error`)
+  }
+
+  throw new Error(`Export: Failed to fetch ${options.url}: ${error.message}`)
+}

package/src/stringifyStream.ts

@@ -0,0 +1,7 @@
+import type {Transform} from 'node:stream'
+
+import {throughObj} from './util/streamHelpers.js'
+
+export function stringifyStream(): Transform {
+  return throughObj((doc: unknown, _enc, callback) => callback(null, `${JSON.stringify(doc)}\n`))
+}

package/src/{tryParseJson.js → tryParseJson.ts}

@@ -1,30 +1,30 @@
-/**
- * Safe JSON parser that is able to handle lines interrupted by an error object.
- *
- * This may occur when streaming NDJSON from the Export HTTP API.
- *
- * @internal
- * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request}
- */
-export const tryParseJson = createSafeJsonParser({
-  errorLabel: 'Error streaming dataset',
-})
+interface SafeJsonParserOptions {
+  errorLabel: string
+}
 
-function createSafeJsonParser({errorLabel}) {
-  return function safeJsonParser(line) {
+interface ParsedErrorLine {
+  error?: {
+    description?: string
+  }
+}
+
+function createSafeJsonParser({errorLabel}: SafeJsonParserOptions) {
+  return function safeJsonParser(line: string): unknown {
     try {
-      return JSON.parse(line)
+      return JSON.parse(line) as unknown
     } catch (err) {
       // Catch half-done lines with an error at the end
       const errorPosition = line.lastIndexOf('{"error":')
       if (errorPosition === -1) {
-        err.message = `${err.message} (${line})`
+        if (err instanceof Error) {
+          err.message = `${err.message} (${line})`
+        }
         throw err
       }
 
       const errorJson = line.slice(errorPosition)
-      const errorLine = JSON.parse(errorJson)
-      const error = errorLine && errorLine.error
+      const errorLine = JSON.parse(errorJson) as ParsedErrorLine
+      const error = errorLine.error
       if (error && error.description) {
         throw new Error(`${errorLabel}: ${error.description}\n\n${errorJson}\n`, {cause: err})
       }
@@ -33,3 +33,15 @@ function createSafeJsonParser({errorLabel}) {
     }
   }
 }
+
+/**
+ * Safe JSON parser that is able to handle lines interrupted by an error object.
+ *
+ * This may occur when streaming NDJSON from the Export HTTP API.
+ *
+ * @internal
+ * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request}
+ */
+export const tryParseJson = createSafeJsonParser({
+  errorLabel: 'Error streaming dataset',
+})

package/src/types.ts

@@ -0,0 +1,274 @@
+import type {Writable} from 'node:stream'
+
+/**
+ * The mode to use when exporting documents.
+ *
+ * - `"stream"` mode uses a continuous stream of documents from the source and will
+ *   give a consistent snapshot of the dataset at the time the export started, but
+ *   will struggle with very large datasets.
+ * - `"cursor"` mode uses paginated requests to fetch documents using a cursor, and
+ *   will handle very large datasets better, but may give an inconsistent snapshot if
+ *   the dataset is being modified during the export.
+ *
+ * @public
+ */
+export type ExportMode = 'stream' | 'cursor'
+
+/**
+ * Minimal representation of a Sanity document.
+ *
+ * @public
+ */
+export interface SanityDocument {
+  _id: string
+  _type: string
+  _rev?: string
+  _createdAt?: string
+  _updatedAt?: string
+  [key: string]: unknown
+}
+
+/**
+ * @public
+ */
+export interface ExportProgress {
+  /** Description of the current export step */
+  step: string
+
+  /** Number of documents processed so far */
+  current?: number
+
+  /** Total number of documents, if known - otherwise `?` */
+  total?: number | '?'
+
+  /** Set to `true` if the progress update is a repeat of a previous one but with new values */
+  update?: boolean
+}
+
+/**
+ * @public
+ */
+export interface SanityClientLike {
+  getUrl: (path: string) => string
+  config: () => {token?: string}
+}
+
+/**
+ * The options used to configure an export operation.
+ *
+ * @public
+ */
+export type ExportOptions = {
+  /**
+   * An instance of `@sanity/client`, configured with the project ID and authentication
+   * token to be used for the export operation.
+   */
+  client: SanityClientLike
+
+  /**
+   * Either a filesystem path to write the output `tar.gz` file to, or a writable stream
+   */
+  outputPath: string | Writable
+
+  /**
+   * Whether or not to include asset files in the export
+   */
+  assets?: boolean
+
+  /**
+   * Whether or not to export the documents "raw", meaning asset documents will be
+   * included as-is, referencing asset file URLs in the source dataset and project ID or
+   * media library ID. Also skips downloading assets. Generally this is only useful if
+   * importing to the same source and you do not want to download the assets and
+   * re-upload them to the same source.
+   *
+   * @note This will usually cause undesirable results if imported into another project
+   * or media library!
+   */
+  raw?: boolean
+
+  /**
+   * Whether or not to include draft documents in the export.
+   */
+  drafts?: boolean
+
+  /**
+   * An array of document type names to include in the export. If not specified, all types
+   * will be included.
+   */
+  types?: string[] | undefined
+
+  /**
+   * How many asset downloads to perform concurrently. Must be between 1 and 24 if specified.
+   */
+  assetConcurrency?: number
+
+  /**
+   * Maximum number of times to retry downloading a failed asset.
+   *
+   * @note Only certain errors are retried (like network errors and 5xx responses).
+   */
+  maxAssetRetries?: number
+
+  /**
+   * Maximum number of times to retry fetching documents from the source.
+   *
+   * @note Only certain errors are retried (like network errors and 5xx responses).
+   */
+  maxRetries?: number
+
+  /**
+   * Delay between retry attempts in milliseconds.
+   */
+  retryDelayMs?: number
+
+  /**
+   * Timeout for read operations in milliseconds.
+   */
+  readTimeout?: number
+
+  /**
+   * The mode to use when exporting documents, either `"stream"` or `"cursor"`.
+   */
+  mode?: ExportMode
+
+  /**
+   * Whether or not to compress the output tarball (gzip). Note that the output will
+   * still be a gzipped tarball even if this is set to `false`, setting it to `false`
+   * only sets the gzip compression level to 0 (no compression).
+   */
+  compress?: boolean
+
+  /**
+   * Whether or not to include an `assets.json` file in the export, mapping asset IDs to
+   * their custom metadata (like original filename, etc).
+   */
+  assetsMap?: boolean
+
+  /**
+   * Optional filter function to determine whether or not a document should be included
+   * in the export. Note that this is run after any built-in document filtering such as
+   * draft exclusion, document type filtering, etc.
+   *
+   * @param doc - The document to evaluate
+   * @returns Whether or not to include the document in the export
+   */
+  filterDocument?: (doc: SanityDocument) => boolean
+
+  /**
+   * Optional transform function to modify a document before it is included in the export.
+   * Note that this is run post-filtering, and post asset document processing.
+   *
+   * @param doc - The document to transform
+   * @returns The transformed document
+   */
+  transformDocument?: (doc: SanityDocument) => SanityDocument
+
+  /**
+   * Optional progress callback that will be called periodically during the export.
+   *
+   * @param progress - The current export progress
+   */
+  onProgress?: (progress: ExportProgress) => void
+} & ExportSource
+
+/**
+ * The source of data to export, either a dataset name or a media library ID.
+ *
+ * @public
+ */
+export type ExportSource =
+  | {
+      /**
+       * The name of the dataset to export from.
+       */
+      dataset: string
+    }
+  | {
+      /**
+       * The ID of the media library to export from.
+       */
+      mediaLibraryId: string
+    }
+
+/**
+ * @public
+ */
+export interface ExportResult<T = string | Writable> {
+  /**
+   * The filesystem path or writable stream that was passed as `options.outputPath`.
+   */
+  outputPath: T
+
+  /**
+   * The number of documents exported.
+   */
+  documentCount: number
+
+  /**
+   * The number of assets exported.
+   */
+  assetCount: number
+}
+
+/**
+ * @internal
+ */
+export type NormalizedExportOptions = ExportOptions & {
+  assets: boolean
+  raw: boolean
+  drafts: boolean
+  maxAssetRetries: number
+  maxRetries: number
+  readTimeout: number
+  mode: ExportMode
+  compress: boolean
+  assetsMap: boolean
+  filterDocument: (doc: SanityDocument) => boolean
+  transformDocument: (doc: SanityDocument) => SanityDocument
+}
+
+/**
+ * @internal
+ */
+export interface AssetMetadata {
+  [key: string]: unknown
+}
+
+/**
+ * @internal
+ */
+export interface AssetMap {
+  [assetId: string]: AssetMetadata
+}
+
+/**
+ * @internal
+ */
+export interface RequestStreamOptions {
+  url: string
+  headers?: Record<string, string>
+  maxRetries?: number
+  retryDelayMs?: number
+  readTimeout?: number
+}
+
+/**
+ * @internal
+ */
+export interface ResponseStream extends NodeJS.ReadableStream {
+  statusCode?: number
+  headers?: Record<string, string | string[] | undefined>
+}
+
+/**
+ * @internal
+ */
+export interface AssetDocument extends SanityDocument {
+  _type: 'sanity.imageAsset' | 'sanity.fileAsset'
+  url?: string
+  path?: string
+  assetId?: string
+  extension?: string
+  mimeType?: string
+}

package/src/util/{delay.js → delay.ts}

@@ -1,3 +1,3 @@
-export function delay(ms) {
+export function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }

package/src/util/extractFirstError.ts

@@ -0,0 +1,31 @@
+interface AggregateErrorLike {
+  name: string
+  errors: Array<{message: string}>
+}
+
+function isAggregateError(err: unknown): err is AggregateErrorLike {
+  if (typeof err !== 'object' || err === null) {
+    return false
+  }
+
+  if (err instanceof AggregateError) {
+    return true
+  }
+
+  const record = err as Record<string, unknown>
+  return (
+    record.name === 'AggregateError' &&
+    Array.isArray(record.errors) &&
+    record.errors.length > 0 &&
+    typeof record.errors[0] === 'object' &&
+    record.errors[0] !== null &&
+    'message' in record.errors[0]
+  )
+}
+
+export function extractFirstError(err: unknown): unknown {
+  if (isAggregateError(err)) {
+    return err.errors[0]
+  }
+  return err
+}