@sanity/export 5.0.0 → 6.0.0

package/src/{export.js → export.ts}

@@ -1,17 +1,18 @@
 import {createWriteStream} from 'node:fs'
-import {mkdir} from 'node:fs/promises'
+import {mkdir, rm} from 'node:fs/promises'
 import {tmpdir} from 'node:os'
 import {join as joinPath} from 'node:path'
-import {PassThrough} from 'node:stream'
+import {PassThrough, type Writable} from 'node:stream'
 import {finished, pipeline} from 'node:stream/promises'
+import {deprecate} from 'node:util'
 import {constants as zlib} from 'node:zlib'
 
 import archiver from 'archiver'
 import {JsonStreamStringify} from 'json-stream-stringify'
-import {rimraf} from 'rimraf'
 
+import {isWritableStream, split, throughObj} from './util/streamHelpers.js'
 import {AssetHandler} from './AssetHandler.js'
-import {DOCUMENT_STREAM_DEBUG_INTERVAL, MODE_CURSOR, MODE_STREAM} from './constants.js'
+import {DOCUMENT_STREAM_DEBUG_INTERVAL, MODE_STREAM} from './constants.js'
 import {debug} from './debug.js'
 import {filterDocuments} from './filterDocuments.js'
 import {filterDocumentTypes} from './filterDocumentTypes.js'
@@ -21,14 +22,34 @@ import {logFirstChunk} from './logFirstChunk.js'
 import {rejectOnApiError} from './rejectOnApiError.js'
 import {stringifyStream} from './stringifyStream.js'
 import {tryParseJson} from './tryParseJson.js'
-import {isWritableStream, split, throughObj} from './util/streamHelpers.js'
-import {validateOptions} from './validateOptions.js'
-
-const noop = () => null
-
-export async function exportDataset(opts) {
+import type {
+  ExportOptions,
+  NormalizedExportOptions,
+  ExportResult,
+  ResponseStream,
+  SanityDocument,
+} from './types.js'
+import {getSource, validateOptions} from './options.js'
+
+const noop = (): null => null
+
+/**
+ * Export the dataset with the given options.
+ *
+ * @param opts - Export options
+ * @returns The export result
+ * @public
+ */
+export async function exportDataset(
+  opts: ExportOptions & {outputPath: Writable},
+): Promise<ExportResult<Writable>>
+export async function exportDataset(
+  opts: ExportOptions & {outputPath: string},
+): Promise<ExportResult<string>>
+export async function exportDataset(opts: ExportOptions): Promise<ExportResult>
+export async function exportDataset(opts: ExportOptions): Promise<ExportResult> {
   const options = validateOptions(opts)
-  const onProgress = options.onProgress || noop
+  const onProgress = options.onProgress ?? noop
   const archive = archiver('tar', {
     gzip: true,
     gzipOptions: {
@@ -47,66 +68,70 @@ export async function exportDataset(opts) {
     .replace(/[^a-z0-9]/gi, '-')
     .toLowerCase()
 
-  const prefix = `${opts.dataset ?? opts.mediaLibraryId}-export-${slugDate}`
+  const source = getSource(opts)
+  const prefix = `${source.id}-export-${slugDate}`
   const tmpDir = joinPath(tmpdir(), prefix)
   await mkdir(tmpDir, {recursive: true})
   const dataPath = joinPath(tmpDir, 'data.ndjson')
   const assetsPath = joinPath(tmpDir, 'assets.json')
 
   const cleanup = () =>
-    rimraf(tmpDir).catch((err) => {
-      debug(`Error while cleaning up temporary files: ${err.message}`)
+    rm(tmpDir, {recursive: true, force: true}).catch((err: unknown) => {
+      debug(`Error while cleaning up temporary files: ${err instanceof Error ? err.message : err}`)
+      return false
     })
 
   const assetHandler = new AssetHandler({
     client: options.client,
     tmpDir,
     prefix,
-    concurrency: options.assetConcurrency,
+    ...(options.assetConcurrency !== undefined && {concurrency: options.assetConcurrency}),
+    ...(options.retryDelayMs !== undefined && {retryDelayMs: options.retryDelayMs}),
     maxRetries: options.maxAssetRetries,
-    retryDelayMs: options.retryDelayMs,
   })
 
   debug('Downloading assets (temporarily) to %s', tmpDir)
-  debug('Downloading to %s', options.outputPath === '-' ? 'stdout' : options.outputPath)
-
-  let outputStream
-  if (isWritableStream(options.outputPath)) {
-    outputStream = options.outputPath
-  } else {
-    outputStream =
-      options.outputPath === '-' ? process.stdout : createWriteStream(options.outputPath)
-  }
+  debug('Downloading to %s', isWritableStream(options.outputPath) ? 'stream' : options.outputPath)
+
+  const outputStream: Writable = isWritableStream(options.outputPath)
+    ? options.outputPath
+    : createWriteStream(options.outputPath)
 
   let assetStreamHandler = assetHandler.noop
   if (!options.raw) {
     assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets
   }
 
-  let resolve
-  let reject
-  const result = new Promise((res, rej) => {
+  let resolve: (value: ExportResult) => void
+  let reject: (reason: Error) => void
+  const result = new Promise<ExportResult>((res, rej) => {
     resolve = res
     reject = rej
   })
 
   finished(archive)
-    .then(async () => {
+    .then(() => {
       debug('Archive finished')
     })
-    .catch(async (archiveErr) => {
-      debug('Archiving errored: %s', archiveErr.stack)
-      await cleanup()
-      reject(archiveErr)
+    .catch(async (archiveErr: unknown) => {
+      const err = archiveErr instanceof Error ? archiveErr : new Error(`${archiveErr}`)
+      debug('Archiving errored: %s', err.stack)
+      // Try cleanup, but let original error be the main rejection reason, not the cleanup
+      await cleanup().catch(noop)
+      reject(err)
     })
 
   debug('Getting dataset export stream, mode: "%s"', options.mode)
   onProgress({step: 'Exporting documents...'})
 
   let documentCount = 0
-  let lastDocumentID = null
+  let lastDocumentID: string | null = null
   let lastReported = Date.now()
-  const reportDocumentCount = (doc, enc, cb) => {
+  const reportDocumentCount = (
+    doc: SanityDocument,
+    _enc: BufferEncoding,
+    cb: (err: Error | null, doc: SanityDocument) => void,
+  ): void => {
     ++documentCount
 
     const now = Date.now()
@@ -129,15 +154,15 @@ export async function exportDataset(opts) {
   }
 
   const inputStream = await getDocumentInputStream(options)
-  if (inputStream.statusCode) {
+  if ('statusCode' in inputStream) {
     debug('Got HTTP %d', inputStream.statusCode)
   }
-  if (inputStream.headers) {
+  if ('headers' in inputStream) {
     debug('Response headers: %o', inputStream.headers)
   }
 
-  let debugTimer = null
-  function scheduleDebugTimer() {
+  let debugTimer: ReturnType<typeof setTimeout> | null = null
+  function scheduleDebugTimer(): void {
     debugTimer = setTimeout(() => {
       debug('Still streaming documents', {
         documentCount,
@@ -151,28 +176,24 @@ export async function exportDataset(opts) {
 
   scheduleDebugTimer()
 
-  const filterTransform = throughObj((doc, _enc, callback) => {
-    if (!options.filterDocument) {
-      return callback(null, doc)
-    }
-
+  const filterTransform = throughObj((doc: SanityDocument, _enc: BufferEncoding, callback) => {
     try {
       const include = options.filterDocument(doc)
-      return include ? callback(null, doc) : callback()
+      if (include) {
+        callback(null, doc)
+      } else {
+        callback()
+      }
     } catch (err) {
-      return callback(err)
+      callback(err instanceof Error ? err : new Error(`${err}`))
     }
   })
 
-  const transformTransform = throughObj((doc, _enc, callback) => {
-    if (!options.transformDocument) {
-      return callback(null, doc)
-    }
-
+  const transformTransform = throughObj((doc: SanityDocument, _enc: BufferEncoding, callback) => {
     try {
-      return callback(null, options.transformDocument(doc))
+      callback(null, options.transformDocument(doc))
     } catch (err) {
-      return callback(err)
+      callback(err instanceof Error ? err : new Error(`${err}`))
     }
   })
 
@@ -182,7 +203,7 @@ export async function exportDataset(opts) {
   const jsonStream = new PassThrough()
   finished(jsonStream)
     .then(() => debug('JSON stream finished'))
-    .catch((err) => reject(err))
+    .catch((err: unknown) => reject(err instanceof Error ? err : new Error(`${err}`)))
 
   pipeline(
     inputStream,
@@ -197,10 +218,10 @@ export async function exportDataset(opts) {
     reportTransform,
     stringifyStream(),
     jsonStream,
-  ).catch((err) => {
+  ).catch((err: unknown) => {
     if (debugTimer !== null) clearTimeout(debugTimer)
     debug(`Export stream error @ ${lastDocumentID}/${documentCount}: `, err)
-    reject(err)
+    reject(err instanceof Error ? err : new Error(`${err}`))
   })
 
   pipeline(jsonStream, createWriteStream(dataPath))
@@ -262,30 +283,31 @@ export async function exportDataset(opts) {
         clearInterval(progressInterval)
       } catch (assetErr) {
         clearInterval(progressInterval)
-        await cleanup()
-        reject(assetErr)
+        await cleanup().catch(noop) // Try to clean up, but ignore errors here
+        reject(assetErr instanceof Error ? assetErr : new Error(`${assetErr}`))
         return
       }
 
       // Add all downloaded assets to archive
-      archive.directory(joinPath(tmpDir, 'files'), `${prefix}/files`, {store: true})
-      archive.directory(joinPath(tmpDir, 'images'), `${prefix}/images`, {store: true})
+      archive.directory(joinPath(tmpDir, 'files'), `${prefix}/files`)
+      archive.directory(joinPath(tmpDir, 'images'), `${prefix}/images`)
 
       debug('Finalizing archive, flushing streams')
       onProgress({step: 'Adding assets to archive...'})
       await archive.finalize()
     })
-    .catch(async (err) => {
+    .catch(async (err: unknown) => {
       if (debugTimer !== null) clearTimeout(debugTimer)
       debug(`Export stream error @ ${lastDocumentID}/${documentCount}: `, err)
-      reject(err)
+      await cleanup().catch(noop)
+      reject(err instanceof Error ? err : new Error(`${err}`))
     })
 
   pipeline(archive, outputStream)
     .then(() => onComplete())
     .catch(onComplete)
 
-  async function onComplete(err) {
+  async function onComplete(err?: Error): Promise<void> {
     onProgress({step: 'Clearing temporary files...'})
     await cleanup()
 
@@ -307,13 +329,47 @@ export async function exportDataset(opts) {
   return result
 }
 
-function getDocumentInputStream(options) {
-  if (options.mode === MODE_STREAM) {
-    return getDocumentsStream(options)
-  }
-  if (options.mode === MODE_CURSOR) {
-    return getDocumentCursorStream(options)
-  }
+function getDocumentInputStream(options: NormalizedExportOptions): Promise<ResponseStream> {
+  return options.mode === MODE_STREAM
+    ? getDocumentsStream(options)
+    : getDocumentCursorStream(options)
+}
 
-  throw new Error(`Invalid mode: ${options.mode}`)
+type MediaLibraryExportOptions = Omit<ExportOptions, 'dataset' | 'mediaLibraryId'> & {
+  mediaLibraryId: string
 }
+
+/**
+ * Export the media library with the given `mediaLibraryId`.
+ *
+ * @param options - Export options
+ * @returns The export result
+ * @public
+ */
+export async function exportMediaLibrary(
+  options: MediaLibraryExportOptions & {outputPath: Writable},
+): Promise<ExportResult<Writable>>
+export async function exportMediaLibrary(
+  options: MediaLibraryExportOptions & {outputPath: string},
+): Promise<ExportResult<string>>
+export async function exportMediaLibrary(options: MediaLibraryExportOptions): Promise<ExportResult>
+export async function exportMediaLibrary(
+  options: MediaLibraryExportOptions,
+): Promise<ExportResult> {
+  return exportDataset(options as ExportOptions)
+}
+
+/**
+ * Alias for `exportDataset`, for backwards compatibility.
+ * Use named `exportDataset` instead.
+ *
+ * @deprecated Default export is deprecated and will be removed in a future release. Use named "exportDataset" function instead.
+ * @public
+ */
+export default deprecate(
+  function deprecatedExport(opts: ExportOptions): Promise<ExportResult> {
+    return exportDataset(opts)
+  },
+  `Default export of "@sanity/export" is deprecated and will be removed in a future release. Please use the named "exportDataset" function instead.`,
+  'DEP_SANITY_EXPORT_DEFAULT',
+)

package/src/filterDocumentTypes.ts

@@ -0,0 +1,21 @@
+import type {Transform} from 'node:stream'
+
+import {throughObj} from './util/streamHelpers.js'
+import type {SanityDocument} from './types.js'
+
+export function filterDocumentTypes(allowedTypes: string[] | undefined): Transform {
+  if (!allowedTypes || allowedTypes.length === 0) {
+    // Pass-through
+    return throughObj((doc: SanityDocument, _enc, callback) => callback(null, doc))
+  }
+
+  return throughObj(function docTypesFilter(doc: SanityDocument, _enc, callback) {
+    const type = doc._type
+    if (allowedTypes.includes(type)) {
+      callback(null, doc)
+      return
+    }
+
+    callback()
+  })
+}

package/src/filterDocuments.ts

@@ -0,0 +1,55 @@
+import type {Transform} from 'node:stream'
+
+import {throughObj} from './util/streamHelpers.js'
+import {debug} from './debug.js'
+import type {SanityDocument} from './types.js'
+
+interface CursorDocument {
+  nextCursor?: string
+}
+
+const isDraftOrVersion = (doc: SanityDocument): boolean =>
+  Boolean(doc._id && (doc._id.indexOf('drafts.') === 0 || doc._id.indexOf('versions.') === 0))
+
+const isSystemDocument = (doc: SanityDocument): boolean =>
+  Boolean(doc._id && doc._id.indexOf('_.') === 0)
+
+const isReleaseDocument = (doc: SanityDocument): boolean =>
+  Boolean(doc._id && doc._id.indexOf('_.releases.') === 0)
+
+const isCursor = (doc: unknown): doc is CursorDocument =>
+  typeof doc === 'object' &&
+  doc !== null &&
+  !('_id' in doc) &&
+  'nextCursor' in doc &&
+  (doc as CursorDocument).nextCursor !== undefined
+
+export function filterDocuments(drafts: boolean): Transform {
+  return throughObj(function filterDocs(doc: SanityDocument | CursorDocument, _enc, callback) {
+    if (isCursor(doc)) {
+      debug('%o is a cursor, skipping', doc)
+      callback()
+      return
+    }
+
+    const sanityDoc = doc
+
+    if (!drafts && isDraftOrVersion(sanityDoc)) {
+      debug('%s is a draft or version, skipping', sanityDoc._id)
+      callback()
+      return
+    }
+
+    if (isSystemDocument(sanityDoc)) {
+      if (drafts && isReleaseDocument(sanityDoc)) {
+        callback(null, sanityDoc)
+        return
+      }
+      debug('%s is a system document, skipping', sanityDoc._id)
+      callback()
+      return
+    }
+
+    callback(null, sanityDoc)
+  })
+}

package/src/{getDocumentCursorStream.js → getDocumentCursorStream.ts}

@@ -1,15 +1,24 @@
-import {Transform} from 'node:stream'
+import {Transform, type TransformCallback} from 'node:stream'
 
 import {debug} from './debug.js'
 import {getUserAgent} from './getUserAgent.js'
 import {requestStream} from './requestStream.js'
+import type {NormalizedExportOptions, ResponseStream} from './types.js'
+import {getSource} from './options.js'
 
 // same regex as split2 is using by default: https://github.com/mcollina/split2/blob/53432f54bd5bf422bd55d91d38f898b6c9496fc1/index.js#L86
 const splitRegex = /\r?\n/
 
-export async function getDocumentCursorStream(options) {
+interface CursorChunk {
+  nextCursor?: string
+  _id?: string
+}
+
+export async function getDocumentCursorStream(
+  options: NormalizedExportOptions,
+): Promise<Transform> {
   let streamsInflight = 0
-  function decrementInflight(stream) {
+  function decrementInflight(stream: Transform): void {
     streamsInflight--
     if (streamsInflight === 0) {
       stream.end()
@@ -17,25 +26,30 @@ export async function getDocumentCursorStream(options) {
   }
 
   const stream = new Transform({
-    async transform(chunk, encoding, callback) {
-      if (encoding !== 'buffer' && encoding !== 'string') {
+    transform(
+      this: Transform,
+      chunk: Buffer,
+      encoding: BufferEncoding,
+      callback: TransformCallback,
+    ) {
+      if (encoding !== ('buffer' as BufferEncoding) && encoding !== ('string' as BufferEncoding)) {
         callback(null, chunk)
         return
       }
       this.push(chunk, encoding)
 
-      let parsedChunk = null
+      let parsedChunk: CursorChunk | null = null
       for (const chunkStr of chunk.toString().split(splitRegex)) {
         if (chunkStr.trim() === '') {
           continue
         }
 
         try {
-          parsedChunk = JSON.parse(chunkStr)
-        } catch (err) {
+          parsedChunk = JSON.parse(chunkStr) as CursorChunk
+        } catch {
           // Ignore JSON parse errors
           // this can happen if the chunk is not a JSON object. We just pass it through and let the caller handle it.
-          debug('Failed to parse JSON chunk, ignoring', err, chunkStr)
+          debug('Failed to parse JSON chunk, ignoring', chunkStr)
         }
 
         if (
@@ -48,9 +62,10 @@ export async function getDocumentCursorStream(options) {
           debug('Got next cursor "%s", fetching next stream', parsedChunk.nextCursor)
           streamsInflight++
 
-          const reqStream = await startStream(options, parsedChunk.nextCursor)
-          reqStream.on('end', () => decrementInflight(this))
-          reqStream.pipe(this, {end: false})
+          void startStream(options, parsedChunk.nextCursor).then((reqStream) => {
+            reqStream.on('end', () => decrementInflight(this))
+            reqStream.pipe(this, {end: false})
+          })
         }
       }
 
@@ -65,11 +80,15 @@ export async function getDocumentCursorStream(options) {
   return stream
 }
 
-function startStream(options, nextCursor) {
+function startStream(
+  options: NormalizedExportOptions,
+  nextCursor: string,
+): Promise<ResponseStream> {
+  const source = getSource(options)
   const baseUrl = options.client.getUrl(
-    options.dataset
-      ? `/data/export/${options.dataset}`
-      : `/media-libraries/${options.mediaLibraryId}/export`,
+    source.type === 'dataset'
+      ? `/data/export/${source.id}`
+      : `/media-libraries/${source.id}/export`,
   )
 
   const url = new URL(baseUrl)
@@ -79,7 +98,7 @@ function startStream(options, nextCursor) {
     url.searchParams.set('types', options.types.join())
   }
   const token = options.client.config().token
-  const headers = {
+  const headers: Record<string, string> = {
     'User-Agent': getUserAgent(),
     ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
@@ -90,7 +109,7 @@ function startStream(options, nextCursor) {
     url: url.toString(),
     headers,
     maxRetries: options.maxRetries,
-    retryDelayMs: options.retryDelayMs,
+    ...(options.retryDelayMs !== undefined ? {retryDelayMs: options.retryDelayMs} : {}),
     readTimeout: options.readTimeout,
   }).then((res) => {
     debug('Got stream with HTTP %d', res.statusCode)

package/src/{getDocumentsStream.js → getDocumentsStream.ts}

@@ -1,13 +1,23 @@
 import {getUserAgent} from './getUserAgent.js'
+import {getSource} from './options.js'
 import {requestStream} from './requestStream.js'
+import type {ExportSource, NormalizedExportOptions, ResponseStream} from './types.js'
 
-export function getDocumentsStream(options) {
+type GetDocumentStreamOptions = Partial<NormalizedExportOptions> &
+  Pick<
+    NormalizedExportOptions,
+    'client' | 'types' | 'maxRetries' | 'retryDelayMs' | 'readTimeout'
+  > &
+  ExportSource
+
+export function getDocumentsStream(options: GetDocumentStreamOptions): Promise<ResponseStream> {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
+  const source = getSource(options)
   const baseUrl = options.client.getUrl(
-    options.dataset
-      ? `/data/export/${options.dataset}`
-      : `/media-libraries/${options.mediaLibraryId}/export`,
+    source.type === 'dataset'
+      ? `/data/export/${source.id}`
+      : `/media-libraries/${source.id}/export`,
   )
 
   const url = new URL(baseUrl)
@@ -16,7 +26,7 @@ export function getDocumentsStream(options) {
   }
 
   const token = options.client.config().token
-  const headers = {
+  const headers: Record<string, string> = {
     'User-Agent': getUserAgent(),
     ...(token ? {Authorization: `Bearer ${token}`} : {}),
   }
@@ -25,7 +35,7 @@ export function getDocumentsStream(options) {
     url: url.toString(),
     headers,
     maxRetries: options.maxRetries,
-    retryDelayMs: options.retryDelayMs,
+    ...(options.retryDelayMs !== undefined ? {retryDelayMs: options.retryDelayMs} : {}),
     readTimeout: options.readTimeout,
   })
 }

package/src/{getUserAgent.js → getUserAgent.ts}

@@ -1,12 +1,17 @@
 import {readFileSync} from 'node:fs'
 import {join as joinPath} from 'node:path'
 
-let ua = null
+interface PackageJson {
+  name: string
+  version: string
+}
+
+let ua: string | null = null
 
-export function getUserAgent() {
+export function getUserAgent(): string {
   if (!ua) {
     const data = readFileSync(joinPath(import.meta.dirname, '..', 'package.json'), 'utf-8')
-    const pkg = JSON.parse(data)
+    const pkg = JSON.parse(data) as PackageJson
     ua = `${pkg.name}@${pkg.version}`
   }
 

package/src/index.ts

@@ -0,0 +1,11 @@
+// eslint-disable-next-line @typescript-eslint/no-deprecated
+export {exportDataset, exportMediaLibrary, default} from './export.js'
+export type {
+  ExportMode,
+  ExportOptions,
+  ExportResult,
+  SanityClientLike,
+  SanityDocument,
+  ExportProgress,
+  ExportSource,
+} from './types.js'

package/src/{logFirstChunk.js → logFirstChunk.ts}

@@ -1,12 +1,14 @@
-import {debug} from './debug.js'
+import type {Transform} from 'node:stream'
+
 import {through} from './util/streamHelpers.js'
+import {debug} from './debug.js'
 
-export function logFirstChunk() {
+export function logFirstChunk(): Transform {
   let firstChunk = true
-  return through((chunk, enc, callback) => {
+  return through((chunk, _enc, callback) => {
     if (firstChunk) {
       const string = chunk.toString('utf8').split('\n')[0]
-      debug('First chunk received: %s', string.slice(0, 300))
+      debug('First chunk received: %s', string?.slice(0, 300))
       firstChunk = false
     }