@transloadit/node 4.1.9 → 4.3.0

package/src/Transloadit.ts

@@ -1,7 +1,8 @@
 import * as assert from 'node:assert'
 import { randomUUID } from 'node:crypto'
 import { constants, createReadStream } from 'node:fs'
-import { access } from 'node:fs/promises'
+import { access, stat } from 'node:fs/promises'
+import { basename } from 'node:path'
 import type { Readable } from 'node:stream'
 import { setTimeout as delay } from 'node:timers/promises'
 import { getSignedSmartCdnUrl, signParamsSync } from '@transloadit/utils/node'
@@ -44,6 +45,11 @@ import type {
   TemplateResponse,
 } from './apiTypes.ts'
 import InconsistentResponseError from './InconsistentResponseError.ts'
+import type {
+  LintAssemblyInstructionsInput,
+  LintAssemblyInstructionsResult,
+} from './lintAssemblyInstructions.ts'
+import { lintAssemblyInstructions as lintAssemblyInstructionsInternal } from './lintAssemblyInstructions.ts'
 import PaginationStream from './PaginationStream.ts'
 import PollingTimeoutError from './PollingTimeoutError.ts'
 import type { Stream } from './tus.ts'
@@ -64,6 +70,7 @@ export {
 export type { AssemblyStatus } from './alphalib/types/assemblyStatus.ts'
 export * from './apiTypes.ts'
 export { InconsistentResponseError, ApiError }
+export type { LintAssemblyInstructionsResult, LintFatalLevel } from './lintAssemblyInstructions.ts'
 
 const log = debug('transloadit')
 const logWarn = debug('transloadit:warn')
@@ -77,8 +84,73 @@ const { version } = packageJson
 
 export type AssemblyProgress = (assembly: AssemblyStatus) => void
 
-export interface CreateAssemblyOptions {
-  params?: CreateAssemblyParams
+type UploadDescriptor = {
+  label: string
+  filename: string
+  size?: number
+  path?: string
+  value?: Readable | IntoStreamInput
+}
+
+const getUploadKey = (
+  fieldname: string | null | undefined,
+  filename: string | null | undefined,
+  size: number | null | undefined,
+): string | null => {
+  if (!fieldname || !filename || size == null) return null
+  return JSON.stringify([fieldname, filename, size])
+}
+
+const getSizeFromValue = (value: Readable | IntoStreamInput): number | undefined => {
+  if (typeof value === 'string') return Buffer.byteLength(value)
+  if (Buffer.isBuffer(value)) return value.length
+  if (value instanceof ArrayBuffer) return value.byteLength
+  if (ArrayBuffer.isView(value)) return value.byteLength
+  return undefined
+}
+
+const toReadableUpload = (label: string, value: Readable | IntoStreamInput): Readable => {
+  const readable = isReadableStream(value)
+  if (!readable && isStream(value)) {
+    throw new Error(`Upload named "${label}" is not a Readable stream`)
+  }
+  return readable ? value : intoStream(value)
+}
+
+const buildStreamsMap = (descriptors: UploadDescriptor[]): Record<string, Stream> =>
+  Object.fromEntries(
+    descriptors.map((descriptor) => {
+      if (descriptor.path) {
+        const stream = createReadStream(descriptor.path)
+        return [descriptor.label, { stream, path: descriptor.path }]
+      }
+
+      const value = descriptor.value
+      if (value == null) {
+        throw new Error(`Upload named "${descriptor.label}" has no data`)
+      }
+      const stream = toReadableUpload(descriptor.label, value)
+      return [descriptor.label, { stream }]
+    }),
+  )
+
+const pauseStreams = (streamsMap: Record<string, Stream>): void => {
+  for (const { stream } of Object.values(streamsMap)) {
+    stream.pause()
+  }
+}
+
+const createStreamErrorPromise = (streamsMap: Record<string, Stream>): Promise<never> => {
+  const promise = new Promise<never>((_resolve, reject) => {
+    for (const { stream } of Object.values(streamsMap)) {
+      stream.on('error', reject)
+    }
+  })
+  promise.catch(() => {})
+  return promise
+}
+
+interface AssemblyUploadOptions {
   files?: {
     [name: string]: string
   }
@@ -91,19 +163,32 @@ export interface CreateAssemblyOptions {
   timeout?: number
   onUploadProgress?: (uploadProgress: UploadProgress) => void
   onAssemblyProgress?: AssemblyProgress
-  assemblyId?: string
   /**
-   * Optional AbortSignal to cancel the assembly creation and upload.
+   * Optional AbortSignal to cancel the upload and any follow-up polling.
    * When aborted, any in-flight HTTP requests and TUS uploads will be cancelled.
    */
   signal?: AbortSignal
 }
 
+export interface CreateAssemblyOptions extends AssemblyUploadOptions {
+  params?: CreateAssemblyParams
+  assemblyId?: string
+}
+
+export interface ResumeAssemblyUploadsOptions extends AssemblyUploadOptions {
+  assemblyUrl: string
+}
+
 export interface AwaitAssemblyCompletionOptions {
   onAssemblyProgress?: AssemblyProgress
   timeout?: number
   interval?: number
   startTimeMs?: number
+  /**
+   * Optional assembly URL to poll instead of the configured client endpoint.
+   * Useful when resuming an Assembly created on a different host/region.
+   */
+  assemblyUrl?: string
   /**
    * Optional AbortSignal to cancel polling.
    * When aborted, the polling loop will stop and throw an AbortError.
@@ -117,6 +202,14 @@ export interface AwaitAssemblyCompletionOptions {
   onPoll?: () => boolean | undefined
 }
 
+export interface LintAssemblyInstructionsOptions
+  extends Omit<LintAssemblyInstructionsInput, 'template'> {
+  /**
+   * Template ID to merge with the provided instructions before linting.
+   */
+  templateId?: string
+}
+
 export interface SmartCDNUrlOptions {
   /**
    * Workspace slug
@@ -159,6 +252,14 @@ function getHrTimeMs(): number {
   return Number(process.hrtime.bigint() / 1000000n)
 }
 
+function getAssemblyIdFromUrl(assemblyUrl: string): string {
+  const match = assemblyUrl.match(/\/assemblies\/([^/?#]+)/)
+  if (!match) {
+    throw new Error(`Invalid assembly URL: ${assemblyUrl}`)
+  }
+  return match[1] ?? ''
+}
+
 function checkResult<T>(result: T | { error: string }): asserts result is T {
   // In case server returned a successful HTTP status code, but an `error` in the JSON object
   // This happens sometimes, for example when createAssembly with an invalid file (IMPORT_FILE_ERROR)
@@ -276,36 +377,25 @@ export class Transloadit {
         { concurrency: 5 },
       )
 
-      // Convert uploads to streams
-      const streamsMap = Object.fromEntries(
-        Object.entries(uploads).map(([label, value]) => {
-          const isReadable = isReadableStream(value)
-          if (!isReadable && isStream(value)) {
-            // https://github.com/transloadit/node-sdk/issues/92
-            throw new Error(`Upload named "${label}" is not a Readable stream`)
-          }
-
-          return [label, isReadableStream(value) ? value : intoStream(value)]
-        }),
-      )
-
-      // Wrap in object structure (so we can store whether it's a pathless stream or not)
-      const allStreamsMap = Object.fromEntries<Stream>(
-        Object.entries(streamsMap).map(([label, stream]) => [label, { stream }]),
-      )
-
-      // Create streams from files too
-      for (const [label, path] of Object.entries(files)) {
-        const stream = createReadStream(path)
-        allStreamsMap[label] = { stream, path } // File streams have path
-      }
+      const descriptors: UploadDescriptor[] = [
+        ...Object.entries(files).map(([label, path]) => ({
+          label,
+          path,
+          filename: basename(path),
+        })),
+        ...Object.entries(uploads).map(([label, value]) => ({
+          label,
+          filename: label,
+          value,
+        })),
+      ]
+
+      const allStreamsMap = buildStreamsMap(descriptors)
 
       const allStreams = Object.values(allStreamsMap)
 
       // Pause all streams
-      for (const { stream } of allStreams) {
-        stream.pause()
-      }
+      pauseStreams(allStreamsMap)
 
       // If any stream emits error, we want to handle this and exit with error.
       // This promise races against createAssemblyAndUpload() below via Promise.race().
@@ -313,12 +403,7 @@ export class Transloadit {
       // it's no longer awaited, but stream error handlers remain attached.
       // The no-op catch prevents Node's unhandled rejection warning if a stream
       // errors after the race is already won.
-      const streamErrorPromise = new Promise<AssemblyStatus>((_resolve, reject) => {
-        for (const { stream } of allStreams) {
-          stream.on('error', reject)
-        }
-      })
-      streamErrorPromise.catch(() => {})
+      const streamErrorPromise = createStreamErrorPromise(allStreamsMap)
 
       const createAssemblyAndUpload = async () => {
         const result: AssemblyStatus = await this._remoteJson({
@@ -368,6 +453,149 @@ export class Transloadit {
     return Object.assign(promise, { assemblyId: effectiveAssemblyId })
   }
 
+  /**
+   * Lint Assembly Instructions locally.
+   *
+   * If a templateId is provided, the template content is merged with the instructions,
+   * just like the API. When a template sets `allow_steps_override=false`, providing
+   * `steps` will throw a TEMPLATE_DENIES_STEPS_OVERRIDE error.
+   *
+   * The `assemblyInstructions` input may be a JSON string, a full instructions object,
+   * or a steps-only object (missing the `steps` property).
+   */
+  async lintAssemblyInstructions(
+    options: LintAssemblyInstructionsOptions,
+  ): Promise<LintAssemblyInstructionsResult> {
+    const { templateId, ...rest } = options
+    if (!templateId) {
+      return await lintAssemblyInstructionsInternal(rest)
+    }
+
+    const template = await this.getTemplate(templateId)
+    return await lintAssemblyInstructionsInternal({
+      ...rest,
+      template: template.content,
+    })
+  }
+
+  async resumeAssemblyUploads(opts: ResumeAssemblyUploadsOptions): Promise<AssemblyStatus> {
+    const {
+      assemblyUrl,
+      files = {},
+      uploads = {},
+      chunkSize: requestedChunkSize = Number.POSITIVE_INFINITY,
+      uploadConcurrency = 10,
+      timeout = 24 * 60 * 60 * 1000, // 1 day
+      waitForCompletion = false,
+      onUploadProgress = () => {},
+      onAssemblyProgress = () => {},
+      signal,
+    } = opts
+
+    const startTimeMs = getHrTimeMs()
+
+    getAssemblyIdFromUrl(assemblyUrl)
+    const assembly = await this._fetchAssemblyStatus({ url: assemblyUrl, signal })
+    const statusUrl = assembly.assembly_ssl_url ?? assembly.assembly_url ?? assemblyUrl
+
+    const finishedKeys = new Set<string>()
+    for (const upload of assembly.uploads ?? []) {
+      const key = getUploadKey(upload.field ?? null, upload.basename ?? null, upload.size)
+      if (key) finishedKeys.add(key)
+    }
+    for (const upload of assembly.tus_uploads ?? []) {
+      if (!upload.finished) continue
+      const key = getUploadKey(upload.fieldname, upload.filename, upload.size)
+      if (key) finishedKeys.add(key)
+    }
+
+    const resumeUrls = new Map<string, string>()
+    for (const upload of assembly.tus_uploads ?? []) {
+      if (upload.finished) continue
+      if (!upload.upload_url) continue
+      const key = getUploadKey(upload.fieldname, upload.filename, upload.size)
+      if (key) resumeUrls.set(key, upload.upload_url)
+    }
+
+    const descriptors: UploadDescriptor[] = []
+
+    await pMap(
+      Object.entries(files),
+      async ([label, path]) => {
+        await access(path, constants.F_OK | constants.R_OK)
+        const info = await stat(path)
+        descriptors.push({
+          label,
+          path,
+          filename: basename(path),
+          size: info.size,
+        })
+      },
+      { concurrency: 5 },
+    )
+
+    for (const [label, value] of Object.entries(uploads)) {
+      descriptors.push({
+        label,
+        filename: label,
+        size: isReadableStream(value) ? undefined : getSizeFromValue(value),
+        value,
+      })
+    }
+
+    const descriptorsToUpload = descriptors.filter((descriptor) => {
+      const key = getUploadKey(descriptor.label, descriptor.filename, descriptor.size ?? null)
+      return key ? !finishedKeys.has(key) : true
+    })
+
+    const uploadUrlsByLabel: Record<string, string> = {}
+    for (const descriptor of descriptorsToUpload) {
+      if (!descriptor.path) continue
+      const key = getUploadKey(descriptor.label, descriptor.filename, descriptor.size ?? null)
+      if (!key) continue
+      const uploadUrl = resumeUrls.get(key)
+      if (uploadUrl) uploadUrlsByLabel[descriptor.label] = uploadUrl
+    }
+
+    const streamsMap = buildStreamsMap(descriptorsToUpload)
+    pauseStreams(streamsMap)
+
+    if (Object.keys(streamsMap).length > 0) {
+      const streamErrorPromise = createStreamErrorPromise(streamsMap)
+
+      const uploadPromise = sendTusRequest({
+        streamsMap,
+        assembly,
+        requestedChunkSize,
+        uploadConcurrency,
+        onProgress: onUploadProgress,
+        signal,
+        uploadUrls: uploadUrlsByLabel,
+      })
+
+      await Promise.race([uploadPromise, streamErrorPromise])
+    }
+
+    const latestAssembly = await this._fetchAssemblyStatus({ url: statusUrl, signal })
+    if (!waitForCompletion) return latestAssembly
+
+    if (latestAssembly.assembly_id == null) {
+      throw new InconsistentResponseError(
+        'Server returned an assembly response without an assembly_id after resuming uploads',
+      )
+    }
+
+    const awaitResult = await this.awaitAssemblyCompletion(latestAssembly.assembly_id, {
+      timeout,
+      onAssemblyProgress,
+      startTimeMs,
+      assemblyUrl: statusUrl,
+      signal,
+    })
+    checkResult(awaitResult)
+    return awaitResult
+  }
+
   async awaitAssemblyCompletion(
     assemblyId: string,
     {
@@ -375,6 +603,7 @@ export class Transloadit {
       timeout,
       startTimeMs = getHrTimeMs(),
       interval = 1000,
+      assemblyUrl,
       signal,
       onPoll,
     }: AwaitAssemblyCompletionOptions = {},
@@ -383,6 +612,12 @@ export class Transloadit {
 
     let lastResult: AssemblyStatus | undefined
 
+    const fetchAssemblyStatus = (): Promise<AssemblyStatus> => {
+      return assemblyUrl
+        ? this._fetchAssemblyStatus({ url: assemblyUrl, signal })
+        : this.getAssembly(assemblyId, { signal })
+    }
+
     while (true) {
       // Check if caller wants to stop polling early
       if (onPoll?.() === false && lastResult) {
@@ -394,7 +629,7 @@ export class Transloadit {
         throw signal.reason ?? new DOMException('Aborted', 'AbortError')
       }
 
-      const result = await this.getAssembly(assemblyId, { signal })
+      const result = await fetchAssemblyStatus()
       lastResult = result
 
       // If 'ok' is not in result, it implies a terminal state (e.g., error, completed, canceled).
@@ -576,16 +811,33 @@ export class Transloadit {
     assemblyId: string,
     options?: { signal?: AbortSignal },
   ): Promise<AssemblyStatus> {
-    const rawResult = await this._remoteJson<Record<string, unknown>, OptionalAuthParams>({
-      urlSuffix: `/assemblies/${assemblyId}`,
+    return await this._fetchAssemblyStatus({
+      assemblyId,
       signal: options?.signal,
     })
+  }
+
+  private async _fetchAssemblyStatus({
+    assemblyId,
+    url,
+    signal,
+  }: {
+    assemblyId?: string
+    url?: string
+    signal?: AbortSignal
+  }): Promise<AssemblyStatus> {
+    const rawResult = await this._remoteJson<Record<string, unknown>, OptionalAuthParams>({
+      url,
+      urlSuffix: url ? undefined : `/assemblies/${assemblyId}`,
+      signal,
+    })
 
     const parsedResult = zodParseWithContext(assemblyStatusSchema, rawResult)
 
     if (!parsedResult.success) {
+      const label = assemblyId ?? url ?? 'unknown'
       this.maybeThrowInconsistentResponseError(
-        `The API responded with data that does not match the expected schema while getting Assembly: ${assemblyId}.\n${parsedResult.humanReadable}`,
+        `The API responded with data that does not match the expected schema while getting Assembly: ${label}.\n${parsedResult.humanReadable}`,
       )
     }
 
@@ -915,23 +1167,40 @@ export class Transloadit {
 
           // check whether we should retry
           // https://transloadit.com/blog/2012/04/introducing-rate-limiting/
-          if (
+          const retryAfterHeader = err.response?.headers?.['retry-after']
+          const retryAfterSeconds =
+            typeof retryAfterHeader === 'string' ? Number(retryAfterHeader) : undefined
+          const retryInFromInfo =
             typeof body === 'object' &&
             body != null &&
-            'error' in body &&
             'info' in body &&
             typeof body.info === 'object' &&
             body.info != null &&
             'retryIn' in body.info &&
             typeof body.info.retryIn === 'number' &&
-            Boolean(body.info.retryIn) &&
+            body.info.retryIn > 0
+              ? body.info.retryIn
+              : undefined
+          const retryInSec =
+            retryInFromInfo ??
+            (typeof retryAfterSeconds === 'number' && retryAfterSeconds > 0
+              ? retryAfterSeconds
+              : undefined)
+          const shouldRetry =
             retryCount < this._maxRetries && // 413 taken from https://transloadit.com/blog/2012/04/introducing-rate-limiting/
             // todo can 413 be removed?
-            ((statusCode === 413 && body.error === 'RATE_LIMIT_REACHED') || statusCode === 429)
-          ) {
-            const { retryIn: retryInSec } = body.info
-            logWarn(`Rate limit reached, retrying request in approximately ${retryInSec} seconds.`)
-            const retryInMs = 1000 * (retryInSec * (1 + 0.1 * Math.random()))
+            ((statusCode === 413 &&
+              body &&
+              typeof body === 'object' &&
+              body.error === 'RATE_LIMIT_REACHED') ||
+              statusCode === 429)
+
+          if (shouldRetry) {
+            const retryDelaySec = retryInSec ?? 1
+            logWarn(
+              `Rate limit reached, retrying request in approximately ${retryDelaySec} seconds.`,
+            )
+            const retryInMs = 1000 * (retryDelaySec * (1 + 0.1 * Math.random()))
             await delay(retryInMs)
             // Retry
           } else {