@transloadit/node 4.3.0 → 4.4.0

package/src/inputFiles.ts

@@ -0,0 +1,278 @@
+import { createWriteStream } from 'node:fs'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { isIP } from 'node:net'
+import { tmpdir } from 'node:os'
+import { basename, join } from 'node:path'
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import got from 'got'
+import type { Input as IntoStreamInput } from 'into-stream'
+import type { CreateAssemblyParams } from './apiTypes.ts'
+
+export type InputFile =
+  | {
+      kind: 'path'
+      field: string
+      path: string
+    }
+  | {
+      kind: 'base64'
+      field: string
+      base64: string
+      filename: string
+      contentType?: string
+    }
+  | {
+      kind: 'url'
+      field: string
+      url: string
+      filename?: string
+      contentType?: string
+    }
+
+export type UploadInput = Readable | IntoStreamInput
+
+export type Base64Strategy = 'buffer' | 'tempfile'
+export type UrlStrategy = 'import' | 'download' | 'import-if-present'
+
+export type PrepareInputFilesOptions = {
+  inputFiles?: InputFile[]
+  params?: CreateAssemblyParams
+  fields?: Record<string, unknown>
+  base64Strategy?: Base64Strategy
+  urlStrategy?: UrlStrategy
+  maxBase64Bytes?: number
+  allowPrivateUrls?: boolean
+  tempDir?: string
+}
+
+export type PrepareInputFilesResult = {
+  params: CreateAssemblyParams
+  files: Record<string, string>
+  uploads: Record<string, UploadInput>
+  cleanup: Array<() => Promise<void>>
+}
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null
+
+const ensureUnique = (field: string, used: Set<string>): void => {
+  if (used.has(field)) {
+    throw new Error(`Duplicate file field: ${field}`)
+  }
+  used.add(field)
+}
+
+const ensureUniqueStepName = (baseName: string, used: Set<string>): string => {
+  let name = baseName
+  let counter = 1
+  while (used.has(name)) {
+    name = `${baseName}_${counter}`
+    counter += 1
+  }
+  used.add(name)
+  return name
+}
+
+const decodeBase64 = (value: string): Buffer => Buffer.from(value, 'base64')
+
+const estimateBase64DecodedBytes = (value: string): number => {
+  const trimmed = value.trim()
+  if (!trimmed) return 0
+  let padding = 0
+  if (trimmed.endsWith('==')) padding = 2
+  else if (trimmed.endsWith('=')) padding = 1
+  return Math.floor((trimmed.length * 3) / 4) - padding
+}
+
+const getFilenameFromUrl = (value: string): string | null => {
+  try {
+    const pathname = new URL(value).pathname
+    const base = basename(pathname)
+    if (base && base !== '/' && base !== '.') return base
+  } catch {
+    return null
+  }
+  return null
+}
+
+const isHttpImportStep = (value: unknown): value is Record<string, unknown> =>
+  isRecord(value) && value.robot === '/http/import'
+
+const findImportStepName = (field: string, steps: Record<string, unknown>): string | null => {
+  if (isHttpImportStep(steps[field])) return field
+  const matches = Object.entries(steps).filter(([, step]) => isHttpImportStep(step))
+  if (matches.length === 1) return matches[0]?.[0] ?? null
+  return null
+}
+
+const downloadUrlToFile = async (url: string, filePath: string): Promise<void> => {
+  await pipeline(got.stream(url), createWriteStream(filePath))
+}
+
+const isPrivateIp = (address: string): boolean => {
+  if (address === 'localhost') return true
+  const family = isIP(address)
+  if (family === 4) {
+    const parts = address.split('.').map((chunk) => Number(chunk))
+    const [a, b] = parts
+    if (a === 10) return true
+    if (a === 127) return true
+    if (a === 0) return true
+    if (a === 169 && b === 254) return true
+    if (a === 172 && b >= 16 && b <= 31) return true
+    if (a === 192 && b === 168) return true
+    return false
+  }
+  if (family === 6) {
+    const normalized = address.toLowerCase()
+    if (normalized === '::1') return true
+    if (normalized.startsWith('fe80:')) return true
+    if (normalized.startsWith('fc') || normalized.startsWith('fd')) return true
+    return false
+  }
+  return false
+}
+
+const assertPublicDownloadUrl = (value: string): void => {
+  const parsed = new URL(value)
+  if (!['http:', 'https:'].includes(parsed.protocol)) {
+    throw new Error(`URL downloads are limited to http/https: ${value}`)
+  }
+  if (isPrivateIp(parsed.hostname)) {
+    throw new Error(`URL downloads are limited to public hosts: ${value}`)
+  }
+}
+
+export const prepareInputFiles = async (
+  options: PrepareInputFilesOptions = {},
+): Promise<PrepareInputFilesResult> => {
+  const {
+    inputFiles = [],
+    params = {},
+    fields,
+    base64Strategy = 'buffer',
+    urlStrategy = 'import',
+    maxBase64Bytes,
+    allowPrivateUrls = true,
+    tempDir,
+  } = options
+
+  let nextParams: CreateAssemblyParams = { ...params }
+  const files: Record<string, string> = {}
+  const uploads: Record<string, UploadInput> = {}
+  const cleanup: Array<() => Promise<void>> = []
+
+  if (fields && Object.keys(fields).length > 0) {
+    nextParams = {
+      ...nextParams,
+      fields: {
+        ...(isRecord(nextParams.fields) ? nextParams.fields : {}),
+        ...fields,
+      },
+    }
+  }
+
+  const steps = isRecord(nextParams.steps) ? { ...nextParams.steps } : {}
+  const usedSteps = new Set(Object.keys(steps))
+  const usedFields = new Set<string>()
+  const importUrlsByStep = new Map<string, string[]>()
+  const importStepNames = Object.keys(steps).filter((name) => isHttpImportStep(steps[name]))
+  const sharedImportStep = importStepNames.length === 1 ? importStepNames[0] : null
+
+  let tempRoot: string | null = null
+  const ensureTempRoot = async (): Promise<string> => {
+    if (!tempRoot) {
+      const root = await mkdtemp(join(tempDir ?? tmpdir(), 'transloadit-input-'))
+      tempRoot = root
+      cleanup.push(() => rm(root, { recursive: true, force: true }))
+    }
+    return tempRoot
+  }
+
+  try {
+    for (const file of inputFiles) {
+      ensureUnique(file.field, usedFields)
+      if (file.kind === 'path') {
+        files[file.field] = file.path
+        continue
+      }
+      if (file.kind === 'base64') {
+        if (maxBase64Bytes) {
+          const estimated = estimateBase64DecodedBytes(file.base64)
+          if (estimated > maxBase64Bytes) {
+            throw new Error(`Base64 payload exceeds ${maxBase64Bytes} bytes.`)
+          }
+        }
+        const buffer = decodeBase64(file.base64)
+        if (maxBase64Bytes && buffer.length > maxBase64Bytes) {
+          throw new Error(`Base64 payload exceeds ${maxBase64Bytes} bytes.`)
+        }
+        if (base64Strategy === 'tempfile') {
+          const root = await ensureTempRoot()
+          const filename = file.filename ? basename(file.filename) : `${file.field}.bin`
+          const filePath = join(root, filename)
+          await writeFile(filePath, buffer)
+          files[file.field] = filePath
+        } else {
+          uploads[file.field] = buffer
+        }
+        continue
+      }
+      if (file.kind === 'url') {
+        const matchedStep = findImportStepName(file.field, steps)
+        const targetStep = matchedStep ?? sharedImportStep
+        const shouldImport =
+          urlStrategy === 'import' || (urlStrategy === 'import-if-present' && targetStep)
+
+        if (shouldImport) {
+          const stepName = targetStep ?? ensureUniqueStepName(file.field, usedSteps)
+          const urls = importUrlsByStep.get(stepName) ?? []
+          urls.push(file.url)
+          importUrlsByStep.set(stepName, urls)
+          continue
+        }
+
+        const root = await ensureTempRoot()
+        const filename =
+          (file.filename ? basename(file.filename) : null) ??
+          getFilenameFromUrl(file.url) ??
+          `${file.field}.bin`
+        const filePath = join(root, filename)
+        if (!allowPrivateUrls) {
+          assertPublicDownloadUrl(file.url)
+        }
+        await downloadUrlToFile(file.url, filePath)
+        files[file.field] = filePath
+      }
+    }
+  } catch (error) {
+    await Promise.all(cleanup.map((fn) => fn()))
+    throw error
+  }
+
+  if (Object.keys(steps).length > 0 || importUrlsByStep.size > 0) {
+    if (importUrlsByStep.size > 0) {
+      for (const [stepName, urls] of importUrlsByStep.entries()) {
+        const existing = isRecord(steps[stepName]) ? steps[stepName] : {}
+        steps[stepName] = {
+          ...existing,
+          robot: '/http/import',
+          url: urls.length === 1 ? urls[0] : urls,
+        }
+      }
+    }
+
+    nextParams = {
+      ...nextParams,
+      steps,
+    }
+  }
+
+  return {
+    params: nextParams,
+    files,
+    uploads,
+    cleanup,
+  }
+}

package/src/robots.ts

@@ -0,0 +1,317 @@
+import type { z } from 'zod'
+import { robotsMeta, robotsSchema } from './alphalib/types/robots/_index.ts'
+
+export type RobotListOptions = {
+  category?: string
+  search?: string
+  limit?: number
+  cursor?: string
+}
+
+export type RobotListItem = {
+  name: string
+  title?: string
+  summary: string
+  category?: string
+}
+
+export type RobotListResult = {
+  robots: RobotListItem[]
+  nextCursor?: string
+}
+
+export type RobotParamHelp = {
+  name: string
+  type: string
+  description?: string
+}
+
+export type RobotHelp = {
+  name: string
+  summary: string
+  requiredParams: RobotParamHelp[]
+  optionalParams: RobotParamHelp[]
+  examples?: Array<{ description: string; snippet: Record<string, unknown> }>
+}
+
+export type RobotHelpOptions = {
+  robotName: string
+  detailLevel?: 'summary' | 'params' | 'examples'
+}
+
+type RobotsMetaMap = typeof robotsMeta
+type RobotMeta = RobotsMetaMap[keyof RobotsMetaMap]
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null
+
+const getDef = (schema: z.ZodTypeAny): Record<string, unknown> =>
+  (schema as unknown as { _def?: Record<string, unknown>; def?: Record<string, unknown> })._def ??
+  (schema as unknown as { def?: Record<string, unknown> }).def ??
+  {}
+
+const getDefType = (def: Record<string, unknown>): string | undefined =>
+  (def.type as string | undefined) ?? (def.typeName as string | undefined)
+
+const robotNameToPath = (name: string): string => {
+  const base = name.replace(/Robot$/, '')
+  const spaced = base
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z0-9])/g, '$1 $2')
+  const parts = spaced.split(/\s+/).filter(Boolean)
+  return `/${parts.map((part) => part.toLowerCase()).join('/')}`
+}
+
+const selectSummary = (meta: RobotMeta): string =>
+  meta.purpose_sentence ?? meta.purpose_words ?? meta.purpose_word ?? meta.title ?? meta.name
+
+const resolveRobotPath = (robotName: string): string =>
+  robotName.startsWith('/') ? robotName : robotNameToPath(robotName)
+
+const unwrapSchema = (schema: z.ZodTypeAny): { base: z.ZodTypeAny; optional: boolean } => {
+  let base = schema
+  let optional = typeof base.isOptional === 'function' ? base.isOptional() : false
+
+  while (true) {
+    const def = getDef(base)
+    const defType = getDefType(def)
+    if (
+      defType === 'optional' ||
+      defType === 'default' ||
+      defType === 'nullable' ||
+      defType === 'catch' ||
+      defType === 'ZodOptional' ||
+      defType === 'ZodDefault' ||
+      defType === 'ZodNullable' ||
+      defType === 'ZodCatch'
+    ) {
+      const inner = def.innerType as z.ZodTypeAny | undefined
+      if (inner) {
+        base = inner
+        if (defType !== 'nullable' && defType !== 'ZodNullable') {
+          optional = true
+        }
+        continue
+      }
+    }
+    break
+  }
+
+  return { base, optional }
+}
+
+const describeSchemaType = (schema: z.ZodTypeAny): string => {
+  const { base } = unwrapSchema(schema)
+  const def = getDef(base)
+  const defType = getDefType(def)
+
+  switch (defType) {
+    case 'string':
+    case 'ZodString':
+      return 'string'
+    case 'number':
+    case 'ZodNumber':
+      return 'number'
+    case 'boolean':
+    case 'ZodBoolean':
+      return 'boolean'
+    case 'bigint':
+    case 'ZodBigInt':
+      return 'bigint'
+    case 'literal':
+    case 'ZodLiteral': {
+      const value = (def.values as unknown[] | undefined)?.[0] ?? def.value
+      return value === undefined ? 'literal' : JSON.stringify(value)
+    }
+    case 'enum':
+    case 'ZodEnum': {
+      const values = Array.isArray(def.values) ? def.values : []
+      return values.length ? `enum(${values.join(' | ')})` : 'enum'
+    }
+    case 'array':
+    case 'ZodArray': {
+      const element = def.element as z.ZodTypeAny | undefined
+      const inner = element ? describeSchemaType(element) : 'unknown'
+      return `array<${inner}>`
+    }
+    case 'object':
+    case 'ZodObject':
+      return 'object'
+    case 'record':
+    case 'ZodRecord':
+      return 'record'
+    case 'union':
+    case 'ZodUnion': {
+      const options = Array.isArray(def.options) ? def.options : []
+      const rendered = options
+        .map((option) => describeSchemaType(option as z.ZodTypeAny))
+        .join(' | ')
+      return rendered ? `union<${rendered}>` : 'union'
+    }
+    case 'ZodDiscriminatedUnion':
+      return 'object'
+    default:
+      return defType ?? 'unknown'
+  }
+}
+
+const getParamDescription = (schema: z.ZodTypeAny): string | undefined => {
+  if (schema.description?.trim()) {
+    return schema.description.trim()
+  }
+  const inner = unwrapSchema(schema).base
+  return inner.description?.trim()
+}
+
+const getShape = (schema: z.ZodTypeAny): Record<string, z.ZodTypeAny> => {
+  const { base } = unwrapSchema(schema)
+  const def = getDef(base)
+  const shape = def.shape as
+    | Record<string, z.ZodTypeAny>
+    | (() => Record<string, z.ZodTypeAny>)
+    | undefined
+  if (typeof shape === 'function') {
+    return shape()
+  }
+  return shape ?? {}
+}
+
+const getRobotParams = (
+  schema: z.ZodTypeAny,
+): { required: RobotParamHelp[]; optional: RobotParamHelp[] } => {
+  const shape = getShape(schema)
+  const required: RobotParamHelp[] = []
+  const optional: RobotParamHelp[] = []
+
+  for (const [key, value] of Object.entries(shape)) {
+    if (key === 'robot') continue
+    const { optional: isOptional } = unwrapSchema(value)
+    const param: RobotParamHelp = {
+      name: key,
+      type: describeSchemaType(value),
+      description: getParamDescription(value),
+    }
+
+    if (isOptional) {
+      optional.push(param)
+    } else {
+      required.push(param)
+    }
+  }
+
+  return { required, optional }
+}
+
+const getRobotsMetaIndex = (): {
+  byName: Map<string, RobotMeta>
+  byPath: Map<string, RobotMeta>
+} => {
+  const byName = new Map<string, RobotMeta>()
+  const byPath = new Map<string, RobotMeta>()
+
+  for (const meta of Object.values(robotsMeta)) {
+    byName.set(meta.name, meta)
+    byPath.set(robotNameToPath(meta.name), meta)
+  }
+
+  return { byName, byPath }
+}
+
+const getRobotSchemaIndex = (): Map<string, z.ZodTypeAny> => {
+  const index = new Map<string, z.ZodTypeAny>()
+  for (const option of robotsSchema.options) {
+    const shape = getShape(option)
+    const robotSchema = shape.robot
+    if (!robotSchema) continue
+    const robotDef = getDef(robotSchema)
+    const robotLiteral = (robotDef.values as unknown[] | undefined)?.[0] ?? robotDef.value
+    if (typeof robotLiteral === 'string') {
+      index.set(robotLiteral, option)
+    }
+  }
+  return index
+}
+
+let cachedMetaIndex: ReturnType<typeof getRobotsMetaIndex> | null = null
+let cachedSchemaIndex: ReturnType<typeof getRobotSchemaIndex> | null = null
+
+const getMetaIndex = (): ReturnType<typeof getRobotsMetaIndex> => {
+  if (!cachedMetaIndex) {
+    cachedMetaIndex = getRobotsMetaIndex()
+  }
+  return cachedMetaIndex
+}
+
+const getSchemaIndex = (): ReturnType<typeof getRobotSchemaIndex> => {
+  if (!cachedSchemaIndex) {
+    cachedSchemaIndex = getRobotSchemaIndex()
+  }
+  return cachedSchemaIndex
+}
+
+export const listRobots = (options: RobotListOptions = {}): RobotListResult => {
+  const normalizedSearch = options.search?.toLowerCase()
+  const normalizedCategory = options.category?.toLowerCase()
+  const { byPath } = getMetaIndex()
+
+  const allRobots: RobotListItem[] = Array.from(byPath.entries()).map(([path, meta]) => ({
+    name: path,
+    title: meta.title,
+    summary: selectSummary(meta),
+    category: meta.service_slug,
+  }))
+
+  const filtered = allRobots
+    .filter((robot) => {
+      if (normalizedCategory && robot.category?.toLowerCase() !== normalizedCategory) {
+        return false
+      }
+      if (!normalizedSearch) return true
+      const haystack = `${robot.name} ${robot.title ?? ''} ${robot.summary}`.toLowerCase()
+      return haystack.includes(normalizedSearch)
+    })
+    .sort((a, b) => a.name.localeCompare(b.name))
+
+  const start = options.cursor ? Number.parseInt(options.cursor, 10) : 0
+  const safeStart = Number.isFinite(start) && start > 0 ? start : 0
+  const safeLimit = options.limit && options.limit > 0 ? options.limit : 20
+  const page = filtered.slice(safeStart, safeStart + safeLimit)
+  const nextCursor =
+    safeStart + safeLimit < filtered.length ? String(safeStart + safeLimit) : undefined
+
+  return {
+    robots: page,
+    nextCursor,
+  }
+}
+
+export const getRobotHelp = (options: RobotHelpOptions): RobotHelp => {
+  const detailLevel = options.detailLevel ?? 'summary'
+  const { byPath, byName } = getMetaIndex()
+  const schemaIndex = getSchemaIndex()
+
+  const path = resolveRobotPath(options.robotName)
+  const meta = byPath.get(path) ?? byName.get(options.robotName) ?? null
+  const summary = meta ? selectSummary(meta) : `Robot ${path}`
+  const schema = schemaIndex.get(path)
+  const params = schema ? getRobotParams(schema) : { required: [], optional: [] }
+
+  const help: RobotHelp = {
+    name: path,
+    summary,
+    requiredParams: detailLevel === 'params' ? params.required : [],
+    optionalParams: detailLevel === 'params' ? params.optional : [],
+  }
+
+  if (detailLevel === 'examples' && meta?.example_code) {
+    const snippet = isRecord(meta.example_code) ? meta.example_code : {}
+    help.examples = [
+      {
+        description: meta.example_code_description ?? 'Example',
+        snippet,
+      },
+    ]
+  }
+
+  return help
+}