@zfadhli/tokrec 0.13.0 → 0.15.0

package/src/io/http.ts

@@ -0,0 +1,167 @@
+/**
+ * HTTP — the transport layer in one file: the wreq-js session (TLS fingerprint
+ * impersonation, cookie jar, proxy) and the token-bucket rate limiter that
+ * keeps polling from tripping TikTok's WAF.
+ *
+ * This file owns the `HttpClient` contract: it is the seam every adapter and
+ * `test/utils/mock-http.ts` implement. There is no second definition.
+ */
+
+import { createSession, type Session } from "wreq-js"
+import type { RecorderConfig } from "../app/config"
+
+// BodyInit is only available inside declare module "bun" in bun-types, not as a global
+// under the ES2022 lib. We explicitly import it to keep core/ focused on ES2022.
+export type HttpBody = import("bun").BodyInit
+
+export interface HttpClient {
+  get: (url: string, signal?: AbortSignal) => Promise<Response>
+  post: (
+    url: string,
+    body?: HttpBody,
+    headers?: Record<string, string>,
+    signal?: AbortSignal,
+  ) => Promise<Response>
+  close: () => Promise<void>
+}
+
+const REQUEST_TIMEOUT_MS = 15_000
+
+/** Standard browser headers to avoid TikTok WAF detection. */
+const BROWSER_HEADERS: Record<string, string> = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0",
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.5",
+  "Accept-Encoding": "gzip, deflate, br",
+  "Upgrade-Insecure-Requests": "1",
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+}
+
+export async function createHttpClient(config: RecorderConfig): Promise<HttpClient> {
+  const session: Session = await createSession({
+    browser: "firefox_149",
+    os: "windows",
+    proxy: config.proxy,
+  })
+
+  // Seed cookies into the session jar so all subsequent requests are authenticated.
+  // NOTE: Using a Cookie header does NOT populate wreq-js's cookie jar —
+  // we must use session.setCookie() instead.
+  // Seed for all TikTok domains since cookies may not propagate across
+  // subdomains automatically in wreq-js's cookie jar.
+  if (config.cookies && "sessionid_ss" in config.cookies) {
+    const domains = ["https://www.tiktok.com", "https://webcast.tiktok.com", "https://m.tiktok.com"]
+    for (const domain of domains) {
+      for (const [name, value] of Object.entries(config.cookies)) {
+        session.setCookie(name, value, domain)
+      }
+    }
+  }
+
+  const rateLimiter = createRateLimiter(config.ratePerSecond ?? 5)
+
+  async function rateLimitedFetch(
+    url: string,
+    options?: { method?: string; body?: unknown; headers?: Record<string, string> },
+    external?: AbortSignal,
+  ): Promise<Response> {
+    const controller = new AbortController()
+    const onAbort = () => controller.abort(external?.reason)
+    if (external?.aborted) controller.abort(external.reason)
+    else external?.addEventListener("abort", onAbort, { once: true })
+    const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS)
+    try {
+      await rateLimiter.acquire(controller.signal)
+      const res = await session.fetch(url, {
+        method: options?.method,
+        body: options?.body as never,
+        headers: {
+          ...BROWSER_HEADERS,
+          ...options?.headers,
+        },
+        signal: controller.signal,
+      })
+      // SAFETY: wreq-js returns its own Response-compatible object; the port only
+      // needs the standard `status`, `ok`, and `text()` surface.
+      return res as unknown as Response
+    } finally {
+      clearTimeout(timeout)
+      external?.removeEventListener("abort", onAbort)
+    }
+  }
+
+  return {
+    get: async (url: string, signal?: AbortSignal) => rateLimitedFetch(url, undefined, signal),
+    post: async (
+      url: string,
+      body?: HttpBody,
+      headers?: Record<string, string>,
+      signal?: AbortSignal,
+    ) =>
+      rateLimitedFetch(
+        url,
+        {
+          method: "POST",
+          body: body as never,
+          headers: {
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            ...headers,
+          },
+        },
+        signal,
+      ),
+    close: async () => {
+      await session.close()
+    },
+  }
+}
+
+// ─── rate limiting ─────────────────────────────────────────
+
+interface RateLimiter {
+  /** Wait until a rate-limit token is available, then consume it. */
+  acquire(signal?: AbortSignal): Promise<void>
+}
+
+/**
+ * Token bucket: a minimum interval between requests. `0` or less is unlimited.
+ * Abort-aware, so a stop does not wait out a pending token.
+ */
+function createRateLimiter(tokensPerSecond: number): RateLimiter {
+  if (tokensPerSecond <= 0) {
+    return { acquire: async () => {} }
+  }
+
+  const MIN_INTERVAL_MS = 1000 / tokensPerSecond
+  let lastTokenTime = 0
+
+  return {
+    async acquire(signal?: AbortSignal): Promise<void> {
+      if (signal?.aborted) return
+
+      const now = Date.now()
+      const waitTime = Math.max(0, lastTokenTime + MIN_INTERVAL_MS - now)
+
+      if (waitTime > 0) {
+        await new Promise<void>((resolve) => {
+          const timer = setTimeout(resolve, waitTime)
+          if (signal) {
+            signal.addEventListener(
+              "abort",
+              () => {
+                clearTimeout(timer)
+                resolve()
+              },
+              { once: true },
+            )
+          }
+        })
+      }
+
+      lastTokenTime = Date.now()
+    },
+  }
+}

package/src/io/media/convert.ts

@@ -0,0 +1,69 @@
+/**
+ * Converter — the media-processing capability `RecordingJob` depends on.
+ *
+ * Remux (TS → MP4) and segment both delegate to `ffmpeg-utils.runFfmpeg`, so
+ * the job never builds an FFmpeg command line.
+ */
+
+import { unlinkSync } from "node:fs"
+import type { Logger } from "../../core/log"
+import { runFfmpeg } from "./ffmpeg"
+
+export interface Converter {
+  /** Convert a TS file to MP4 (stream copy). Deletes the original on success. */
+  convert: (input: string) => Promise<string>
+  /** Split a file into `<name>_partN.mp4`; returns FFmpeg's CSV segment listing. */
+  segment: (input: string, segmentSeconds: number, outputPattern: string) => Promise<string>
+}
+
+export function createConverter(logger?: Logger, signal?: AbortSignal): Converter {
+  async function convert(input: string): Promise<string> {
+    const output = input.replace(/\.(flv|ts)$/i, ".mp4")
+
+    logger?.info(`Converting: ${input} → ${output}`)
+
+    await runFfmpeg(["-i", input, "-c", "copy", "-y", output], signal)
+
+    // Delete original TS
+    try {
+      unlinkSync(input)
+      logger?.info(`Deleted original: ${input}`)
+    } catch {
+      logger?.warn(`Could not delete original: ${input}`)
+    }
+
+    logger?.info(`Converted: ${output}`)
+    return output
+  }
+
+  async function segment(
+    input: string,
+    segmentSeconds: number,
+    outputPattern: string,
+  ): Promise<string> {
+    return runFfmpeg(
+      [
+        "-i",
+        input,
+        "-c",
+        "copy",
+        "-f",
+        "segment",
+        "-segment_time",
+        String(segmentSeconds),
+        "-reset_timestamps",
+        "1",
+        "-segment_start_number",
+        "1",
+        "-segment_list",
+        "pipe:1",
+        "-segment_list_type",
+        "csv",
+        outputPattern,
+      ],
+      signal,
+    )
+  }
+
+  return { convert, segment }
+}

package/src/io/media/download.ts

@@ -0,0 +1,120 @@
+/**
+ * Live stream downloader — downloads a live stream via FFmpeg stdout pipe.
+ *
+ * Outputs MPEG-TS (.ts) format, which is crash-safe and append-friendly.
+ * FFmpeg handles HTTP reconnection via -reconnect flags. When TikTok's
+ * short-lived stream URL expires, FFmpeg exits and the outer loop fetches
+ * a fresh URL via getNextUrl.
+ *
+ * Both FLV and HLS streams are handled by the same download function —
+ * the only difference is an optional label in the log messages.
+ */
+
+import { createWriteStream, mkdirSync, statSync } from "node:fs"
+import { join } from "node:path"
+import type { DownloadProgress, RecordedFile } from "../../app/config"
+import { TikTokError } from "../../core/errors"
+import { bytesToHuman, formatDuration, formatFilename } from "../../core/format"
+import type { Logger } from "../../core/log"
+import { findFfmpegPath, pipeFfmpegSegment } from "./ffmpeg"
+
+export async function downloadStream(
+  liveUrl: string,
+  user: string,
+  outputDir: string,
+  maxDuration: number,
+  onProgress: ((info: DownloadProgress) => void) | undefined,
+  getNextUrl: (() => Promise<string | null>) | undefined,
+  signal: AbortSignal,
+  logger?: Logger,
+  cookies?: Record<string, string>,
+): Promise<RecordedFile> {
+  mkdirSync(outputDir, { recursive: true })
+  const filename = formatFilename(user, "ts")
+  const filepath = join(outputDir, filename)
+  // FLV is the default container — an .m3u8 URL means the CDN served HLS.
+  const prefix = liveUrl.includes(".m3u8") ? " (HLS)" : ""
+  logger?.info(`Recording${prefix}: ${filename}`)
+
+  const startTime = Date.now()
+
+  /** File size read directly from disk — authoritative, avoids in-memory counter drift. */
+  let fileSize = 0
+  let lastProgressTime = 0
+
+  const ffmpegPath = findFfmpegPath()
+  if (!ffmpegPath) {
+    throw new TikTokError(
+      "ffmpeg-not-found",
+      "FFmpeg not found — required for stream download. Install it:\n" +
+        "  Linux:  apt install ffmpeg\n  macOS:  brew install ffmpeg",
+    )
+  }
+
+  function elapsed(): number {
+    return (Date.now() - startTime) / 1000
+  }
+
+  const writer = createWriteStream(filepath)
+
+  try {
+    while (!signal.aborted) {
+      const remaining = maxDuration > 0 ? Math.max(1, maxDuration - elapsed()) : undefined
+      await pipeFfmpegSegment(ffmpegPath, liveUrl, writer, signal, remaining, cookies)
+
+      // Read actual file size from disk — always authoritative
+      try {
+        fileSize = statSync(filepath).size
+      } catch {
+        // File may not be accessible; leave fileSize at last known value
+      }
+
+      // Report progress after each segment
+      const now = Date.now()
+      const e = elapsed()
+      if (onProgress && now - lastProgressTime >= 1000) {
+        lastProgressTime = now
+        const speed = e > 0 ? fileSize / e : 0
+        onProgress({ bytes: fileSize, elapsed: e, speed })
+      }
+
+      if (signal.aborted) break
+      if (maxDuration > 0 && e >= maxDuration) {
+        logger?.info(`Duration limit reached (${maxDuration}s)`)
+        break
+      }
+      if (!getNextUrl) break
+
+      const nextUrl = await getNextUrl()
+      if (!nextUrl) {
+        logger?.info("No new stream URL — recording finished")
+        break
+      }
+      liveUrl = nextUrl
+    }
+  } catch (err) {
+    logger?.error(`Recording error: ${err instanceof Error ? err.message : String(err)}`)
+  } finally {
+    await new Promise<void>((resolve) => writer.end(resolve))
+  }
+
+  const duration = elapsed()
+  // Final authoritative size after flush
+  try {
+    fileSize = statSync(filepath).size
+  } catch {
+    // keep last known value
+  }
+  if (onProgress) {
+    onProgress({
+      bytes: fileSize,
+      elapsed: duration,
+      speed: duration > 0 ? fileSize / duration : 0,
+    })
+  }
+  logger?.info(
+    `Recording finished: ${filename} (${formatDuration(duration)}, ${bytesToHuman(fileSize)})`,
+  )
+
+  return { file: filepath, duration, size: fileSize }
+}

package/src/io/media/ffmpeg.ts

@@ -0,0 +1,201 @@
+/**
+ * FFmpeg — the single home for process invocation.
+ *
+ * Two modes: `pipeFfmpegSegment` streams stdout into a `WriteStream` (live
+ * capture), `runFfmpeg` runs to completion and returns stdout (remux,
+ * segment). Locating the binary lives here too, so no caller reimplements a
+ * PATH lookup.
+ */
+
+import { type ChildProcess, spawn } from "node:child_process"
+import type { WriteStream } from "node:fs"
+import { statSync } from "node:fs"
+import { execa } from "execa"
+import { TikTokError } from "../../core/errors"
+
+/** Kill FFmpeg if no output is received within this window after spawn. */
+export const FFMPEG_STARTUP_TIMEOUT = 30_000
+
+/** Find the FFmpeg binary via `Bun.which()` or a PATH search. */
+export function findFfmpegPath(): string | null {
+  const bunWhich = (globalThis as { Bun?: { which?: (cmd: string) => string | null } }).Bun?.which
+  if (typeof bunWhich === "function") {
+    return (bunWhich("ffmpeg") as string) ?? null
+  }
+  const paths = process.env.PATH?.split(":") ?? []
+  for (const dir of paths) {
+    try {
+      const full = `${dir}/ffmpeg`
+      statSync(full)
+      return full
+    } catch {
+      // not found in this directory
+    }
+  }
+  return null
+}
+
+/**
+ * Run FFmpeg to completion and return stdout — for output that is a file
+ * rather than a pipe (the segment muxer's `-segment_list pipe:1`).
+ */
+export async function runFfmpeg(args: string[], signal?: AbortSignal): Promise<string> {
+  const ffmpegPath = findFfmpegPath()
+  if (!ffmpegPath) {
+    throw new TikTokError(
+      "ffmpeg-not-found",
+      "FFmpeg not found. Install it:\n" +
+        "  Linux:  apt install ffmpeg / brew install ffmpeg / pacman -S ffmpeg\n" +
+        "  macOS:  brew install ffmpeg\n" +
+        "  Windows: choco install ffmpeg",
+    )
+  }
+  try {
+    const result = await execa(ffmpegPath, args, {
+      cancelSignal: signal,
+      stdio: ["ignore", "pipe", "pipe"],
+    })
+    return result.stdout ?? ""
+  } catch (err: unknown) {
+    if (signal?.aborted) {
+      throw new TikTokError("aborted", "Aborted by user")
+    }
+    const message = err instanceof Error ? err.message : String(err)
+    throw new TikTokError("ffmpeg-error", `FFmpeg failed: ${message}`)
+  }
+}
+
+/**
+ * Build FFmpeg HTTP option arguments for cookies and headers.
+ * TikTok's CDN requires session cookies and a Referer to serve the stream.
+ */
+function buildHttpArgs(cookies?: Record<string, string>): string[] {
+  const args: string[] = []
+
+  // ponytail: user-agent header, CDNs tend to be picky
+  args.push(
+    "-user_agent",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0",
+  )
+  args.push("-headers", "Referer: https://www.tiktok.com/\r\n")
+
+  if (cookies && Object.keys(cookies).length > 0) {
+    const cookieStr = Object.entries(cookies)
+      .map(([k, v]) => `${k}=${v}`)
+      .join("\r\n")
+    args.push("-cookies", cookieStr)
+  }
+
+  return args
+}
+
+/** Standard FFmpeg reconnect + mpegts pipe arguments. */
+const FFMPEG_BASE_ARGS = [
+  "-reconnect",
+  "1",
+  "-reconnect_at_eof",
+  "1",
+  "-reconnect_streamed",
+  "1",
+  "-reconnect_delay_max",
+  "5",
+  "-c",
+  "copy",
+  "-f",
+  "mpegts",
+  "pipe:1",
+]
+
+/**
+ * Spawn an FFmpeg process to download from `url` and pipe its stdout to
+ * `writer` with backpressure handling.
+ *
+ * Uses child_process.spawn directly (not execa) because execa v9 defaults
+ * to encoding stdout as UTF-8, which corrupts binary MPEG-TS data.
+ */
+export async function pipeFfmpegSegment(
+  ffmpegPath: string,
+  url: string,
+  writer: WriteStream,
+  signal: AbortSignal,
+  maxDuration?: number,
+  cookies?: Record<string, string>,
+): Promise<void> {
+  const durationArgs = maxDuration ? ["-t", String(maxDuration)] : []
+  const httpArgs = buildHttpArgs(cookies)
+  // ponytail: input options (-headers, -cookies, -user_agent) must come before -i
+  const args = [...httpArgs, "-i", url, ...durationArgs, ...FFMPEG_BASE_ARGS]
+  const proc: ChildProcess = spawn(ffmpegPath, args, {
+    stdio: ["ignore", "pipe", "pipe"],
+  })
+
+  let stderr = ""
+  let firstDataTimer: ReturnType<typeof setTimeout> | null = null
+
+  const clearStartupTimer = () => {
+    if (firstDataTimer) {
+      clearTimeout(firstDataTimer)
+      firstDataTimer = null
+    }
+  }
+
+  // Forward abort signal to process
+  const onAbort = () => {
+    if (!proc.killed) proc.kill("SIGTERM")
+  }
+  signal.addEventListener("abort", onAbort, { once: true })
+
+  proc.stderr?.on("data", (chunk: Buffer) => {
+    clearStartupTimer()
+    stderr += chunk.toString()
+    if (stderr.length > 10000) stderr = stderr.slice(-5000)
+  })
+
+  proc.stdout?.on("data", (chunk: Buffer) => {
+    clearStartupTimer()
+    const canContinue = writer.write(chunk)
+    if (!canContinue) {
+      proc.stdout?.pause()
+      writer.once("drain", () => proc.stdout?.resume())
+    }
+  })
+
+  await new Promise<void>((resolve, reject) => {
+    firstDataTimer = setTimeout(() => {
+      if (!proc.killed) {
+        proc.kill("SIGTERM")
+        setTimeout(() => {
+          if (!proc.killed) proc.kill("SIGKILL")
+        }, 2000)
+      }
+      reject(
+        new TikTokError(
+          "ffmpeg-error",
+          `FFmpeg startup timed out after ${FFMPEG_STARTUP_TIMEOUT / 1000}s\n${stderr.slice(-500)}`,
+        ),
+      )
+    }, FFMPEG_STARTUP_TIMEOUT)
+
+    proc.on("close", (code) => {
+      clearStartupTimer()
+      signal.removeEventListener("abort", onAbort)
+      if (signal.aborted) {
+        resolve()
+        return
+      }
+      if (code === 0 || code === null) {
+        resolve()
+        return
+      }
+      reject(
+        new TikTokError("ffmpeg-error", `FFmpeg exited with code ${code}\n${stderr.slice(-500)}`),
+      )
+    })
+
+    proc.on("error", (err) => {
+      clearStartupTimer()
+      signal.removeEventListener("abort", onAbort)
+      reject(err)
+    })
+  })
+}

package/src/io/media/normalize.ts

@@ -0,0 +1,64 @@
+/**
+ * Audio normalizer — wraps peaknorm's EBU R128 loudness normalization.
+ * Runs after conversion/segmenting to normalize the final MP4 audio.
+ */
+
+import type { NormalizeOptions, NormalizeResult } from "peaknorm"
+import { normalizeFile } from "peaknorm"
+import type { Logger } from "../../core/log"
+
+export interface AudioNormalizer {
+  /** Normalize a single audio/video file. Returns the normalize result. */
+  normalize(file: string): Promise<NormalizeResult>
+}
+
+export interface AudioNormalizerOptions {
+  /** Target loudness in LUFS (default: -14). */
+  loudness: number
+  /** Audio codec for output (default: peaknorm's default, "libopus"). */
+  audioCodec?: string
+  /** Audio bitrate for output (default: peaknorm's default, "96k"). */
+  audioBitrate?: string
+  /** Called when a file starts normalization. */
+  onStart?: (file: string) => void
+  /** Called with progress updates (percent 0-100, phase "analyzing"|"normalizing"). */
+  onProgress?: (file: string, percent: number, phase: string) => void
+  /** Called when a file completes normalization successfully. */
+  onComplete?: (result: NormalizeResult) => void
+  /** Called when a file errors during normalization. */
+  onError?: (file: string, error: Error) => void
+}
+
+export function createAudioNormalizer(
+  opts: AudioNormalizerOptions,
+  logger?: Logger,
+): AudioNormalizer {
+  return {
+    async normalize(file: string): Promise<NormalizeResult> {
+      logger?.info(`Normalizing audio: ${file}`)
+      opts.onStart?.(file)
+
+      try {
+        const normalizeOpts: NormalizeOptions = {
+          loudness: opts.loudness,
+          backup: false,
+          onFileProgress: (f, percent, phase) => {
+            opts.onProgress?.(f, percent, phase)
+          },
+        }
+        if (opts.audioCodec !== undefined) normalizeOpts.audioCodec = opts.audioCodec
+        if (opts.audioBitrate !== undefined) normalizeOpts.audioBitrate = opts.audioBitrate
+        const result = await normalizeFile(file, normalizeOpts)
+
+        logger?.info(`Audio normalized: ${result.output}`)
+        opts.onComplete?.(result)
+        return result
+      } catch (err) {
+        const error = err instanceof Error ? err : new Error(String(err))
+        logger?.error(`Audio normalization failed: ${error.message}`)
+        opts.onError?.(file, error)
+        throw error
+      }
+    },
+  }
+}