cloud-ytdl 1.0.0-rc

package/lib/subtitle.js

@@ -0,0 +1,106 @@
+'use strict'
+
+const { request } = require('undici')
+const { loadCookieHeader } = require('./load')
+const xmlToSrt = require('./xmlToSrt')
+
+async function getInnertubeConfig() {
+  const res = await request('https://www.youtube.com', {
+    headers: {
+      'user-agent':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
+    }
+  })
+
+  const html = await res.body.text()
+
+  const apiKey =
+    html.match(/"INNERTUBE_API_KEY":"([^"]+)"/)?.[1]
+
+  const clientVersion =
+    html.match(/"clientVersion":"([^"]+)"/)?.[1]
+
+  if (!apiKey || !clientVersion)
+    throw Error('Innertube config not found')
+
+  return { apiKey, clientVersion }
+}
+
+async function getPlayer(videoId, cookie) {
+  const { apiKey, clientVersion } =
+    await getInnertubeConfig()
+
+  const headers = {
+    'content-type': 'application/json',
+    'user-agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
+  }
+
+  if (cookie) headers.cookie = cookie
+
+  const res = await request(
+    `https://www.youtube.com/youtubei/v1/player?key=${apiKey}`,
+    {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        videoId,
+        context: {
+          client: {
+            clientName: 'WEB',
+            clientVersion
+          }
+        }
+      })
+    }
+  )
+
+  return res.body.json()
+}
+
+async function downloadSubtitle(track, cookie) {
+  const headers = {
+    'user-agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
+    accept: 'application/xml'
+  }
+
+  if (cookie) headers.cookie = cookie
+
+  const res = await request(track.baseUrl, { headers })
+  return res.body.text()
+}
+
+async function getSubtitles(videoId, opts = {}) {
+  const cookie = opts.cookie
+    ? loadCookieHeader(opts.cookie)
+    : null
+
+  const lang = opts.lang || 'id'
+  const format = opts.format || 'xml'
+
+  const player = await getPlayer(videoId, cookie)
+
+  const tracks =
+    player?.captions
+      ?.playerCaptionsTracklistRenderer
+      ?.captionTracks
+
+  if (!tracks) return null
+
+  let track =
+    tracks.find(t => t.languageCode === lang && !t.kind) ||
+    tracks.find(t => t.languageCode === lang && t.kind === 'asr')
+
+  if (!track) return null
+
+  const xml = await downloadSubtitle(track, cookie)
+
+  if (!xml.includes('<transcript')) return null
+
+  if (format === 'srt') return xmlToSrt(xml)
+
+  return xml
+}
+
+module.exports = { getSubtitles }

package/lib/url-utils.js

@@ -0,0 +1,58 @@
+const validQueryDomains = new Set([
+  'youtube.com',
+  'www.youtube.com',
+  'm.youtube.com',
+  'music.youtube.com',
+  'gaming.youtube.com'
+])
+
+const validPathDomains = /^https?:\/\/(youtu\.be\/|(www\.)?youtube\.com\/(embed|v|shorts|live)\/)/
+const urlRegex = /^https?:\/\//
+const idRegex = /^[a-zA-Z0-9-_]{11}$/
+
+const validateID = id => idRegex.test(id.trim())
+
+const getURLVideoID = link => {
+  const parsed = new URL(link.trim())
+  let id = parsed.searchParams.get('v')
+
+  if (validPathDomains.test(link.trim()) && !id) {
+    const paths = parsed.pathname.split('/')
+    id = parsed.host === 'youtu.be' ? paths[1] : paths[2]
+  } else if (parsed.hostname && !validQueryDomains.has(parsed.hostname)) {
+    throw Error('Not a YouTube domain')
+  }
+
+  if (!id) throw Error(`No video id found: "${link}"`)
+
+  id = id.slice(0, 11)
+
+  if (!validateID(id)) {
+    throw TypeError(`Video id (${id}) does not match expected format (${idRegex})`)
+  }
+
+  return id
+}
+
+const getVideoID = str => {
+  const s = str.trim()
+  if (validateID(s)) return s
+  if (urlRegex.test(s)) return getURLVideoID(s)
+  throw Error(`No video id found: ${str}`)
+}
+
+const validateURL = str => {
+  try {
+    getURLVideoID(str)
+    return true
+  } catch {
+    return false
+  }
+}
+
+module.exports = {
+  getVideoID,
+  getURLVideoID,
+  validateID,
+  validateURL
+}

package/lib/utils.js

@@ -0,0 +1,294 @@
+const { request: undiciRequest } = require('undici')
+const { writeFileSync } = require('fs')
+const AGENT = require('./agents.js')
+const zlib = require('zlib')
+
+const between = (haystack, left, right) => {
+  let pos
+  if (left instanceof RegExp) {
+    const m = haystack.match(left)
+    if (!m) return ''
+    pos = m.index + m[0].length
+  } else {
+    pos = haystack.indexOf(left)
+    if (pos === -1) return ''
+    pos += left.length
+  }
+  haystack = haystack.slice(pos)
+  pos = haystack.indexOf(right)
+  if (pos === -1) return ''
+  return haystack.slice(0, pos)
+}
+
+function parseTime(text) {
+  if (!text) return null
+
+  const parts = text.split(':').map(Number)
+  let seconds = 0
+
+  for (const part of parts) {
+    seconds = seconds * 60 + part
+  }
+
+  return seconds
+}
+const tryParseBetween = (body, left, right, prepend = '', append = '') => {
+  try {
+    let data = between(body, left, right)
+    if (!data) return null
+    data = data.trim().replace(/[,;]$/, '')
+    if (data.endsWith('}}') && !data.endsWith('}}}')) data += '}'
+    let json = `${prepend}${data}${append}`.replace(/,\s*}/g, '}').replace(/,\s*]/g, ']')
+    return JSON.parse(json)
+  } catch {
+    try {
+      let data = between(body, left, right)
+      if (!data) return null
+      data = data
+        .replace(/"/g, '"')
+        .replace(/&/g, '&')
+        .replace(/'/g, "'")
+        .replace(/\\"/g, '"')
+        .replace(/\\[nt]/g, '')
+      const a = data.indexOf('{')
+      const b = data.lastIndexOf('}')
+      if (a !== -1 && b !== -1) return JSON.parse(data.slice(a, b + 1))
+      return null
+    } catch {
+      return null
+    }
+  }
+}
+
+const extractYouTubeJSON = (body, name) => {
+  const patterns = [
+    new RegExp(`var ${name}\\s*=\\s*({.+?});`, 'i'),
+    new RegExp(`"${name}"\\s*:\\s*({.+?})(?:,|$)`, 'i'),
+    new RegExp(`${name}\\s*[":=]\\s*({.+?})(?:[,;}]|$)`, 'i'),
+    new RegExp(`["']${name}["']\\s*:\\s*({.+?})(?:,|$)`, 'i')
+  ]
+
+  for (const p of patterns) {
+    const m = body.match(p)
+    if (!m) continue
+    try {
+      let j = m[1]
+      const o = (j.match(/{/g) || []).length
+      const c = (j.match(/}/g) || []).length
+      if (o > c) j += '}'.repeat(o - c)
+      j = j
+        .replace(/,\s*}/g, '}')
+        .replace(/,\s*]/g, ']')
+        .replace(/([{,]\s*)(\w+):/g, '$1"$2":')
+        .replace(/:\s*'([^']*)'/g, ':"$1"')
+      return JSON.parse(j)
+    } catch {}
+  }
+  return null
+}
+
+const parseAbbreviatedNumber = s => {
+  const m = s.replace(',', '.').replace(' ', '').match(/([\d.]+)([MK]?)/)
+  if (!m) return null
+  const n = parseFloat(m[1])
+  return Math.round(m[2] === 'M' ? n * 1e6 : m[2] === 'K' ? n * 1e3 : n)
+}
+
+const ESC = [
+  { start: '"', end: '"' },
+  { start: "'", end: "'" },
+  { start: '`', end: '`' },
+  { start: '/', end: '/', startPrefix: /(^|[[{:;,/])\s?$/ }
+]
+
+const cutAfterJS = src => {
+  const open = src[0] === '[' ? '[' : src[0] === '{' ? '{' : null
+  const close = open === '[' ? ']' : '}'
+  if (!open) throw Error(`Can't cut JSON starting with ${src[0]}`)
+
+  let esc = null
+  let slash = false
+  let depth = 0
+
+  for (let i = 0; i < src.length; i++) {
+    if (!slash && esc && src[i] === esc.end) {
+      esc = null
+      continue
+    }
+    if (!slash && !esc) {
+      for (const e of ESC) {
+        if (src[i] === e.start && (!e.startPrefix || src.slice(i - 10, i).match(e.startPrefix))) {
+          esc = e
+          break
+        }
+      }
+      if (esc) continue
+    }
+    slash = src[i] === '\\' && !slash
+    if (esc) continue
+    if (src[i] === open) depth++
+    if (src[i] === close) depth--
+    if (depth === 0) return src.slice(0, i + 1)
+  }
+  throw Error('Unclosed JSON')
+}
+
+class UnrecoverableError extends Error {}
+
+const playError = r => {
+  const p = r?.playabilityStatus
+  if (!p) return null
+  if (['ERROR', 'LOGIN_REQUIRED', 'LIVE_STREAM_OFFLINE', 'UNPLAYABLE'].includes(p.status)) {
+    return new UnrecoverableError(p.reason || p.messages?.[0])
+  }
+  return null
+}
+
+const useFetch = async (fetch, url, opts) => {
+  if (opts?.query) {
+    const u = new URL(url)
+    for (const k in opts.query) u.searchParams.append(k, opts.query[k])
+    url = u.toString()
+  }
+  const r = await fetch(url, opts)
+  return {
+    statusCode: r.status,
+    headers: Object.fromEntries(r.headers.entries()),
+    body: Object.assign(r, r.body || {})
+  }
+}
+
+const request = async (url, options = {}) => {
+  let { requestOptions = {}, rewriteRequest, fetch } = options
+
+  if (rewriteRequest) {
+    const r = rewriteRequest(url, requestOptions)
+    url = r.url || url
+    requestOptions = r.requestOptions || requestOptions
+  }
+
+  requestOptions.headers ||= {}
+  requestOptions.headers['Accept-Encoding'] ||= 'identity'
+
+  const res = fetch
+    ? await useFetch(fetch, url, requestOptions)
+    : await undiciRequest(url, requestOptions)
+
+  const code = String(res.statusCode)
+  if (code.startsWith('2')) {
+    if (res.headers['content-type']?.includes('json')) return res.body.json()
+    let text = await res.body.text()
+    if (/[\x00-\x1F\x7F-\xFF]/.test(text.slice(0, 100)) && res.body.arrayBuffer) {
+      try {
+        const buf = Buffer.from(await res.body.arrayBuffer())
+        if (res.headers['content-encoding'] === 'gzip') text = zlib.gunzipSync(buf).toString()
+        if (res.headers['content-encoding'] === 'deflate') text = zlib.inflateSync(buf).toString()
+      } catch {}
+    }
+    return text
+  }
+  if (code.startsWith('3')) return request(res.headers.location, options)
+  const e = Error(`Status code: ${code}`)
+  e.statusCode = res.statusCode
+  throw e
+}
+
+const deprecate = (obj, prop, val) =>
+  Object.defineProperty(obj, prop, { get: () => val })
+
+const getRandomIPv6 = ip => {
+  const [addr, maskRaw] = ip.split('/')
+  const mask = +maskRaw
+  if (!isIPv6(ip) || mask < 1 || mask > 128) throw Error('Invalid IPv6')
+  const base = normalizeIP(addr)
+  const full = Math.floor(mask / 16)
+  const rem = mask % 16
+  const out = new Array(8).fill(0)
+  for (let i = 0; i < 8; i++) {
+    if (i < full) out[i] = base[i]
+    else if (i === full && rem) {
+      const m = 0xffff << (16 - rem)
+      out[i] = (base[i] & m) | Math.floor(Math.random() * (1 << (16 - rem)))
+    } else out[i] = Math.floor(Math.random() * 0x10000)
+  }
+  return out.map(v => v.toString(16).padStart(4, '0')).join(':')
+}
+
+const isIPv6 = ip =>
+  /^(?:[0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}(?:\/\d{1,3})?$/.test(ip)
+
+const normalizeIP = ip => {
+  const [a, b] = ip.split('::')
+  const s = a ? a.split(':') : []
+  const e = b ? b.split(':') : []
+  const z = new Array(8 - s.length - e.length).fill('0')
+  return [...s, ...z, ...e].map(p => parseInt(p || '0', 16))
+}
+
+const saveDebugFile = (name, body) => {
+  if (process.env.YTDL_NO_DEBUG_FILE) return body
+  const p = process.env.YTDL_DEBUG_PATH || '.'
+  const f = `${Date.now()}-${name}`
+  writeFileSync(`${p}/${f}`, body)
+  return f
+}
+
+const findKeyI = (o, p) => Object.keys(o).find(k => k.toLowerCase() === p.toLowerCase())
+const getPropInsensitive = (o, p) => o[findKeyI(o, p)]
+const setPropInsensitive = (o, p, v) => (o[findKeyI(o, p) || p] = v)
+
+const applyDefaultAgent = o => {
+  if (o.agent) return
+  const { jar } = AGENT.defaultAgent
+  const c = getPropInsensitive(o.requestOptions.headers, 'cookie')
+  if (c) {
+    jar.removeAllCookiesSync()
+    AGENT.addCookiesFromString(jar, c)
+  }
+  o.agent = AGENT.defaultAgent
+}
+
+const applyIPv6Rotations = o => {
+  if (o.IPv6Block) {
+    o.requestOptions.localAddress = getRandomIPv6(o.IPv6Block)
+  }
+}
+
+const applyDefaultHeaders = o => {
+  o.requestOptions = { ...o.requestOptions }
+  o.requestOptions.headers = { 'Accept-Encoding': 'identity', ...o.requestOptions.headers }
+}
+
+const generateClientPlaybackNonce = l => {
+  const c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
+  let r = ''
+  while (l--) r += c[Math.random() * c.length | 0]
+  return r
+}
+
+const applyPlayerClients = o => {
+  if (!o.playerClients?.length) o.playerClients = ['WEB_EMBEDDED', 'IOS', 'ANDROID', 'TV']
+}
+const applyOldLocalAddress = () => {}
+
+module.exports = {
+  between,
+  tryParseBetween,
+  extractYouTubeJSON,
+  applyOldLocalAddress,
+  parseAbbreviatedNumber,
+  parseTime,
+  cutAfterJS,
+  UnrecoverableError,
+  playError,
+  request,
+  deprecate,
+  saveDebugFile,
+  getPropInsensitive,
+  setPropInsensitive,
+  applyDefaultAgent,
+  applyIPv6Rotations,
+  applyDefaultHeaders,
+  generateClientPlaybackNonce,
+  applyPlayerClients
+}

package/lib/xmlTosrt.js

@@ -0,0 +1,49 @@
+'use strict'
+
+function pad(num, size = 2) {
+  return String(num).padStart(size, '0')
+}
+
+function secToTime(sec) {
+  const ms = Math.floor((sec % 1) * 1000)
+  sec = Math.floor(sec)
+
+  const s = sec % 60
+  const m = Math.floor(sec / 60) % 60
+  const h = Math.floor(sec / 3600)
+
+  return (
+    pad(h) + ':' +
+    pad(m) + ':' +
+    pad(s) + ',' +
+    pad(ms, 3)
+  )
+}
+
+module.exports = function xmlToSrt(xml) {
+  const entries = [...xml.matchAll(
+    /<text start="([\d.]+)" dur="([\d.]+)">(.*?)<\/text>/gs
+  )]
+
+  let i = 1
+  let out = ''
+
+  for (const [, start, dur, text] of entries) {
+    const s = Number(start)
+    const e = s + Number(dur)
+
+    out +=
+      i++ + '\n' +
+      secToTime(s) + ' --> ' + secToTime(e) + '\n' +
+      text
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&#39;/g, "'")
+        .replace(/&quot;/g, '"')
+        .trim() +
+      '\n\n'
+  }
+
+  return out.trim()
+}

package/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "cloud-ytdl",
+  "version": "1.0.0-rc",
+  "description": "Reliable YouTube downloader and scraper using InnerTube clients. Includes video, stream, and Community Post extraction. Production-ready for Node.js 18+.",
+  "keywords": [
+    "youtube",
+    "ytdl",
+    "ytdl-core",
+    "downloader",
+    "video",
+    "audio",
+    "stream",
+    "innertube",
+    "signature-decoding",
+    "cipher",
+    "hls",
+    "dash",
+    "m3u8",
+    "community-post",
+    "poll",
+    "cjs",
+    "node18"
+  ],
+  "author": "AlfiDev (https://github.com/cloudkuimages)",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/cloudkuimages/ytdl-cloud.git"
+  },
+  "homepage": "https://github.com/cloudkuimages/ytdl-cloud",
+  "bugs": {
+    "url": "https://github.com/cloudkuimages/ytdl-cloud/issues"
+  },
+  "type": "commonjs",
+  "main": "./lib/index.js",
+  "types": "./types/index.d.ts",
+  "exports": {
+    ".": {
+      "require": "./lib/index.js",
+      "types": "./types/index.d.ts"
+    },
+    "./agents": "./lib/agents.js",
+    "./cache": "./lib/cache.js",
+    "./format": "./lib/format.js",
+    "./format-utils": "./lib/format-utils.js",
+    "./info": "./lib/info.js",
+    "./innertube": "./lib/innertube.js",
+    "./post": "./lib/post.js",
+    "./sig-decoder": "./lib/sig-decoder.js",
+    "./url-utils": "./lib/url-utils.js",
+    "./utils": "./lib/utils.js"
+  },
+  "files": [
+    "lib",
+    "types"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "test": "mocha --timeout 10000"
+  },
+  "dependencies": {
+    "http-cookie-agent": "^6.0.8",
+    "https-proxy-agent": "^7.0.6",
+    "m3u8stream": "^0.8.6",
+    "miniget": "^4.2.3",
+    "sax": "^1.4.1",
+    "tough-cookie": "^4.1.4",
+    "undici": "^6.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.0",
+    "eslint": "^8.56.0",
+    "mocha": "^10.3.0",
+    "typescript": "^5.3.3"
+  }
+}