atto-downloaders 3.0.0 → 3.0.1

package/README.md

@@ -86,6 +86,8 @@ Everything is initiated by pure JS — no shell script, no manual install, no ad
 5. Resolves ffmpeg: path → winget/static zip (Windows) → `pkg` (Termux) → `brew`/`apt`/`dnf`/`pacman`/`apk` → static `tar.xz` build (Linux, no root).
 6. Writes `state.json` — later calls take ~30 ms.
 
+**Alpine/musl note:** musl containers get python from official Alpine apks (extracted user-space, no root) — yt-dlp runs fully. The PO-token server's `canvas` has no musl prebuilt, so PO-token stays optional there: the ladder leans on client rotation and closes at the ATTO providers. A glibc image (`ghcr.io/parkervcp/yolks:nodejs_22`) enables the complete path.
+
 In an npm **global** install the package dir is root-owned, so the whole bootstrap lands in `~/.atto/` instead (override with `ATTO_HOME`).
 
 ## API reference

package/atto-ytdlp.js

@@ -335,6 +335,16 @@ function findBinary(o) {
   return which('yt-dlp') || which('yt-dlp.exe') || which('yt-dlp.cmd') || which('yt-dlp.bat');
 }
 
+// musl (Alpine) containers cannot run glibc portable builds — detect before wasting a download
+function isMusl() {
+  if (process.platform !== 'linux') return false;
+  try { return !process.report.getReport().header.glibcVersionRuntime; } catch (_) { /* fall through */ }
+  try {
+    const r = spawnSync('ldd', ['--version'], { encoding: 'utf8', windowsHide: true });
+    return /musl/i.test(`${r.stderr || ''}${r.stdout || ''}`);
+  } catch (_) { return true; }
+}
+
 let pyCache = null;
 function resolvePython() {
   if (pyCache !== null) return pyCache;
@@ -1031,7 +1041,7 @@ async function setupPot(o) {
     if (!fs.existsSync(path.join(dir, 'server', 'package.json'))) {
       if (fs.existsSync(dir)) { say.err(`${dir} exists but has no server/ — remove it or pass --pot-dir`); return; }
       say.info(`cloning bgutil provider @ ${POT_BRANCH}`);
-      const clone = runTool('git', ['clone', '--depth', '1', '--branch', POT_BRANCH, POT_REPO, dir],
+      const clone = runTool('git', ['-c', 'advice.detachedHead=false', 'clone', '--depth', '1', '--branch', POT_BRANCH, POT_REPO, dir],
         { stdio: 'inherit' });
       if (clone.status !== 0) { say.err('git clone failed'); return; }
     }
@@ -1052,7 +1062,13 @@ async function setupPot(o) {
       const serverDir = path.join(dir, 'server');
       say.info('npm ci (this pulls botguard + canvas)');
       if (runTool('npm', ['ci'], { cwd: serverDir, stdio: 'inherit' }).status !== 0) {
-        say.err('npm ci failed'); return;
+        if (isMusl()) {
+          say.err('npm ci failed: canvas ships no musl prebuilt — the PO-token server needs a glibc image.');
+          say.err('optional: switch the egg image to ghcr.io/parkervcp/yolks:nodejs_22. Without PO-token the ladder runs pot-less (client rotation) and still closes at ATTO providers.');
+        } else {
+          say.err('npm ci failed');
+        }
+        return;
       }
       say.info('npx tsc');
       if (runTool('npx', ['tsc'], { cwd: serverDir, stdio: 'inherit' }).status !== 0) {
@@ -1103,7 +1119,7 @@ async function main() {
 module.exports = {
   parseArgs, runUrl, doctor, setupPot, ensurePot, probePot,
   classify, findBinary, buildStrategies, applyTypePreset,
-  spawnSpec, resolvePython, have, which, run, say,
+  spawnSpec, resolvePython, have, which, run, say, isMusl,
   AUDIO_TYPES, RENDER_TYPES, SOURCE_TYPES, POT_PORT,
 };
 

package/package.json

@@ -1,69 +1,72 @@
-{
-  "name": "atto-downloaders",
-  "version": "3.0.0",
-  "description": "Typed media downloader for sites and bots - yt-dlp ladder with PO-token provider, stream-copy exports (zero re-encode), provider fallback, zero-setup bootstrap on Windows/Linux/macOS/Termux",
-  "main": "index.js",
-  "types": "index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./index.d.ts",
-      "require": "./index.js",
-      "default": "./index.js"
-    },
-    "./providers": "./atto-downloaders.js",
-    "./ytdlp": "./atto-ytdlp.js",
-    "./package.json": "./package.json"
-  },
-  "bin": {
-    "atto": "start-atto.js"
-  },
-  "files": [
-    "index.js",
-    "index.d.ts",
-    "start-atto.js",
-    "atto-ytdlp.js",
-    "atto-downloaders.js",
-    "build-ytdlp.sh",
-    "README.md"
-  ],
-  "scripts": {
-    "start": "node start-atto.js",
-    "setup": "node start-atto.js --setup",
-    "atto": "node start-atto.js",
-    "test": "node --check index.js && node --check start-atto.js && node --check atto-ytdlp.js && node --check atto-downloaders.js && node -e \"require('./index.js')\"",
-    "help": "node start-atto.js --help",
-    "providers": "node atto-downloaders.js --list-providers",
-    "build": "bash build-ytdlp.sh",
-    "doctor": "node start-atto.js --doctor",
-    "smoke": "node atto-ytdlp.js --doctor --smoke https://www.youtube.com/watch?v=dQw4w9WgXcQ",
-    "pot": "node atto-ytdlp.js --setup-pot",
-    "ytdl": "node atto-ytdlp.js",
-    "prepublishOnly": "npm test"
-  },
-  "keywords": [
-    "yt-dlp",
-    "youtube-dl",
-    "downloader",
-    "media",
-    "video",
-    "audio",
-    "mp3",
-    "mp4",
-    "bot",
-    "scraper",
-    "savetube",
-    "ytultra",
-    "saveclip",
-    "po-token",
-    "cli",
-    "library"
-  ],
-  "author": "Zyon",
-  "license": "MIT",
-  "optionalDependencies": {
-    "playwright-core": "^1.64.1"
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  }
-}
+{
+  "name": "atto-downloaders",
+  "version": "3.0.1",
+  "description": "Typed media downloader for sites and bots - yt-dlp ladder with PO-token provider, stream-copy exports (zero re-encode), provider fallback, zero-setup bootstrap on Windows/Linux/macOS/Termux",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "require": "./index.js",
+      "default": "./index.js"
+    },
+    "./providers": "./atto-downloaders.js",
+    "./ytdlp": "./atto-ytdlp.js",
+    "./package.json": "./package.json"
+  },
+  "bin": {
+    "atto": "start-atto.js"
+  },
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "start-atto.js",
+    "atto-ytdlp.js",
+    "atto-downloaders.js",
+    "build-ytdlp.sh",
+    "README.md"
+  ],
+  "scripts": {
+    "start": "node start-atto.js",
+    "setup": "node start-atto.js --setup",
+    "atto": "node start-atto.js",
+    "test": "node --check index.js && node --check start-atto.js && node --check atto-ytdlp.js && node --check atto-downloaders.js && node -e \"require('./index.js')\"",
+    "help": "node start-atto.js --help",
+    "providers": "node atto-downloaders.js --list-providers",
+    "build": "bash build-ytdlp.sh",
+    "doctor": "node start-atto.js --doctor",
+    "smoke": "node atto-ytdlp.js --doctor --smoke https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+    "pot": "node atto-ytdlp.js --setup-pot",
+    "ytdl": "node atto-ytdlp.js",
+    "prepublishOnly": "npm test"
+  },
+  "keywords": [
+    "yt-dlp",
+    "youtube-dl",
+    "downloader",
+    "media",
+    "video",
+    "audio",
+    "mp3",
+    "mp4",
+    "bot",
+    "scraper",
+    "savetube",
+    "ytultra",
+    "saveclip",
+    "po-token",
+    "cli",
+    "library"
+  ],
+  "author": "Zyon",
+  "license": "MIT",
+  "optionalDependencies": {
+    "playwright-core": "^1.64.1"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "atto-downloaders": "^3.0.0"
+  }
+}

package/start-atto.js

@@ -42,6 +42,11 @@ const PYTHON_DIR = path.join(DIST, 'python');
 const PBS_VER = process.env.ATTO_PY_VER || '3.13.16';
 const PBS_TAG = process.env.ATTO_PY_TAG || '20261013';
 const PBS_URL = triple => `https://github.com/astral-sh/python-build-standalone/releases/download/${PBS_TAG}/cpython-${PBS_VER}+${PBS_TAG}-${triple}-install_only.tar.gz`;
+// musl containers: python from official Alpine apks (extracted user-space, no root);
+// repo paths are per-architecture: /alpine/v<ver>/main/<arch>/
+const ALPINE_VER = process.env.ATTO_ALPINE_VER || '3.22';
+const ALPINE_ARCH = { x64: 'x86_64', arm64: 'aarch64', arm: 'armv7' }[process.arch] || process.arch;
+const ALPINE_REPO = `https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VER}/main/${ALPINE_ARCH}`;
 const YTDLP_PIN = process.env.ATTO_YTDLP_VERSION || '2026.08.19';
 const POT_BRANCH = process.env.ATTO_POT_BRANCH || '2.0.2';
 const POT_TARBALL = `https://codeload.github.com/Brainicism/bgutil-ytdlp-pot-provider/tar.gz/refs/heads/${POT_BRANCH}`;
@@ -74,6 +79,8 @@ function findSystemPython() {
 
 // provisioned portable python wins over PATH — pinned, deterministic, zero admin
 function portablePythonCmd() {
+  const alpineWrap = path.join(PYTHON_DIR, 'bin', 'atto-python');
+  if (fs.existsSync(alpineWrap)) return { cmd: alpineWrap, prefix: [], name: 'alpine', portable: true };
   const exe = process.platform === 'win32'
     ? path.join(PYTHON_DIR, 'python.exe')
     : path.join(PYTHON_DIR, 'bin', 'python3');
@@ -200,6 +207,13 @@ function findPyRoot(dir, depth = 0) {
 async function ensurePortablePython() {
   const have = portablePythonCmd();
   if (have) return have;
+  if (ytdlp.isMusl()) {
+    // glibc builds never run here — pull python from official Alpine apks instead
+    const alpine = await ensureAlpinePython();
+    if (alpine) return alpine;
+    say.warn('alpine python bootstrap failed — a glibc image (ghcr.io/parkervcp/yolks:nodejs_22) removes this problem');
+    return null;
+  }
   const triple = pbsTriple();
   if (!triple) { say.warn(`no portable python build for ${process.arch}/${process.platform}`); return null; }
   const tar = ytdlp.which('tar');
@@ -235,6 +249,108 @@ async function ensurePortablePython() {
   return port;
 }
 
+// musl containers: official Alpine python3 + dependency closure, extracted from .apk
+// tarballs into dist/python (apks ARE gzipped tars) — no root, no apk tooling. The
+// container's musl loader serves the binaries; a wrapper exports LD_LIBRARY_PATH and
+// PYTHONHOME so the relocated prefix resolves its own stdlib.
+function parseApkIndex(text) {
+  const pkgs = new Map();
+  let cur = null;
+  for (const line of text.split(/\r?\n/)) {
+    if (!line) { if (cur && cur.name) pkgs.set(cur.name, cur); cur = null; continue; }
+    const k = line.slice(0, 2), v = line.slice(2);
+    if (k === 'P:') cur = { name: v, ver: '', deps: [], provides: [] };
+    else if (!cur) continue;
+    else if (k === 'V:') cur.ver = v;
+    else if (k === 'D:') cur.deps = v.split(/\s+/).filter(Boolean);
+    else if (k === 'p:') cur.provides = v.split(/\s+/).filter(Boolean);
+  }
+  if (cur && cur.name) pkgs.set(cur.name, cur);
+  return pkgs;
+}
+
+// resolve the dependency closure of a root package; so: providers map to their package,
+// cmd:/pc:/path: tokens live in busybox/toolchain the container already has — skipped
+function apkClosure(pkgs, root) {
+  const provider = new Map();
+  for (const p of pkgs.values()) for (const pr of p.provides) {
+    const tok = pr.split('=')[0];
+    if (!provider.has(tok)) provider.set(tok, p.name);
+  }
+  const out = new Map();
+  const queue = [root];
+  while (queue.length) {
+    const name = queue.shift();
+    if (out.has(name)) continue;
+    const p = pkgs.get(name);
+    if (!p) continue;
+    out.set(name, p);
+    for (let d of p.deps) {
+      d = d.split('=')[0];
+      if (!d || d.startsWith('!') || d === 'musl' || d.startsWith('cmd:') || d.startsWith('pc:') || d.startsWith('path:')) continue;
+      if (d.startsWith('so:')) {
+        const hit = provider.get(d);
+        if (hit && hit !== 'musl') queue.push(hit);
+      } else if (pkgs.has(d)) queue.push(d);
+      else if (provider.has(d)) queue.push(provider.get(d));
+    }
+  }
+  return [...out.values()];
+}
+
+async function ensureAlpinePython() {
+  const wrapper = path.join(PYTHON_DIR, 'bin', 'atto-python');
+  if (fs.existsSync(wrapper)) return { cmd: wrapper, prefix: [], name: 'alpine', portable: true };
+  const tar = ytdlp.which('tar');
+  if (!tar) { say.warn('tar missing — cannot unpack Alpine python'); return null; }
+  const idx = path.join(CACHE, `APKINDEX-${ALPINE_VER}.tar.gz`);
+  if (!fs.existsSync(idx)) {
+    say.info(`fetching Alpine ${ALPINE_VER} package index`);
+    try { await downloadFile(`${ALPINE_REPO}/APKINDEX.tar.gz`, idx, 'APKINDEX'); }
+    catch (e) { say.warn(`alpine index download failed: ${e.message}`); return null; }
+  }
+  const tmp = path.join(CACHE, 'apkindex-x');
+  fs.rmSync(tmp, { recursive: true, force: true });
+  fs.mkdirSync(tmp, { recursive: true });
+  if (runTool(tar, ['-xzf', idx, '-C', tmp], { stdio: 'ignore' }).status !== 0) {
+    say.warn('APKINDEX unpack failed'); return null;
+  }
+  let text = '';
+  try { text = fs.readFileSync(path.join(tmp, 'APKINDEX'), 'utf8'); } catch (_) { say.warn('APKINDEX missing'); return null; }
+  const pkgs = parseApkIndex(text);
+  if (!pkgs.has('python3')) { say.warn('python3 not in alpine main repo'); return null; }
+
+  const closure = apkClosure(pkgs, 'python3');
+  say.info(`installing alpine python ${pkgs.get('python3').ver} + ${closure.length - 1} deps (user dir, no root)`);
+  fs.mkdirSync(PYTHON_DIR, { recursive: true });
+  for (const p of closure) {
+    const apk = path.join(CACHE, `${p.name}-${p.ver}.apk`);
+    if (!fs.existsSync(apk)) {
+      try { await downloadFile(`${ALPINE_REPO}/${p.name}-${p.ver}.apk`, apk, p.name); }
+      catch (e) { say.warn(`${p.name}: download failed (${e.message}) — continuing`); continue; }
+    }
+    if (runTool(tar, ['-xzf', apk, '-C', PYTHON_DIR], { stdio: 'ignore' }).status !== 0) {
+      say.warn(`${p.name}: unpack failed — continuing`);
+    }
+  }
+  const pyBin = path.join(PYTHON_DIR, 'usr', 'bin', 'python3');
+  if (!fs.existsSync(pyBin)) { say.warn('python3 missing after apk extraction'); return null; }
+  fs.mkdirSync(path.dirname(wrapper), { recursive: true });
+  fs.writeFileSync(wrapper, '#!/bin/sh\n' +
+    'HERE=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)\n' +
+    'export LD_LIBRARY_PATH="$HERE/usr/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"\n' +
+    'export PYTHONHOME="$HERE/usr"\n' +
+    'exec "$HERE/usr/bin/python3" "$@"\n');
+  fs.chmodSync(wrapper, 0o755);
+  const probe = spawnSync(wrapper, ['-V'], { encoding: 'utf8', windowsHide: true });
+  if (probe.status !== 0) {
+    say.warn(`alpine python failed to start: ${(probe.stderr || probe.stdout || '').trim().slice(0, 200)}`);
+    return null;
+  }
+  say.ok(`alpine python ready: ${(probe.stdout || '').trim()}`);
+  return { cmd: wrapper, prefix: [], name: 'alpine', portable: true };
+}
+
 // ---------------- setup steps ----------------
 
 async function stepPython(opts) {
@@ -251,6 +367,9 @@ async function stepPython(opts) {
   if (port) return { ok: true, how: 'portable', at: port.cmd };
   const sys = findSystemPython();
   if (sys) return { ok: true, how: 'found', at: sys.cmd };
+  if (ytdlp.isMusl()) {
+    return { ok: false, msg: 'musl: alpine python bootstrap failed — a glibc image (ghcr.io/parkervcp/yolks:nodejs_22) fixes this' };
+  }
   const plat = platInfo();
   say.warn('portable python unavailable — trying the package manager');
   if (plat.name === 'android') {
@@ -341,8 +460,17 @@ async function stepYtDlp(prevMinor) {
   // phase 1 — core (must succeed)
   let pr = pipRun([`yt-dlp==${want}`, 'yt-dlp-ejs']);
   if (pr.status !== 0) {
-    say.warn('pip missing or failed — running ensurepip, then retrying');
+    say.warn('pip missing or failed — bootstrapping pip, then retrying');
     runTool(py.cmd, [...py.prefix, '-m', 'ensurepip', '--upgrade'], { encoding: 'utf8', windowsHide: true });
+    if (runTool(py.cmd, [...py.prefix, '-m', 'pip', '--version'], { encoding: 'utf8', windowsHide: true }).status !== 0) {
+      // Alpine pythons ship without pip/ensurepip — get-pip.py is pure python
+      try {
+        const gp = path.join(CACHE, 'get-pip.py');
+        if (!fs.existsSync(gp)) await downloadFile('https://bootstrap.pypa.io/get-pip.py', gp, 'get-pip');
+        runTool(py.cmd, [...py.prefix, gp, '--disable-pip-version-check', '--no-warn-script-location'],
+          { encoding: 'utf8', windowsHide: true });
+      } catch (e) { say.warn(`get-pip failed: ${e.message}`); }
+    }
     pr = pipRun([`yt-dlp==${want}`, 'yt-dlp-ejs']);
     if (pr.status !== 0) {
       const tail = pr.out.trim().split(/\r?\n/).slice(-6).join('\n');
@@ -815,7 +943,7 @@ async function cli() {
   process.exit(failed ? 1 : 0);
 }
 
-module.exports = { setup, download, links, formats, doctor, platInfo, TYPES, ROOT };
+module.exports = { setup, download, links, formats, doctor, platInfo, TYPES, ROOT, parseApkIndex, apkClosure };
 
 if (require.main === module) {
   cli().catch(e => { say.err(e.stack || e.message); process.exit(1); });