@koda-sl/baker-cli 0.192.4 → 0.196.0-dev.6d2f498f5

package/dist/{chunk-ISZWNERZ.js → chunk-SSDYBIZB.js}

@@ -1,15 +1,21 @@
 import {
   debugLogHttp,
   readBodyForLog
-} from "./chunk-K3PWXVF7.js";
+} from "./chunk-YUTDQ4PV.js";
 import {
   __commonJS,
-  __toESM
-} from "./chunk-YL3HDEIJ.js";
+  __toESM,
+  captureProxyCredentials,
+  childEnvWith,
+  isChallengeBody,
+  isProxyFailure,
+  plannedRoutes,
+  shouldEscalate
+} from "./chunk-F6OHDJIX.js";
 
-// ../../node_modules/.pnpm/safe-stable-stringify@2.5.0/node_modules/safe-stable-stringify/index.js
+// ../../.pnpm-store/v10/links/@/safe-stable-stringify/2.5.0/810146e81bae4e3a061fe487864f2fde80c4b03b886877dc0f1fffbc6480b67e/node_modules/safe-stable-stringify/index.js
 var require_safe_stable_stringify = __commonJS({
-  "../../node_modules/.pnpm/safe-stable-stringify@2.5.0/node_modules/safe-stable-stringify/index.js"(exports, module) {
+  "../../.pnpm-store/v10/links/@/safe-stable-stringify/2.5.0/810146e81bae4e3a061fe487864f2fde80c4b03b886877dc0f1fffbc6480b67e/node_modules/safe-stable-stringify/index.js"(exports, module) {
     "use strict";
     var { hasOwnProperty } = Object.prototype;
     var stringify = configure2();
@@ -1083,7 +1089,7 @@ function resolveAdaptFormats(params) {
   return params.formats ?? [];
 }
 
-// ../../node_modules/.pnpm/safe-stable-stringify@2.5.0/node_modules/safe-stable-stringify/esm/wrapper.js
+// ../../.pnpm-store/v10/links/@/safe-stable-stringify/2.5.0/810146e81bae4e3a061fe487864f2fde80c4b03b886877dc0f1fffbc6480b67e/node_modules/safe-stable-stringify/esm/wrapper.js
 var import__ = __toESM(require_safe_stable_stringify(), 1);
 var configure = import__.default.configure;
 var wrapper_default = import__.default;
@@ -4909,6 +4915,227 @@ import path4 from "path";
 import { promisify } from "util";
 import { z as z5 } from "zod";
 
+// ../proxy/src/fetch.ts
+import { ProxyAgent, Response, fetch as undiciFetch } from "undici";
+
+// ../proxy/src/fetchDecision.ts
+function classifyFetchOutcome(signal) {
+  if (isProxyFailure(signal)) return "proxy_unavailable";
+  if (shouldEscalate(signal)) return "blocked";
+  if (signal.status != null && signal.status >= 200 && signal.status < 400 && !signal.challenge) return "ok";
+  return "failed";
+}
+function shouldTryNextRoute(outcome) {
+  return outcome === "blocked" || outcome === "proxy_unavailable";
+}
+var TIMEOUT_CODES = /* @__PURE__ */ new Set([
+  "UND_ERR_CONNECT_TIMEOUT",
+  "UND_ERR_HEADERS_TIMEOUT",
+  "UND_ERR_BODY_TIMEOUT",
+  "ETIMEDOUT"
+]);
+var RESET_CODES = /* @__PURE__ */ new Set(["ECONNRESET", "UND_ERR_SOCKET", "EPIPE"]);
+var PROXY_RESPONSE = /Proxy response \((\d+)\)/;
+function errorCode(error) {
+  if (typeof error !== "object" || error === null) return null;
+  const code = error.code;
+  if (typeof code === "string") return code;
+  const name = error.name;
+  return typeof name === "string" ? name : null;
+}
+function errorMessage(error) {
+  if (typeof error === "object" && error !== null) {
+    const message2 = error.message;
+    if (typeof message2 === "string") return message2;
+  }
+  return String(error);
+}
+function errorBlockSignal(error, proxied) {
+  const proxyResponse = PROXY_RESPONSE.exec(errorMessage(error));
+  if (proxyResponse) {
+    const status = Number(proxyResponse[1]);
+    return status === 407 ? { status: 407 } : { netError: "ERR_TUNNEL_CONNECTION_FAILED" };
+  }
+  const code = errorCode(error);
+  if (code !== null && TIMEOUT_CODES.has(code)) return { timedOut: true };
+  if (code === "AbortError" || code === "TimeoutError") return { timedOut: true };
+  if (proxied) {
+    return { netError: "ERR_PROXY_CONNECTION_FAILED" };
+  }
+  if (code !== null && RESET_CODES.has(code)) return { netError: "ERR_CONNECTION_RESET" };
+  return { netError: code ?? "ERR_FAILED" };
+}
+function shouldSniffChallenge(contentType, contentLength, maxSniffBytes) {
+  if (contentLength !== null && contentLength > maxSniffBytes) return false;
+  if (contentType === null || contentType.trim() === "") return true;
+  const type = contentType.toLowerCase();
+  if (type.startsWith("image/") || type.startsWith("video/") || type.startsWith("audio/")) return false;
+  return type.includes("text/") || type.includes("html") || type.includes("json") || type.includes("xml");
+}
+
+// ../proxy/src/fetch.ts
+var DEFAULT_TIMEOUT_MS = 6e4;
+var MAX_SNIFF_BYTES = 64 * 1024;
+function tierOf(route) {
+  return route.kind === "proxy" ? route.tier : "direct";
+}
+function agentFor(route) {
+  if (route.kind !== "proxy") return null;
+  const token = Buffer.from(`${route.username}:${route.password}`).toString("base64");
+  return new ProxyAgent({ uri: route.server, token: `Basic ${token}` });
+}
+var NULL_BODY_STATUS = /* @__PURE__ */ new Set([101, 103, 204, 205, 304]);
+function rebuild(response, body) {
+  if (response.status < 200 || response.status > 599 || NULL_BODY_STATUS.has(response.status)) {
+    return new Response(null, { status: response.status, headers: response.headers });
+  }
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers: response.headers
+  });
+}
+function declaredLength(response) {
+  const raw = response.headers.get("content-length");
+  if (raw === null) return null;
+  const value = Number(raw);
+  return Number.isFinite(value) ? value : null;
+}
+async function readOnce(response, sniff, maxBytes) {
+  const declared = declaredLength(response);
+  if (maxBytes !== void 0 && declared !== null && declared > maxBytes) {
+    await response.body?.cancel();
+    return { response: rebuild(response, new ArrayBuffer(0)), challenge: false };
+  }
+  const body = await response.arrayBuffer();
+  const wantsSniff = sniff && shouldSniffChallenge(response.headers.get("content-type"), body.byteLength, MAX_SNIFF_BYTES);
+  const challenge = wantsSniff ? isChallengeBody(new TextDecoder().decode(body)) : false;
+  return { response: rebuild(response, body), challenge };
+}
+async function attemptRoute(url, init, route, options) {
+  const tier = tierOf(route);
+  const agent = agentFor(route);
+  const startedAt = Date.now();
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const signal = init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs);
+  try {
+    const raw = await undiciFetch(url, { ...init, dispatcher: agent ?? void 0, signal });
+    const read = await readOnce(raw, options.sniffChallenge ?? true, options.maxBytes);
+    const outcome = classifyFetchOutcome({ status: raw.status, challenge: read.challenge });
+    return {
+      attempt: { tier, status: raw.status, ms: Date.now() - startedAt, outcome },
+      response: outcome === "proxy_unavailable" ? null : read.response
+    };
+  } catch (error) {
+    const blockSignal = errorBlockSignal(error, route.kind === "proxy");
+    return {
+      attempt: {
+        tier,
+        status: blockSignal.status ?? void 0,
+        ms: Date.now() - startedAt,
+        outcome: classifyFetchOutcome(blockSignal)
+      },
+      response: null,
+      error
+    };
+  } finally {
+    await agent?.close();
+  }
+}
+async function fetchWithEscalation(url, init, options) {
+  const attempts = [];
+  let originResponse = null;
+  let originTier = "direct";
+  let lastError;
+  for (const route of plannedRoutes(url, options.credentials, options.routeOptions)) {
+    const rung = await attemptRoute(url, init, route, options);
+    attempts.push(rung.attempt);
+    if (rung.response !== null) {
+      originResponse = rung.response;
+      originTier = rung.attempt.tier;
+    }
+    if (rung.error !== void 0) lastError = rung.error;
+    if (!shouldTryNextRoute(rung.attempt.outcome)) break;
+  }
+  return {
+    response: originResponse,
+    attempts,
+    finalTier: originResponse !== null ? originTier : lastAttempt(attempts)?.tier ?? "direct",
+    ...originResponse === null && lastError !== void 0 ? { error: lastError } : {}
+  };
+}
+function lastAttempt(attempts) {
+  return attempts[attempts.length - 1];
+}
+function describeLadder(result) {
+  const rungs = result.attempts.map((a) => `${a.tier}:${a.outcome}${a.status ? `(${a.status})` : ""}@${a.ms}ms`);
+  return `${rungs.join(" \u2192 ")} | final=${result.finalTier}`;
+}
+function rescued(result) {
+  return result.attempts.some((attempt) => attempt.outcome === "ok");
+}
+function endedBlocked(result) {
+  if (rescued(result)) return false;
+  return result.attempts.some((attempt) => attempt.outcome === "blocked");
+}
+function endedProxyUnavailable(result) {
+  if (rescued(result) || endedBlocked(result)) return false;
+  return result.attempts.some((attempt) => attempt.outcome === "proxy_unavailable");
+}
+
+// src/lib/externalFetch.ts
+var BROWSER_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+  Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9"
+};
+var ExternalFetchError = class extends Error {
+  /** True when the fault was ours — a route we could not open, not a dead host. */
+  ourFault;
+  constructor(message2, ourFault) {
+    super(message2);
+    this.name = "ExternalFetchError";
+    this.ourFault = ourFault;
+  }
+};
+async function fetchExternal(url, options = {}) {
+  const result = await fetchWithEscalation(
+    url,
+    { headers: { ...BROWSER_HEADERS, ...options.headers } },
+    {
+      credentials: captureProxyCredentials(),
+      ...options.timeoutMs === void 0 ? {} : { timeoutMs: options.timeoutMs },
+      ...options.maxBytes === void 0 ? {} : { maxBytes: options.maxBytes },
+      ...options.sniffChallenge === void 0 ? {} : { sniffChallenge: options.sniffChallenge }
+    }
+  );
+  if (result.attempts.length > 1) {
+    process.stderr.write(`fetch ladder ${url} \u2014 ${describeLadder(result)}
+`);
+  }
+  if (result.response === null) {
+    const ourFault = endedProxyUnavailable(result);
+    throw new ExternalFetchError(
+      ourFault ? "Could not open a route to that address." : "That address did not answer.",
+      ourFault
+    );
+  }
+  return { response: result.response, blocked: endedBlocked(result) };
+}
+async function fetchExternalBytes(url, options = {}) {
+  const { response, blocked } = await fetchExternal(url, options);
+  if (!response.ok) {
+    throw new ExternalFetchError(`${response.status} ${response.statusText}`, false);
+  }
+  if (blocked) {
+    throw new ExternalFetchError("That host served a security check instead of the file.", false);
+  }
+  return {
+    buffer: Buffer.from(await response.arrayBuffer()),
+    contentType: response.headers.get("content-type")
+  };
+}
+
 // src/engine/schema/refs.ts
 import { z as z4 } from "zod";
 
@@ -5104,6 +5331,25 @@ function mapClientError(ctx, e) {
   return new NodeExecutionError(ctx.nodeId, ctx.nodeType, { kind: "local", cause: e });
 }
 
+// src/engine/nodes/ytDlpSignal.ts
+var HTTP_ERROR = /HTTP Error (\d{3})/i;
+var CHALLENGE_PHRASES = [
+  "sign in to confirm you're not a bot",
+  "sign in to confirm you\u2019re not a bot",
+  "confirm you are not a bot",
+  "unusual traffic",
+  "captcha"
+];
+var NOT_A_BLOCK_PHRASES = ["private video", "video unavailable", "members-only", "this video is available to"];
+function ytDlpBlockSignal(stderr) {
+  const text = stderr.toLowerCase();
+  if (NOT_A_BLOCK_PHRASES.some((phrase) => text.includes(phrase))) return {};
+  const status = HTTP_ERROR.exec(stderr);
+  if (status) return { status: Number(status[1]) };
+  if (CHALLENGE_PHRASES.some((phrase) => text.includes(phrase))) return { challenge: true };
+  return {};
+}
+
 // src/engine/nodes/ingest.ts
 var execFile = promisify(execFileCb);
 var ExpectEnum = z5.enum(["image", "video", "audio", "text", "json", "font"]);
@@ -5221,17 +5467,18 @@ async function execDirectFetch(params, ctx) {
   return assertAssetOutput(result, params.expect);
 }
 async function ingestImageUrl(url, ctx) {
-  const res = await fetch(url);
-  if (!res.ok) {
-    throw localExecError(ctx, `fetch ${url} \u2192 ${res.status}`);
+  let bytes;
+  try {
+    ({ buffer: bytes } = await fetchExternalBytes(url, { maxBytes: MAX_ASSET_BYTES }));
+  } catch (e) {
+    throw localExecError(ctx, `fetch ${url} \u2192 ${e.message}`);
   }
-  const ab = await res.arrayBuffer();
-  if (ab.byteLength > MAX_ASSET_BYTES) {
-    throw localExecError(ctx, `file_too_large: ${url} is ${ab.byteLength} bytes (limit ${MAX_ASSET_BYTES})`);
+  if (bytes.byteLength > MAX_ASSET_BYTES) {
+    throw localExecError(ctx, `file_too_large: ${url} is ${bytes.byteLength} bytes (limit ${MAX_ASSET_BYTES})`);
   }
   let normalized;
   try {
-    normalized = await toModelSafeImage(Buffer.from(ab));
+    normalized = await toModelSafeImage(bytes);
   } catch (e) {
     throw localExecError(ctx, `${url}: ${e.message}`);
   }
@@ -5567,6 +5814,29 @@ async function execYtDlp(params, ctx) {
     });
   }
 }
+async function runYtDlpViaLadder(url, argv, ctx) {
+  const routes = plannedRoutes(url, captureProxyCredentials());
+  let last = { code: -1, stderr: "" };
+  for (const route of routes) {
+    if (route.kind === "proxy") ctx.log(`ingest: retrying yt-dlp via ${route.tier}`);
+    last = await spawnAndWait(YT_DLP_BIN, argv, YT_DLP_TIMEOUT_MS, proxyEnvFor(route));
+    if (last.code === 0) return last;
+    if (!shouldEscalate(ytDlpBlockSignal(last.stderr))) return last;
+  }
+  return last;
+}
+function proxyEnvFor(route) {
+  if (route.kind !== "proxy") return void 0;
+  const uri = new URL(route.server);
+  uri.username = encodeURIComponent(route.username);
+  uri.password = encodeURIComponent(route.password);
+  return {
+    HTTP_PROXY: uri.toString(),
+    HTTPS_PROXY: uri.toString(),
+    http_proxy: uri.toString(),
+    https_proxy: uri.toString()
+  };
+}
 async function runYtDlp(args) {
   const outTemplate = path4.join(args.workDir, "out.%(ext)s");
   const infoPath = path4.join(args.workDir, "out.info.json");
@@ -5586,7 +5856,7 @@ async function runYtDlp(args) {
     argv.push("--format", "bv*+ba/b", "--merge-output-format", "mp4");
   }
   args.ctx.log(`ingest: spawning yt-dlp (${args.audioOnly ? "audio" : "video"})`);
-  const { code, stderr } = await spawnAndWait(YT_DLP_BIN, argv, YT_DLP_TIMEOUT_MS);
+  const { code, stderr } = await runYtDlpViaLadder(args.url, argv, args.ctx);
   if (code !== 0) {
     throw new NodeExecutionError(args.ctx.nodeId, args.ctx.nodeType, {
       kind: "local",
@@ -5633,9 +5903,12 @@ async function uploadAndIngest(args) {
   });
   return { ...localRef, url: publicUrl };
 }
-function spawnAndWait(bin, argv, timeoutMs) {
+function spawnAndWait(bin, argv, timeoutMs, env) {
   return new Promise((resolve, reject) => {
-    const child = spawn(bin, argv, { stdio: ["ignore", "pipe", "pipe"] });
+    const child = spawn(bin, argv, {
+      stdio: ["ignore", "pipe", "pipe"],
+      ...env ? { env: childEnvWith(env) } : {}
+    });
     const stderrChunks = [];
     let stderrLen = 0;
     const MAX_STDERR = 256 * 1024;
@@ -5700,7 +5973,7 @@ import { tmpdir as tmpdir2 } from "os";
 import path5 from "path";
 import { promisify as promisify2 } from "util";
 var execFile2 = promisify2(execFileCb2);
-var DEFAULT_TIMEOUT_MS = 10 * 60 * 1e3;
+var DEFAULT_TIMEOUT_MS2 = 10 * 60 * 1e3;
 var URL_SCHEME_RE = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//;
 var MIME_BY_EXT = {
   png: "image/png",
@@ -5852,7 +6125,7 @@ function tailLines(text, maxLines) {
   return lines.slice(-maxLines).join("\n");
 }
 async function runCli(opts) {
-  const { bin, args, inputs, outputs, ctx, timeoutMs = DEFAULT_TIMEOUT_MS } = opts;
+  const { bin, args, inputs, outputs, ctx, timeoutMs = DEFAULT_TIMEOUT_MS2 } = opts;
   const tmpDir = await mkdtemp2(path5.join(tmpdir2(), `cli-${bin.replace(/[^a-z0-9]/gi, "")}-`));
   try {
     const { lookup, stagedInputs, outputPaths } = planPlaceholders(tmpDir, inputs, outputs);
@@ -8365,6 +8638,7 @@ export {
   spineInputFlags,
   spineInputOps,
   elementMentionKeywords,
+  fetchExternalBytes,
   toModelSafeImage,
   BackendClient2,
   Engine2 as Engine,
@@ -8376,4 +8650,4 @@ export {
   defaultRegistry,
   createEngineFromEnv
 };
-//# sourceMappingURL=chunk-ISZWNERZ.js.map
+//# sourceMappingURL=chunk-SSDYBIZB.js.map