@zenalexa/unicli 0.212.1 → 0.213.0

package/dist/engine/steps/exec.js

@@ -0,0 +1,161 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { registerStep } from "../step-registry.js";
+import { PipelineError } from "../executor.js";
+import { evalTemplate } from "../template.js";
+import { matchSensitivePathRealpath, buildSensitivePathDenial, } from "../../permissions/sensitive-paths.js";
+const execFileAsync = promisify(execFile);
+export async function stepExec(ctx, config) {
+    const cmd = evalTemplate(config.command, ctx);
+    const execArgs = (config.args ?? []).map((a) => evalTemplate(String(a), ctx));
+    const timeout = config.timeout ?? 30000;
+    // Sensitive-path deny list — realpath-aware so symlink smuggling is
+    // blocked too. Cannot be overridden by permission mode.
+    for (const arg of execArgs) {
+        if (typeof arg !== "string" || arg.length === 0)
+            continue;
+        if (!arg.startsWith("/") && !arg.startsWith("~/"))
+            continue;
+        const expanded = arg.startsWith("~/") ? join(homedir(), arg.slice(2)) : arg;
+        const matched = matchSensitivePathRealpath(expanded);
+        if (matched) {
+            const denial = buildSensitivePathDenial(expanded);
+            throw new PipelineError("sensitive_path_denied", {
+                step: -1,
+                action: "exec",
+                config: {
+                    command: cmd,
+                    args: execArgs,
+                    denial_path: denial.path,
+                    denial_pattern: denial.pattern,
+                },
+                errorType: "assertion_failed",
+                suggestion: denial.hint,
+                retryable: false,
+                alternatives: [],
+            });
+        }
+    }
+    let envOption;
+    if (config.env) {
+        const resolved = {};
+        for (const [k, v] of Object.entries(config.env)) {
+            resolved[k] = evalTemplate(String(v), ctx);
+        }
+        envOption = { ...process.env, ...resolved };
+    }
+    const stdinContent = config.stdin
+        ? evalTemplate(config.stdin, ctx)
+        : undefined;
+    const outputFile = config.output_file
+        ? evalTemplate(config.output_file, ctx)
+        : undefined;
+    try {
+        let stdout;
+        if (stdinContent !== undefined) {
+            const { spawn } = await import("node:child_process");
+            stdout = await new Promise((resolve, reject) => {
+                const child = spawn(cmd, execArgs, {
+                    timeout,
+                    env: envOption,
+                    stdio: ["pipe", "pipe", "pipe"],
+                });
+                const chunks = [];
+                const errChunks = [];
+                child.stdout.on("data", (c) => chunks.push(c));
+                child.stderr.on("data", (c) => errChunks.push(c));
+                child.on("error", (err) => reject(err));
+                child.on("close", (code) => {
+                    if (code !== 0) {
+                        const stderr = Buffer.concat(errChunks).toString("utf8");
+                        reject(new Error(`Process exited with code ${code}${stderr ? `: ${stderr}` : ""}`));
+                    }
+                    else {
+                        resolve(Buffer.concat(chunks).toString("utf8"));
+                    }
+                });
+                child.stdin.write(stdinContent);
+                child.stdin.end();
+            });
+        }
+        else {
+            const opts = {
+                timeout,
+                maxBuffer: 10 * 1024 * 1024,
+            };
+            if (envOption)
+                opts.env = envOption;
+            ({ stdout } = await execFileAsync(cmd, execArgs, opts));
+        }
+        if (outputFile) {
+            const { stat } = await import("node:fs/promises");
+            try {
+                const info = await stat(outputFile);
+                return { ...ctx, data: { file: outputFile, size: info.size } };
+            }
+            catch {
+                throw new PipelineError(`exec "${cmd}" did not produce expected output file: ${outputFile}`, {
+                    step: -1,
+                    action: "exec",
+                    config: { command: cmd, args: execArgs },
+                    errorType: "parse_error",
+                    suggestion: `Check that the command writes to "${outputFile}". Verify the path is correct.`,
+                    retryable: false,
+                    alternatives: [],
+                });
+            }
+        }
+        let data;
+        switch (config.parse ?? "lines") {
+            case "json":
+                data = JSON.parse(stdout);
+                break;
+            case "lines":
+                data = stdout
+                    .split("\n")
+                    .filter(Boolean)
+                    .map((line) => ({ line }));
+                break;
+            case "csv": {
+                const lines = stdout.split("\n").filter(Boolean);
+                if (lines.length < 2) {
+                    data = [];
+                    break;
+                }
+                const headers = lines[0].split(",").map((h) => h.trim());
+                data = lines.slice(1).map((line) => {
+                    const vals = line.split(",");
+                    const row = {};
+                    headers.forEach((h, i) => {
+                        row[h] = (vals[i] ?? "").trim();
+                    });
+                    return row;
+                });
+                break;
+            }
+            case "text":
+            default:
+                data = stdout;
+        }
+        return { ...ctx, data };
+    }
+    catch (err) {
+        if (err instanceof PipelineError)
+            throw err;
+        const msg = err instanceof Error ? err.message : String(err);
+        const isExecTransient = /timeout|ETIMEDOUT|ECONNREFUSED|ECONNRESET/i.test(msg);
+        throw new PipelineError(`exec "${cmd}" failed: ${msg}`, {
+            step: -1,
+            action: "exec",
+            config: { command: cmd, args: execArgs },
+            errorType: isExecTransient ? "timeout" : "parse_error",
+            suggestion: `Check that "${cmd}" is installed and accessible. Run: which ${cmd}`,
+            retryable: isExecTransient,
+            alternatives: [],
+        });
+    }
+}
+registerStep("exec", stepExec);
+//# sourceMappingURL=exec.js.map

package/dist/engine/steps/exec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec.js","sourceRoot":"","sources":["../../../src/engine/steps/exec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAwB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,0BAA0B,EAC1B,wBAAwB,GACzB,MAAM,sCAAsC,CAAC;AAE9C,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAY1C,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAoB,EACpB,MAAkB;IAElB,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IAExC,oEAAoE;IACpE,wDAAwD;IACxD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC1D,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAC5D,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5E,MAAM,OAAO,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,IAAI,aAAa,CAAC,uBAAuB,EAAE;gBAC/C,IAAI,EAAE,CAAC,CAAC;gBACR,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE;oBACN,OAAO,EAAE,GAAG;oBACZ,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,MAAM,CAAC,IAAI;oBACxB,cAAc,EAAE,MAAM,CAAC,OAAO;iBAC/B;gBACD,SAAS,EAAE,kBAAkB;gBAC7B,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,SAAS,EAAE,KAAK;gBAChB,YAAY,EAAE,EAAE;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,SAAwC,CAAC;IAC7C,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;QACf,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,QAAQ,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;QACD,SAAS,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK;QAC/B,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC;QACjC,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW;QACnC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC;QACvC,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,CAAC;QACH,IAAI,MAAc,CAAC;QAEnB,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACrD,MAAM,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACrD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE;oBACjC,OAAO;oBACP,GAAG,EAAE,SAAS;oBACd,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;iBAChC,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAa,EAAE,CAAC;gBAE/B,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE1D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACzB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;wBACzD,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACjE,CACF,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;oBAClD,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAChC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACpB,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAIN;gBACF,OAAO;gBACP,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;aAC5B,CAAC;YACF,IAAI,SAAS;gBAAE,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;YACpC,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAClD,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;gBACpC,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjE,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,aAAa,CACrB,SAAS,GAAG,2CAA2C,UAAU,EAAE,EACnE;oBACE,IAAI,EAAE,CAAC,CAAC;oBACR,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxC,SAAS,EAAE,aAAa;oBACxB,UAAU,EAAE,qCAAqC,UAAU,gCAAgC;oBAC3F,SAAS,EAAE,KAAK;oBAChB,YAAY,EAAE,EAAE;iBACjB,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,IAAa,CAAC;QAClB,QAAQ,MAAM,CAAC,KAAK,IAAI,OAAO,EAAE,CAAC;YAChC,KAAK,MAAM;gBACT,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC1B,MAAM;YACR,KAAK,OAAO;gBACV,IAAI,GAAG,MAAM;qBACV,KAAK,CAAC,IAAI,CAAC;qBACX,MAAM,CAAC,OAAO,CAAC;qBACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM;YACR,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACjD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,IAAI,GAAG,EAAE,CAAC;oBACV,MAAM;gBACR,CAAC;gBACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC7B,MAAM,GAAG,GAA2B,EAAE,CAAC;oBACvC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;wBACvB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;oBAClC,CAAC,CAAC,CAAC;oBACH,OAAO,GAAG,CAAC;gBACb,CAAC,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YACD,KAAK,MAAM,CAAC;YACZ;gBACE,IAAI,GAAG,MAAM,CAAC;QAClB,CAAC;QAED,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,aAAa;YAAE,MAAM,GAAG,CAAC;QAC5C,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,eAAe,GAAG,4CAA4C,CAAC,IAAI,CACvE,GAAG,CACJ,CAAC;QACF,MAAM,IAAI,aAAa,CAAC,SAAS,GAAG,aAAa,GAAG,EAAE,EAAE;YACtD,IAAI,EAAE,CAAC,CAAC;YACR,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;YACtD,UAAU,EAAE,eAAe,GAAG,6CAA6C,GAAG,EAAE;YAChF,SAAS,EAAE,eAAe;YAC1B,YAAY,EAAE,EAAE;SACjB,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,YAAY,CAAC,MAAM,EAAE,QAAuB,CAAC,CAAC"}

package/dist/engine/steps/extract.d.ts

@@ -0,0 +1,14 @@
+import type { PipelineContext } from "../executor.js";
+interface FieldDef {
+    selector: string;
+    type?: "text" | "number" | "html" | "attribute";
+    attribute?: string;
+    pattern?: string;
+}
+export interface ExtractConfig {
+    from: string;
+    fields: Record<string, FieldDef>;
+}
+export declare function stepExtract(ctx: PipelineContext, config: ExtractConfig): Promise<PipelineContext>;
+export {};
+//# sourceMappingURL=extract.d.ts.map

package/dist/engine/steps/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/extract.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAItD,UAAU,QAAQ;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;CAClC;AAED,wBAAsB,WAAW,CAC/B,GAAG,EAAE,eAAe,EACpB,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,eAAe,CAAC,CA8C1B"}

package/dist/engine/steps/extract.js

@@ -0,0 +1,51 @@
+import { registerStep } from "../step-registry.js";
+import { evalTemplate } from "../template.js";
+import { acquirePage } from "./browser-helpers.js";
+export async function stepExtract(ctx, config) {
+    const page = await acquirePage(ctx);
+    const containerSelector = evalTemplate(config.from, ctx);
+    // Build a JS expression that extracts structured data
+    const fieldEntries = Object.entries(config.fields);
+    const fieldJs = fieldEntries
+        .map(([key, def]) => {
+        const sel = JSON.stringify(def.selector);
+        const attr = def.attribute ? JSON.stringify(def.attribute) : null;
+        const pattern = def.pattern ? JSON.stringify(def.pattern) : null;
+        const type = def.type ?? "text";
+        if (type === "attribute" || attr) {
+            return `${JSON.stringify(key)}: (() => { const el = item.querySelector(${sel}); return el ? el.getAttribute(${attr ?? JSON.stringify("href")}) : null; })()`;
+        }
+        else if (type === "number") {
+            return `${JSON.stringify(key)}: (() => { const el = item.querySelector(${sel}); if (!el) return null; const txt = el.textContent || ''; ${pattern ? `const m = txt.match(new RegExp(${pattern})); return m ? parseFloat(m[0]) : null;` : `return parseFloat(txt.replace(/[^\\d.-]/g, '')) || null;`} })()`;
+        }
+        else if (type === "html") {
+            return `${JSON.stringify(key)}: (() => { const el = item.querySelector(${sel}); return el ? el.innerHTML : null; })()`;
+        }
+        else {
+            // text (default)
+            if (pattern) {
+                return `${JSON.stringify(key)}: (() => { const el = item.querySelector(${sel}); if (!el) return null; const txt = el.textContent || ''; const m = txt.match(new RegExp(${pattern})); return m ? (m[1] || m[0]) : txt.trim(); })()`;
+            }
+            return `${JSON.stringify(key)}: (() => { const el = item.querySelector(${sel}); return el ? el.textContent.trim() : null; })()`;
+        }
+    })
+        .join(",\n      ");
+    const extractJs = `
+    JSON.stringify(
+      Array.from(document.querySelectorAll(${JSON.stringify(containerSelector)})).map(item => ({
+        ${fieldJs}
+      }))
+    )
+  `;
+    const resultStr = (await page.evaluate(extractJs));
+    let data;
+    try {
+        data = JSON.parse(resultStr);
+    }
+    catch {
+        data = [];
+    }
+    return { ...ctx, data, page };
+}
+registerStep("extract", stepExtract);
+//# sourceMappingURL=extract.js.map

package/dist/engine/steps/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../src/engine/steps/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAcnD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAoB,EACpB,MAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,iBAAiB,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAEzD,sDAAsD;IACtD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,YAAY;SACzB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC;QAEhC,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,EAAE,CAAC;YACjC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4CAA4C,GAAG,kCAAkC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAC/J,CAAC;aAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4CAA4C,GAAG,8DAA8D,OAAO,CAAC,CAAC,CAAC,kCAAkC,OAAO,yCAAyC,CAAC,CAAC,CAAC,0DAA0D,OAAO,CAAC;QAC7S,CAAC;aAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4CAA4C,GAAG,0CAA0C,CAAC;QACzH,CAAC;aAAM,CAAC;YACN,iBAAiB;YACjB,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4CAA4C,GAAG,6FAA6F,OAAO,kDAAkD,CAAC;YACrO,CAAC;YACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4CAA4C,GAAG,mDAAmD,CAAC;QAClI,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CAAC,WAAW,CAAC,CAAC;IAErB,MAAM,SAAS,GAAG;;6CAEyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC;UACpE,OAAO;;;GAGd,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAW,CAAC;IAC7D,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAc,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAChC,CAAC;AAED,YAAY,CAAC,SAAS,EAAE,WAA0B,CAAC,CAAC"}

package/dist/engine/steps/fetch-text.d.ts

@@ -0,0 +1,4 @@
+import { type PipelineContext } from "../executor.js";
+import type { FetchConfig } from "./fetch.js";
+export declare function stepFetchText(ctx: PipelineContext, config: FetchConfig): Promise<PipelineContext>;
+//# sourceMappingURL=fetch-text.d.ts.map

package/dist/engine/steps/fetch-text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-text.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/fetch-text.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,gBAAgB,CAAC;AAIrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,wBAAsB,aAAa,CACjC,GAAG,EAAE,eAAe,EACpB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC,CAsD1B"}

package/dist/engine/steps/fetch-text.js

@@ -0,0 +1,53 @@
+import { USER_AGENT } from "../../constants.js";
+import { registerStep } from "../step-registry.js";
+import { PipelineError } from "../executor.js";
+import { assertSafeRequestUrl } from "../ssrf.js";
+import { evalTemplate } from "../template.js";
+import { getProxyAgent } from "../proxy.js";
+export async function stepFetchText(ctx, config) {
+    let url = evalTemplate(config.url, ctx);
+    assertSafeRequestUrl(url);
+    if (config.params) {
+        const params = new URLSearchParams();
+        for (const [k, v] of Object.entries(config.params)) {
+            params.set(k, evalTemplate(String(v), ctx));
+        }
+        url += (url.includes("?") ? "&" : "?") + params.toString();
+    }
+    const method = config.method ?? "GET";
+    const headers = {
+        "User-Agent": USER_AGENT,
+        ...config.headers,
+    };
+    if (ctx.cookieHeader) {
+        headers["Cookie"] = ctx.cookieHeader;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- dispatcher from undici not in standard RequestInit
+    const fetchInit = { method, headers };
+    const ftAgent = getProxyAgent();
+    if (ftAgent)
+        fetchInit.dispatcher = ftAgent;
+    const resp = await fetch(url, fetchInit);
+    if (!resp.ok) {
+        throw new PipelineError(`HTTP ${resp.status} ${resp.statusText} from ${url}`, {
+            step: -1,
+            action: "fetch_text",
+            config: { url, method },
+            errorType: "http_error",
+            url,
+            statusCode: resp.status,
+            suggestion: `Check if the URL is still valid: ${url}`,
+            retryable: resp.status === 429 ||
+                resp.status === 500 ||
+                resp.status === 502 ||
+                resp.status === 503,
+            alternatives: resp.status === 401 || resp.status === 403
+                ? ["unicli auth setup <site>"]
+                : [],
+        });
+    }
+    const text = await resp.text();
+    return { ...ctx, data: text };
+}
+registerStep("fetch_text", stepFetchText);
+//# sourceMappingURL=fetch-text.js.map

package/dist/engine/steps/fetch-text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-text.js","sourceRoot":"","sources":["../../../src/engine/steps/fetch-text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAwB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAoB,EACpB,MAAmB;IAEnB,IAAI,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAE1B,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC;IACtC,MAAM,OAAO,GAA2B;QACtC,YAAY,EAAE,UAAU;QACxB,GAAG,MAAM,CAAC,OAAO;KAClB,CAAC;IAEF,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,OAAO,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,YAAY,CAAC;IACvC,CAAC;IAED,oHAAoH;IACpH,MAAM,SAAS,GAAwB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC3D,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;IAChC,IAAI,OAAO;QAAE,SAAS,CAAC,UAAU,GAAG,OAAO,CAAC;IAE5C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,SAAwB,CAAC,CAAC;IACxD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,aAAa,CACrB,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,SAAS,GAAG,EAAE,EACpD;YACE,IAAI,EAAE,CAAC,CAAC;YACR,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE;YACvB,SAAS,EAAE,YAAY;YACvB,GAAG;YACH,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,UAAU,EAAE,oCAAoC,GAAG,EAAE;YACrD,SAAS,EACP,IAAI,CAAC,MAAM,KAAK,GAAG;gBACnB,IAAI,CAAC,MAAM,KAAK,GAAG;gBACnB,IAAI,CAAC,MAAM,KAAK,GAAG;gBACnB,IAAI,CAAC,MAAM,KAAK,GAAG;YACrB,YAAY,EACV,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG;gBACxC,CAAC,CAAC,CAAC,0BAA0B,CAAC;gBAC9B,CAAC,CAAC,EAAE;SACT,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAChC,CAAC;AAED,YAAY,CAAC,YAAY,EAAE,aAA4B,CAAC,CAAC"}

package/dist/engine/steps/fetch.d.ts

@@ -1,10 +1,13 @@
-/**
- * Fetch / RSS / HTML → Markdown step handlers.
- *
- * Per-concern module extracted as part of the v0.212 engine restructure.
- * The implementations remain in `src/engine/yaml-runner.ts` (the legacy
- * orchestrator); this module is the stable import boundary that future
- * phases will migrate bodies into.
- */
-export { stepFetch as handleFetch, stepFetchText as handleFetchText, stepParseRss as handleParseRss, stepHtmlToMd as handleHtmlToMd, type FetchConfig, type RssConfig, } from "../yaml-runner.js";
+import { type PipelineContext } from "../executor.js";
+export interface FetchConfig {
+    url: string;
+    method?: string;
+    params?: Record<string, unknown>;
+    headers?: Record<string, string>;
+    body?: unknown;
+    retry?: number;
+    backoff?: number;
+    cache?: number;
+}
+export declare function stepFetch(ctx: PipelineContext, config: FetchConfig): Promise<PipelineContext>;
 //# sourceMappingURL=fetch.d.ts.map

package/dist/engine/steps/fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,SAAS,IAAI,WAAW,EACxB,aAAa,IAAI,eAAe,EAChC,YAAY,IAAI,cAAc,EAC9B,YAAY,IAAI,cAAc,EAC9B,KAAK,WAAW,EAChB,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/fetch.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,gBAAgB,CAAC;AAOrE,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,SAAS,CAC7B,GAAG,EAAE,eAAe,EACpB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC,CAgE1B"}

package/dist/engine/steps/fetch.js

@@ -1,10 +1,186 @@
-/**
- * Fetch / RSS / HTML → Markdown step handlers.
- *
- * Per-concern module extracted as part of the v0.212 engine restructure.
- * The implementations remain in `src/engine/yaml-runner.ts` (the legacy
- * orchestrator); this module is the stable import boundary that future
- * phases will migrate bodies into.
- */
-export { stepFetch as handleFetch, stepFetchText as handleFetchText, stepParseRss as handleParseRss, stepHtmlToMd as handleHtmlToMd, } from "../yaml-runner.js";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { createHash } from "node:crypto";
+import { USER_AGENT } from "../../constants.js";
+import { registerStep } from "../step-registry.js";
+import { PipelineError } from "../executor.js";
+import { assertSafeRequestUrl } from "../ssrf.js";
+import { evalTemplate, resolveTemplateDeep } from "../template.js";
+import { formatCookieHeader, loadCookiesWithCDP } from "../cookies.js";
+import { mapConcurrent } from "../download.js";
+import { getProxyAgent } from "../proxy.js";
+export async function stepFetch(ctx, config) {
+    let url = evalTemplate(config.url, ctx);
+    assertSafeRequestUrl(url);
+    // Fan-out with concurrency limit when data is an array of items.
+    if (Array.isArray(ctx.data)) {
+        const items = ctx.data;
+        const concurrency = config
+            .concurrency
+            ? Number(config.concurrency)
+            : 5;
+        const results = await mapConcurrent(items, concurrency, async (item) => {
+            const itemCtx = { ...ctx, data: item };
+            const itemUrl = evalTemplate(config.url, itemCtx);
+            assertSafeRequestUrl(itemUrl);
+            const resolvedConfig = config.body
+                ? { ...config, body: resolveTemplateDeep(config.body, itemCtx) }
+                : config;
+            return fetchJson(itemUrl, resolvedConfig, ctx.cookieHeader);
+        });
+        return { ...ctx, data: results };
+    }
+    if (config.params) {
+        const params = new URLSearchParams();
+        for (const [k, v] of Object.entries(config.params)) {
+            const val = evalTemplate(String(v), ctx);
+            params.set(k, val);
+        }
+        url += (url.includes("?") ? "&" : "?") + params.toString();
+    }
+    const resolvedConfig = config.body
+        ? { ...config, body: resolveTemplateDeep(config.body, ctx) }
+        : config;
+    try {
+        const data = await fetchJson(url, resolvedConfig, ctx.cookieHeader);
+        return { ...ctx, data };
+    }
+    catch (err) {
+        if (err instanceof PipelineError &&
+            (err.detail.statusCode === 401 || err.detail.statusCode === 403) &&
+            !ctx.cookieHeader) {
+            try {
+                const hostname = new URL(url).hostname;
+                const siteName = hostname
+                    .replace(/^www\./, "")
+                    .split(".")
+                    .slice(0, -1)
+                    .join("-");
+                const cookies = await loadCookiesWithCDP(siteName);
+                if (cookies) {
+                    const fallbackCookie = formatCookieHeader(cookies);
+                    const data = await fetchJson(url, resolvedConfig, fallbackCookie);
+                    return { ...ctx, data, cookieHeader: fallbackCookie };
+                }
+            }
+            catch {
+                // Cookie fallback also failed — throw original
+            }
+        }
+        throw err;
+    }
+}
+const CACHE_DIR = join(homedir(), ".unicli", "cache");
+function fetchCacheKey(url, method) {
+    return createHash("sha256")
+        .update(`${method}:${url}`)
+        .digest("hex")
+        .slice(0, 16);
+}
+function readFetchCache(url, method, ttlSeconds) {
+    const key = fetchCacheKey(url, method);
+    const filePath = join(CACHE_DIR, `${key}.json`);
+    if (!existsSync(filePath))
+        return null;
+    try {
+        const raw = readFileSync(filePath, "utf-8");
+        const entry = JSON.parse(raw);
+        if (Date.now() - entry.ts > ttlSeconds * 1000)
+            return null;
+        return entry.data;
+    }
+    catch {
+        return null;
+    }
+}
+const MAX_CACHE_ENTRY_BYTES = 10 * 1024 * 1024;
+function writeFetchCache(url, method, data) {
+    try {
+        const payload = JSON.stringify({ ts: Date.now(), url, data });
+        if (payload.length > MAX_CACHE_ENTRY_BYTES)
+            return;
+        mkdirSync(CACHE_DIR, { recursive: true });
+        const key = fetchCacheKey(url, method);
+        writeFileSync(join(CACHE_DIR, `${key}.json`), payload);
+    }
+    catch {
+        /* cache write failure is non-fatal */
+    }
+}
+async function fetchJson(url, config, cookieHeader) {
+    const method = config.method ?? "GET";
+    if (config.cache && config.cache > 0) {
+        const cached = readFetchCache(url, method, config.cache);
+        if (cached !== null)
+            return cached;
+    }
+    const headers = {
+        Accept: "application/json",
+        "User-Agent": USER_AGENT,
+        ...config.headers,
+    };
+    if (cookieHeader) {
+        headers["Cookie"] = cookieHeader;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- dispatcher from undici not in standard RequestInit
+    const init = { method, headers };
+    if (config.body && method !== "GET") {
+        headers["Content-Type"] = "application/json";
+        init.body = JSON.stringify(config.body);
+    }
+    const proxyAgent = getProxyAgent();
+    if (proxyAgent)
+        init.dispatcher = proxyAgent;
+    const maxAttempts = config.retry ?? 1;
+    const baseDelay = config.backoff ?? 1000;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        const resp = await fetch(url, init);
+        if (resp.ok) {
+            const data = await resp.json();
+            if (config.cache && config.cache > 0)
+                writeFetchCache(url, method, data);
+            return data;
+        }
+        const isRetryable = resp.status === 429 || resp.status >= 500;
+        const isLastAttempt = attempt === maxAttempts;
+        if (isRetryable && !isLastAttempt) {
+            await new Promise((r) => setTimeout(r, baseDelay * 2 ** (attempt - 1)));
+            continue;
+        }
+        let preview = "";
+        try {
+            preview = (await resp.text()).slice(0, 200);
+        }
+        catch {
+            /* ignore */
+        }
+        const isRetryableStatus = resp.status === 429 ||
+            resp.status === 500 ||
+            resp.status === 502 ||
+            resp.status === 503;
+        throw new PipelineError(`HTTP ${resp.status} ${resp.statusText} from ${url}`, {
+            step: -1,
+            action: "fetch",
+            config: { url, method },
+            errorType: "http_error",
+            url,
+            statusCode: resp.status,
+            responsePreview: preview,
+            suggestion: resp.status === 403
+                ? "The API is blocking requests. The endpoint may require authentication (cookie strategy) or the User-Agent may need updating."
+                : resp.status === 404
+                    ? "The API endpoint was not found. The URL path may have changed — check the target site for the current API."
+                    : resp.status === 429
+                        ? "Rate limited. Add a delay between requests or reduce the limit parameter."
+                        : `HTTP ${resp.status} error. Check if the API endpoint is still valid.`,
+            retryable: isRetryableStatus,
+            alternatives: resp.status === 401 || resp.status === 403
+                ? ["unicli auth setup <site>"]
+                : [],
+        });
+    }
+    throw new Error("fetchJson: unreachable");
+}
+registerStep("fetch", stepFetch);
 //# sourceMappingURL=fetch.js.map

package/dist/engine/steps/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/engine/steps/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,SAAS,IAAI,WAAW,EACxB,aAAa,IAAI,eAAe,EAChC,YAAY,IAAI,cAAc,EAC9B,YAAY,IAAI,cAAc,GAG/B,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/engine/steps/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAwB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAa5C,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAAoB,EACpB,MAAmB;IAEnB,IAAI,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAE1B,iEAAiE;IACjE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,IAAsC,CAAC;QACzD,MAAM,WAAW,GAAI,MAA6C;aAC/D,WAAW;YACZ,CAAC,CAAC,MAAM,CAAE,MAA6C,CAAC,WAAW,CAAC;YACpE,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrE,MAAM,OAAO,GAAG,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAClD,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI;gBAChC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE;gBAChE,CAAC,CAAC,MAAM,CAAC;YACX,OAAO,SAAS,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrB,CAAC;QACD,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI;QAChC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;QAC5D,CAAC,CAAC,MAAM,CAAC;IAEX,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QACpE,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IACE,GAAG,YAAY,aAAa;YAC5B,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,GAAG,CAAC;YAChE,CAAC,GAAG,CAAC,YAAY,EACjB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;gBACvC,MAAM,QAAQ,GAAG,QAAQ;qBACtB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;qBACrB,KAAK,CAAC,GAAG,CAAC;qBACV,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;qBACZ,IAAI,CAAC,GAAG,CAAC,CAAC;gBACb,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBACnD,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,cAAc,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;oBACnD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC;oBAClE,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;gBACxD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,+CAA+C;YACjD,CAAC;QACH,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AAEtD,SAAS,aAAa,CAAC,GAAW,EAAE,MAAc;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;SAC1B,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CACrB,GAAW,EACX,MAAc,EACd,UAAkB;IAElB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAkC,CAAC;QAC/D,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,GAAG,UAAU,GAAG,IAAI;YAAE,OAAO,IAAI,CAAC;QAC3D,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,qBAAqB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAE/C,SAAS,eAAe,CAAC,GAAW,EAAE,MAAc,EAAE,IAAa;IACjE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO;QACnD,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;IACxC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,MAAmB,EACnB,YAAqB;IAErB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC;IAEtC,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IAED,MAAM,OAAO,GAA2B;QACtC,MAAM,EAAE,kBAAkB;QAC1B,YAAY,EAAE,UAAU;QACxB,GAAG,MAAM,CAAC,OAAO;KAClB,CAAC;IAEF,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,CAAC,QAAQ,CAAC,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,oHAAoH;IACpH,MAAM,IAAI,GAAwB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACtD,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACpC,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,UAAU;QAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAE7C,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;IAEzC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,IAAmB,CAAC,CAAC;QAEnD,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC;gBAAE,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;QAC9D,MAAM,aAAa,GAAG,OAAO,KAAK,WAAW,CAAC;QAE9C,IAAI,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC;YAClC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QAED,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;QACD,MAAM,iBAAiB,GACrB,IAAI,CAAC,MAAM,KAAK,GAAG;YACnB,IAAI,CAAC,MAAM,KAAK,GAAG;YACnB,IAAI,CAAC,MAAM,KAAK,GAAG;YACnB,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC;QACtB,MAAM,IAAI,aAAa,CACrB,QAAQ,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,SAAS,GAAG,EAAE,EACpD;YACE,IAAI,EAAE,CAAC,CAAC;YACR,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE;YACvB,SAAS,EAAE,YAAY;YACvB,GAAG;YACH,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,eAAe,EAAE,OAAO;YACxB,UAAU,EACR,IAAI,CAAC,MAAM,KAAK,GAAG;gBACjB,CAAC,CAAC,8HAA8H;gBAChI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,GAAG;oBACnB,CAAC,CAAC,4GAA4G;oBAC9G,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,GAAG;wBACnB,CAAC,CAAC,2EAA2E;wBAC7E,CAAC,CAAC,QAAQ,IAAI,CAAC,MAAM,mDAAmD;YAChF,SAAS,EAAE,iBAAiB;YAC5B,YAAY,EACV,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG;gBACxC,CAAC,CAAC,CAAC,0BAA0B,CAAC;gBAC9B,CAAC,CAAC,EAAE;SACT,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAED,YAAY,CAAC,OAAO,EAAE,SAAwB,CAAC,CAAC"}

package/dist/engine/steps/filter.d.ts

@@ -0,0 +1,3 @@
+import type { PipelineContext } from "../executor.js";
+export declare function stepFilter(ctx: PipelineContext, expr: string): PipelineContext;
+//# sourceMappingURL=filter.d.ts.map

package/dist/engine/steps/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/filter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAGtD,wBAAgB,UAAU,CACxB,GAAG,EAAE,eAAe,EACpB,IAAI,EAAE,MAAM,GACX,eAAe,CAUjB"}

package/dist/engine/steps/filter.js

@@ -0,0 +1,14 @@
+import { registerStep } from "../step-registry.js";
+import { evalExpression } from "../template.js";
+export function stepFilter(ctx, expr) {
+    if (!Array.isArray(ctx.data))
+        return ctx;
+    const items = ctx.data;
+    const filtered = items.filter((item, index) => {
+        const result = evalExpression(expr, { item, index, args: ctx.args });
+        return Boolean(result);
+    });
+    return { ...ctx, data: filtered };
+}
+registerStep("filter", stepFilter);
+//# sourceMappingURL=filter.js.map

package/dist/engine/steps/filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.js","sourceRoot":"","sources":["../../../src/engine/steps/filter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,UAAU,UAAU,CACxB,GAAoB,EACpB,IAAY;IAEZ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IAEzC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAiB,CAAC;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AACpC,CAAC;AAED,YAAY,CAAC,QAAQ,EAAE,UAAyB,CAAC,CAAC"}

package/dist/engine/steps/html-to-md.d.ts

@@ -0,0 +1,3 @@
+import type { PipelineContext } from "../executor.js";
+export declare function stepHtmlToMd(ctx: PipelineContext): PipelineContext;
+//# sourceMappingURL=html-to-md.d.ts.map

package/dist/engine/steps/html-to-md.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-md.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/html-to-md.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtD,wBAAgB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,eAAe,CAQlE"}

package/dist/engine/steps/html-to-md.js

@@ -0,0 +1,13 @@
+import TurndownService from "turndown";
+import { registerStep } from "../step-registry.js";
+export function stepHtmlToMd(ctx) {
+    const html = String(ctx.data ?? "");
+    const turndown = new TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+    });
+    const md = turndown.turndown(html);
+    return { ...ctx, data: md };
+}
+registerStep("html_to_md", stepHtmlToMd);
+//# sourceMappingURL=html-to-md.js.map

package/dist/engine/steps/html-to-md.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-md.js","sourceRoot":"","sources":["../../../src/engine/steps/html-to-md.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAGrE,MAAM,UAAU,YAAY,CAAC,GAAoB;IAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;KACzB,CAAC,CAAC;IACH,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AAC9B,CAAC;AAED,YAAY,CAAC,YAAY,EAAE,YAA2B,CAAC,CAAC"}

package/dist/engine/steps/if.d.ts

@@ -0,0 +1,8 @@
+import { type PipelineContext } from "../executor.js";
+import type { PipelineStep } from "../../types.js";
+export declare function stepIf(ctx: PipelineContext, config: {
+    if: string;
+    then?: PipelineStep[];
+    else?: PipelineStep[];
+}, stepIndex: number, depth?: number): Promise<PipelineContext>;
+//# sourceMappingURL=if.d.ts.map

package/dist/engine/steps/if.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"if.d.ts","sourceRoot":"","sources":["../../../src/engine/steps/if.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,eAAe,EAIrB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,wBAAsB,MAAM,CAC1B,GAAG,EAAE,eAAe,EACpB,MAAM,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAA;CAAE,EACpE,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,MAAU,GAChB,OAAO,CAAC,eAAe,CAAC,CAsC1B"}