lobster-cli 0.1.0

package/dist/pipeline/index.js

@@ -0,0 +1,802 @@
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+
+// src/pipeline/template.ts
+var EXPR_RE = /\$\{\{\s*(.*?)\s*\}\}/g;
+function renderTemplate(template, ctx) {
+  if (typeof template !== "string") {
+    if (typeof template === "object" && template !== null) {
+      if (Array.isArray(template)) return template.map((v) => renderTemplate(v, ctx));
+      const result = {};
+      for (const [k, v] of Object.entries(template)) {
+        result[k] = renderTemplate(v, ctx);
+      }
+      return result;
+    }
+    return template;
+  }
+  const fullMatch = template.match(/^\$\{\{\s*(.*?)\s*\}\}$/);
+  if (fullMatch) {
+    return evaluateExpression(fullMatch[1], ctx);
+  }
+  return template.replace(EXPR_RE, (_, expr) => {
+    const val = evaluateExpression(expr, ctx);
+    return val === null || val === void 0 ? "" : String(val);
+  });
+}
+function evaluateExpression(expr, ctx) {
+  const parts = expr.split(/\s*\|\s*/);
+  let value = resolveValue(parts[0].trim(), ctx);
+  for (let i = 1; i < parts.length; i++) {
+    value = applyFilter(value, parts[i].trim());
+  }
+  return value;
+}
+function resolveValue(path, ctx) {
+  const arithMatch = path.match(/^(\w[\w.]*)\s*([+\-*])\s*(\d+)$/);
+  if (arithMatch) {
+    const base = Number(resolvePath(arithMatch[1], ctx));
+    const op = arithMatch[2];
+    const num = Number(arithMatch[3]);
+    if (op === "+") return base + num;
+    if (op === "-") return base - num;
+    if (op === "*") return base * num;
+  }
+  const orMatch = path.match(/^(.+?)\s*\|\|\s*(.+)$/);
+  if (orMatch) {
+    const left = resolvePath(orMatch[1].trim(), ctx);
+    if (left !== null && left !== void 0 && left !== "" && left !== false) return left;
+    const right = orMatch[2].trim();
+    if (right.startsWith("'") && right.endsWith("'") || right.startsWith('"') && right.endsWith('"')) {
+      return right.slice(1, -1);
+    }
+    return resolvePath(right, ctx);
+  }
+  if (path.startsWith("'") && path.endsWith("'") || path.startsWith('"') && path.endsWith('"')) {
+    return path.slice(1, -1);
+  }
+  if (!isNaN(Number(path)) && path !== "") return Number(path);
+  return resolvePath(path, ctx);
+}
+function resolvePath(path, ctx) {
+  if (path === "index") return ctx.index ?? 0;
+  const parts = path.split(".");
+  let root;
+  if (parts[0] === "args") {
+    root = ctx.args;
+    parts.shift();
+  } else if (parts[0] === "item") {
+    root = ctx.item;
+    parts.shift();
+  } else if (parts[0] === "data") {
+    root = ctx.data;
+    parts.shift();
+  } else {
+    root = getNestedValue(ctx.item, parts);
+    if (root !== void 0) return root;
+    root = getNestedValue(ctx.args, parts);
+    if (root !== void 0) return root;
+    root = getNestedValue(ctx.data, parts);
+    if (root !== void 0) return root;
+    return void 0;
+  }
+  return getNestedValue(root, parts);
+}
+function getNestedValue(obj, parts) {
+  let current = obj;
+  for (const part of parts) {
+    if (current === null || current === void 0) return void 0;
+    if (typeof current === "object") {
+      current = current[part];
+    } else {
+      return void 0;
+    }
+  }
+  return current;
+}
+function applyFilter(value, filter) {
+  const match = filter.match(/^(\w+)(?:\((.+)\))?$/);
+  if (!match) return value;
+  const name = match[1];
+  const arg = match[2]?.replace(/^['"]|['"]$/g, "");
+  switch (name) {
+    case "default":
+      return value === null || value === void 0 || value === "" ? arg : value;
+    case "join":
+      return Array.isArray(value) ? value.join(arg || ", ") : value;
+    case "upper":
+      return typeof value === "string" ? value.toUpperCase() : value;
+    case "lower":
+      return typeof value === "string" ? value.toLowerCase() : value;
+    case "trim":
+      return typeof value === "string" ? value.trim() : value;
+    case "truncate": {
+      const len = parseInt(arg || "100");
+      if (typeof value === "string" && value.length > len) return value.slice(0, len) + "...";
+      return value;
+    }
+    case "replace": {
+      if (typeof value !== "string" || !arg) return value;
+      const [from, to] = arg.split(",").map((s) => s.trim().replace(/^['"]|['"]$/g, ""));
+      return value.replaceAll(from, to || "");
+    }
+    case "keys":
+      return typeof value === "object" && value !== null ? Object.keys(value) : [];
+    case "length":
+      return Array.isArray(value) ? value.length : typeof value === "string" ? value.length : 0;
+    case "first":
+      return Array.isArray(value) ? value[0] : value;
+    case "last":
+      return Array.isArray(value) ? value[value.length - 1] : value;
+    case "json":
+      return JSON.stringify(value);
+    case "slugify":
+      return typeof value === "string" ? value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") : value;
+    case "sanitize":
+      return typeof value === "string" ? value.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_").replace(/__+/g, "_").trim() : value;
+    case "ext": {
+      if (typeof value !== "string") return "";
+      const extMatch = value.match(/\.([a-zA-Z0-9]+)(?:\?.*)?$/);
+      return extMatch ? extMatch[1] : "";
+    }
+    case "basename": {
+      if (typeof value !== "string") return "";
+      try {
+        return new URL(value).pathname.split("/").pop() || "";
+      } catch {
+      }
+      return value.split("/").pop() || "";
+    }
+    default:
+      return value;
+  }
+}
+
+// src/pipeline/registry.ts
+var stepHandlers = /* @__PURE__ */ new Map();
+function registerStep(name, handler) {
+  stepHandlers.set(name, handler);
+}
+function getStep(name) {
+  return stepHandlers.get(name);
+}
+function getStepNames() {
+  return [...stepHandlers.keys()];
+}
+
+// src/pipeline/steps/fetch.ts
+async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
+  const headersJs = JSON.stringify(headers);
+  const urlsJs = JSON.stringify(urls);
+  return page.evaluate(`
+    (async () => {
+      const urls = ${urlsJs};
+      const method = ${JSON.stringify(method)};
+      const headers = ${headersJs};
+      const concurrency = ${concurrency};
+
+      const results = new Array(urls.length);
+      let idx = 0;
+
+      async function worker() {
+        while (idx < urls.length) {
+          const i = idx++;
+          try {
+            const resp = await fetch(urls[i], { method, headers, credentials: "include" });
+            results[i] = await resp.json();
+          } catch (e) {
+            results[i] = { error: e.message };
+          }
+        }
+      }
+
+      const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
+      await Promise.all(workers);
+      return results;
+    })()
+  `);
+}
+async function mapConcurrent(items, limit, fn) {
+  const results = new Array(items.length);
+  let index = 0;
+  async function worker() {
+    while (index < items.length) {
+      const i = index++;
+      results[i] = await fn(items[i], i);
+    }
+  }
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+async function fetchSingle(page, url, method, headers) {
+  if (page) {
+    const headersJs = JSON.stringify(headers);
+    const urlJs = JSON.stringify(url);
+    const methodJs = JSON.stringify(method);
+    return page.evaluate(`
+      (async () => {
+        const resp = await fetch(${urlJs}, {
+          method: ${methodJs}, headers: ${headersJs}, credentials: "include"
+        });
+        return await resp.json();
+      })()
+    `);
+  }
+  const resp = await fetch(url, { method, headers });
+  return resp.json();
+}
+registerStep("fetch", async (ctx, params) => {
+  const data = ctx.data;
+  const urlOrObj = typeof params === "string" ? params : params?.url ?? "";
+  const method = params?.method || "GET";
+  const rawHeaders = params?.headers ?? {};
+  const rawParams = params?.params ?? {};
+  const urlTemplate = String(urlOrObj);
+  const headers = {};
+  for (const [k, v] of Object.entries(rawHeaders)) {
+    headers[k] = String(renderTemplate(v, { args: ctx.args, data }));
+  }
+  if (Array.isArray(data) && urlTemplate.includes("item")) {
+    const concurrency = typeof params?.concurrency === "number" ? params.concurrency : 5;
+    const renderedParams2 = {};
+    for (const [k, v] of Object.entries(rawParams)) {
+      renderedParams2[k] = String(renderTemplate(v, { args: ctx.args, data }));
+    }
+    const urls = data.map((item, index) => {
+      let url2 = String(renderTemplate(urlTemplate, { args: ctx.args, data, item, index }));
+      if (Object.keys(renderedParams2).length > 0) {
+        const qs = new URLSearchParams(renderedParams2).toString();
+        url2 = `${url2}${url2.includes("?") ? "&" : "?"}${qs}`;
+      }
+      return url2;
+    });
+    if (ctx.page) {
+      return fetchBatchInBrowser(ctx.page, urls, method.toUpperCase(), headers, concurrency);
+    }
+    return mapConcurrent(urls, concurrency, async (url2) => {
+      return fetchSingle(null, url2, method.toUpperCase(), headers);
+    });
+  }
+  let url = String(renderTemplate(urlOrObj, { args: ctx.args, data }));
+  const renderedParams = {};
+  for (const [k, v] of Object.entries(rawParams)) {
+    renderedParams[k] = String(renderTemplate(v, { args: ctx.args, data }));
+  }
+  if (Object.keys(renderedParams).length > 0) {
+    const qs = new URLSearchParams(renderedParams).toString();
+    url = `${url}${url.includes("?") ? "&" : "?"}${qs}`;
+  }
+  return fetchSingle(ctx.page, url, method.toUpperCase(), headers);
+});
+
+// src/pipeline/steps/browser.ts
+registerStep("navigate", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for navigate step");
+  const url = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  await ctx.page.goto(url);
+  return ctx.data;
+});
+registerStep("click", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for click step");
+  const ref = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  await ctx.page.click(ref);
+  return ctx.data;
+});
+registerStep("type", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for type step");
+  const p = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  await ctx.page.typeText(p.ref, p.text);
+  if (p.submit) await ctx.page.pressKey("Enter");
+  return ctx.data;
+});
+registerStep("wait", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for wait step");
+  const rendered = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  if (typeof rendered === "number") {
+    await ctx.page.wait(rendered);
+  } else {
+    await ctx.page.wait(rendered);
+  }
+  return ctx.data;
+});
+registerStep("press", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for press step");
+  const key = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  await ctx.page.pressKey(key);
+  return ctx.data;
+});
+registerStep("snapshot", async (ctx, _params) => {
+  if (!ctx.page) throw new Error("Browser page required for snapshot step");
+  return ctx.page.snapshot();
+});
+registerStep("evaluate", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for evaluate step");
+  const js = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  const result = await ctx.page.evaluate(js);
+  if (typeof result === "string") {
+    try {
+      return JSON.parse(result);
+    } catch {
+      return result;
+    }
+  }
+  return result;
+});
+
+// src/pipeline/steps/transform.ts
+registerStep("select", async (ctx, params) => {
+  const path = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  const parts = path.split(".");
+  let current = ctx.data;
+  for (const part of parts) {
+    if (current === null || current === void 0) return void 0;
+    const indexMatch = part.match(/^(\w+)\[(\d+)\]$/);
+    if (indexMatch) {
+      current = current[indexMatch[1]];
+      if (Array.isArray(current)) current = current[Number(indexMatch[2])];
+      else return void 0;
+      continue;
+    }
+    const wildcardMatch = part.match(/^(\w+)\[\*\]$/);
+    if (wildcardMatch) {
+      current = current[wildcardMatch[1]];
+      if (!Array.isArray(current)) return void 0;
+      const remaining = parts.slice(parts.indexOf(part) + 1);
+      if (remaining.length > 0) {
+        return current.map((item) => {
+          let val = item;
+          for (const r of remaining) {
+            if (val === null || val === void 0) return void 0;
+            val = val[r];
+          }
+          return val;
+        });
+      }
+      continue;
+    }
+    current = current[part];
+  }
+  return current;
+});
+registerStep("map", async (ctx, params) => {
+  if (!Array.isArray(ctx.data)) throw new Error("map requires array data");
+  const template = params;
+  return ctx.data.map(
+    (item, index) => renderTemplate(template, { args: ctx.args, item, data: ctx.data, index })
+  );
+});
+registerStep("filter", async (ctx, params) => {
+  if (!Array.isArray(ctx.data)) throw new Error("filter requires array data");
+  const expr = params;
+  return ctx.data.filter((item, index) => {
+    const result = renderTemplate(`\${{ ${expr} }}`, { args: ctx.args, item, data: ctx.data, index });
+    return Boolean(result);
+  });
+});
+registerStep("sort", async (ctx, params) => {
+  if (!Array.isArray(ctx.data)) throw new Error("sort requires array data");
+  const p = params;
+  const sorted = [...ctx.data].sort((a, b) => {
+    const va = a[p.by];
+    const vb = b[p.by];
+    if (typeof va === "number" && typeof vb === "number") return va - vb;
+    return String(va).localeCompare(String(vb));
+  });
+  return p.order === "desc" ? sorted.reverse() : sorted;
+});
+registerStep("limit", async (ctx, params) => {
+  if (!Array.isArray(ctx.data)) return ctx.data;
+  const n = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  return ctx.data.slice(0, Number(n) || 20);
+});
+
+// src/pipeline/steps/intercept.ts
+registerStep("intercept", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for intercept step");
+  const p = params;
+  const pattern = renderTemplate(p.pattern, { args: ctx.args, data: ctx.data });
+  await ctx.page.installInterceptor(pattern);
+  if (p.trigger) {
+    const trigger = renderTemplate(p.trigger, { args: ctx.args, data: ctx.data });
+    const [action, value] = trigger.split(":");
+    switch (action) {
+      case "navigate":
+        await ctx.page.goto(value);
+        break;
+      case "click":
+        await ctx.page.click(value);
+        break;
+      case "evaluate":
+        await ctx.page.evaluate(value);
+        break;
+      case "scroll":
+        await ctx.page.scroll("down");
+        break;
+    }
+  }
+  const timeout = p.timeout || 10;
+  await ctx.page.wait(timeout);
+  const requests = await ctx.page.getInterceptedRequests();
+  if (requests.length === 0) return ctx.data;
+  let result = requests.map((r) => r.body);
+  if (result && Array.isArray(result) && result.length === 1) result = result[0];
+  if (p.select && result) {
+    const parts = p.select.split(".");
+    let current = result;
+    for (const part of parts) {
+      if (current && typeof current === "object") {
+        current = current[part];
+      }
+    }
+    result = current;
+  }
+  return result;
+});
+
+// src/pipeline/steps/download.ts
+import { writeFileSync, mkdirSync, existsSync } from "fs";
+import { join, basename } from "path";
+import { execSync } from "child_process";
+import { tmpdir } from "os";
+var DownloadProgressTracker = class {
+  total;
+  completed = 0;
+  failed = 0;
+  totalBytes = 0;
+  constructor(total) {
+    this.total = total;
+  }
+  success(bytes) {
+    this.completed++;
+    this.totalBytes += bytes;
+  }
+  fail() {
+    this.completed++;
+    this.failed++;
+  }
+  summary() {
+    return `Downloaded ${this.completed - this.failed}/${this.total} files (${formatBytes(this.totalBytes)})${this.failed > 0 ? `, ${this.failed} failed` : ""}`;
+  }
+};
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+async function exportCookiesNetscape(ctx, domain) {
+  if (!ctx.page) return null;
+  try {
+    const cookies = await ctx.page.getCookies({ domain });
+    if (cookies.length === 0) return null;
+    const lines = ["# Netscape HTTP Cookie File"];
+    for (const c of cookies) {
+      const httpOnly = c.httpOnly ? "TRUE" : "FALSE";
+      const secure = c.secure ? "TRUE" : "FALSE";
+      const expires = c.expires ? Math.floor(c.expires) : 0;
+      lines.push(`${c.domain || domain}	TRUE	${c.path || "/"}	${secure}	${expires}	${c.name}	${c.value}`);
+    }
+    const tmpFile = join(tmpdir(), `lobster-cookies-${Date.now()}.txt`);
+    writeFileSync(tmpFile, lines.join("\n"));
+    return tmpFile;
+  } catch {
+    return null;
+  }
+}
+function hasYtDlp() {
+  try {
+    execSync("yt-dlp --version", { stdio: "pipe" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function downloadWithYtDlp(url, dir, opts) {
+  const args = ["yt-dlp", "-o", join(dir, opts.filename || "%(title)s.%(ext)s")];
+  if (opts.format) args.push("-f", opts.format);
+  if (opts.cookieFile) args.push("--cookies", opts.cookieFile);
+  args.push("--no-warnings", "--no-progress", url);
+  try {
+    execSync(args.join(" "), { stdio: "pipe", timeout: 3e5 });
+    return { url, file: dir, success: true };
+  } catch (err) {
+    return { url, file: "", success: false, error: err.message?.slice(0, 200) };
+  }
+}
+async function downloadPool(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let idx = 0;
+  async function worker() {
+    while (idx < items.length) {
+      const i = idx++;
+      results[i] = await fn(items[i], i);
+    }
+  }
+  const workers = Array.from(
+    { length: Math.min(concurrency, items.length) },
+    () => worker()
+  );
+  await Promise.all(workers);
+  return results;
+}
+registerStep("download", async (ctx, params) => {
+  const p = params;
+  const dir = renderTemplate(p.dir || "./downloads", { args: ctx.args, data: ctx.data });
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const concurrency = p.concurrency || 3;
+  const skipExisting = p.skip_existing ?? false;
+  const timeout = (p.timeout || 60) * 1e3;
+  const items = [];
+  if (p.url) {
+    items.push({
+      url: renderTemplate(p.url, { args: ctx.args, data: ctx.data }),
+      index: 0
+    });
+  } else if (Array.isArray(ctx.data)) {
+    for (let i = 0; i < ctx.data.length; i++) {
+      const item = ctx.data[i];
+      const url = typeof item === "string" ? item : item.url;
+      if (url) items.push({ url, item, index: i });
+    }
+  }
+  if (items.length === 0) return [];
+  const tracker = new DownloadProgressTracker(items.length);
+  if (p.video) {
+    if (!hasYtDlp()) {
+      throw new Error("yt-dlp not found. Install: brew install yt-dlp (mac) or pip install yt-dlp");
+    }
+    const cookieFile = await exportCookiesNetscape(ctx, new URL(items[0].url).hostname);
+    const results2 = await downloadPool(items, Math.min(concurrency, 2), async (entry) => {
+      const result = downloadWithYtDlp(entry.url, dir, {
+        format: p.format,
+        cookieFile,
+        filename: p.filename ? renderTemplate(p.filename, { args: ctx.args, item: entry.item, data: ctx.data, index: entry.index }) : void 0
+      });
+      if (result.success) tracker.success(0);
+      else tracker.fail();
+      return result;
+    });
+    if (cookieFile) try {
+      __require("fs").unlinkSync(cookieFile);
+    } catch {
+    }
+    if (ctx.debug) console.log(tracker.summary());
+    return results2;
+  }
+  if (p.content) {
+    const results2 = await downloadPool(items, concurrency, async (entry) => {
+      try {
+        const resp = await fetch(entry.url, {
+          signal: AbortSignal.timeout(timeout)
+        });
+        if (!resp.ok) return { url: entry.url, file: "", success: false, error: `HTTP ${resp.status}` };
+        let content;
+        const html = await resp.text();
+        if (p.content === "html") {
+          content = html;
+        } else if (p.content === "text") {
+          content = html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+        } else if (p.content === "json") {
+          try {
+            content = JSON.stringify(JSON.parse(html), null, 2);
+          } catch {
+            content = html;
+          }
+        } else {
+          content = html.replace(/<h[1-6][^>]*>(.*?)<\/h[1-6]>/gi, "\n## $1\n").replace(/<p[^>]*>(.*?)<\/p>/gi, "\n$1\n").replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, "[$2]($1)").replace(/<[^>]+>/g, "").replace(/\n{3,}/g, "\n\n").trim();
+        }
+        const filename = p.filename ? renderTemplate(p.filename, { args: ctx.args, item: entry.item, data: ctx.data, index: entry.index }) : basename(new URL(entry.url).pathname).replace(/\.[^.]+$/, "") + (p.content === "json" ? ".json" : ".md");
+        const filepath = join(dir, filename);
+        if (p.metadata) {
+          const meta = renderTemplate(p.metadata, { args: ctx.args, item: entry.item, data: ctx.data, index: entry.index });
+          const header = Object.entries(meta).map(([k, v]) => `${k}: ${v}`).join("\n");
+          content = `---
+${header}
+---
+
+${content}`;
+        }
+        writeFileSync(filepath, content, "utf-8");
+        tracker.success(Buffer.byteLength(content));
+        return { url: entry.url, file: filepath, success: true, size: Buffer.byteLength(content), content: content.slice(0, 200) };
+      } catch (err) {
+        tracker.fail();
+        return { url: entry.url, file: "", success: false, error: err.message };
+      }
+    });
+    if (ctx.debug) console.log(tracker.summary());
+    return results2;
+  }
+  const results = await downloadPool(items, concurrency, async (entry) => {
+    try {
+      const filename = p.filename ? renderTemplate(p.filename, { args: ctx.args, item: entry.item, data: ctx.data, index: entry.index }) : decodeURIComponent(basename(new URL(entry.url).pathname)) || `download-${entry.index}`;
+      const filepath = join(dir, filename);
+      if (skipExisting && existsSync(filepath)) {
+        tracker.success(0);
+        return { url: entry.url, file: filepath, success: true, size: 0 };
+      }
+      const resp = await fetch(entry.url, {
+        signal: AbortSignal.timeout(timeout)
+      });
+      if (!resp.ok) {
+        tracker.fail();
+        return { url: entry.url, file: "", success: false, error: `HTTP ${resp.status}` };
+      }
+      const buffer = Buffer.from(await resp.arrayBuffer());
+      writeFileSync(filepath, buffer);
+      tracker.success(buffer.length);
+      return { url: entry.url, file: filepath, success: true, size: buffer.length };
+    } catch (err) {
+      tracker.fail();
+      return { url: entry.url, file: "", success: false, error: err.message };
+    }
+  });
+  if (ctx.debug) console.log(tracker.summary());
+  return results;
+});
+
+// src/pipeline/steps/tap.ts
+registerStep("tap", async (ctx, params) => {
+  if (!ctx.page) throw new Error("Browser page required for tap step");
+  const p = renderTemplate(params, { args: ctx.args, data: ctx.data });
+  const storeName = p.store;
+  const actionName = p.action;
+  const capturePattern = p.capture;
+  const actionArgs = p.args || [];
+  const timeoutSec = p.timeout || 5;
+  const selectPath = p.select;
+  const result = await ctx.page.evaluate(`
+    (async () => {
+      let captured = null;
+      let captureResolve;
+      const capturePromise = new Promise(r => { captureResolve = r; });
+      const capturePattern = ${JSON.stringify(capturePattern)};
+      const timeoutMs = ${timeoutSec * 1e3};
+
+      // 1. Patch fetch
+      const origFetch = window.fetch;
+      window.fetch = async function(...args) {
+        const resp = await origFetch.apply(this, args);
+        try {
+          const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || '';
+          if (url.includes(capturePattern) && !captured) {
+            captured = await resp.clone().json();
+            captureResolve();
+          }
+        } catch {}
+        return resp;
+      };
+
+      // 2. Patch XHR
+      const origSend = XMLHttpRequest.prototype.send;
+      const origOpen = XMLHttpRequest.prototype.open;
+      XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+        this.__tapUrl = url;
+        return origOpen.call(this, method, url, ...rest);
+      };
+      XMLHttpRequest.prototype.send = function(...args) {
+        this.addEventListener('load', function() {
+          if (this.__tapUrl?.includes(capturePattern) && !captured) {
+            try {
+              captured = JSON.parse(this.responseText);
+              captureResolve();
+            } catch {}
+          }
+        });
+        return origSend.apply(this, args);
+      };
+
+      try {
+        // 3. Find the store
+        const app = document.querySelector('#app');
+        let store = null;
+
+        // Try Pinia via __vue_app__
+        if (app?.__vue_app__) {
+          const pinia = app.__vue_app__.config?.globalProperties?.$pinia;
+          if (pinia?._s) {
+            store = pinia._s.get(${JSON.stringify(storeName)});
+          }
+          // Try Vuex
+          if (!store) {
+            const vuex = app.__vue_app__.config?.globalProperties?.$store;
+            if (vuex) store = vuex;
+          }
+        }
+
+        // Fallback: global pinia
+        if (!store && window.__pinia?._s) {
+          store = window.__pinia._s.get(${JSON.stringify(storeName)});
+        }
+
+        if (!store) {
+          return { error: 'Store not found: ' + ${JSON.stringify(storeName)} };
+        }
+
+        // 4. Call the action
+        const actionFn = store[${JSON.stringify(actionName)}];
+        if (typeof actionFn !== 'function') {
+          return { error: 'Action not found: ' + ${JSON.stringify(actionName)} };
+        }
+
+        await actionFn.apply(store, ${JSON.stringify(actionArgs)});
+
+        // 5. Wait for capture
+        if (!captured) {
+          await Promise.race([
+            capturePromise,
+            new Promise(r => setTimeout(r, timeoutMs)),
+          ]);
+        }
+      } finally {
+        // 6. Restore originals
+        window.fetch = origFetch;
+        XMLHttpRequest.prototype.send = origSend;
+        XMLHttpRequest.prototype.open = origOpen;
+      }
+
+      return captured;
+    })()
+  `);
+  if (!result) return null;
+  if (result?.error) {
+    throw new Error(result.error);
+  }
+  if (selectPath) {
+    let current = result;
+    for (const part of selectPath.split(".")) {
+      if (current === null || current === void 0) return null;
+      current = current[part];
+    }
+    return current;
+  }
+  return result;
+});
+
+// src/utils/logger.ts
+import chalk from "chalk";
+var log = {
+  info: (msg) => console.log(chalk.blue("\u2139"), msg),
+  success: (msg) => console.log(chalk.green("\u2713"), msg),
+  warn: (msg) => console.log(chalk.yellow("\u26A0"), msg),
+  error: (msg) => console.error(chalk.red("\u2717"), msg),
+  debug: (msg) => {
+    if (process.env.LOBSTER_DEBUG) console.log(chalk.gray("\u22EF"), msg);
+  },
+  step: (n, msg) => console.log(chalk.cyan(`[${n}]`), msg),
+  dim: (msg) => console.log(chalk.dim(msg))
+};
+
+// src/pipeline/executor.ts
+async function executePipeline(steps, page, args, debug = false) {
+  const ctx = { page, args, data: null, debug };
+  for (let i = 0; i < steps.length; i++) {
+    const stepDef = steps[i];
+    const [stepName, params] = Object.entries(stepDef)[0];
+    const handler = getStep(stepName);
+    if (!handler) {
+      throw new Error(`Unknown pipeline step: ${stepName}`);
+    }
+    if (debug) {
+      log.step(i + 1, `${stepName}`);
+    }
+    ctx.data = await handler(ctx, params);
+    if (debug && ctx.data !== void 0) {
+      const preview = JSON.stringify(ctx.data)?.slice(0, 200);
+      log.dim(`  \u2192 ${preview}...`);
+    }
+  }
+  return ctx.data;
+}
+export {
+  executePipeline,
+  getStep,
+  getStepNames,
+  registerStep,
+  renderTemplate
+};
+//# sourceMappingURL=index.js.map