imperium-crawl 1.1.8 → 1.5.0

package/README.md

@@ -79,6 +79,30 @@ npm i rebrowser-playwright
 npx playwright install chromium
 ```
 
+### AI Agent Guide (SKILL.md)
+
+imperium-crawl ships with [`SKILL.md`](./SKILL.md) — a structured guide that teaches AI agents (Claude, GPT, etc.) how to use all 16 tools effectively. It includes 6 proven workflows, decision trees, error recovery strategies, and advanced patterns like manual skill refinement.
+
+**Without SKILL.md**, agents can call tools but won't know which tool to try first, when to fallback, or how to chain tools together optimally.
+
+**With SKILL.md**, agents follow battle-tested workflows — readability → scrape → extract fallback chains, auto-detect → manual refinement for skills, search → select → deep-scrape for research, and more.
+
+**Two ways to connect SKILL.md to any agent:**
+
+| Method | Setup | Works with |
+|--------|-------|-----------|
+| **MCP + SKILL.md** | Add imperium-crawl as MCP server + SKILL.md in agent context | Claude Code, Cursor, Windsurf, any MCP client |
+| **CLI + SKILL.md** | `npm i -g imperium-crawl` + SKILL.md in agent context | **Any agent with bash access** — OpenClaw, ChatGPT, GPT agents, custom agents, anything |
+
+The CLI approach is universal — any agent that can run shell commands can use all 16 tools. No MCP required.
+
+| AI Agent | How to add SKILL.md |
+|----------|-------------------|
+| **Claude Code** | Copy `SKILL.md` to your project root — Claude Code reads it automatically |
+| **Cursor / Windsurf** | Add `SKILL.md` to project rules or include in system prompt |
+| **OpenClaw / custom agents** | Include SKILL.md content in your system prompt or context window |
+| **ChatGPT / GPT agents** | Paste SKILL.md content into custom instructions |
+
 ---
 
 ## CLI Mode
@@ -332,27 +356,22 @@ Every tool tested against production websites with real anti-bot defenses:
 
 | Tool | Target | Result |
 |------|--------|--------|
-| 🕷️ **extract** | Amazon (AirPods Pro 2) | Product title, 45,297 reviews, brand extracted |
-| 🔓 **discover_apis** | Spotify | **8 hidden APIs** — access token exposed, client ID, dealer servers, analytics |
-| 🕷️ **extract** | Stack Overflow | **15 top questions** — #1 with 27,520 votes |
-| 📡 **monitor_websocket** | Binance BTC/USDT | **3 WebSocket connections, 23 live messages** — real-time price $69,390 |
-| 🔓 **discover_apis** | Airbnb Paris | **34 hidden APIs** — DataDome anti-bot, Google Maps key exposed, internal search/polygon/viewport APIs |
-| 🕷️ **extract** | Hacker News | **30 front-page posts** — titles + URLs extracted |
-| 🔓 **discover_apis** | Netflix | **5 APIs** — OneTrust consent, geolocation (detected country: Serbia 🇷🇸) |
 | 📄 **scrape** | BBC News | Full markdown content, stealth level 3 auto-escalation |
 | 🕸️ **crawl** | Cloudflare Blog | **213K characters** crawled with depth control |
 | 🗺️ **map** | BBC | Full URL discovery via sitemap + page link extraction |
+| 🕷️ **extract** | Amazon (AirPods Pro 2) | Product title, 45,297 reviews, brand extracted |
 | 📖 **readability** | Medium article | Clean extraction — title, author, content, publish date |
 | 📸 **screenshot** | ProductHunt | Captured Cloudflare Turnstile challenge page |
-| 🔓 **discover_apis** | weather.com | **11 hidden APIs** — main weather API with exposed key |
-| ⚡ **query_api** | jsonplaceholder | Direct JSON API call with stealth headers |
 | 🔍 **search** | Brave Web Search | Web results with snippets and URLs |
 | 📰 **news_search** | Brave News Search | News results with freshness ranking |
 | 🖼️ **image_search** | Brave Image Search | Images with thumbnails and source URLs |
 | 🎬 **video_search** | Brave Video Search | Video results across platforms |
-| 🛠️ **create_skill** | Any page | Auto-detects repeating patterns, generates CSS selectors |
+| 🛠️ **create_skill** | Hacker News | Auto-detected 30 repeating stories with CSS selectors |
 | ▶️ **run_skill** | Saved skill | Fresh structured data from saved extraction config |
 | 📋 **list_skills** | — | Lists all saved skills with configurations |
+| 🔓 **discover_apis** | Airbnb Paris | **34 hidden APIs** — DataDome anti-bot, Google Maps key, internal APIs |
+| ⚡ **query_api** | jsonplaceholder | Direct JSON API call with stealth headers |
+| 📡 **monitor_websocket** | Binance BTC/USDT | **3 WebSocket connections, 23 live messages** — BTC price live |
 
 > 🏆 **16/16 tools working. 58 hidden APIs discovered. Live crypto feed captured. Zero API keys needed for scraping.**
 

package/dist/batch/index.d.ts

@@ -0,0 +1,3 @@
+export type { BatchJob, BatchJobResult } from "./types.js";
+export { JobStore, getJobStore, resetJobStore } from "./job-store.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/batch/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/batch/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/batch/index.js

@@ -0,0 +1,2 @@
+export { JobStore, getJobStore, resetJobStore } from "./job-store.js";
+//# sourceMappingURL=index.js.map

package/dist/batch/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/batch/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/batch/job-store.d.ts

@@ -0,0 +1,15 @@
+import type { BatchJob } from "./types.js";
+export declare class JobStore {
+    private cache;
+    private dir;
+    constructor(dir?: string);
+    private jobPath;
+    save(job: BatchJob): Promise<void>;
+    load(id: string): Promise<BatchJob | null>;
+    delete(id: string): Promise<void>;
+    list(): Promise<string[]>;
+}
+export declare function getJobStore(): JobStore;
+/** Reset singleton (for testing) */
+export declare function resetJobStore(): void;
+//# sourceMappingURL=job-store.d.ts.map

package/dist/batch/job-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-store.d.ts","sourceRoot":"","sources":["../../src/batch/job-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,GAAG,CAAS;gBAER,GAAG,CAAC,EAAE,MAAM;IAIxB,OAAO,CAAC,OAAO;IAMT,IAAI,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlC,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAwB1C,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASjC,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAUhC;AAMD,wBAAgB,WAAW,IAAI,QAAQ,CAKtC;AAED,oCAAoC;AACpC,wBAAgB,aAAa,IAAI,IAAI,CAEpC"}

package/dist/batch/job-store.js

@@ -0,0 +1,77 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { getJobsDir } from "../config.js";
+export class JobStore {
+    cache = new Map();
+    dir;
+    constructor(dir) {
+        this.dir = dir ?? getJobsDir();
+    }
+    jobPath(id) {
+        // Sanitize id to prevent path traversal
+        const safe = id.replace(/[^a-zA-Z0-9_\-]/g, "_");
+        return path.join(this.dir, `${safe}.json`);
+    }
+    async save(job) {
+        const updated = { ...job, updated_at: new Date().toISOString() };
+        this.cache.set(job.id, updated);
+        await fs.mkdir(this.dir, { recursive: true });
+        const filePath = this.jobPath(job.id);
+        const tmpPath = filePath + ".tmp";
+        await fs.writeFile(tmpPath, JSON.stringify(updated, null, 2), "utf-8");
+        await fs.rename(tmpPath, filePath);
+    }
+    async load(id) {
+        if (this.cache.has(id))
+            return this.cache.get(id);
+        try {
+            const data = await fs.readFile(this.jobPath(id), "utf-8");
+            const job = JSON.parse(data);
+            this.cache.set(id, job);
+            return job;
+        }
+        catch (err) {
+            const isEnoent = err &&
+                typeof err === "object" &&
+                "code" in err &&
+                err.code === "ENOENT";
+            if (!isEnoent) {
+                console.error("[batch] Failed to load job:", err instanceof Error ? err.message : String(err));
+            }
+            return null;
+        }
+    }
+    async delete(id) {
+        this.cache.delete(id);
+        try {
+            await fs.unlink(this.jobPath(id));
+        }
+        catch {
+            // Job didn't exist on disk — fine
+        }
+    }
+    async list() {
+        try {
+            const files = await fs.readdir(this.dir);
+            return files
+                .filter((f) => f.endsWith(".json") && !f.endsWith(".tmp.json"))
+                .map((f) => f.replace(/\.json$/, ""));
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Singleton ──
+let store = null;
+export function getJobStore() {
+    if (!store) {
+        store = new JobStore();
+    }
+    return store;
+}
+/** Reset singleton (for testing) */
+export function resetJobStore() {
+    store = null;
+}
+//# sourceMappingURL=job-store.js.map

package/dist/batch/job-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-store.js","sourceRoot":"","sources":["../../src/batch/job-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,MAAM,OAAO,QAAQ;IACX,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IACpC,GAAG,CAAS;IAEpB,YAAY,GAAY;QACtB,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,UAAU,EAAE,CAAC;IACjC,CAAC;IAEO,OAAO,CAAC,EAAU;QACxB,wCAAwC;QACxC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAa;QACtB,MAAM,OAAO,GAAa,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAC3E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAEhC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAC;QAClC,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACvE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAU;QACnB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QAEnD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAa,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACxB,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,QAAQ,GACZ,GAAG;gBACH,OAAO,GAAG,KAAK,QAAQ;gBACvB,MAAM,IAAI,GAAG;gBACZ,GAA6B,CAAC,IAAI,KAAK,QAAQ,CAAC;YACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CACX,6BAA6B,EAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACjD,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzC,OAAO,KAAK;iBACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;iBAC9D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAED,kBAAkB;AAElB,IAAI,KAAK,GAAoB,IAAI,CAAC;AAElC,MAAM,UAAU,WAAW;IACzB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,IAAI,QAAQ,EAAE,CAAC;IACzB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,aAAa;IAC3B,KAAK,GAAG,IAAI,CAAC;AACf,CAAC"}

package/dist/batch/types.d.ts

@@ -0,0 +1,20 @@
+export interface BatchJobResult {
+    url: string;
+    success: boolean;
+    content?: string;
+    data?: unknown;
+    error?: string;
+    status_code?: number;
+    duration_ms: number;
+}
+export interface BatchJob {
+    id: string;
+    status: "running" | "completed" | "failed";
+    urls_total: number;
+    urls_completed: number;
+    urls_failed: number;
+    results: BatchJobResult[];
+    created_at: string;
+    updated_at: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/batch/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/batch/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB"}

package/dist/batch/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/batch/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/batch/types.ts"],"names":[],"mappings":""}

package/dist/cli-config.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Config file management for imperium-crawl CLI.
+ *
+ * Saves API keys to ~/.imperium-crawl/config.json so users don't
+ * need to set environment variables manually.
+ *
+ * Priority: process.env (system) > config.json
+ * applyCliConfig() fills in env vars from config only if not already set.
+ */
+export declare function getCliConfigPath(): string;
+export declare function loadCliConfig(): Record<string, string>;
+export declare function saveCliConfig(config: Record<string, string>): void;
+/**
+ * Apply config.json values to process.env.
+ * System env vars take priority — config values are only applied
+ * when the key is not already set.
+ *
+ * Call this once at startup, before any tool initialization.
+ */
+export declare function applyCliConfig(): void;
+//# sourceMappingURL=cli-config.d.ts.map

package/dist/cli-config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-config.d.ts","sourceRoot":"","sources":["../src/cli-config.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC;AAED,wBAAgB,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAWtD;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAIlE;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,IAAI,IAAI,CAOrC"}

package/dist/cli-config.js

@@ -0,0 +1,51 @@
+/**
+ * Config file management for imperium-crawl CLI.
+ *
+ * Saves API keys to ~/.imperium-crawl/config.json so users don't
+ * need to set environment variables manually.
+ *
+ * Priority: process.env (system) > config.json
+ * applyCliConfig() fills in env vars from config only if not already set.
+ */
+import path from "node:path";
+import os from "node:os";
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { SKILLS_DIR_NAME } from "./constants.js";
+const CONFIG_FILENAME = "config.json";
+export function getCliConfigPath() {
+    return path.join(os.homedir(), SKILLS_DIR_NAME, CONFIG_FILENAME);
+}
+export function loadCliConfig() {
+    try {
+        const content = readFileSync(getCliConfigPath(), "utf-8");
+        const parsed = JSON.parse(content);
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+            return parsed;
+        }
+    }
+    catch {
+        // File doesn't exist or invalid JSON — return empty config
+    }
+    return {};
+}
+export function saveCliConfig(config) {
+    const configPath = getCliConfigPath();
+    mkdirSync(path.dirname(configPath), { recursive: true });
+    writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n", "utf-8");
+}
+/**
+ * Apply config.json values to process.env.
+ * System env vars take priority — config values are only applied
+ * when the key is not already set.
+ *
+ * Call this once at startup, before any tool initialization.
+ */
+export function applyCliConfig() {
+    const config = loadCliConfig();
+    for (const [key, value] of Object.entries(config)) {
+        if (typeof value === "string" && !process.env[key]) {
+            process.env[key] = value;
+        }
+    }
+}
+//# sourceMappingURL=cli-config.js.map

package/dist/cli-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-config.js","sourceRoot":"","sources":["../src/cli-config.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjD,MAAM,eAAe,GAAG,aAAa,CAAC;AAEtC,MAAM,UAAU,gBAAgB;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,gBAAgB,EAAE,EAAE,OAAO,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,OAAO,MAAgC,CAAC;QAC1C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAA8B;IAC1D,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc;IAC5B,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/cli-onboarding.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Interactive setup wizard for imperium-crawl CLI.
+ *
+ * Usage: imperium-crawl setup
+ *
+ * Guides the user through configuring API keys and saves them to
+ * ~/.imperium-crawl/config.json for persistent use.
+ */
+export declare function runSetup(): Promise<void>;
+//# sourceMappingURL=cli-onboarding.d.ts.map

package/dist/cli-onboarding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-onboarding.d.ts","sourceRoot":"","sources":["../src/cli-onboarding.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAeH,wBAAsB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CA2H9C"}

package/dist/cli-onboarding.js

@@ -0,0 +1,128 @@
+/**
+ * Interactive setup wizard for imperium-crawl CLI.
+ *
+ * Usage: imperium-crawl setup
+ *
+ * Guides the user through configuring API keys and saves them to
+ * ~/.imperium-crawl/config.json for persistent use.
+ */
+import chalk from "chalk";
+import { input, select, confirm } from "@inquirer/prompts";
+import { loadCliConfig, saveCliConfig, getCliConfigPath } from "./cli-config.js";
+const BANNER = chalk.cyan(`
+  ██╗███╗   ███╗██████╗ ███████╗██████╗ ██╗██╗   ██╗███╗   ███╗
+  ██║████╗ ████║██╔══██╗██╔════╝██╔══██╗██║██║   ██║████╗ ████║
+  ██║██╔████╔██║██████╔╝█████╗  ██████╔╝██║██║   ██║██╔████╔██║
+  ██║██║╚██╔╝██║██╔═══╝ ██╔══╝  ██╔══██╗██║██║   ██║██║╚██╔╝██║
+  ██║██║ ╚═╝ ██║██║     ███████╗██║  ██║██║╚██████╔╝██║ ╚═╝ ██║
+  ╚═╝╚═╝     ╚═╝╚═╝     ╚══════╝╚═╝  ╚═╝╚═╝ ╚═════╝ ╚═╝     ╚═╝
+`);
+export async function runSetup() {
+    console.log(BANNER);
+    console.log(chalk.bold("  API Key Setup\n"));
+    const existing = loadCliConfig();
+    const config = { ...existing };
+    // ── Brave Search ──────────────────────────────────────────────────
+    const hasBrave = !!(process.env.BRAVE_API_KEY || existing.BRAVE_API_KEY);
+    if (hasBrave) {
+        console.log(chalk.green("  ✓ BRAVE_API_KEY") +
+            chalk.dim(" — already configured (search, news_search, image_search, video_search)"));
+    }
+    else {
+        console.log(chalk.dim("  Brave Search enables 4 search tools. Free tier: https://brave.com/search/api/\n"));
+        const braveKey = await input({
+            message: "Brave Search API key (press Enter to skip):",
+        });
+        if (braveKey.trim())
+            config.BRAVE_API_KEY = braveKey.trim();
+    }
+    console.log();
+    // ── LLM Provider ─────────────────────────────────────────────────
+    const hasLLM = !!(process.env.LLM_API_KEY || existing.LLM_API_KEY);
+    if (hasLLM) {
+        const currentProvider = process.env.LLM_PROVIDER || existing.LLM_PROVIDER || "anthropic";
+        console.log(chalk.green(`  ✓ LLM_API_KEY (${currentProvider})`) +
+            chalk.dim(" — already configured (ai_extract tool)"));
+    }
+    else {
+        console.log(chalk.dim("  LLM key enables the ai_extract tool — natural language data extraction.\n"));
+        const provider = await select({
+            message: "LLM provider (for ai_extract tool):",
+            choices: [
+                {
+                    name: "Anthropic (Claude Haiku — default)",
+                    value: "anthropic",
+                    description: "Fast, affordable. Get key: https://console.anthropic.com",
+                },
+                {
+                    name: "OpenAI (GPT-4o mini — default)",
+                    value: "openai",
+                    description: "Widely used. Get key: https://platform.openai.com",
+                },
+                {
+                    name: "MiniMax (M2.5 — 200K context, reasoning)",
+                    value: "minimax",
+                    description: "Strong model, OpenAI-compatible API.",
+                },
+                {
+                    name: "Skip for now",
+                    value: "skip",
+                    description: "Configure later via env vars or run setup again.",
+                },
+            ],
+        });
+        if (provider !== "skip") {
+            config.LLM_PROVIDER = provider;
+            const providerLabel = provider === "anthropic" ? "Anthropic" : provider === "openai" ? "OpenAI" : "MiniMax";
+            const llmKey = await input({
+                message: `${providerLabel} API key:`,
+            });
+            if (llmKey.trim())
+                config.LLM_API_KEY = llmKey.trim();
+        }
+    }
+    console.log();
+    // ── 2Captcha ─────────────────────────────────────────────────────
+    const hasCaptcha = !!(process.env.TWOCAPTCHA_API_KEY ||
+        process.env.TWO_CAPTCHA_API_KEY ||
+        existing.TWOCAPTCHA_API_KEY);
+    if (hasCaptcha) {
+        console.log(chalk.green("  ✓ TWOCAPTCHA_API_KEY") +
+            chalk.dim(" — already configured (auto CAPTCHA solving in stealth level 3)"));
+    }
+    else {
+        const wantCaptcha = await confirm({
+            message: "Configure 2Captcha for automatic CAPTCHA solving? (optional)",
+            default: false,
+        });
+        if (wantCaptcha) {
+            const captchaKey = await input({
+                message: "2Captcha API key (https://2captcha.com):",
+            });
+            if (captchaKey.trim())
+                config.TWOCAPTCHA_API_KEY = captchaKey.trim();
+        }
+    }
+    // ── Save + Summary ────────────────────────────────────────────────
+    saveCliConfig(config);
+    console.log("\n" + chalk.bold("  ─────────────────────────────────────────"));
+    const enabledTools = [];
+    if (config.BRAVE_API_KEY || process.env.BRAVE_API_KEY) {
+        enabledTools.push("search, news_search, image_search, video_search");
+    }
+    if (config.LLM_API_KEY || process.env.LLM_API_KEY) {
+        enabledTools.push("ai_extract");
+    }
+    if (config.TWOCAPTCHA_API_KEY || process.env.TWOCAPTCHA_API_KEY) {
+        enabledTools.push("CAPTCHA auto-solve (stealth lvl 3)");
+    }
+    if (enabledTools.length > 0) {
+        console.log(chalk.green(`\n  🚀 Ready! Extra tools enabled: ${enabledTools.join(", ")}`));
+    }
+    else {
+        console.log(chalk.yellow("\n  ⚠ No API keys configured.") +
+            chalk.dim(" Basic scraping tools work without keys."));
+    }
+    console.log(chalk.dim(`\n  Config saved → ${getCliConfigPath()}\n`));
+}
+//# sourceMappingURL=cli-onboarding.js.map

package/dist/cli-onboarding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-onboarding.js","sourceRoot":"","sources":["../src/cli-onboarding.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEjF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;;;;;;;CAOzB,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,QAAQ;IAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAE7C,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC;IACjC,MAAM,MAAM,GAA2B,EAAE,GAAG,QAAQ,EAAE,CAAC;IAEvD,qEAAqE;IACrE,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa,CAAC,CAAC;IACzE,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,mBAAmB,CAAC;YAC9B,KAAK,CAAC,GAAG,CAAC,yEAAyE,CAAC,CACvF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mFAAmF,CAAC,CAAC,CAAC;QAC5G,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,6CAA6C;SACvD,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,IAAI,EAAE;YAAE,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC9D,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;IACnE,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,IAAI,WAAW,CAAC;QACzF,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,oBAAoB,eAAe,GAAG,CAAC;YACjD,KAAK,CAAC,GAAG,CAAC,yCAAyC,CAAC,CACvD,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC,CAAC;QACtG,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC;YAC5B,OAAO,EAAE,qCAAqC;YAC9C,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,oCAAoC;oBAC1C,KAAK,EAAE,WAAW;oBAClB,WAAW,EAAE,0DAA0D;iBACxE;gBACD;oBACE,IAAI,EAAE,gCAAgC;oBACtC,KAAK,EAAE,QAAQ;oBACf,WAAW,EAAE,mDAAmD;iBACjE;gBACD;oBACE,IAAI,EAAE,0CAA0C;oBAChD,KAAK,EAAE,SAAS;oBAChB,WAAW,EAAE,sCAAsC;iBACpD;gBACD;oBACE,IAAI,EAAE,cAAc;oBACpB,KAAK,EAAE,MAAM;oBACb,WAAW,EAAE,kDAAkD;iBAChE;aACF;SACF,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YACxB,MAAM,CAAC,YAAY,GAAG,QAAQ,CAAC;YAC/B,MAAM,aAAa,GACjB,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YACxF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC;gBACzB,OAAO,EAAE,GAAG,aAAa,WAAW;aACrC,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,IAAI,EAAE;gBAAE,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QACxD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,oEAAoE;IACpE,MAAM,UAAU,GAAG,CAAC,CAAC,CACnB,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAC9B,OAAO,CAAC,GAAG,CAAC,mBAAmB;QAC/B,QAAQ,CAAC,kBAAkB,CAC5B,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,wBAAwB,CAAC;YACnC,KAAK,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAC/E,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC;YAChC,OAAO,EAAE,8DAA8D;YACvE,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC;gBAC7B,OAAO,EAAE,0CAA0C;aACpD,CAAC,CAAC;YACH,IAAI,UAAU,CAAC,IAAI,EAAE;gBAAE,MAAM,CAAC,kBAAkB,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;QACvE,CAAC;IACH,CAAC;IAED,qEAAqE;IACrE,aAAa,CAAC,MAAM,CAAC,CAAC;IAEtB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC,CAAC;IAE9E,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,MAAM,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QACtD,YAAY,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;QAClD,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,MAAM,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;QAChE,YAAY,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sCAAsC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5F,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,MAAM,CAAC,+BAA+B,CAAC;YAC3C,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,CACxD,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC;AACvE,CAAC"}

package/dist/cli-tui.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Full TUI (Terminal User Interface) for imperium-crawl — v3
+ *
+ * Slash-command-driven UX (Claude Code aesthetic).
+ * readline for main prompt, @clack/prompts for param collection only.
+ *
+ * Activated when: no CLI args AND process.stdout.isTTY.
+ * Non-TTY mode (pipe/CI/agents) is unaffected — MCP server runs as before.
+ * CLI subcommands (scrape, crawl, etc.) are unaffected — they bypass this.
+ */
+export declare function runTui(): Promise<void>;
+//# sourceMappingURL=cli-tui.d.ts.map

package/dist/cli-tui.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-tui.d.ts","sourceRoot":"","sources":["../src/cli-tui.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAgjCH,wBAAsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAsB5C"}