firecrawl-cli 1.17.0 → 1.18.0

package/dist/commands/experimental/workflows/knowledge-base.js

@@ -1,319 +0,0 @@
-"use strict";
-/**
- * Workflow: Knowledge Base
- *
- * A single command that adapts based on the user's goal: local reference docs,
- * RAG-ready chunks, fine-tuning datasets, or full doc site mirrors. All output
- * follows the `.firecrawl/<hostname>/<path>/index.md` convention.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.register = register;
-const backends_1 = require("../backends");
-const shared_1 = require("../shared");
-// ─── Input gathering ────────────────────────────────────────────────────────
-async function gatherInputs(prefill) {
-    const { input, select } = await Promise.resolve().then(() => __importStar(require('@inquirer/prompts')));
-    const rawSource = prefill?.source ||
-        (await input({
-            message: 'What do you want to build a knowledge base from? (URL or topic)',
-            validate: (0, shared_1.validateRequired)('URL or topic'),
-        }));
-    const goal = await select({
-        message: 'What are you building this for?',
-        choices: [
-            {
-                name: 'Local reference (organized markdown files)',
-                value: 'reference',
-            },
-            {
-                name: 'RAG / embedding pipeline (chunked, with metadata)',
-                value: 'rag',
-            },
-            { name: 'Fine-tuning dataset (JSONL training data)', value: 'train' },
-            { name: 'Documentation scrape (mirror a doc site)', value: 'docs' },
-        ],
-    });
-    let trainFormat = '';
-    let trainExamples = '';
-    if (goal === 'train') {
-        trainFormat = (await select({
-            message: 'Training data format?',
-            choices: [
-                { name: 'OpenAI JSONL (messages array)', value: 'openai' },
-                { name: 'Alpaca (instruction/input/output)', value: 'alpaca' },
-                { name: 'ShareGPT (conversations)', value: 'sharegpt' },
-            ],
-        }));
-        trainExamples = await input({
-            message: 'Roughly how many training examples?',
-            default: '100',
-        });
-    }
-    const depth = await select({
-        message: 'How thorough?',
-        choices: [
-            { name: 'Quick (5-10 sources)', value: 'quick' },
-            { name: 'Thorough (15-25 sources)', value: 'thorough' },
-            { name: 'Exhaustive (25+ sources)', value: 'exhaustive' },
-        ],
-    });
-    const context = await input({
-        message: 'Any specific focus or instructions? (leave blank to skip)',
-        default: '',
-    });
-    const outputDir = await input({
-        message: 'Output directory?',
-        default: '.firecrawl/',
-    });
-    return {
-        source: (0, shared_1.normalizeSource)(rawSource),
-        goal,
-        depth,
-        context,
-        outputDir,
-        trainFormat,
-        trainExamples,
-    };
-}
-// ─── System prompt ──────────────────────────────────────────────────────────
-const FILE_CONVENTION = `## File Organization
-
-**IMPORTANT:** Follow the same structure as \`firecrawl download\`. Save all files under \`.firecrawl/\` using nested directories that mirror each URL's hostname and path:
-
-\`\`\`
-.firecrawl/
-  <hostname>/
-    <path>/
-      index.md          # Page content as clean markdown
-\`\`\`
-
-For example, \`https://docs.stripe.com/api/charges\` becomes:
-\`\`\`
-.firecrawl/docs.stripe.com/api/charges/index.md
-\`\`\`
-
-Strip \`www.\` from hostnames. Each page gets its own directory with an \`index.md\` inside it.`;
-function buildGoalInstructions(opts) {
-    switch (opts.goal) {
-        case 'reference':
-            return `${FILE_CONVENTION}
-
-Also create these at the root of \`${opts.outputDir}\`:
-- \`index.md\` -- Table of contents with links to all scraped pages
-- \`sources.json\` -- All URLs scraped with metadata (title, type, url)
-
-Each markdown file should have frontmatter:
-\`\`\`yaml
----
-title: "Page Title"
-url: "https://..."
-source: "Source Name"
-type: "docs | article | tutorial | reference | discussion"
----
-\`\`\`
-
-Focus on clean, readable markdown. Preserve code examples and formatting.`;
-        case 'rag':
-            return `${FILE_CONVENTION}
-
-After scraping, chunk each page into embedding-ready pieces (500-1500 tokens). Save chunks alongside the source:
-\`\`\`
-.firecrawl/<hostname>/<path>/
-  index.md              # Full page content
-  chunks/
-    001.md              # Chunk 1
-    002.md              # Chunk 2
-\`\`\`
-
-Each chunk file should have frontmatter:
-\`\`\`yaml
----
-title: "Page Title"
-url: "https://..."
-chunk: 1
-total_chunks: 5
-section: "Section Name"
----
-\`\`\`
-
-Also create \`${opts.outputDir}/manifest.json\` listing every chunk with its metadata for easy ingestion into a vector store.`;
-        case 'train':
-            return `${FILE_CONVENTION}
-
-Scrape source pages into the \`.firecrawl/\` directory structure first, then generate training data from the scraped content.
-
-## Training Data Format
-
-${opts.trainFormat === 'openai'
-                ? `OpenAI fine-tuning JSONL. Each line:
-\`\`\`json
-{"messages": [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
-\`\`\``
-                : opts.trainFormat === 'alpaca'
-                    ? `Alpaca format JSONL. Each line:
-\`\`\`json
-{"instruction": "...", "input": "...", "output": "..."}
-\`\`\``
-                    : `ShareGPT conversation JSONL. Each line:
-\`\`\`json
-{"conversations": [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]}
-\`\`\``}
-
-Target ~${opts.trainExamples} examples.
-
-Save the dataset to \`training-data.jsonl\` in the current directory.
-
-Also save \`training-metadata.json\` with:
-- Total examples generated
-- Sources used (URLs)
-- Topic coverage breakdown
-- Format used
-
-### Quality Guidelines
-
-- Each example should be self-contained and accurate
-- Vary the instruction style (questions, commands, scenarios)
-- Include code examples where relevant
-- Remove boilerplate, navigation, and ads from scraped content
-- Cite the source URL in a metadata field for traceability
-- Deduplicate similar examples`;
-        case 'docs':
-            return `${FILE_CONVENTION}
-
-Also create \`${opts.outputDir}/index.md\` as a table of contents linking to all scraped pages, organized by section.
-
-Each markdown file should have frontmatter:
-\`\`\`yaml
----
-title: "Page Title"
-url: "https://..."
-section: "Section Name"
----
-\`\`\`
-
-Be thorough. Scrape every page, preserve all code examples. This content will be used as LLM context, so accuracy matters.`;
-        default:
-            return FILE_CONVENTION;
-    }
-}
-function buildAgentStrategy(goal) {
-    switch (goal) {
-        case 'docs':
-            return `## Agent Assignments
-
-Spawn agents based on the doc structure:
-1. **Section Agent** (one per major section) -- Scrape all pages in the section. Save each page as clean markdown. Preserve code examples and formatting.
-
-Start by mapping the site with \`firecrawl map\` to discover all pages, then divide by section.`;
-        case 'train':
-            return `## Agent Assignments
-
-Spawn agents by source type:
-1. **Documentation Agent** -- Scrape official docs. Generate instruction/response pairs from doc sections (e.g., "How do I X?" with the answer from docs).
-2. **Tutorial Agent** -- Scrape tutorials and how-to articles. Generate step-by-step instruction pairs.
-3. **Q&A Agent** -- Scrape Stack Overflow, GitHub discussions, forums. Extract real question/answer pairs.
-4. **Reference Agent** -- Scrape reference material. Generate factual Q&A pairs.`;
-        default:
-            return `## Agent Assignments
-
-Spawn agents by source type:
-1. **Official Docs Agent** -- Find and scrape official documentation, reference material, specs.
-2. **Articles & Tutorials Agent** -- Find and scrape the best articles, blog posts, tutorials.
-3. **Community & Discussions Agent** -- Find and scrape relevant forum posts, Stack Overflow answers, GitHub discussions.
-4. **Reference Agent** -- Wikipedia, glossaries, standards documents, whitepapers.
-
-Adjust agents based on what sources exist for the topic.`;
-    }
-}
-function buildSystemPrompt(opts) {
-    const depthInstructions = {
-        quick: 'Find and scrape 5-10 of the best sources.',
-        thorough: 'Find and scrape 15-25 sources covering different perspectives.',
-        exhaustive: 'Find and scrape 25+ sources including primary docs, articles, tutorials, and reference material.',
-    };
-    return `You are a knowledge base team lead powered by Firecrawl. You scrape web content and organize it into structured, LLM-ready formats.
-
-${shared_1.FIRECRAWL_TOOLS_BLOCK}
-
-## Depth
-
-${depthInstructions[opts.depth]}
-
-## Your Strategy
-
-You are a **team lead**. Your job is to:
-
-1. **Find the best sources** -- ${opts.goal === 'docs' ? 'Map the documentation site to discover all pages.' : 'Search broadly to identify the most valuable sources on the topic.'}
-2. **Spawn parallel subagents** -- Divide the work across agents. Each scrapes their assigned sources.
-3. **Collect and organize** -- Build the final output structure from all agent results.
-
-${buildAgentStrategy(opts.goal)}
-
-${shared_1.SUBAGENT_INSTRUCTIONS}
-
-${buildGoalInstructions(opts)}
-
----
-
-Tell the user the output path when done.
-
-Start immediately.`;
-}
-// ─── Command registration ───────────────────────────────────────────────────
-function register(parentCmd, backend) {
-    const config = backends_1.BACKENDS[backend];
-    parentCmd
-        .command('knowledge-base')
-        .description('Build a knowledge base from web content (docs, RAG, fine-tuning)')
-        .argument('[source]', 'URL or topic to build from')
-        .option('-y, --yes', 'Auto-approve all tool permissions')
-        .action(async (source, options) => {
-        const inputs = await gatherInputs(source ? { source } : undefined);
-        const skipPermissions = options.yes || (await (0, shared_1.askPermissionMode)(backend));
-        console.log(`\nLaunching ${config.displayName}...\n`);
-        (0, backends_1.launchAgent)(backend, buildSystemPrompt({
-            goal: inputs.goal,
-            depth: inputs.depth,
-            outputDir: inputs.outputDir,
-            trainFormat: inputs.trainFormat,
-            trainExamples: inputs.trainExamples,
-        }), (0, shared_1.buildMessage)([
-            `Build a knowledge base from: ${inputs.source}`,
-            inputs.context,
-        ]), skipPermissions);
-    });
-}
-//# sourceMappingURL=knowledge-base.js.map

package/dist/commands/experimental/workflows/knowledge-base.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"knowledge-base.js","sourceRoot":"","sources":["../../../../src/commands/experimental/workflows/knowledge-base.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2TH,4BAgCC;AAxVD,0CAAkE;AAClE,sCAOmB;AAcnB,+EAA+E;AAE/E,KAAK,UAAU,YAAY,CAAC,OAA6B;IACvD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,wDAAa,mBAAmB,GAAC,CAAC;IAE5D,MAAM,SAAS,GACb,OAAO,EAAE,MAAM;QACf,CAAC,MAAM,KAAK,CAAC;YACX,OAAO,EACL,iEAAiE;YACnE,QAAQ,EAAE,IAAA,yBAAgB,EAAC,cAAc,CAAC;SAC3C,CAAC,CAAC,CAAC;IAEN,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC;QACxB,OAAO,EAAE,iCAAiC;QAC1C,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,4CAA4C;gBAClD,KAAK,EAAE,WAAW;aACnB;YACD;gBACE,IAAI,EAAE,mDAAmD;gBACzD,KAAK,EAAE,KAAK;aACb;YACD,EAAE,IAAI,EAAE,2CAA2C,EAAE,KAAK,EAAE,OAAO,EAAE;YACrE,EAAE,IAAI,EAAE,0CAA0C,EAAE,KAAK,EAAE,MAAM,EAAE;SACpE;KACF,CAAC,CAAC;IAEH,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,aAAa,GAAG,EAAE,CAAC;IAEvB,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrB,WAAW,GAAG,CAAC,MAAM,MAAM,CAAC;YAC1B,OAAO,EAAE,uBAAuB;YAChC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,+BAA+B,EAAE,KAAK,EAAE,QAAQ,EAAE;gBAC1D,EAAE,IAAI,EAAE,mCAAmC,EAAE,KAAK,EAAE,QAAQ,EAAE;gBAC9D,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,UAAU,EAAE;aACxD;SACF,CAAC,CAAW,CAAC;QAEd,aAAa,GAAG,MAAM,KAAK,CAAC;YAC1B,OAAO,EAAE,qCAAqC;YAC9C,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;QACzB,OAAO,EAAE,eAAe;QACxB,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,OAAO,EAAE;YAChD,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,UAAU,EAAE;YACvD,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,YAAY,EAAE;SAC1D;KACF,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC;QAC1B,OAAO,EAAE,2DAA2D;QACpE,OAAO,EAAE,EAAE;KACZ,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC;QAC5B,OAAO,EAAE,mBAAmB;QAC5B,OAAO,EAAE,aAAa;KACvB,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,IAAA,wBAAe,EAAC,SAAS,CAAC;QAClC,IAAI;QACJ,KAAK;QACL,OAAO;QACP,SAAS;QACT,WAAW;QACX,aAAa;KACd,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;gGAgBwE,CAAC;AAEjG,SAAS,qBAAqB,CAAC,IAK9B;IACC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,WAAW;YACd,OAAO,GAAG,eAAe;;qCAEM,IAAI,CAAC,SAAS;;;;;;;;;;;;;;0EAcuB,CAAC;QAEvE,KAAK,KAAK;YACR,OAAO,GAAG,eAAe;;;;;;;;;;;;;;;;;;;;;;gBAsBf,IAAI,CAAC,SAAS,gGAAgG,CAAC;QAE3H,KAAK,OAAO;YACV,OAAO,GAAG,eAAe;;;;;;EAO7B,IAAI,CAAC,WAAW,KAAK,QAAQ;gBAC3B,CAAC,CAAC;;;OAGC;gBACH,CAAC,CAAC,IAAI,CAAC,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC;;;OAGD;oBACD,CAAC,CAAC;;;OAIR;;UAEU,IAAI,CAAC,aAAa;;;;;;;;;;;;;;;;;+BAiBG,CAAC;QAE5B,KAAK,MAAM;YACT,OAAO,GAAG,eAAe;;gBAEf,IAAI,CAAC,SAAS;;;;;;;;;;;2HAW6F,CAAC;QAExH;YACE,OAAO,eAAe,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,MAAM;YACT,OAAO;;;;;gGAKmF,CAAC;QAE7F,KAAK,OAAO;YACV,OAAO;;;;;;iFAMoE,CAAC;QAE9E;YACE,OAAO;;;;;;;;yDAQ4C,CAAC;IACxD,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,IAM1B;IACC,MAAM,iBAAiB,GAA2B;QAChD,KAAK,EAAE,2CAA2C;QAClD,QAAQ,EAAE,gEAAgE;QAC1E,UAAU,EACR,kGAAkG;KACrG,CAAC;IAEF,OAAO;;EAEP,8BAAqB;;;;EAIrB,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;;;;;;kCAMG,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,mDAAmD,CAAC,CAAC,CAAC,oEAAoE;;;;EAIjL,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;;EAE7B,8BAAqB;;EAErB,qBAAqB,CAAC,IAAI,CAAC;;;;;;mBAMV,CAAC;AACpB,CAAC;AAED,+EAA+E;AAE/E,SAAgB,QAAQ,CAAC,SAAkB,EAAE,OAAgB;IAC3D,MAAM,MAAM,GAAG,mBAAQ,CAAC,OAAO,CAAC,CAAC;IAEjC,SAAS;SACN,OAAO,CAAC,gBAAgB,CAAC;SACzB,WAAW,CACV,kEAAkE,CACnE;SACA,QAAQ,CAAC,UAAU,EAAE,4BAA4B,CAAC;SAClD,MAAM,CAAC,WAAW,EAAE,mCAAmC,CAAC;SACxD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAEnE,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,OAAO,CAAC,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,OAAO,CAAC,CAAC;QAEtD,IAAA,sBAAW,EACT,OAAO,EACP,iBAAiB,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC,EACF,IAAA,qBAAY,EAAC;YACX,gCAAgC,MAAM,CAAC,MAAM,EAAE;YAC/C,MAAM,CAAC,OAAO;SACf,CAAC,EACF,eAAe,CAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/experimental/workflows/knowledge-ingest.d.ts

@@ -1,12 +0,0 @@
-/**
- * Workflow: Knowledge Base Ingestion
- *
- * Navigates auth-gated documentation portals using saved browser profiles,
- * paginates through articles and sections, and extracts everything into
- * structured JSON. Built for portals that require login, have pagination,
- * or use JS-heavy rendering that static scraping can't handle.
- */
-import { Command } from 'commander';
-import { type Backend } from '../backends';
-export declare function register(parentCmd: Command, backend: Backend): void;
-//# sourceMappingURL=knowledge-ingest.d.ts.map

package/dist/commands/experimental/workflows/knowledge-ingest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"knowledge-ingest.d.ts","sourceRoot":"","sources":["../../../../src/commands/experimental/workflows/knowledge-ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,KAAK,OAAO,EAAyB,MAAM,aAAa,CAAC;AAuNlE,wBAAgB,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CA+BnE"}

package/dist/commands/experimental/workflows/knowledge-ingest.js

@@ -1,251 +0,0 @@
-"use strict";
-/**
- * Workflow: Knowledge Base Ingestion
- *
- * Navigates auth-gated documentation portals using saved browser profiles,
- * paginates through articles and sections, and extracts everything into
- * structured JSON. Built for portals that require login, have pagination,
- * or use JS-heavy rendering that static scraping can't handle.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.register = register;
-const backends_1 = require("../backends");
-const shared_1 = require("../shared");
-// ─── Input gathering ────────────────────────────────────────────────────────
-async function gatherInputs(prefill) {
-    if (prefill?.url) {
-        return {
-            url: prefill.url,
-            profile: '',
-            format: 'json',
-            maxPages: '100',
-            context: '',
-        };
-    }
-    const { input, select } = await Promise.resolve().then(() => __importStar(require('@inquirer/prompts')));
-    const url = await input({
-        message: 'URL of the docs portal or knowledge base?',
-        validate: (0, shared_1.validateRequired)('URL'),
-    });
-    const profile = await input({
-        message: 'Browser profile for auth? (leave blank for public/anonymous access)',
-        default: '',
-    });
-    const format = await select({
-        message: 'Output format?',
-        choices: [
-            {
-                name: 'Structured JSON (articles with metadata)',
-                value: 'json',
-            },
-            {
-                name: 'Markdown files (one per article, .firecrawl/ convention)',
-                value: 'markdown',
-            },
-            {
-                name: 'Single merged file (all content in one document)',
-                value: 'merged',
-            },
-        ],
-    });
-    const maxPages = await input({
-        message: 'Max pages to extract?',
-        default: '100',
-    });
-    const context = await input({
-        message: 'Any specific sections or topics to focus on? (leave blank for everything)',
-        default: '',
-    });
-    return { url, profile, format, maxPages, context };
-}
-// ─── System prompt ──────────────────────────────────────────────────────────
-function buildSystemPrompt(opts) {
-    const profileBlock = opts.profile
-        ? `\n### Authentication\n\nUse the saved browser profile \`${opts.profile}\` to access auth-gated content:\n\`\`\`bash\nfirecrawl browser "open <url>" --profile ${opts.profile}\n\`\`\`\nAfter the first \`open\` with \`--profile\`, subsequent browser commands don't need the flag.`
-        : '';
-    const outputInstructions = {
-        json: `Save results to \`knowledge-base.json\` in the current directory. Tell the user the file path when done.
-
-Use this schema:
-\`\`\`json
-{
-  "source": "string (portal name)",
-  "url": "string (base URL)",
-  "extractedAt": "ISO-8601",
-  "totalArticles": 0,
-  "sections": [
-    {
-      "name": "string",
-      "articles": [
-        {
-          "title": "string",
-          "url": "string",
-          "section": "string",
-          "content": "string (full markdown content)",
-          "metadata": {
-            "lastUpdated": "string",
-            "author": "string",
-            "tags": ["string"]
-          }
-        }
-      ]
-    }
-  ]
-}
-\`\`\``,
-        markdown: `Save each article as a separate markdown file following the .firecrawl/ convention:
-\`\`\`
-.firecrawl/<hostname>/<path>/index.md
-\`\`\`
-
-Each file should have frontmatter:
-\`\`\`yaml
----
-title: "Article Title"
-url: "https://..."
-section: "Section Name"
-lastUpdated: "date if available"
----
-\`\`\`
-
-Also create \`.firecrawl/index.md\` as a table of contents. Tell the user the output path when done.`,
-        merged: `Save all content to a single \`knowledge-base.md\` file in the current directory with a table of contents at the top. Each article should be a section with its title as a heading. Tell the user the file path when done.`,
-    };
-    return `You are a knowledge base ingestion agent powered by Firecrawl. You use a real cloud browser to navigate documentation portals -- including auth-gated ones -- paginate through all articles, and extract content into structured formats.
-
-## STEP 1: Launch Browser and Open Live View
-
-Before anything else, launch a browser session so the user can watch:
-
-\`\`\`bash
-firecrawl browser launch-session --json
-\`\`\`
-
-Extract the \`interactiveLiveViewUrl\` from the JSON output and open it (NOT the regular \`liveViewUrl\` -- the interactive one lets the user click and interact):
-
-\`\`\`bash
-open "<interactiveLiveViewUrl>"          # macOS
-xdg-open "<interactiveLiveViewUrl>"     # Linux
-\`\`\`
-
-If the \`open\` command fails, print the URL clearly.
-${profileBlock}
-
-## STEP 2: Map the Portal Structure
-
-1. Open the portal's main page / table of contents / sidebar
-2. Snapshot to see the navigation structure
-3. Identify all sections, categories, or sidebar nav items
-4. Build a list of all article URLs to visit
-
-If the portal has a sitemap or API docs index, use that. Otherwise, click through sidebar/nav items to discover pages.
-
-\`\`\`bash
-firecrawl browser "open <url>"
-firecrawl browser "snapshot"
-firecrawl browser "scrape"
-\`\`\`
-
-Also try \`firecrawl map <url>\` to discover URLs programmatically -- combine with browser navigation for auth-gated content.
-
-## STEP 3: Extract Articles
-
-For each article/page:
-
-1. **Navigate** to the article URL
-2. **Wait** for content to fully render (some portals are JS-heavy)
-3. **Scrape** the full page content as markdown
-4. **Extract metadata** -- title, section, last updated date, author, tags
-5. **Handle pagination** within articles (multi-page docs, "Next" buttons)
-6. **Navigate** to the next article
-
-### Pagination strategies:
-- **Sidebar navigation**: Click through each sidebar item systematically
-- **"Next article" links**: Follow sequential article links
-- **Paginated lists**: Click page numbers or "Load More"
-- **Infinite scroll**: Scroll down and snapshot to load more items
-- **Search/filter**: If the portal has search, use it to find specific sections
-
-### Browser commands:
-\`\`\`bash
-firecrawl browser "open <url>"
-firecrawl browser "snapshot"
-firecrawl browser "click @<ref>"
-firecrawl browser "scroll down"
-firecrawl browser "scrape"
-\`\`\`
-
-## Limits
-
-Extract up to ${opts.maxPages} pages. Prioritize breadth (cover all sections) over depth (every sub-article) if you're approaching the limit.
-
-## Output Format
-
-${outputInstructions[opts.format]}
-
-## Quality Guidelines
-
-- Preserve code examples, tables, and formatting
-- Strip navigation chrome, headers, footers -- extract only article content
-- Note any pages that failed to load or were access-restricted
-- Track progress: print "Extracted X/Y articles..." periodically
-
-Do everything sequentially. Start immediately.`;
-}
-// ─── Command registration ───────────────────────────────────────────────────
-function register(parentCmd, backend) {
-    const config = backends_1.BACKENDS[backend];
-    parentCmd
-        .command('knowledge-ingest')
-        .description('Extract auth-gated docs portals into structured JSON or markdown')
-        .argument('[url]', 'URL of the docs portal or knowledge base')
-        .option('-y, --yes', 'Auto-approve all tool permissions')
-        .action(async (url, options) => {
-        const inputs = await gatherInputs(url ? { url } : undefined);
-        const parts = [`Ingest knowledge base from: ${inputs.url}`];
-        if (inputs.context)
-            parts.push(`Focus on: ${inputs.context}`);
-        const userMessage = parts.join('. ') + '.';
-        const skipPermissions = true;
-        console.log(`\nLaunching ${config.displayName}...\n`);
-        (0, backends_1.launchAgent)(backend, buildSystemPrompt({
-            profile: inputs.profile,
-            format: inputs.format,
-            maxPages: inputs.maxPages,
-        }), userMessage, skipPermissions);
-    });
-}
-//# sourceMappingURL=knowledge-ingest.js.map

package/dist/commands/experimental/workflows/knowledge-ingest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"knowledge-ingest.js","sourceRoot":"","sources":["../../../../src/commands/experimental/workflows/knowledge-ingest.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0NH,4BA+BC;AAtPD,0CAAkE;AAClE,sCAA6C;AAY7C,+EAA+E;AAE/E,KAAK,UAAU,YAAY,CAAC,OAA0B;IACpD,IAAI,OAAO,EAAE,GAAG,EAAE,CAAC;QACjB,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,wDAAa,mBAAmB,GAAC,CAAC;IAE5D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC;QACtB,OAAO,EAAE,2CAA2C;QACpD,QAAQ,EAAE,IAAA,yBAAgB,EAAC,KAAK,CAAC;KAClC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC;QAC1B,OAAO,EACL,qEAAqE;QACvE,OAAO,EAAE,EAAE;KACZ,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;QAC1B,OAAO,EAAE,gBAAgB;QACzB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,0CAA0C;gBAChD,KAAK,EAAE,MAAM;aACd;YACD;gBACE,IAAI,EAAE,0DAA0D;gBAChE,KAAK,EAAE,UAAU;aAClB;YACD;gBACE,IAAI,EAAE,kDAAkD;gBACxD,KAAK,EAAE,QAAQ;aAChB;SACF;KACF,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC;QAC3B,OAAO,EAAE,uBAAuB;QAChC,OAAO,EAAE,KAAK;KACf,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC;QAC1B,OAAO,EACL,2EAA2E;QAC7E,OAAO,EAAE,EAAE;KACZ,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AACrD,CAAC;AAED,+EAA+E;AAE/E,SAAS,iBAAiB,CAAC,IAI1B;IACC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO;QAC/B,CAAC,CAAC,2DAA2D,IAAI,CAAC,OAAO,0FAA0F,IAAI,CAAC,OAAO,yGAAyG;QACxR,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,kBAAkB,GAA2B;QACjD,IAAI,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BH;QACH,QAAQ,EAAE;;;;;;;;;;;;;;;qGAeuF;QACjG,MAAM,EAAE,4NAA4N;KACrO,CAAC;IAEF,OAAO;;;;;;;;;;;;;;;;;;EAkBP,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gBAgDE,IAAI,CAAC,QAAQ;;;;EAI3B,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC;;;;;;;;;+CASc,CAAC;AAChD,CAAC;AAED,+EAA+E;AAE/E,SAAgB,QAAQ,CAAC,SAAkB,EAAE,OAAgB;IAC3D,MAAM,MAAM,GAAG,mBAAQ,CAAC,OAAO,CAAC,CAAC;IAEjC,SAAS;SACN,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CACV,kEAAkE,CACnE;SACA,QAAQ,CAAC,OAAO,EAAE,0CAA0C,CAAC;SAC7D,MAAM,CAAC,WAAW,EAAE,mCAAmC,CAAC;SACxD,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAE7D,MAAM,KAAK,GAAG,CAAC,+BAA+B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QAE3C,MAAM,eAAe,GAAG,IAAI,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,OAAO,CAAC,CAAC;QAEtD,IAAA,sBAAW,EACT,OAAO,EACP,iBAAiB,CAAC;YAChB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC,EACF,WAAW,EACX,eAAe,CAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/experimental/workflows/lead-gen.d.ts

@@ -1,11 +0,0 @@
-/**
- * Workflow: Lead Generation
- *
- * Uses a cloud browser to fill search forms on prospect databases (Apollo,
- * LinkedIn Sales Nav, ZoomInfo, etc.), apply filters, paginate through
- * results, and extract contact details at scale into structured formats.
- */
-import { Command } from 'commander';
-import { type Backend } from '../backends';
-export declare function register(parentCmd: Command, backend: Backend): void;
-//# sourceMappingURL=lead-gen.d.ts.map

package/dist/commands/experimental/workflows/lead-gen.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"lead-gen.d.ts","sourceRoot":"","sources":["../../../../src/commands/experimental/workflows/lead-gen.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,KAAK,OAAO,EAAyB,MAAM,aAAa,CAAC;AAkOlE,wBAAgB,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CAwCnE"}