@render-harness/cap-scrape-firecrawl 0.1.1 → 0.1.4

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"names":[],"mappings":";;;;;;AAqCA,IAAM,IAAA,GAAO,OAAA,CAAQ,aAAA,CAAc,MAAA,CAAA,IAAA,CAAY,GAAG,CAAC,CAAA;AACnD,IAAM,UAAA,GAAa,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,QAAQ,CAAA;AAQ5C,SAAS,WAAW,GAAA,EAAkB;AACpC,EAAA,MAAM,MAAM,GAAA,CAAI,MAAA;AAChB,EAAA,OAAO;AAAA,IACL,SAAA,EAAW,IAAI,SAAA,IAAa,mBAAA;AAAA,IAC5B,OAAA,EAAS,IAAI,OAAA,IAAW,IAAA;AAAA,IACxB,OAAA,EAAS,IAAI,OAAA,IAAW;AAAA,GAC1B;AACF;AAEA,IAAM,oBAAA,GAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAAA;AAc7B,IAAI,YAAA,GAAe,KAAA;AACnB,eAAe,YAAA,GAA8B;AAC3C,EAAA,IAAI,YAAA,EAAc;AAClB,EAAA,MAAM,OAAO,OAAA,EAAQ;AACrB,EAAA,MAAM,IAAA,CAAK,MAAM,oBAAoB,CAAA;AACrC,EAAA,YAAA,GAAe,IAAA;AACjB;AAEA,IAAM,OAAO,UAAA,CAAW;AAAA,EACtB,IAAA,EAAM,sBAAA;AAAA,EACN,OAAA,EAAS,OAAA;AAAA,EACT,SAAA,EAAW;AAAA,IACT;AAAA,MACE,IAAA,EAAM,mBAAA;AAAA,MACN,QAAA,EAAU,IAAA;AAAA,MACV,MAAA,EAAQ,IAAA;AAAA,MACR,WAAA,EAAa;AAAA;AACf,GACF;AAAA,EACA,WAAW,GAAA,EAAqC;AAC9C,IAAA,MAAM,GAAA,GAAM,WAAW,GAAG,CAAA;AAC1B,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAI,GAAA,CAAI,SAAS,CAAA;AACpC,IAAA,IAAI,CAAC,MAAA,EAAQ;AACX,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,8BAAA,EAAiC,IAAI,SAAS,CAAA,0DAAA;AAAA,OAChD;AAAA,IACF;AACA,IAAA,OAAO;AAAA,MACL;AAAA,QACE,IAAA,EAAM,WAAA;AAAA,QACN,SAAA,EAAW,OAAA;AAAA,QACX,OAAA,EAAS,KAAA;AAAA,QACT,IAAA,EAAM,CAAC,IAAA,EAAM,eAAe,CAAA;AAAA,QAC5B,GAAA,EAAK,EAAE,iBAAA,EAAmB,MAAA;AAAO;AACnC,KACF;AAAA,EACF,CAAA;AAAA,EACA,WAAW,GAAA,EAAsC;AAC/C,IAAA,MAAM,GAAA,GAAM,WAAW,GAAG,CAAA;AAC1B,IAAA,IAAI,CAAC,GAAA,CAAI,OAAA,EAAS,OAAO,EAAC;AAC1B,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAI,GAAA,CAAI,SAAS,CAAA;AACpC,IAAA,IAAI,CAAC,MAAA,EAAQ,OAAO,EAAC;AACrB,IAAA,MAAM,UAAU,GAAA,CAAI,OAAA;AAEpB,IAAA,MAAM,cAAA,GAAmC;AAAA,MACvC,UAAA,EAAY;AAAA,QACV,IAAA,EAAM,kBAAA;AAAA,QACN,WAAA,EACE,gMAAA;AAAA,QACF,MAAA,EAAQ,2BAAA;AAAA,QACR,WAAA,EAAa;AAAA,UACX,IAAA,EAAM,QAAA;AAAA,UACN,oBAAA,EAAsB,KAAA;AAAA,UACtB,UAAA,EAAY;AAAA,YACV,GAAA,EAAK,EAAE,IAAA,EAAM,QAAA,EAAU,aAAa,oBAAA,EAAqB;AAAA,YACzD,UAAA,EAAY;AAAA,cACV,IAAA,EAAM,SAAA;AAAA,cACN,WAAA,EAAa,yDAAA;AAAA,cACb,OAAA,EAAS,GAAA;AAAA,cACT,OAAA,EAAS;AAAA;AACX,WACF;AAAA,UACA,QAAA,EAAU,CAAC,KAAK;AAAA;AAClB,OACF;AAAA,MACA,MAAM,OAAA,CAAQ,EAAE,OAAO,KAAA,EAAO,MAAA,EAAQ,QAAO,EAAG;AAC9C,QAAA,MAAM,IAAA,GAAQ,SAAS,EAAC;AACxB,QAAA,IAAI,CAAC,IAAA,CAAK,GAAA,IAAO,OAAO,IAAA,CAAK,QAAQ,QAAA,EAAU;AAC7C,UAAA,OAAO,EAAE,OAAA,EAAS,4CAAA,EAA8C,OAAA,EAAS,IAAA,EAAK;AAAA,QAChF;AACA,QAAA,MAAM,YAAA,EAAa;AACnB,QAAA,MAAM,SAAA,GAAY,KAAK,UAAA,IAAc,GAAA;AACrC,QAAA,MAAM,IAAA,GAAO,IAAI,eAAA,EAAgB;AACjC,QAAA,MAAM,QAAQ,UAAA,CAAW,MAAM,IAAA,CAAK,KAAA,IAAS,SAAS,CAAA;AACtD,QAAA,MAAM,eAAA,GAAkB,MAAM,IAAA,CAAK,KAAA,EAAM;AACzC,QAAA,MAAA,CAAO,iBAAiB,OAAA,EAAS,eAAA,EAAiB,EAAE,IAAA,EAAM,MAAM,CAAA;AAChE,QAAA,IAAI;AACF,UAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,CAAA,EAAG,OAAO,CAAA,UAAA,CAAA,EAAc;AAAA,YAC9C,MAAA,EAAQ,MAAA;AAAA,YACR,OAAA,EAAS;AAAA,cACP,aAAA,EAAe,UAAU,MAAM,CAAA,CAAA;AAAA,cAC/B,cAAA,EAAgB;AAAA,aAClB;AAAA,YACA,IAAA,EAAM,IAAA,CAAK,SAAA,CAAU,EAAE,GAAA,EAAK,IAAA,CAAK,GAAA,EAAK,OAAA,EAAS,CAAC,UAAU,CAAA,EAAG,CAAA;AAAA,YAC7D,QAAQ,IAAA,CAAK;AAAA,WACd,CAAA;AACD,UAAA,IAAI,CAAC,IAAI,EAAA,EAAI;AACX,YAAA,MAAM,OAAO,MAAM,GAAA,CAAI,MAAK,CAAE,KAAA,CAAM,MAAM,EAAE,CAAA;AAC5C,YAAA,MAAA,CAAO,KAAK,EAAE,MAAA,EAAQ,IAAI,MAAA,EAAQ,IAAA,IAAQ,yBAAyB,CAAA;AACnE,YAAA,OAAO;AAAA,cACL,OAAA,EAAS,wCAAwC,GAAA,CAAI,MAAM,KAAK,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,cAClF,OAAA,EAAS;AAAA,aACX;AAAA,UACF;AACA,UAAA,MAAM,IAAA,GAAQ,MAAM,GAAA,CAAI,IAAA,EAAK;AAG7B,UAAA,MAAM,QAAA,GAAW,IAAA,CAAK,IAAA,EAAM,QAAA,IAAY,EAAA;AACxC,UAAA,MAAM,UAAA,GAAa,IAAA,CAAK,IAAA,EAAM,QAAA,EAAU,UAAA,IAAc,IAAA;AACtD,UAAA,MAAM,OAAO,OAAA,EAAQ;AACrB,UAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,KAAA;AAAA,YACxB,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAAA,CAAA;AAAA,YAQA,CAAC,KAAA,EAAO,IAAA,CAAK,GAAA,EAAK,UAAA,EAAY,UAAU,IAAI;AAAA,WAC9C;AACA,UAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,IAAA,CAAK,CAAC,GAAG,EAAA,IAAM,GAAA;AACpC,UAAA,MAAM,OAAA,GAAU,QAAA,CAAS,KAAA,CAAM,CAAA,EAAG,GAAI,CAAA;AACtC,UAAA,OAAO;AAAA,YACL,OAAA,EAAS,CAAA,+BAAA,EAAkC,KAAK,CAAA,SAAA,EAAY,cAAc,GAAG,CAAA;;AAAA,EAAiB,OAAO,CAAA;AAAA,WACvG;AAAA,QACF,SAAS,GAAA,EAAK;AACZ,UAAA,IAAK,GAAA,CAAc,SAAS,YAAA,EAAc;AACxC,YAAA,OAAO,EAAE,OAAA,EAAS,6BAAA,EAA+B,OAAA,EAAS,IAAA,EAAK;AAAA,UACjE;AACA,UAAA,OAAO;AAAA,YACL,OAAA,EAAS,CAAA,kBAAA,EAAsB,GAAA,CAAc,OAAO,CAAA,CAAA;AAAA,YACpD,OAAA,EAAS;AAAA,WACX;AAAA,QACF,CAAA,SAAE;AACA,UAAA,YAAA,CAAa,KAAK,CAAA;AAClB,UAAA,MAAA,CAAO,mBAAA,CAAoB,SAAS,eAAe,CAAA;AAAA,QACrD;AAAA,MACF;AAAA,KACF;AACA,IAAA,OAAO,CAAC,cAAc,CAAA;AAAA,EACxB,CAAA;AAAA,EACA,OAAO,IAAA,EAAoC;AACzC,IAAA,OAAO;AAAA,MACL;AAAA,QACE,IAAA,EAAM,kBAAA;AAAA,QACN,WAAA,EAAa,iEAAA;AAAA,QACb,SAAA,EACE,+MAAA;AAAA,QACF,WAAA,EAAa,IAAA,CAAK,UAAA,EAAY,qBAAqB;AAAA;AACrD,KACF;AAAA,EACF;AACF,CAAC,CAAA;AAED,IAAO,WAAA,GAAQ","file":"index.js","sourcesContent":["/**\n * cap-scrape-firecrawl — wires Firecrawl's MCP server AND adds a\n * `scrape_and_store` LocalToolHandler that calls Firecrawl's REST API\n * directly and persists the result in Postgres for later retrieval.\n *\n * Usage in render-harness.yaml:\n *\n *   capabilities:\n *     - pack: \"@render-harness/cap-scrape-firecrawl\"\n *       config:\n *         persist: true   # default\n *\n * Surfaces:\n *   - One MCP server `firecrawl` (stdio transport).\n *   - One LocalToolHandler `scrape_and_store` (when `persist: true`).\n *   - One skill (skills/firecrawl-scrape.md).\n *   - envSchema entry for FIRECRAWL_API_KEY.\n *\n * Config keys:\n *   - `apiKeyEnv` (string, default \"FIRECRAWL_API_KEY\")\n *   - `persist` (boolean, default true) — disable to skip the Postgres\n *     bootstrap and the scrape_and_store tool. Useful when the entry\n *     wants Firecrawl tools but already has its own storage.\n *   - `apiBase` (string, default \"https://api.firecrawl.dev\") — the\n *     REST base URL. Override for self-hosted Firecrawl.\n */\n\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport {\n  type LocalToolHandler,\n  type McpServerConfig,\n  type SkillMetadata,\n  getPool,\n} from \"@render-harness/core\";\nimport { definePack, type PackContext } from \"@render-harness/registry\";\n\nconst HERE = dirname(fileURLToPath(import.meta.url));\nconst SKILLS_DIR = join(HERE, \"..\", \"skills\");\n\ninterface FirecrawlConfig {\n  apiKeyEnv?: string;\n  persist?: boolean;\n  apiBase?: string;\n}\n\nfunction readConfig(ctx: PackContext) {\n  const cfg = ctx.config as FirecrawlConfig;\n  return {\n    apiKeyEnv: cfg.apiKeyEnv ?? \"FIRECRAWL_API_KEY\",\n    persist: cfg.persist ?? true,\n    apiBase: cfg.apiBase ?? \"https://api.firecrawl.dev\",\n  };\n}\n\nconst SCHEMA_BOOTSTRAP_SQL = `\nCREATE TABLE IF NOT EXISTS firecrawl_scrapes (\n  id BIGSERIAL PRIMARY KEY,\n  run_id UUID NOT NULL,\n  url TEXT NOT NULL,\n  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n  status_code INTEGER,\n  markdown TEXT,\n  raw JSONB,\n  CONSTRAINT firecrawl_scrapes_url_run UNIQUE (run_id, url)\n);\nCREATE INDEX IF NOT EXISTS firecrawl_scrapes_run_idx ON firecrawl_scrapes (run_id);\n`;\n\nlet bootstrapped = false;\nasync function ensureSchema(): Promise<void> {\n  if (bootstrapped) return;\n  const pool = getPool();\n  await pool.query(SCHEMA_BOOTSTRAP_SQL);\n  bootstrapped = true;\n}\n\nconst pack = definePack({\n  name: \"cap-scrape-firecrawl\",\n  version: \"0.1.0\",\n  envSchema: [\n    {\n      name: \"FIRECRAWL_API_KEY\",\n      required: true,\n      secret: true,\n      description: \"API key for Firecrawl (https://firecrawl.dev).\",\n    },\n  ],\n  mcpServers(ctx: PackContext): McpServerConfig[] {\n    const cfg = readConfig(ctx);\n    const apiKey = ctx.env(cfg.apiKeyEnv);\n    if (!apiKey) {\n      throw new Error(\n        `cap-scrape-firecrawl: env var ${cfg.apiKeyEnv} is not set. Set it before building or starting the agent.`,\n      );\n    }\n    return [\n      {\n        name: \"firecrawl\",\n        transport: \"stdio\",\n        command: \"npx\",\n        args: [\"-y\", \"firecrawl-mcp\"],\n        env: { FIRECRAWL_API_KEY: apiKey },\n      },\n    ];\n  },\n  localTools(ctx: PackContext): LocalToolHandler[] {\n    const cfg = readConfig(ctx);\n    if (!cfg.persist) return [];\n    const apiKey = ctx.env(cfg.apiKeyEnv);\n    if (!apiKey) return [];\n    const apiBase = cfg.apiBase;\n\n    const scrapeAndStore: LocalToolHandler = {\n      definition: {\n        name: \"scrape_and_store\",\n        description:\n          \"Scrape a URL via Firecrawl and persist the rendered markdown + raw JSON to Postgres. Returns the row id and a short excerpt. Use this when you need to come back to the page later in the run.\",\n        source: \"pack:cap-scrape-firecrawl\",\n        inputSchema: {\n          type: \"object\",\n          additionalProperties: false,\n          properties: {\n            url: { type: \"string\", description: \"The URL to scrape.\" },\n            timeout_ms: {\n              type: \"integer\",\n              description: \"Per-request timeout in milliseconds. Defaults to 60000.\",\n              minimum: 1000,\n              maximum: 300000,\n            },\n          },\n          required: [\"url\"],\n        },\n      },\n      async handler({ input, runId, signal, logger }) {\n        const args = (input ?? {}) as { url?: string; timeout_ms?: number };\n        if (!args.url || typeof args.url !== \"string\") {\n          return { content: \"scrape_and_store: missing or invalid `url`\", isError: true };\n        }\n        await ensureSchema();\n        const timeoutMs = args.timeout_ms ?? 60_000;\n        const ctrl = new AbortController();\n        const timer = setTimeout(() => ctrl.abort(), timeoutMs);\n        const onUpstreamAbort = () => ctrl.abort();\n        signal.addEventListener(\"abort\", onUpstreamAbort, { once: true });\n        try {\n          const res = await fetch(`${apiBase}/v1/scrape`, {\n            method: \"POST\",\n            headers: {\n              Authorization: `Bearer ${apiKey}`,\n              \"content-type\": \"application/json\",\n            },\n            body: JSON.stringify({ url: args.url, formats: [\"markdown\"] }),\n            signal: ctrl.signal,\n          });\n          if (!res.ok) {\n            const body = await res.text().catch(() => \"\");\n            logger.warn({ status: res.status, body }, \"firecrawl scrape failed\");\n            return {\n              content: `scrape_and_store: firecrawl returned ${res.status}: ${body.slice(0, 500)}`,\n              isError: true,\n            };\n          }\n          const json = (await res.json()) as {\n            data?: { markdown?: string; metadata?: { statusCode?: number } };\n          };\n          const markdown = json.data?.markdown ?? \"\";\n          const statusCode = json.data?.metadata?.statusCode ?? null;\n          const pool = getPool();\n          const insert = await pool.query<{ id: string }>(\n            `INSERT INTO firecrawl_scrapes (run_id, url, status_code, markdown, raw)\n             VALUES ($1, $2, $3, $4, $5)\n             ON CONFLICT (run_id, url) DO UPDATE SET\n               fetched_at = now(),\n               status_code = EXCLUDED.status_code,\n               markdown = EXCLUDED.markdown,\n               raw = EXCLUDED.raw\n             RETURNING id`,\n            [runId, args.url, statusCode, markdown, json],\n          );\n          const rowId = insert.rows[0]?.id ?? \"?\";\n          const excerpt = markdown.slice(0, 1000);\n          return {\n            content: `Stored as firecrawl_scrapes.id=${rowId}. status=${statusCode ?? \"?\"}. excerpt:\\n\\n${excerpt}`,\n          };\n        } catch (err) {\n          if ((err as Error).name === \"AbortError\") {\n            return { content: \"scrape_and_store: timed out\", isError: true };\n          }\n          return {\n            content: `scrape_and_store: ${(err as Error).message}`,\n            isError: true,\n          };\n        } finally {\n          clearTimeout(timer);\n          signal.removeEventListener(\"abort\", onUpstreamAbort);\n        }\n      },\n    };\n    return [scrapeAndStore];\n  },\n  skills(_ctx: PackContext): SkillMetadata[] {\n    return [\n      {\n        name: \"firecrawl-scrape\",\n        description: \"Use Firecrawl to render a URL into clean markdown and store it.\",\n        whenToUse:\n          \"When the user gives you a URL whose content you'll need later in the run, OR you've already searched and want to read the top result. Prefer scrape_and_store over the raw MCP tool when persistence matters.\",\n        contentPath: join(SKILLS_DIR, \"firecrawl-scrape.md\"),\n      },\n    ];\n  },\n});\n\nexport default pack;\n"]}
+{"version":3,"sources":["../src/index.ts"],"names":[],"mappings":";;;;;;AAqCA,IAAM,IAAA,GAAO,OAAA,CAAQ,aAAA,CAAc,MAAA,CAAA,IAAA,CAAY,GAAG,CAAC,CAAA;AACnD,IAAM,UAAA,GAAa,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,QAAQ,CAAA;AAQ5C,SAAS,WAAW,GAAA,EAAkB;AACpC,EAAA,MAAM,MAAM,GAAA,CAAI,MAAA;AAChB,EAAA,OAAO;AAAA,IACL,SAAA,EAAW,IAAI,SAAA,IAAa,mBAAA;AAAA,IAC5B,OAAA,EAAS,IAAI,OAAA,IAAW,IAAA;AAAA,IACxB,OAAA,EAAS,IAAI,OAAA,IAAW;AAAA,GAC1B;AACF;AAEA,IAAM,oBAAA,GAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAAA;AAc7B,IAAI,YAAA,GAAe,KAAA;AACnB,eAAe,YAAA,GAA8B;AAC3C,EAAA,IAAI,YAAA,EAAc;AAClB,EAAA,MAAM,OAAO,OAAA,EAAQ;AACrB,EAAA,MAAM,IAAA,CAAK,MAAM,oBAAoB,CAAA;AACrC,EAAA,YAAA,GAAe,IAAA;AACjB;AAEA,IAAM,OAAO,UAAA,CAAW;AAAA,EACtB,IAAA,EAAM,sBAAA;AAAA,EACN,OAAA,EAAS,OAAA;AAAA,EACT,SAAA,EAAW;AAAA,IACT;AAAA,MACE,IAAA,EAAM,mBAAA;AAAA,MACN,QAAA,EAAU,IAAA;AAAA,MACV,MAAA,EAAQ,IAAA;AAAA,MACR,WAAA,EAAa;AAAA;AACf,GACF;AAAA,EACA,WAAW,GAAA,EAAqC;AAC9C,IAAA,MAAM,GAAA,GAAM,WAAW,GAAG,CAAA;AAC1B,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAI,GAAA,CAAI,SAAS,CAAA;AACpC,IAAA,IAAI,CAAC,MAAA,EAAQ;AACX,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,8BAAA,EAAiC,IAAI,SAAS,CAAA,0DAAA;AAAA,OAChD;AAAA,IACF;AACA,IAAA,OAAO;AAAA,MACL;AAAA,QACE,IAAA,EAAM,WAAA;AAAA,QACN,SAAA,EAAW,OAAA;AAAA,QACX,OAAA,EAAS,KAAA;AAAA,QACT,IAAA,EAAM,CAAC,IAAA,EAAM,eAAe,CAAA;AAAA,QAC5B,GAAA,EAAK,EAAE,iBAAA,EAAmB,MAAA;AAAO;AACnC,KACF;AAAA,EACF,CAAA;AAAA,EACA,WAAW,GAAA,EAAsC;AAC/C,IAAA,MAAM,GAAA,GAAM,WAAW,GAAG,CAAA;AAC1B,IAAA,IAAI,CAAC,GAAA,CAAI,OAAA,EAAS,OAAO,EAAC;AAC1B,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAI,GAAA,CAAI,SAAS,CAAA;AACpC,IAAA,IAAI,CAAC,MAAA,EAAQ,OAAO,EAAC;AACrB,IAAA,MAAM,UAAU,GAAA,CAAI,OAAA;AAEpB,IAAA,MAAM,cAAA,GAAmC;AAAA,MACvC,UAAA,EAAY;AAAA,QACV,IAAA,EAAM,kBAAA;AAAA,QACN,WAAA,EACE,gMAAA;AAAA,QACF,MAAA,EAAQ,2BAAA;AAAA,QACR,WAAA,EAAa;AAAA,UACX,IAAA,EAAM,QAAA;AAAA,UACN,oBAAA,EAAsB,KAAA;AAAA,UACtB,UAAA,EAAY;AAAA,YACV,GAAA,EAAK,EAAE,IAAA,EAAM,QAAA,EAAU,aAAa,oBAAA,EAAqB;AAAA,YACzD,UAAA,EAAY;AAAA,cACV,IAAA,EAAM,SAAA;AAAA,cACN,WAAA,EAAa,yDAAA;AAAA,cACb,OAAA,EAAS,GAAA;AAAA,cACT,OAAA,EAAS;AAAA;AACX,WACF;AAAA,UACA,QAAA,EAAU,CAAC,KAAK;AAAA;AAClB,OACF;AAAA,MACA,MAAM,OAAA,CAAQ,EAAE,OAAO,KAAA,EAAO,MAAA,EAAQ,QAAO,EAAG;AAC9C,QAAA,MAAM,IAAA,GAAQ,SAAS,EAAC;AACxB,QAAA,IAAI,CAAC,IAAA,CAAK,GAAA,IAAO,OAAO,IAAA,CAAK,QAAQ,QAAA,EAAU;AAC7C,UAAA,OAAO,EAAE,OAAA,EAAS,4CAAA,EAA8C,OAAA,EAAS,IAAA,EAAK;AAAA,QAChF;AACA,QAAA,MAAM,YAAA,EAAa;AACnB,QAAA,MAAM,SAAA,GAAY,KAAK,UAAA,IAAc,GAAA;AACrC,QAAA,MAAM,IAAA,GAAO,IAAI,eAAA,EAAgB;AACjC,QAAA,MAAM,QAAQ,UAAA,CAAW,MAAM,IAAA,CAAK,KAAA,IAAS,SAAS,CAAA;AACtD,QAAA,MAAM,eAAA,GAAkB,MAAM,IAAA,CAAK,KAAA,EAAM;AACzC,QAAA,MAAA,CAAO,iBAAiB,OAAA,EAAS,eAAA,EAAiB,EAAE,IAAA,EAAM,MAAM,CAAA;AAChE,QAAA,IAAI;AACF,UAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,CAAA,EAAG,OAAO,CAAA,UAAA,CAAA,EAAc;AAAA,YAC9C,MAAA,EAAQ,MAAA;AAAA,YACR,OAAA,EAAS;AAAA,cACP,aAAA,EAAe,UAAU,MAAM,CAAA,CAAA;AAAA,cAC/B,cAAA,EAAgB;AAAA,aAClB;AAAA,YACA,IAAA,EAAM,IAAA,CAAK,SAAA,CAAU,EAAE,GAAA,EAAK,IAAA,CAAK,GAAA,EAAK,OAAA,EAAS,CAAC,UAAU,CAAA,EAAG,CAAA;AAAA,YAC7D,QAAQ,IAAA,CAAK;AAAA,WACd,CAAA;AACD,UAAA,IAAI,CAAC,IAAI,EAAA,EAAI;AACX,YAAA,MAAM,OAAO,MAAM,GAAA,CAAI,MAAK,CAAE,KAAA,CAAM,MAAM,EAAE,CAAA;AAC5C,YAAA,MAAA,CAAO,KAAK,EAAE,MAAA,EAAQ,IAAI,MAAA,EAAQ,IAAA,IAAQ,yBAAyB,CAAA;AACnE,YAAA,OAAO;AAAA,cACL,OAAA,EAAS,wCAAwC,GAAA,CAAI,MAAM,KAAK,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,CAAA;AAAA,cAClF,OAAA,EAAS;AAAA,aACX;AAAA,UACF;AACA,UAAA,MAAM,IAAA,GAAQ,MAAM,GAAA,CAAI,IAAA,EAAK;AAG7B,UAAA,MAAM,QAAA,GAAW,IAAA,CAAK,IAAA,EAAM,QAAA,IAAY,EAAA;AACxC,UAAA,MAAM,UAAA,GAAa,IAAA,CAAK,IAAA,EAAM,QAAA,EAAU,UAAA,IAAc,IAAA;AACtD,UAAA,MAAM,OAAO,OAAA,EAAQ;AACrB,UAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,KAAA;AAAA,YACxB,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAAA,CAAA;AAAA,YAQA,CAAC,KAAA,EAAO,IAAA,CAAK,GAAA,EAAK,UAAA,EAAY,UAAU,IAAI;AAAA,WAC9C;AACA,UAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,IAAA,CAAK,CAAC,GAAG,EAAA,IAAM,GAAA;AACpC,UAAA,MAAM,OAAA,GAAU,QAAA,CAAS,KAAA,CAAM,CAAA,EAAG,GAAI,CAAA;AACtC,UAAA,OAAO;AAAA,YACL,OAAA,EAAS,CAAA,+BAAA,EAAkC,KAAK,CAAA,SAAA,EAAY,cAAc,GAAG,CAAA;;AAAA,EAAiB,OAAO,CAAA;AAAA,WACvG;AAAA,QACF,SAAS,GAAA,EAAK;AACZ,UAAA,IAAK,GAAA,CAAc,SAAS,YAAA,EAAc;AACxC,YAAA,OAAO,EAAE,OAAA,EAAS,6BAAA,EAA+B,OAAA,EAAS,IAAA,EAAK;AAAA,UACjE;AACA,UAAA,OAAO;AAAA,YACL,OAAA,EAAS,CAAA,kBAAA,EAAsB,GAAA,CAAc,OAAO,CAAA,CAAA;AAAA,YACpD,OAAA,EAAS;AAAA,WACX;AAAA,QACF,CAAA,SAAE;AACA,UAAA,YAAA,CAAa,KAAK,CAAA;AAClB,UAAA,MAAA,CAAO,mBAAA,CAAoB,SAAS,eAAe,CAAA;AAAA,QACrD;AAAA,MACF;AAAA,KACF;AACA,IAAA,OAAO,CAAC,cAAc,CAAA;AAAA,EACxB,CAAA;AAAA,EACA,OAAO,IAAA,EAAoC;AACzC,IAAA,OAAO;AAAA,MACL;AAAA,QACE,IAAA,EAAM,kBAAA;AAAA,QACN,WAAA,EAAa,iEAAA;AAAA,QACb,SAAA,EACE,+MAAA;AAAA,QACF,WAAA,EAAa,IAAA,CAAK,UAAA,EAAY,qBAAqB;AAAA;AACrD,KACF;AAAA,EACF;AACF,CAAC,CAAA;AAED,IAAO,WAAA,GAAQ","file":"index.js","sourcesContent":["/**\n * cap-scrape-firecrawl — wires Firecrawl's MCP server AND adds a\n * `scrape_and_store` LocalToolHandler that calls Firecrawl's REST API\n * directly and persists the result in Postgres for later retrieval.\n *\n * Usage in render-harness.yaml:\n *\n *   capabilities:\n *     - pack: \"@render-harness/cap-scrape-firecrawl\"\n *       config:\n *         persist: true   # default\n *\n * Surfaces:\n *   - One MCP server `firecrawl` (stdio transport).\n *   - One LocalToolHandler `scrape_and_store` (when `persist: true`).\n *   - One skill (skills/firecrawl-scrape.md).\n *   - envSchema entry for FIRECRAWL_API_KEY.\n *\n * Config keys:\n *   - `apiKeyEnv` (string, default \"FIRECRAWL_API_KEY\")\n *   - `persist` (boolean, default true) — disable to skip the Postgres\n *     bootstrap and the scrape_and_store tool. Useful when the entry\n *     wants Firecrawl tools but already has its own storage.\n *   - `apiBase` (string, default \"https://api.firecrawl.dev\") — the\n *     REST base URL. Override for self-hosted Firecrawl.\n */\n\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport {\n  getPool,\n  type LocalToolHandler,\n  type McpServerConfig,\n  type SkillMetadata,\n} from \"@render-harness/core\";\nimport { definePack, type PackContext } from \"@render-harness/registry\";\n\nconst HERE = dirname(fileURLToPath(import.meta.url));\nconst SKILLS_DIR = join(HERE, \"..\", \"skills\");\n\ninterface FirecrawlConfig {\n  apiKeyEnv?: string;\n  persist?: boolean;\n  apiBase?: string;\n}\n\nfunction readConfig(ctx: PackContext) {\n  const cfg = ctx.config as FirecrawlConfig;\n  return {\n    apiKeyEnv: cfg.apiKeyEnv ?? \"FIRECRAWL_API_KEY\",\n    persist: cfg.persist ?? true,\n    apiBase: cfg.apiBase ?? \"https://api.firecrawl.dev\",\n  };\n}\n\nconst SCHEMA_BOOTSTRAP_SQL = `\nCREATE TABLE IF NOT EXISTS firecrawl_scrapes (\n  id BIGSERIAL PRIMARY KEY,\n  run_id UUID NOT NULL,\n  url TEXT NOT NULL,\n  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),\n  status_code INTEGER,\n  markdown TEXT,\n  raw JSONB,\n  CONSTRAINT firecrawl_scrapes_url_run UNIQUE (run_id, url)\n);\nCREATE INDEX IF NOT EXISTS firecrawl_scrapes_run_idx ON firecrawl_scrapes (run_id);\n`;\n\nlet bootstrapped = false;\nasync function ensureSchema(): Promise<void> {\n  if (bootstrapped) return;\n  const pool = getPool();\n  await pool.query(SCHEMA_BOOTSTRAP_SQL);\n  bootstrapped = true;\n}\n\nconst pack = definePack({\n  name: \"cap-scrape-firecrawl\",\n  version: \"0.1.0\",\n  envSchema: [\n    {\n      name: \"FIRECRAWL_API_KEY\",\n      required: true,\n      secret: true,\n      description: \"API key for Firecrawl (https://firecrawl.dev).\",\n    },\n  ],\n  mcpServers(ctx: PackContext): McpServerConfig[] {\n    const cfg = readConfig(ctx);\n    const apiKey = ctx.env(cfg.apiKeyEnv);\n    if (!apiKey) {\n      throw new Error(\n        `cap-scrape-firecrawl: env var ${cfg.apiKeyEnv} is not set. Set it before building or starting the agent.`,\n      );\n    }\n    return [\n      {\n        name: \"firecrawl\",\n        transport: \"stdio\",\n        command: \"npx\",\n        args: [\"-y\", \"firecrawl-mcp\"],\n        env: { FIRECRAWL_API_KEY: apiKey },\n      },\n    ];\n  },\n  localTools(ctx: PackContext): LocalToolHandler[] {\n    const cfg = readConfig(ctx);\n    if (!cfg.persist) return [];\n    const apiKey = ctx.env(cfg.apiKeyEnv);\n    if (!apiKey) return [];\n    const apiBase = cfg.apiBase;\n\n    const scrapeAndStore: LocalToolHandler = {\n      definition: {\n        name: \"scrape_and_store\",\n        description:\n          \"Scrape a URL via Firecrawl and persist the rendered markdown + raw JSON to Postgres. Returns the row id and a short excerpt. Use this when you need to come back to the page later in the run.\",\n        source: \"pack:cap-scrape-firecrawl\",\n        inputSchema: {\n          type: \"object\",\n          additionalProperties: false,\n          properties: {\n            url: { type: \"string\", description: \"The URL to scrape.\" },\n            timeout_ms: {\n              type: \"integer\",\n              description: \"Per-request timeout in milliseconds. Defaults to 60000.\",\n              minimum: 1000,\n              maximum: 300000,\n            },\n          },\n          required: [\"url\"],\n        },\n      },\n      async handler({ input, runId, signal, logger }) {\n        const args = (input ?? {}) as { url?: string; timeout_ms?: number };\n        if (!args.url || typeof args.url !== \"string\") {\n          return { content: \"scrape_and_store: missing or invalid `url`\", isError: true };\n        }\n        await ensureSchema();\n        const timeoutMs = args.timeout_ms ?? 60_000;\n        const ctrl = new AbortController();\n        const timer = setTimeout(() => ctrl.abort(), timeoutMs);\n        const onUpstreamAbort = () => ctrl.abort();\n        signal.addEventListener(\"abort\", onUpstreamAbort, { once: true });\n        try {\n          const res = await fetch(`${apiBase}/v1/scrape`, {\n            method: \"POST\",\n            headers: {\n              Authorization: `Bearer ${apiKey}`,\n              \"content-type\": \"application/json\",\n            },\n            body: JSON.stringify({ url: args.url, formats: [\"markdown\"] }),\n            signal: ctrl.signal,\n          });\n          if (!res.ok) {\n            const body = await res.text().catch(() => \"\");\n            logger.warn({ status: res.status, body }, \"firecrawl scrape failed\");\n            return {\n              content: `scrape_and_store: firecrawl returned ${res.status}: ${body.slice(0, 500)}`,\n              isError: true,\n            };\n          }\n          const json = (await res.json()) as {\n            data?: { markdown?: string; metadata?: { statusCode?: number } };\n          };\n          const markdown = json.data?.markdown ?? \"\";\n          const statusCode = json.data?.metadata?.statusCode ?? null;\n          const pool = getPool();\n          const insert = await pool.query<{ id: string }>(\n            `INSERT INTO firecrawl_scrapes (run_id, url, status_code, markdown, raw)\n             VALUES ($1, $2, $3, $4, $5)\n             ON CONFLICT (run_id, url) DO UPDATE SET\n               fetched_at = now(),\n               status_code = EXCLUDED.status_code,\n               markdown = EXCLUDED.markdown,\n               raw = EXCLUDED.raw\n             RETURNING id`,\n            [runId, args.url, statusCode, markdown, json],\n          );\n          const rowId = insert.rows[0]?.id ?? \"?\";\n          const excerpt = markdown.slice(0, 1000);\n          return {\n            content: `Stored as firecrawl_scrapes.id=${rowId}. status=${statusCode ?? \"?\"}. excerpt:\\n\\n${excerpt}`,\n          };\n        } catch (err) {\n          if ((err as Error).name === \"AbortError\") {\n            return { content: \"scrape_and_store: timed out\", isError: true };\n          }\n          return {\n            content: `scrape_and_store: ${(err as Error).message}`,\n            isError: true,\n          };\n        } finally {\n          clearTimeout(timer);\n          signal.removeEventListener(\"abort\", onUpstreamAbort);\n        }\n      },\n    };\n    return [scrapeAndStore];\n  },\n  skills(_ctx: PackContext): SkillMetadata[] {\n    return [\n      {\n        name: \"firecrawl-scrape\",\n        description: \"Use Firecrawl to render a URL into clean markdown and store it.\",\n        whenToUse:\n          \"When the user gives you a URL whose content you'll need later in the run, OR you've already searched and want to read the top result. Prefer scrape_and_store over the raw MCP tool when persistence matters.\",\n        contentPath: join(SKILLS_DIR, \"firecrawl-scrape.md\"),\n      },\n    ];\n  },\n});\n\nexport default pack;\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@render-harness/cap-scrape-firecrawl",
-  "version": "0.1.1",
+  "version": "0.1.4",
   "description": "Firecrawl scrape capability pack with optional Postgres persistence.",
   "type": "module",
   "license": "MIT",
@@ -29,14 +29,9 @@
       "envHint": "FIRECRAWL_API_KEY"
     }
   },
-  "scripts": {
-    "build": "tsup",
-    "typecheck": "tsc --noEmit",
-    "test": "vitest run --passWithNoTests"
-  },
   "dependencies": {
-    "@render-harness/core": "workspace:*",
-    "@render-harness/registry": "workspace:*"
+    "@render-harness/core": "0.1.1",
+    "@render-harness/registry": "0.1.1"
   },
   "devDependencies": {
     "@types/node": "^25.6.2",
@@ -46,5 +41,10 @@
   },
   "publishConfig": {
     "access": "public"
+  },
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run --passWithNoTests"
   }
-}
+}