@narumitw/pi-firecrawl 0.1.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 narumiruna
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,99 @@
+# pi-firecrawl
+
+A public [pi](https://pi.dev) extension package that exposes [Firecrawl](https://www.firecrawl.dev/) web scraping, crawling, URL discovery, and search APIs as native pi tools.
+
+## Install
+
+```bash
+pi install npm:@narumitw/pi-firecrawl
+```
+
+Try without installing:
+
+```bash
+FIRECRAWL_API_KEY=fc-... pi -e npm:@narumitw/pi-firecrawl
+```
+
+Try this package locally from the repository root:
+
+```bash
+FIRECRAWL_API_KEY=fc-... pi -e ./extensions/pi-firecrawl
+```
+
+## Configuration
+
+Set a Firecrawl API key before running pi:
+
+```bash
+export FIRECRAWL_API_KEY=fc-your-key
+```
+
+Optional API endpoint override:
+
+```bash
+export FIRECRAWL_API_URL=https://api.firecrawl.dev/v1
+```
+
+`FIRECRAWL_BASE_URL` is also accepted for compatibility. The extension never logs or displays the API key.
+
+## Tools
+
+- `firecrawl_scrape` — scrape a single URL and return requested formats such as markdown, HTML, links, or JSON.
+- `firecrawl_crawl` — start a site crawl job and return the Firecrawl job id.
+- `firecrawl_crawl_status` — check a crawl job status and retrieve completed crawl data.
+- `firecrawl_map` — discover URLs for a site.
+- `firecrawl_search` — search the web through Firecrawl and optionally scrape result pages.
+
+All tools fail with a clear configuration error when `FIRECRAWL_API_KEY` is missing.
+
+## Command
+
+```text
+/firecrawl
+```
+
+Shows whether the extension sees an API key and which Firecrawl API URL it will call.
+
+## Examples
+
+Scrape a page as markdown:
+
+```json
+{
+  "url": "https://example.com",
+  "formats": ["markdown"]
+}
+```
+
+Map a small site:
+
+```json
+{
+  "url": "https://example.com",
+  "limit": 20
+}
+```
+
+Start a crawl with markdown extraction:
+
+```json
+{
+  "url": "https://example.com",
+  "limit": 10,
+  "scrapeOptions": {
+    "formats": ["markdown"]
+  }
+}
+```
+
+## Package layout
+
+```txt
+extensions/pi-firecrawl/
+├── src/
+│   └── firecrawl.ts
+├── README.md
+├── LICENSE
+├── tsconfig.json
+└── package.json
+```

package/package.json

@@ -0,0 +1,39 @@
+{
+	"name": "@narumitw/pi-firecrawl",
+	"version": "0.1.3",
+	"description": "Pi extension that exposes Firecrawl web scraping and crawling tools.",
+	"type": "module",
+	"license": "MIT",
+	"private": false,
+	"keywords": [
+		"pi-package",
+		"pi-extension",
+		"pi",
+		"firecrawl",
+		"scrape",
+		"crawl"
+	],
+	"files": [
+		"src",
+		"README.md",
+		"LICENSE"
+	],
+	"pi": {
+		"extensions": [
+			"./src/firecrawl.ts"
+		]
+	},
+	"scripts": {
+		"check": "biome check . && npm run typecheck",
+		"format": "biome check --write .",
+		"typecheck": "tsc --noEmit"
+	},
+	"dependencies": {
+		"typebox": "^1.1.37"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "2.4.14",
+		"@mariozechner/pi-coding-agent": "0.73.0",
+		"typescript": "6.0.3"
+	}
+}

package/src/firecrawl.ts

@@ -0,0 +1,297 @@
+import { defineTool, type ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { Type } from "typebox";
+
+const DEFAULT_API_URL = "https://api.firecrawl.dev/v1";
+const STATUS_KEY = "firecrawl";
+
+const StringArray = Type.Array(Type.String());
+
+interface FirecrawlState {
+	apiUrl: string;
+}
+
+const state: FirecrawlState = {
+	apiUrl: normalizeApiUrl(process.env.FIRECRAWL_API_URL ?? process.env.FIRECRAWL_BASE_URL),
+};
+
+const scrapeTool = defineTool({
+	name: "firecrawl_scrape",
+	label: "Firecrawl: Scrape",
+	description: "Scrape a single URL through Firecrawl and return requested formats.",
+	promptSnippet: "Scrape a URL through Firecrawl",
+	promptGuidelines: [
+		"Use firecrawl_scrape when you need clean markdown, HTML, links, screenshots, or structured extraction for one URL.",
+		"If FIRECRAWL_API_KEY is missing, report the configuration error instead of retrying repeatedly.",
+	],
+	parameters: Type.Object({
+		url: Type.String({ description: "URL to scrape." }),
+		formats: Type.Optional(
+			Type.Array(
+				Type.String({
+					description:
+						"Requested Firecrawl output format, such as markdown, html, rawHtml, links, screenshot, or json.",
+				}),
+				{ description: "Firecrawl output formats. Defaults to Firecrawl's API default." },
+			),
+		),
+		onlyMainContent: Type.Optional(
+			Type.Boolean({ description: "Return only the main page content when supported." }),
+		),
+		includeTags: Type.Optional(StringArray),
+		excludeTags: Type.Optional(StringArray),
+		waitFor: Type.Optional(Type.Number({ description: "Milliseconds to wait before scraping." })),
+		timeout: Type.Optional(
+			Type.Number({ description: "Firecrawl request timeout in milliseconds." }),
+		),
+		mobile: Type.Optional(Type.Boolean({ description: "Use a mobile user agent when supported." })),
+		skipTlsVerification: Type.Optional(
+			Type.Boolean({ description: "Skip TLS certificate verification when supported." }),
+		),
+		removeBase64Images: Type.Optional(
+			Type.Boolean({ description: "Remove base64 image data from the response when supported." }),
+		),
+		blockAds: Type.Optional(
+			Type.Boolean({ description: "Block ads while scraping when supported." }),
+		),
+		headers: Type.Optional(
+			Type.Record(Type.String(), Type.String(), {
+				description: "Additional HTTP headers Firecrawl should use while fetching the target URL.",
+			}),
+		),
+		jsonOptions: Type.Optional(
+			Type.Any({ description: "Firecrawl jsonOptions for structured extraction." }),
+		),
+		actions: Type.Optional(
+			Type.Array(Type.Any(), {
+				description: "Firecrawl browser actions to perform before scraping.",
+			}),
+		),
+		location: Type.Optional(Type.Any({ description: "Firecrawl location options." })),
+	}),
+	async execute(_toolCallId, params, signal) {
+		const payload = await firecrawlRequest("POST", "/scrape", cleanObject(params), signal);
+		return jsonResult(payload);
+	},
+});
+
+const crawlTool = defineTool({
+	name: "firecrawl_crawl",
+	label: "Firecrawl: Crawl",
+	description: "Start a Firecrawl crawl job for a website.",
+	promptSnippet: "Start a Firecrawl site crawl job",
+	parameters: Type.Object({
+		url: Type.String({ description: "Starting URL for the crawl." }),
+		limit: Type.Optional(Type.Number({ description: "Maximum number of pages to crawl." })),
+		maxDepth: Type.Optional(Type.Number({ description: "Maximum crawl depth when supported." })),
+		includePaths: Type.Optional(
+			Type.Array(Type.String(), { description: "URL path patterns to include." }),
+		),
+		excludePaths: Type.Optional(
+			Type.Array(Type.String(), { description: "URL path patterns to exclude." }),
+		),
+		allowBackwardLinks: Type.Optional(
+			Type.Boolean({ description: "Allow crawling backward links when supported." }),
+		),
+		allowExternalLinks: Type.Optional(
+			Type.Boolean({ description: "Allow crawling external links when supported." }),
+		),
+		ignoreSitemap: Type.Optional(Type.Boolean({ description: "Ignore sitemap discovery." })),
+		deduplicateSimilarURLs: Type.Optional(
+			Type.Boolean({ description: "Deduplicate similar URLs when supported." }),
+		),
+		scrapeOptions: Type.Optional(
+			Type.Any({ description: "Firecrawl scrapeOptions applied to crawled pages." }),
+		),
+		webhook: Type.Optional(Type.Any({ description: "Firecrawl webhook configuration." })),
+	}),
+	async execute(_toolCallId, params, signal) {
+		const payload = await firecrawlRequest("POST", "/crawl", cleanObject(params), signal);
+		return jsonResult(payload);
+	},
+});
+
+const crawlStatusTool = defineTool({
+	name: "firecrawl_crawl_status",
+	label: "Firecrawl: Crawl Status",
+	description: "Check a Firecrawl crawl job status and retrieve completed crawl data.",
+	promptSnippet: "Check a Firecrawl crawl job status",
+	parameters: Type.Object({
+		id: Type.String({ description: "Crawl job id returned by firecrawl_crawl." }),
+	}),
+	async execute(_toolCallId, params, signal) {
+		const payload = await firecrawlRequest(
+			"GET",
+			`/crawl/${encodeURIComponent(params.id)}`,
+			undefined,
+			signal,
+		);
+		return jsonResult(payload);
+	},
+});
+
+const mapTool = defineTool({
+	name: "firecrawl_map",
+	label: "Firecrawl: Map",
+	description: "Discover URLs for a site through Firecrawl's map endpoint.",
+	promptSnippet: "Map/discover URLs for a site through Firecrawl",
+	parameters: Type.Object({
+		url: Type.String({ description: "Website URL to map." }),
+		search: Type.Optional(
+			Type.String({ description: "Optional search term to filter discovered URLs." }),
+		),
+		ignoreSitemap: Type.Optional(Type.Boolean({ description: "Ignore sitemap discovery." })),
+		sitemapOnly: Type.Optional(
+			Type.Boolean({ description: "Only use sitemap URLs when supported." }),
+		),
+		includeSubdomains: Type.Optional(
+			Type.Boolean({ description: "Include subdomains when supported." }),
+		),
+		limit: Type.Optional(Type.Number({ description: "Maximum number of URLs to return." })),
+	}),
+	async execute(_toolCallId, params, signal) {
+		const payload = await firecrawlRequest("POST", "/map", cleanObject(params), signal);
+		return jsonResult(payload);
+	},
+});
+
+const searchTool = defineTool({
+	name: "firecrawl_search",
+	label: "Firecrawl: Search",
+	description: "Search the web through Firecrawl and optionally scrape search results.",
+	promptSnippet: "Search the web through Firecrawl",
+	parameters: Type.Object({
+		query: Type.String({ description: "Search query." }),
+		limit: Type.Optional(Type.Number({ description: "Maximum number of search results." })),
+		tbs: Type.Optional(
+			Type.String({ description: "Google-style time based search filter when supported." }),
+		),
+		location: Type.Optional(Type.String({ description: "Search location when supported." })),
+		scrapeOptions: Type.Optional(
+			Type.Any({ description: "Firecrawl scrapeOptions for search result pages." }),
+		),
+	}),
+	async execute(_toolCallId, params, signal) {
+		const payload = await firecrawlRequest("POST", "/search", cleanObject(params), signal);
+		return jsonResult(payload);
+	},
+});
+
+export default function firecrawl(pi: ExtensionAPI) {
+	pi.registerTool(scrapeTool);
+	pi.registerTool(crawlTool);
+	pi.registerTool(crawlStatusTool);
+	pi.registerTool(mapTool);
+	pi.registerTool(searchTool);
+
+	pi.registerCommand("firecrawl", {
+		description: "Show Firecrawl extension configuration status",
+		handler: async (_args, ctx) => {
+			ctx.ui.notify(buildStatusMessage(), hasApiKey() ? "info" : "warning");
+			updateStatus(ctx);
+		},
+	});
+
+	pi.on("session_start", (_event, ctx) => {
+		updateStatus(ctx);
+	});
+
+	pi.on("session_shutdown", (_event, ctx) => {
+		ctx.ui.setStatus(STATUS_KEY, undefined);
+	});
+}
+
+async function firecrawlRequest(
+	method: "GET" | "POST",
+	path: string,
+	body: unknown,
+	signal: AbortSignal | undefined,
+) {
+	const apiKey = getApiKey();
+	const response = await fetch(`${state.apiUrl}${path}`, {
+		method,
+		headers: {
+			Authorization: `Bearer ${apiKey}`,
+			...(body === undefined ? {} : { "Content-Type": "application/json" }),
+		},
+		body: body === undefined ? undefined : JSON.stringify(body),
+		signal,
+	});
+
+	const responseText = await response.text();
+	const payload = parseResponseBody(responseText);
+
+	if (!response.ok) {
+		throw new Error(
+			`Firecrawl ${method} ${path} returned ${response.status} ${response.statusText}: ${formatPayload(payload)}`,
+		);
+	}
+
+	return payload;
+}
+
+function getApiKey() {
+	const apiKey = process.env.FIRECRAWL_API_KEY?.trim();
+	if (!apiKey) {
+		throw new Error(
+			"FIRECRAWL_API_KEY is required for pi-firecrawl. Set it in the environment before running pi.",
+		);
+	}
+
+	return apiKey;
+}
+
+function hasApiKey() {
+	return Boolean(process.env.FIRECRAWL_API_KEY?.trim());
+}
+
+function normalizeApiUrl(apiUrl: string | undefined) {
+	return (apiUrl?.trim() || DEFAULT_API_URL).replace(/\/+$/, "");
+}
+
+function parseResponseBody(responseText: string) {
+	if (!responseText) return null;
+
+	try {
+		return JSON.parse(responseText) as unknown;
+	} catch {
+		return responseText;
+	}
+}
+
+function formatPayload(payload: unknown) {
+	if (typeof payload === "string") return payload;
+	return JSON.stringify(payload);
+}
+
+function jsonResult(payload: unknown) {
+	return {
+		content: [{ type: "text" as const, text: JSON.stringify(payload, null, 2) }],
+		details: payload,
+	};
+}
+
+function updateStatus(ctx: {
+	ui: { setStatus: (key: string, value: string | undefined) => void };
+}) {
+	ctx.ui.setStatus(STATUS_KEY, hasApiKey() ? "firecrawl: configured" : "firecrawl: missing key");
+}
+
+function buildStatusMessage() {
+	return hasApiKey()
+		? `Firecrawl configured: ${state.apiUrl} (API key present).`
+		: `Firecrawl missing FIRECRAWL_API_KEY. API URL: ${state.apiUrl}.`;
+}
+
+function cleanObject<T>(value: T): T {
+	if (Array.isArray(value)) {
+		return value.map((item) => cleanObject(item)) as T;
+	}
+
+	if (!value || typeof value !== "object") return value;
+
+	const entries = Object.entries(value)
+		.filter(([, entryValue]) => entryValue !== undefined)
+		.map(([entryKey, entryValue]) => [entryKey, cleanObject(entryValue)]);
+
+	return Object.fromEntries(entries) as T;
+}