@hasna/connectors 0.0.1

package/connectors/connect-firecrawl/README.md

@@ -0,0 +1,95 @@
+# connect-firecrawl
+
+Firecrawl web scraping API connector - Scrape, crawl, and map websites with AI-powered extraction
+
+## Installation
+
+```bash
+bun install -g @hasna/connect-firecrawl
+```
+
+## Quick Start
+
+```bash
+# Set your API key
+connect-firecrawl config set-key YOUR_API_KEY
+
+# Or use environment variable
+export FIRECRAWL_API_KEY=YOUR_API_KEY
+```
+
+## CLI Commands
+
+```bash
+connect-firecrawl config set-key <key>     # Set API key
+connect-firecrawl config show              # Show config
+connect-firecrawl profile list             # List profiles
+connect-firecrawl profile use <name>       # Switch profile
+```
+
+## Profile Management
+
+```bash
+# Create profiles for different accounts
+connect-firecrawl profile create work --api-key xxx --use
+connect-firecrawl profile create personal --api-key yyy
+
+# Switch profiles
+connect-firecrawl profile use work
+
+# Use profile for single command
+connect-firecrawl -p personal <command>
+
+# List profiles
+connect-firecrawl profile list
+```
+
+## Library Usage
+
+```typescript
+import { Firecrawl } from '@hasna/connect-firecrawl';
+
+const client = new Firecrawl({ apiKey: 'YOUR_API_KEY' });
+
+// Use the client
+// See CLAUDE.md for detailed API documentation
+```
+
+## Environment Variables
+
+| Variable | Description |
+|----------|-------------|
+| `FIRECRAWL_API_KEY` | API key (required) |
+| `FIRECRAWL_BASE_URL` | Override base URL (default: https://api.firecrawl.dev/v1) |
+
+## Data Storage
+
+Configuration stored in `~/.connect/connect-firecrawl/`:
+
+```
+~/.connect/connect-firecrawl/
+├── current_profile   # Active profile name
+└── profiles/
+    ├── default.json  # Default profile
+    └── {name}.json   # Named profiles
+```
+
+## Development
+
+```bash
+# Install dependencies
+bun install
+
+# Run CLI in development
+bun run dev
+
+# Build
+bun run build
+
+# Type check
+bun run typecheck
+```
+
+## License
+
+MIT

package/connectors/connect-firecrawl/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@hasna/connect-firecrawl",
+  "version": "0.0.1",
+  "description": "Firecrawl web scraping API connector - Scrape, crawl, and map websites with AI-powered extraction",
+  "type": "module",
+  "bin": {
+    "connect-firecrawl": "./bin/index.js"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "bun build ./src/index.ts --outdir ./dist --target bun && bun build ./src/cli/index.ts --outdir ./bin --target bun",
+    "dev": "bun run ./src/cli/index.ts",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bun run build"
+  },
+  "keywords": [
+    "firecrawl",
+    "web-scraping",
+    "crawler",
+    "api",
+    "connector",
+    "cli",
+    "typescript",
+    "bun",
+    "hasna"
+  ],
+  "author": "Hasna",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5"
+  },
+  "dependencies": {
+    "commander": "^12.1.0",
+    "chalk": "^5.3.0"
+  },
+  "engines": {
+    "bun": ">=1.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hasna/opensource-connectors.git"
+  }
+}

package/connectors/connect-firecrawl/src/api/client.ts

@@ -0,0 +1,123 @@
+import type { FirecrawlConfig } from '../types';
+import { FirecrawlApiError } from '../types';
+
+const DEFAULT_BASE_URL = 'https://api.firecrawl.dev/v1';
+
+export interface RequestOptions {
+  method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
+  params?: Record<string, string | number | boolean | undefined>;
+  body?: Record<string, unknown> | unknown[] | string;
+  headers?: Record<string, string>;
+}
+
+export class FirecrawlClient {
+  private readonly apiKey: string;
+  private readonly baseUrl: string;
+
+  constructor(config: FirecrawlConfig) {
+    if (!config.apiKey) {
+      throw new Error('API key is required');
+    }
+    this.apiKey = config.apiKey;
+    this.baseUrl = config.baseUrl || DEFAULT_BASE_URL;
+  }
+
+  private buildUrl(path: string, params?: Record<string, string | number | boolean | undefined>): string {
+    const url = new URL(`${this.baseUrl}${path}`);
+
+    if (params) {
+      Object.entries(params).forEach(([key, value]) => {
+        if (value !== undefined && value !== null && value !== '') {
+          url.searchParams.append(key, String(value));
+        }
+      });
+    }
+
+    return url.toString();
+  }
+
+  /**
+   * Make an authenticated request to the Firecrawl API
+   * Uses Bearer token authentication
+   */
+  async request<T>(path: string, options: RequestOptions = {}): Promise<T> {
+    const { method = 'GET', params, body, headers = {} } = options;
+
+    const url = this.buildUrl(path, params);
+
+    const requestHeaders: Record<string, string> = {
+      'Authorization': `Bearer ${this.apiKey}`,
+      'Accept': 'application/json',
+      ...headers,
+    };
+
+    if (body && ['POST', 'PUT', 'PATCH'].includes(method)) {
+      requestHeaders['Content-Type'] = 'application/json';
+    }
+
+    const fetchOptions: RequestInit = {
+      method,
+      headers: requestHeaders,
+    };
+
+    if (body && ['POST', 'PUT', 'PATCH'].includes(method)) {
+      fetchOptions.body = typeof body === 'string' ? body : JSON.stringify(body);
+    }
+
+    const response = await fetch(url, fetchOptions);
+
+    // Handle 204 No Content
+    if (response.status === 204) {
+      return {} as T;
+    }
+
+    // Parse response
+    let data: unknown;
+    const contentType = response.headers.get('content-type') || '';
+
+    if (contentType.includes('application/json')) {
+      const text = await response.text();
+      if (text) {
+        try {
+          data = JSON.parse(text);
+        } catch {
+          data = text;
+        }
+      }
+    } else {
+      data = await response.text();
+    }
+
+    // Handle errors
+    if (!response.ok) {
+      const errorMessage = typeof data === 'object' && data !== null
+        ? JSON.stringify(data)
+        : String(data || response.statusText);
+      throw new FirecrawlApiError(errorMessage, response.status);
+    }
+
+    return data as T;
+  }
+
+  async get<T>(path: string, params?: Record<string, string | number | boolean | undefined>): Promise<T> {
+    return this.request<T>(path, { method: 'GET', params });
+  }
+
+  async post<T>(path: string, body?: Record<string, unknown> | unknown[] | string | object, params?: Record<string, string | number | boolean | undefined>): Promise<T> {
+    return this.request<T>(path, { method: 'POST', body: body as Record<string, unknown>, params });
+  }
+
+  async delete<T>(path: string, params?: Record<string, string | number | boolean | undefined>): Promise<T> {
+    return this.request<T>(path, { method: 'DELETE', params });
+  }
+
+  /**
+   * Get a preview of the API key (for display/debugging)
+   */
+  getApiKeyPreview(): string {
+    if (this.apiKey.length > 10) {
+      return `${this.apiKey.substring(0, 6)}...${this.apiKey.substring(this.apiKey.length - 4)}`;
+    }
+    return '***';
+  }
+}

package/connectors/connect-firecrawl/src/api/crawl.ts

@@ -0,0 +1,93 @@
+import type { FirecrawlClient } from './client';
+import type {
+  CrawlRequest,
+  CrawlStartResponse,
+  CrawlStatusResponse,
+  CrawlCancelResponse,
+} from '../types';
+
+/**
+ * Crawl API - Crawl entire websites
+ */
+export class CrawlApi {
+  constructor(private readonly client: FirecrawlClient) {}
+
+  /**
+   * Start a new crawl job
+   * @param url - The starting URL to crawl
+   * @param options - Crawl options
+   * @returns The crawl job ID
+   */
+  async start(url: string, options?: Omit<CrawlRequest, 'url'>): Promise<CrawlStartResponse> {
+    const body: CrawlRequest = {
+      url,
+      ...options,
+    };
+
+    return this.client.post<CrawlStartResponse>('/crawl', body);
+  }
+
+  /**
+   * Get the status of a crawl job
+   * @param jobId - The crawl job ID
+   */
+  async getStatus(jobId: string): Promise<CrawlStatusResponse> {
+    return this.client.get<CrawlStatusResponse>(`/crawl/${jobId}`);
+  }
+
+  /**
+   * Cancel a running crawl job
+   * @param jobId - The crawl job ID
+   */
+  async cancel(jobId: string): Promise<CrawlCancelResponse> {
+    return this.client.delete<CrawlCancelResponse>(`/crawl/${jobId}`);
+  }
+
+  /**
+   * Wait for a crawl job to complete
+   * @param jobId - The crawl job ID
+   * @param pollInterval - Polling interval in milliseconds (default: 2000)
+   * @param timeout - Maximum wait time in milliseconds (default: 300000 = 5 minutes)
+   */
+  async waitForCompletion(
+    jobId: string,
+    pollInterval = 2000,
+    timeout = 300000
+  ): Promise<CrawlStatusResponse> {
+    const startTime = Date.now();
+
+    while (Date.now() - startTime < timeout) {
+      const status = await this.getStatus(jobId);
+
+      if (status.status === 'completed' || status.status === 'failed' || status.status === 'cancelled') {
+        return status;
+      }
+
+      await new Promise(resolve => setTimeout(resolve, pollInterval));
+    }
+
+    throw new Error(`Crawl job ${jobId} timed out after ${timeout}ms`);
+  }
+
+  /**
+   * Start a crawl and wait for completion
+   * @param url - The starting URL to crawl
+   * @param options - Crawl options
+   * @param pollInterval - Polling interval in milliseconds
+   * @param timeout - Maximum wait time in milliseconds
+   */
+  async crawlAndWait(
+    url: string,
+    options?: Omit<CrawlRequest, 'url'>,
+    pollInterval = 2000,
+    timeout = 300000
+  ): Promise<CrawlStatusResponse> {
+    const startResponse = await this.start(url, options);
+
+    if (!startResponse.success || !startResponse.id) {
+      throw new Error(startResponse.error || 'Failed to start crawl job');
+    }
+
+    return this.waitForCompletion(startResponse.id, pollInterval, timeout);
+  }
+}

package/connectors/connect-firecrawl/src/api/index.ts

@@ -0,0 +1,61 @@
+import type { FirecrawlConfig } from '../types';
+import { FirecrawlClient } from './client';
+import { ScrapeApi } from './scrape';
+import { CrawlApi } from './crawl';
+import { MapApi } from './map';
+import { SearchApi } from './search';
+
+/**
+ * Main Firecrawl connector class
+ */
+export class Firecrawl {
+  private readonly client: FirecrawlClient;
+
+  // API modules
+  public readonly scrape: ScrapeApi;
+  public readonly crawl: CrawlApi;
+  public readonly map: MapApi;
+  public readonly search: SearchApi;
+
+  constructor(config: FirecrawlConfig) {
+    this.client = new FirecrawlClient(config);
+    this.scrape = new ScrapeApi(this.client);
+    this.crawl = new CrawlApi(this.client);
+    this.map = new MapApi(this.client);
+    this.search = new SearchApi(this.client);
+  }
+
+  /**
+   * Create a client from environment variables
+   * Looks for FIRECRAWL_API_KEY and optionally FIRECRAWL_BASE_URL
+   */
+  static fromEnv(): Firecrawl {
+    const apiKey = process.env.FIRECRAWL_API_KEY;
+    const baseUrl = process.env.FIRECRAWL_BASE_URL;
+
+    if (!apiKey) {
+      throw new Error('FIRECRAWL_API_KEY environment variable is required');
+    }
+    return new Firecrawl({ apiKey, baseUrl });
+  }
+
+  /**
+   * Get a preview of the API key (for debugging)
+   */
+  getApiKeyPreview(): string {
+    return this.client.getApiKeyPreview();
+  }
+
+  /**
+   * Get the underlying client for direct API access
+   */
+  getClient(): FirecrawlClient {
+    return this.client;
+  }
+}
+
+export { FirecrawlClient } from './client';
+export { ScrapeApi } from './scrape';
+export { CrawlApi } from './crawl';
+export { MapApi } from './map';
+export { SearchApi } from './search';

package/connectors/connect-firecrawl/src/api/map.ts

@@ -0,0 +1,51 @@
+import type { FirecrawlClient } from './client';
+import type { MapRequest, MapResponse } from '../types';
+
+/**
+ * Map API - Discover URLs on a website
+ */
+export class MapApi {
+  constructor(private readonly client: FirecrawlClient) {}
+
+  /**
+   * Map a website to discover all its URLs
+   * @param url - The website URL to map
+   * @param options - Map options
+   */
+  async map(url: string, options?: Omit<MapRequest, 'url'>): Promise<MapResponse> {
+    const body: MapRequest = {
+      url,
+      ...options,
+    };
+
+    return this.client.post<MapResponse>('/map', body);
+  }
+
+  /**
+   * Search for specific URLs on a website
+   * @param url - The website URL
+   * @param search - Search query to filter URLs
+   * @param limit - Maximum number of URLs to return
+   */
+  async search(url: string, search: string, limit?: number): Promise<MapResponse> {
+    return this.map(url, { search, limit });
+  }
+
+  /**
+   * Map only the sitemap URLs
+   * @param url - The website URL
+   * @param limit - Maximum number of URLs to return
+   */
+  async sitemapOnly(url: string, limit?: number): Promise<MapResponse> {
+    return this.map(url, { sitemapOnly: true, limit });
+  }
+
+  /**
+   * Map including subdomains
+   * @param url - The website URL
+   * @param limit - Maximum number of URLs to return
+   */
+  async withSubdomains(url: string, limit?: number): Promise<MapResponse> {
+    return this.map(url, { includeSubdomains: true, limit });
+  }
+}

package/connectors/connect-firecrawl/src/api/scrape.ts

@@ -0,0 +1,71 @@
+import type { FirecrawlClient } from './client';
+import type { ScrapeRequest, ScrapeResponse, ScrapeFormat } from '../types';
+
+/**
+ * Scrape API - Scrape single URLs
+ */
+export class ScrapeApi {
+  constructor(private readonly client: FirecrawlClient) {}
+
+  /**
+   * Scrape a single URL
+   * @param url - The URL to scrape
+   * @param options - Scrape options
+   */
+  async scrape(url: string, options?: Omit<ScrapeRequest, 'url'>): Promise<ScrapeResponse> {
+    const body: ScrapeRequest = {
+      url,
+      ...options,
+    };
+
+    return this.client.post<ScrapeResponse>('/scrape', body);
+  }
+
+  /**
+   * Scrape a URL with screenshot
+   * @param url - The URL to scrape
+   * @param fullPage - Whether to capture full page screenshot
+   */
+  async scrapeWithScreenshot(url: string, fullPage = false): Promise<ScrapeResponse> {
+    const formats: ScrapeFormat[] = fullPage
+      ? ['markdown', 'screenshot@fullPage']
+      : ['markdown', 'screenshot'];
+
+    return this.scrape(url, { formats });
+  }
+
+  /**
+   * Scrape a URL and extract structured data using AI
+   * @param url - The URL to scrape
+   * @param schema - JSON schema for extraction
+   * @param prompt - Optional extraction prompt
+   */
+  async scrapeWithExtraction(
+    url: string,
+    schema: Record<string, unknown>,
+    prompt?: string
+  ): Promise<ScrapeResponse> {
+    return this.scrape(url, {
+      formats: ['markdown'],
+      extract: {
+        schema,
+        prompt,
+      },
+    });
+  }
+
+  /**
+   * Scrape a URL with JavaScript actions
+   * @param url - The URL to scrape
+   * @param actions - Actions to perform before scraping
+   */
+  async scrapeWithActions(
+    url: string,
+    actions: ScrapeRequest['actions']
+  ): Promise<ScrapeResponse> {
+    return this.scrape(url, {
+      formats: ['markdown'],
+      actions,
+    });
+  }
+}

package/connectors/connect-firecrawl/src/api/search.ts

@@ -0,0 +1,61 @@
+import type { FirecrawlClient } from './client';
+import type { SearchRequest, SearchResponse } from '../types';
+
+/**
+ * Search API - Search the web and scrape results (Beta)
+ */
+export class SearchApi {
+  constructor(private readonly client: FirecrawlClient) {}
+
+  /**
+   * Search the web and scrape the results
+   * @param query - The search query
+   * @param options - Search options
+   */
+  async search(query: string, options?: Omit<SearchRequest, 'query'>): Promise<SearchResponse> {
+    const body: SearchRequest = {
+      query,
+      ...options,
+    };
+
+    return this.client.post<SearchResponse>('/search', body);
+  }
+
+  /**
+   * Search with language and country filters
+   * @param query - The search query
+   * @param lang - Language code (e.g., 'en', 'es', 'fr')
+   * @param country - Country code (e.g., 'us', 'gb', 'de')
+   * @param limit - Maximum number of results
+   */
+  async searchLocalized(
+    query: string,
+    lang: string,
+    country: string,
+    limit?: number
+  ): Promise<SearchResponse> {
+    return this.search(query, { lang, country, limit });
+  }
+
+  /**
+   * Search with time filter
+   * @param query - The search query
+   * @param timeFilter - Time filter: 'day', 'week', 'month', 'year'
+   * @param limit - Maximum number of results
+   */
+  async searchRecent(
+    query: string,
+    timeFilter: 'day' | 'week' | 'month' | 'year',
+    limit?: number
+  ): Promise<SearchResponse> {
+    // Google tbs parameter for time filtering
+    const tbsMap = {
+      day: 'qdr:d',
+      week: 'qdr:w',
+      month: 'qdr:m',
+      year: 'qdr:y',
+    };
+
+    return this.search(query, { tbs: tbsMap[timeFilter], limit });
+  }
+}