firecrawl-sdk 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7266e8ff84ad11eebc0312025933594af43172061ea8d2f5959b65d98eb34f64
+  data.tar.gz: 48abbfb695f5f9e688e9b02fe5120cfdb6c55bc9c1c22398fd6bbb32582d606e
+SHA512:
+  metadata.gz: 0b69ffbc921e023aba67107a388b44aa90f1f479f9c62ad494734536beea0e831be9b6988ad50132fcfecce39250745b9e38edf37ef22ac1f7f2fecae761615b
+  data.tar.gz: 48f082ce92fb3bc1f6c48a4cdef5b3d2f4a074185fd2e87d3f661a0983835bf60f16c085b118cd134d132afd418e8bad21b54c17faa2c507261bbe7620cf9e00

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Firecrawl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,197 @@
+# Firecrawl Ruby SDK
+
+Ruby SDK for the [Firecrawl](https://firecrawl.dev) v2 web scraping API.
+
+## Prerequisites
+
+- Ruby >= 3.0
+
+## Installation
+
+Add to your `Gemfile`:
+
+```ruby
+gem "firecrawl-sdk", "~> 1.0"
+```
+
+Or install directly:
+
+```bash
+gem install firecrawl-sdk
+```
+
+## Quick Start
+
+```ruby
+require "firecrawl"
+
+# Create a client
+client = Firecrawl::Client.new(api_key: "fc-your-api-key")
+
+# Or load from FIRECRAWL_API_KEY environment variable
+client = Firecrawl::Client.from_env
+
+# Scrape a single page
+doc = client.scrape("https://example.com")
+puts doc.markdown
+```
+
+## Environment Setup
+
+```bash
+export FIRECRAWL_API_KEY="fc-your-api-key"
+# Optional: custom API URL
+export FIRECRAWL_API_URL="http://localhost:3002"
+```
+
+## API Reference
+
+### Scrape
+
+```ruby
+# Basic scrape
+doc = client.scrape("https://example.com")
+puts doc.markdown
+
+# Scrape with options
+doc = client.scrape("https://example.com",
+  Firecrawl::Models::ScrapeOptions.new(
+    formats: ["markdown", "html"],
+    only_main_content: true,
+    wait_for: 1000
+  ))
+puts doc.html
+```
+
+### Crawl
+
+```ruby
+# Crawl with auto-polling (blocks until complete)
+job = client.crawl("https://example.com",
+  Firecrawl::Models::CrawlOptions.new(limit: 50))
+job.data.each { |doc| puts doc.markdown }
+
+# Async crawl
+response = client.start_crawl("https://example.com",
+  Firecrawl::Models::CrawlOptions.new(limit: 10))
+puts response.id
+
+# Check status
+status = client.get_crawl_status(response.id)
+puts status.status
+
+# Cancel
+client.cancel_crawl(response.id)
+```
+
+### Batch Scrape
+
+```ruby
+urls = ["https://example.com/page1", "https://example.com/page2"]
+
+# Batch scrape with auto-polling
+job = client.batch_scrape(urls,
+  Firecrawl::Models::BatchScrapeOptions.new(
+    options: Firecrawl::Models::ScrapeOptions.new(formats: ["markdown"])
+  ))
+job.data.each { |doc| puts doc.markdown }
+```
+
+### Map
+
+```ruby
+# Discover URLs on a website
+result = client.map("https://example.com")
+result.links.each { |link| puts link["url"] }
+
+# With options
+result = client.map("https://example.com",
+  Firecrawl::Models::MapOptions.new(limit: 100, search: "blog"))
+```
+
+### Search
+
+```ruby
+# Web search
+results = client.search("firecrawl web scraping")
+results.web&.each { |r| puts r["url"] }
+
+# With options
+results = client.search("latest news",
+  Firecrawl::Models::SearchOptions.new(limit: 5, location: "US"))
+```
+
+### Agent
+
+```ruby
+# Run an AI agent task (blocks until complete)
+status = client.agent(
+  Firecrawl::Models::AgentOptions.new(
+    prompt: "Find the pricing information",
+    urls: ["https://example.com"]
+  ))
+puts status.data
+```
+
+### Usage & Metrics
+
+```ruby
+# Check concurrency
+concurrency = client.get_concurrency
+puts concurrency.concurrency
+
+# Check credit usage
+usage = client.get_credit_usage
+puts usage.remaining_credits
+```
+
+## Configuration
+
+```ruby
+client = Firecrawl::Client.new(
+  api_key: "fc-your-api-key",
+  api_url: "https://api.firecrawl.dev",  # custom API URL
+  timeout: 300,                           # HTTP timeout in seconds
+  max_retries: 3,                         # automatic retries
+  backoff_factor: 0.5                     # exponential backoff factor
+)
+```
+
+## Error Handling
+
+```ruby
+begin
+  doc = client.scrape("https://example.com")
+rescue Firecrawl::AuthenticationError => e
+  puts "Invalid API key: #{e.message}"
+rescue Firecrawl::RateLimitError => e
+  puts "Rate limited: #{e.message}"
+rescue Firecrawl::JobTimeoutError => e
+  puts "Job #{e.job_id} timed out after #{e.timeout_seconds}s"
+rescue Firecrawl::FirecrawlError => e
+  puts "Error (#{e.status_code}): #{e.message}"
+end
+```
+
+## Development
+
+### Building from Source
+
+```bash
+cd apps/ruby-sdk
+bundle install
+```
+
+### Running Tests
+
+```bash
+# Unit tests
+bundle exec rake test
+
+# With API key for E2E tests
+FIRECRAWL_API_KEY=fc-your-key bundle exec rake test
+```
+
+## License
+
+MIT License - see [LICENSE](LICENSE).

data/lib/firecrawl/client.rb

@@ -0,0 +1,396 @@
+# frozen_string_literal: true
+
+module Firecrawl
+  # Client for the Firecrawl v2 API.
+  #
+  # @example Quick start
+  #   client = Firecrawl::Client.new(api_key: "fc-your-api-key")
+  #
+  #   # Scrape a single page
+  #   doc = client.scrape("https://example.com",
+  #     Firecrawl::Models::ScrapeOptions.new(formats: ["markdown"]))
+  #
+  #   # Crawl a website
+  #   job = client.crawl("https://example.com",
+  #     Firecrawl::Models::CrawlOptions.new(limit: 50))
+  class Client
+    DEFAULT_API_URL = "https://api.firecrawl.dev"
+    DEFAULT_TIMEOUT = 300 # seconds
+    DEFAULT_MAX_RETRIES = 3
+    DEFAULT_BACKOFF_FACTOR = 0.5
+    DEFAULT_POLL_INTERVAL = 2 # seconds
+    DEFAULT_JOB_TIMEOUT = 300 # seconds
+
+    # Creates a new Firecrawl client.
+    #
+    # @param api_key [String, nil] API key (falls back to FIRECRAWL_API_KEY env var)
+    # @param api_url [String] API base URL
+    # @param timeout [Integer] HTTP request timeout in seconds
+    # @param max_retries [Integer] maximum automatic retries for transient failures
+    # @param backoff_factor [Float] exponential backoff factor in seconds
+    def initialize(
+      api_key: nil,
+      api_url: nil,
+      timeout: DEFAULT_TIMEOUT,
+      max_retries: DEFAULT_MAX_RETRIES,
+      backoff_factor: DEFAULT_BACKOFF_FACTOR
+    )
+      resolved_key = api_key || ENV["FIRECRAWL_API_KEY"]
+      if resolved_key.nil? || resolved_key.strip.empty?
+        raise FirecrawlError, "API key is required. Provide api_key: or set FIRECRAWL_API_KEY environment variable."
+      end
+
+      resolved_url = api_url || ENV["FIRECRAWL_API_URL"] || DEFAULT_API_URL
+      unless resolved_url.match?(%r{\Ahttps?://}i)
+        raise FirecrawlError, "API URL must be a fully qualified HTTP or HTTPS URL (got: #{resolved_url})."
+      end
+
+      @http = HttpClient.new(
+        api_key: resolved_key,
+        base_url: resolved_url,
+        timeout: timeout,
+        max_retries: max_retries,
+        backoff_factor: backoff_factor
+      )
+    end
+
+    # Creates a client from the FIRECRAWL_API_KEY environment variable.
+    #
+    # @return [Client]
+    def self.from_env
+      new
+    end
+
+    # ================================================================
+    # SCRAPE
+    # ================================================================
+
+    # Scrapes a single URL and returns the document.
+    #
+    # @param url [String] the URL to scrape
+    # @param options [Models::ScrapeOptions, nil] scrape configuration
+    # @return [Models::Document]
+    def scrape(url, options = nil)
+      raise ArgumentError, "URL is required" if url.nil?
+
+      body = { "url" => url }
+      body.merge!(options.to_h) if options
+      raw = @http.post("/v2/scrape", body)
+      data = raw["data"] || raw
+      Models::Document.new(data)
+    end
+
+    # Interacts with the scrape-bound browser session for a scrape job.
+    #
+    # @param job_id [String] the scrape job ID
+    # @param code [String] the code to execute
+    # @param language [String] "python", "node", or "bash" (default: "node")
+    # @param timeout [Integer, nil] execution timeout in seconds (1-300)
+    # @return [Hash] execution result with stdout, stderr, exit_code
+    def interact(job_id, code, language: "node", timeout: nil)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+      raise ArgumentError, "Code is required" if code.nil?
+
+      body = { "code" => code, "language" => language }
+      body["timeout"] = timeout if timeout
+      @http.post("/v2/scrape/#{job_id}/interact", body)
+    end
+
+    # Stops the interactive browser session for a scrape job.
+    #
+    # @param job_id [String] the scrape job ID
+    # @return [Hash] stop response
+    def stop_interactive_browser(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      @http.delete("/v2/scrape/#{job_id}/interact")
+    end
+
+    # ================================================================
+    # CRAWL
+    # ================================================================
+
+    # Starts an async crawl job and returns immediately.
+    #
+    # @param url [String] the URL to start crawling from
+    # @param options [Models::CrawlOptions, nil] crawl configuration
+    # @return [Models::CrawlResponse]
+    def start_crawl(url, options = nil)
+      raise ArgumentError, "URL is required" if url.nil?
+
+      body = { "url" => url }
+      body.merge!(options.to_h) if options
+      raw = @http.post("/v2/crawl", body)
+      Models::CrawlResponse.new(raw)
+    end
+
+    # Gets the status and results of a crawl job.
+    #
+    # @param job_id [String] the crawl job ID
+    # @return [Models::CrawlJob]
+    def get_crawl_status(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      raw = @http.get("/v2/crawl/#{job_id}")
+      Models::CrawlJob.new(raw)
+    end
+
+    # Crawls a website and waits for completion (auto-polling).
+    #
+    # @param url [String] the URL to crawl
+    # @param options [Models::CrawlOptions, nil] crawl configuration
+    # @param poll_interval [Integer] seconds between status checks
+    # @param timeout [Integer] maximum seconds to wait
+    # @return [Models::CrawlJob]
+    def crawl(url, options = nil, poll_interval: DEFAULT_POLL_INTERVAL, timeout: DEFAULT_JOB_TIMEOUT)
+      start = start_crawl(url, options)
+      poll_crawl(start.id, poll_interval, timeout)
+    end
+
+    # Cancels a running crawl job.
+    #
+    # @param job_id [String] the crawl job ID
+    # @return [Hash]
+    def cancel_crawl(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      @http.delete("/v2/crawl/#{job_id}")
+    end
+
+    # Gets errors from a crawl job.
+    #
+    # @param job_id [String] the crawl job ID
+    # @return [Hash]
+    def get_crawl_errors(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      @http.get("/v2/crawl/#{job_id}/errors")
+    end
+
+    # ================================================================
+    # BATCH SCRAPE
+    # ================================================================
+
+    # Starts an async batch scrape job.
+    #
+    # @param urls [Array<String>] the URLs to scrape
+    # @param options [Models::BatchScrapeOptions, nil] batch scrape configuration
+    # @return [Models::BatchScrapeResponse]
+    def start_batch_scrape(urls, options = nil)
+      raise ArgumentError, "URLs list is required" if urls.nil?
+
+      body = { "urls" => urls }
+      extra_headers = {}
+      if options
+        opts_hash = options.to_h
+
+        # idempotencyKey goes as a header, not in body
+        if options.idempotency_key && !options.idempotency_key.empty?
+          extra_headers["x-idempotency-key"] = options.idempotency_key
+        end
+
+        # Flatten nested scrape options to top level (API expects this)
+        nested = opts_hash.delete("options")
+        body.merge!(opts_hash)
+        body.merge!(nested) if nested
+      end
+      raw = @http.post("/v2/batch/scrape", body, extra_headers: extra_headers)
+      Models::BatchScrapeResponse.new(raw)
+    end
+
+    # Gets the status and results of a batch scrape job.
+    #
+    # @param job_id [String] the batch scrape job ID
+    # @return [Models::BatchScrapeJob]
+    def get_batch_scrape_status(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      raw = @http.get("/v2/batch/scrape/#{job_id}")
+      Models::BatchScrapeJob.new(raw)
+    end
+
+    # Batch-scrapes URLs and waits for completion (auto-polling).
+    #
+    # @param urls [Array<String>] the URLs to scrape
+    # @param options [Models::BatchScrapeOptions, nil] batch scrape configuration
+    # @param poll_interval [Integer] seconds between status checks
+    # @param timeout [Integer] maximum seconds to wait
+    # @return [Models::BatchScrapeJob]
+    def batch_scrape(urls, options = nil, poll_interval: DEFAULT_POLL_INTERVAL, timeout: DEFAULT_JOB_TIMEOUT)
+      start = start_batch_scrape(urls, options)
+      poll_batch_scrape(start.id, poll_interval, timeout)
+    end
+
+    # Cancels a running batch scrape job.
+    #
+    # @param job_id [String] the batch scrape job ID
+    # @return [Hash]
+    def cancel_batch_scrape(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      @http.delete("/v2/batch/scrape/#{job_id}")
+    end
+
+    # ================================================================
+    # MAP
+    # ================================================================
+
+    # Discovers URLs on a website.
+    #
+    # @param url [String] the URL to map
+    # @param options [Models::MapOptions, nil] map configuration
+    # @return [Models::MapData]
+    def map(url, options = nil)
+      raise ArgumentError, "URL is required" if url.nil?
+
+      body = { "url" => url }
+      body.merge!(options.to_h) if options
+      raw = @http.post("/v2/map", body)
+      data = raw["data"] || raw
+      Models::MapData.new(data)
+    end
+
+    # ================================================================
+    # SEARCH
+    # ================================================================
+
+    # Performs a web search.
+    #
+    # @param query [String] the search query
+    # @param options [Models::SearchOptions, nil] search configuration
+    # @return [Models::SearchData]
+    def search(query, options = nil)
+      raise ArgumentError, "Query is required" if query.nil?
+
+      body = { "query" => query }
+      body.merge!(options.to_h) if options
+      raw = @http.post("/v2/search", body)
+      data = raw["data"] || raw
+      Models::SearchData.new(data)
+    end
+
+    # ================================================================
+    # AGENT
+    # ================================================================
+
+    # Starts an async agent task.
+    #
+    # @param options [Models::AgentOptions] agent configuration
+    # @return [Models::AgentResponse]
+    def start_agent(options)
+      raise ArgumentError, "Agent options are required" if options.nil?
+
+      raw = @http.post("/v2/agent", options.to_h)
+      Models::AgentResponse.new(raw)
+    end
+
+    # Gets the status of an agent task.
+    #
+    # @param job_id [String] the agent job ID
+    # @return [Models::AgentStatusResponse]
+    def get_agent_status(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      raw = @http.get("/v2/agent/#{job_id}")
+      Models::AgentStatusResponse.new(raw)
+    end
+
+    # Runs an agent task and waits for completion (auto-polling).
+    #
+    # @param options [Models::AgentOptions] agent configuration
+    # @param poll_interval [Integer] seconds between status checks
+    # @param timeout [Integer] maximum seconds to wait
+    # @return [Models::AgentStatusResponse]
+    def agent(options, poll_interval: DEFAULT_POLL_INTERVAL, timeout: DEFAULT_JOB_TIMEOUT)
+      start = start_agent(options)
+      raise FirecrawlError, "Agent start did not return a job ID" if start.id.nil?
+
+      deadline = Time.now + timeout
+      while Time.now < deadline
+        status = get_agent_status(start.id)
+        return status if status.done?
+
+        sleep(poll_interval)
+      end
+      raise JobTimeoutError.new(start.id, timeout, "Agent")
+    end
+
+    # Cancels a running agent task.
+    #
+    # @param job_id [String] the agent job ID
+    # @return [Hash]
+    def cancel_agent(job_id)
+      raise ArgumentError, "Job ID is required" if job_id.nil?
+
+      @http.delete("/v2/agent/#{job_id}")
+    end
+
+    # ================================================================
+    # USAGE & METRICS
+    # ================================================================
+
+    # Gets current concurrency usage.
+    #
+    # @return [Models::ConcurrencyCheck]
+    def get_concurrency
+      raw = @http.get("/v2/concurrency-check")
+      Models::ConcurrencyCheck.new(raw)
+    end
+
+    # Gets current credit usage.
+    #
+    # @return [Models::CreditUsage]
+    def get_credit_usage
+      raw = @http.get("/v2/team/credit-usage")
+      Models::CreditUsage.new(raw)
+    end
+
+    private
+
+    def poll_crawl(job_id, poll_interval, timeout)
+      deadline = Time.now + timeout
+      while Time.now < deadline
+        job = get_crawl_status(job_id)
+        return paginate_crawl(job) if job.done?
+
+        sleep(poll_interval)
+      end
+      raise JobTimeoutError.new(job_id, timeout, "Crawl")
+    end
+
+    def poll_batch_scrape(job_id, poll_interval, timeout)
+      deadline = Time.now + timeout
+      while Time.now < deadline
+        job = get_batch_scrape_status(job_id)
+        return paginate_batch_scrape(job) if job.done?
+
+        sleep(poll_interval)
+      end
+      raise JobTimeoutError.new(job_id, timeout, "Batch scrape")
+    end
+
+    def paginate_crawl(job)
+      job.data ||= []
+      current = job
+      while current.next_url && !current.next_url.empty?
+        raw = @http.get_absolute(current.next_url)
+        next_page = Models::CrawlJob.new(raw)
+        job.data.concat(next_page.data) unless next_page.data.empty?
+        current = next_page
+      end
+      job
+    end
+
+    def paginate_batch_scrape(job)
+      job.data ||= []
+      current = job
+      while current.next_url && !current.next_url.empty?
+        raw = @http.get_absolute(current.next_url)
+        next_page = Models::BatchScrapeJob.new(raw)
+        job.data.concat(next_page.data) unless next_page.data.empty?
+        current = next_page
+      end
+      job
+    end
+  end
+end

data/lib/firecrawl/errors.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Firecrawl
+  # Base error class for all Firecrawl SDK errors.
+  class FirecrawlError < StandardError
+    attr_reader :status_code, :error_code, :details
+
+    def initialize(message = nil, status_code: nil, error_code: nil, details: nil)
+      @status_code = status_code
+      @error_code = error_code
+      @details = details
+      super(message)
+    end
+  end
+
+  # Raised on 401 Unauthorized responses.
+  class AuthenticationError < FirecrawlError
+    def initialize(message = nil, error_code: nil, details: nil)
+      super(message, status_code: 401, error_code: error_code, details: details)
+    end
+  end
+
+  # Raised on 429 Too Many Requests responses.
+  class RateLimitError < FirecrawlError
+    def initialize(message = nil, error_code: nil, details: nil)
+      super(message, status_code: 429, error_code: error_code, details: details)
+    end
+  end
+
+  # Raised when an async job exceeds its timeout.
+  class JobTimeoutError < FirecrawlError
+    attr_reader :job_id, :timeout_seconds
+
+    def initialize(job_id, timeout_seconds, label = "Job")
+      @job_id = job_id
+      @timeout_seconds = timeout_seconds
+      super("#{label} #{job_id} timed out after #{timeout_seconds} seconds")
+    end
+  end
+end