firecrawlrb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6750678d7f156f49281a7ee81cebebab83b09a9241c7ab04efe28041d6498498
+  data.tar.gz: 4cbf4932ccb8f642d33627ff8fb4933f37c90888c36ed1ad0de57e129bd76731
+SHA512:
+  metadata.gz: 806574da79377099e9d58f93e873145681bad840355397918099562179fc57d48add2a41f597251d3bb9ddf574d24aa1aa34af487f14f1ff4af9a6f40b61ad55
+  data.tar.gz: 8a4fd4b8f0996211fce4ca0686e7290d75c297dae1ee7b12a1810396e17a3ce8504cbb33c51e3565cd4c15c839092200f79e0e85cc6a060b1beff75e80227aba

data/README.md

@@ -0,0 +1,109 @@
+# Firecrawlrb
+
+A zero-dependency Ruby client for the [Firecrawl API](https://www.firecrawl.dev). Uses only Ruby standard library (`Net::HTTP`, `URI`, `JSON`).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'firecrawlrb', path: '../firecrawlrb'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+## Usage
+
+### Configuration
+
+You can provide your Firecrawl API key via an environment variable:
+
+```bash
+export FIRECRAWL_API_KEY='fc-your_api_key_here'
+```
+
+Or via a configuration block in a Rails initializer (`config/initializers/firecrawlrb.rb`):
+
+```ruby
+require 'firecrawlrb'
+
+Firecrawlrb.configure do |config|
+  config.api_key = ENV['FIRECRAWL_API_KEY']
+end
+```
+
+Or pass it directly when initializing the client:
+
+```ruby
+client = Firecrawlrb::Client.new(api_key: 'fc-your_api_key_here')
+```
+
+### LLM JSON Extraction
+
+You can specify a target URL, schema (concise Ruby Hash/Array or raw JSON Schema), and prompt to receive back JSON matching your schema:
+
+#### Concise Ruby Schema Example
+
+```ruby
+client = Firecrawlrb::Client.new
+
+schema = {
+  products: [
+    {
+      url: :string,
+      image_url: :string,
+      price: :string,
+      name: :string
+    }
+  ]
+}
+
+prompt = "extract the products from the main product list site for https://www.blindster.com"
+
+products = client.extract(
+  url: "https://www.blindster.com",
+  schema: schema,
+  prompt: prompt
+)
+
+puts products
+# => { "products" => [ { "name" => "...", "price" => "...", "image_url" => "...", "url" => "..." } ] }
+```
+
+#### Raw JSON Schema Example
+
+```ruby
+schema = {
+  type: "object",
+  required: [],
+  properties: {
+    products: {
+      type: "array",
+      items: {
+        type: "object",
+        required: [],
+        properties: {
+          url: { type: "string" },
+          image_url: { type: "string" },
+          price: { type: "string" },
+          name: { type: "string" }
+        }
+      }
+    }
+  }
+}
+
+products = client.extract(
+  url: "https://www.blindster.com",
+  schema: schema,
+  prompt: prompt
+)
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/firecrawlrb/client.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Firecrawlrb
+  class Client
+    API_URL = "https://api.firecrawl.dev/v2/scrape".freeze
+
+    attr_reader :api_key, :api_url
+
+    def initialize(api_key: nil, api_url: nil)
+      @api_key = api_key || ENV['FIRECRAWL_API_KEY'] || Firecrawlrb.configuration.api_key
+      @api_url = api_url || Firecrawlrb.configuration.api_url || API_URL
+      raise ArgumentError, "API key is required. Set ENV['FIRECRAWL_API_KEY'], use Firecrawlrb.configure, or pass it to Client.new" if @api_key.nil? || @api_key.empty?
+    end
+
+    # Extract JSON matching a schema using Firecrawl v2 API
+    #
+    # @param url_or_kw [String, nil] Target URL if positional argument
+    # @param schema_arg [Hash, Array, nil] Schema if positional argument
+    # @param prompt_arg [String, nil] Prompt if positional argument
+    # @param url [String, nil] Target URL if keyword argument
+    # @param schema [Hash, Array, nil] Schema if keyword argument
+    # @param prompt [String, nil] Prompt if keyword argument
+    # @param only_main_content [Boolean] Whether to extract main content only (default true)
+    # @param max_age [Integer] Cache max age in ms (default 172800000)
+    # @param options [Hash] Additional options to merge into payload
+    # @return [Hash, Array, nil] The parsed extracted JSON
+    def extract(url_or_kw = nil, schema_arg = nil, prompt_arg = nil, url: nil, schema: nil, prompt: nil, only_main_content: true, max_age: 172800000, options: {})
+      target_url = url || url_or_kw
+      target_schema = schema || schema_arg
+      target_prompt = prompt || prompt_arg
+
+      raise ArgumentError, "url is required" if target_url.nil? || target_url.to_s.empty?
+      raise ArgumentError, "schema is required" if target_schema.nil?
+
+      normalized_schema = Firecrawlrb::Schema.normalize(target_schema)
+
+      format_hash = {
+        type: "json",
+        schema: normalized_schema
+      }
+      format_hash[:prompt] = target_prompt if target_prompt && !target_prompt.to_s.empty?
+
+      payload = {
+        url: target_url.to_s,
+        onlyMainContent: only_main_content,
+        maxAge: max_age,
+        parsers: [],
+        formats: [format_hash]
+      }.merge(options)
+
+      response = request(payload)
+      parsed = parse_response(response)
+
+      if parsed.is_a?(Hash) && parsed.key?("data") && parsed["data"].is_a?(Hash) && parsed["data"].key?("json")
+        parsed["data"]["json"]
+      elsif parsed.is_a?(Hash) && parsed.key?("json")
+        parsed["json"]
+      else
+        parsed
+      end
+    end
+
+    alias scrape extract
+
+    private
+
+    def request(payload)
+      uri = URI.parse(@api_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == "https")
+      http.read_timeout = 180
+      http.open_timeout = 180
+
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req['Authorization'] = "Bearer #{@api_key}"
+      req['Content-Type'] = 'application/json'
+      req.body = JSON.generate(payload)
+
+      http.request(req)
+    end
+
+    def parse_response(response)
+      body = response.body
+      begin
+        parsed = JSON.parse(body)
+      rescue JSON::ParserError
+        raise Error, "Invalid JSON response from Firecrawl API (#{response.code}): #{body.to_s[0..100]}..."
+      end
+
+      unless response.is_a?(Net::HTTPSuccess)
+        error_msg = parsed['error'] || parsed['message'] || response.message
+        raise Error, "Firecrawl API Error (#{response.code}): #{error_msg}"
+      end
+
+      parsed
+    end
+  end
+end

data/lib/firecrawlrb.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative "firecrawlrb/client"
+
+module Firecrawlrb
+  class Error < StandardError; end
+
+  class Configuration
+    attr_accessor :api_key, :api_url
+
+    def initialize
+      @api_key = nil
+      @api_url = "https://api.firecrawl.dev/v2/scrape"
+    end
+  end
+
+  module Schema
+    TYPE_MAPPING = {
+      string: "string",
+      number: "number",
+      float: "number",
+      decimal: "number",
+      integer: "integer",
+      int: "integer",
+      boolean: "boolean",
+      bool: "boolean",
+      array: "array",
+      object: "object"
+    }.freeze
+
+    def self.normalize(input)
+      case input
+      when Hash
+        # If it already looks like a raw JSON Schema (has :type or "type" key)
+        if input.key?(:type) || input.key?("type") || input.key?(:properties) || input.key?("properties")
+          input
+        else
+          properties = {}
+          input.each do |key, value|
+            properties[key.to_s] = normalize(value)
+          end
+          {
+            type: "object",
+            required: [],
+            properties: properties
+          }
+        end
+      when Array
+        if input.empty?
+          { type: "array", items: { type: "string" } }
+        else
+          { type: "array", items: normalize(input.first) }
+        end
+      when Symbol, String
+        type_str = input.to_s.downcase.to_sym
+        json_type = TYPE_MAPPING[type_str] || input.to_s
+        { type: json_type }
+      else
+        { type: "string" }
+      end
+    end
+  end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configuration=(config)
+      @configuration = config
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def reset
+      @configuration = Configuration.new
+    end
+  end
+end

metadata

@@ -0,0 +1,43 @@
+--- !ruby/object:Gem::Specification
+name: firecrawlrb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Firecrawlrb Author
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Simple zero-dependency Ruby gem for Firecrawl API LLM JSON extraction.
+email:
+- author@example.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/firecrawlrb.rb
+- lib/firecrawlrb/client.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: A zero-dependency Ruby client for the Firecrawl API
+test_files: []