firecrawl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d79423650fbec60124bab7710e372f16774987f437e8bb52cb13e72a434bdec6
+  data.tar.gz: bfb0ec5c302e4e646812855d5c954e2fc6c075e4a03af495a6a09815ce588e44
+SHA512:
+  metadata.gz: 46c819135a19388beb434c797e16a3a73f52b1d85cc37f317e03dc159de8ed1a56a6a5170f2e2457e2798fbff25d93cfbf11f7be12682fcb6ef13534aa347f62
+  data.tar.gz: ffe9b239c29138617a1902f8502099b6f0b2019eeed8b266cd8f77e94eee868e8e876ee9f8545cd584d701a8bfffa75f6552508b2949463f2cc187580eeb35fe

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Endless International
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/firecrawl.gemspec

@@ -0,0 +1,37 @@
+Gem::Specification.new do | spec |
+
+  spec.name             = 'firecrawl'
+  spec.version          = '0.0.1'
+  spec.authors          = [ 'Kristoph Cichocki-Romanov' ]
+  spec.email            = [ 'rubygems.org@kristoph.net' ]
+
+  spec.summary          = 
+    "The Firecrawl gem implements a lightweight interface to the Firecrawl.dev API which takes " \
+    "a URL, crawls it and returns html, markdown, or structured data. It is of particular value" \
+    "when used with LLM's for grounding."
+  spec.description      =
+    "The Firecrawl gem implements a lightweight interface to the Firecrawl.dev API. Firecrawl " \
+    "can take a URL, scrape the page contents and return the whole page or principal content " \
+    "as html, markdown, or structured data.\n" \
+    "\n" \
+    "In addition, Firecrawl can crawl an entire site returning the pages it encounters or just " \
+    "the map of the pages, which can be used for subsequent scraping."
+  spec.license          = 'MIT'
+  spec.homepage         = 'https://github.com/EndlessInternational/firecrawl'
+  spec.metadata         = {
+    'source_code_uri'   => 'https://github.com/EndlessInternational/firecrawl',
+    'bug_tracker_uri'   => 'https://github.com/EndlessInternational/firecrawl/issues',
+#    'documentation_uri' => 'https://github.com/EndlessInternational/firecrawl'
+  }
+
+  spec.required_ruby_version = '>= 3.0'
+  spec.files            = Dir[ "lib/**/*.rb", "LICENSE", "README.md", "firecrawl.gemspec" ]
+  spec.require_paths    = [ "lib" ]
+
+  spec.add_runtime_dependency 'faraday', '~> 2.7'
+  spec.add_runtime_dependency 'dynamicschema', '~> 1.0.0.beta03'
+
+  spec.add_development_dependency 'rspec', '~> 3.13'
+  spec.add_development_dependency 'debug', '~> 1.9'
+
+end

data/lib/firecrawl/batch_scrape_request.rb

@@ -0,0 +1,106 @@
+module Firecrawl 
+
+  ##
+  # The +BatchScrapeRequest+ class encapsulates a batch scrape request to the Firecrawl API. 
+  # After creating a new +BatchScrapeRequest+ instance you can begin batch scraping by calling 
+  # the +begin_scraping+ method and then subsequently evaluate the results by calling the  
+  # +continue_scraping' method.
+  #
+  # === examples
+  # 
+  # require 'firecrawl'
+  #
+  # request = Firecrawl::BatchScrapeRequest.new( api_key: ENV[ 'FIRECRAWL_API_KEY' )
+  #
+  # urls = [ 'https://example.com', 'https://icann.org' ]
+  # options = Firecrawl::ScrapeOptions.build do 
+  #   format                [ :markdown, 'screenshot@full_page' ]
+  #   only_main_content     true
+  # end
+  # 
+  # batch_response = request.beging_scraping( urls, options )
+  # while response.success?
+  #   batch_result = batch_response.result 
+  #   if batch_result.success?
+  #     batch_result.scrape_results.each do | result |
+  #       puts response.metadata[ 'title ] 
+  #       puts '---'
+  #       puts response.markdown
+  #       puts "\n\n"
+  #     end
+  #   end
+  #   break unless batch_result.status?( :scraping )
+  # end
+  #
+  # unless response.success? 
+  #   puts response.result.error_description 
+  # end 
+  # 
+  class BatchScrapeRequest < Request 
+
+    ## 
+    # The +start_scraping+ method makes a Firecrawl '/batch/scrape' POST request which will 
+    # initiate batch scraping of the given urls. 
+    # 
+    # The response is always an instance of +Faraday::Response+. If +response.success?+ is true,
+    # then +response.result+ will be an instance +BatchScrapeResult+. If the request is not 
+    # successful then +response.result+ will be an instance of +ErrorResult+.
+    #
+    # Remember that you should call +response.success?+ to valida that the call to the API was
+    # successful and then +response.result.success?+ to validate that the API processed the
+    # request successfuly. 
+    #
+    def start_scraping( urls, options = nil, &block )        
+      if options
+        options = options.is_a?( ScrapeOptions ) ? options : ScrapeOptions.build( options.to_h ) 
+        options = options.to_h
+      else 
+        options = {}
+      end
+      options[ :urls ] = [ urls ].flatten
+
+      response = post( "#{BASE_URI}/batch/scrape", options, &block )
+      result = nil 
+      if response.success?
+        attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
+        attributes ||= { success: false, status: :failed  }
+        result = BatchScrapeResult.new( attributes[ :success ], attributes )
+      else 
+        result = ErrorResult.new( response.status, attributes )
+      end
+
+      ResponseMethods.install( response, result )  
+    end 
+
+    ## 
+    # The +retrieve_scraping+ method makes a Firecrawl '/batch/scrape' GET request which will 
+    # retrieve batch scraping results. Note that there is no guarantee that there are any batch 
+    # scraping results at the time of the call and you may need to call this method multiple 
+    # times.
+    # 
+    # The response is always an instance of +Faraday::Response+. If +response.success?+ is +true+, 
+    # then +response.result+ will be an instance +BatchScrapeResult+. If the request is not 
+    # successful then +response.result+ will be an instance of +ErrorResult+.
+    #
+    # Remember that you should call +response.success?+ to valida that the call to the API was
+    # successful and then +response.result.success?+ to validate that the API processed the
+    # request successfuly. 
+    #
+    def retrieve_scraping( batch_result, &block )
+      raise ArgumentError, "The first argument must be an instance of BatchScrapeResult." \
+        unless batch_result.is_a?( BatchScrapeResult )
+      response = get( batch_result.next_url, &block )  
+      result = nil 
+      if response.success? 
+        attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
+        attributes ||= { success: false, status: :failed  }
+        result = batch_result.merge( attributes  )
+      else 
+        result = ErrorResult.new( response.status, attributes )
+      end 
+
+      ResponseMethods.install( response, result )     
+    end
+
+  end 
+end

data/lib/firecrawl/batch_scrape_result.rb

@@ -0,0 +1,63 @@
+module Firecrawl 
+  class BatchScrapeResult 
+
+    def initialize( success, attributes )
+      @success = success
+      @attributes = attributes || {}
+    end
+
+    def success?
+      @success || false 
+    end
+
+    def status
+      # the initial Firecrawl response does not have a status so we synthesize a 'scraping'
+      # status if the operation was otherwise successful 
+      @attributes[ :status ]&.to_sym || ( @success ? :scraping : :failed )
+    end 
+
+    def status?( status )
+      self.status == status 
+    end 
+
+    def id
+      @attributes[ :id ]
+    end
+
+    def total 
+      @attributes[ :total ] || 0
+    end
+
+    def completed
+      @attributes[ :completed ] || 0
+    end 
+
+    def credits_used 
+      @attributes[ :creditsUsed ] || 0
+    end 
+
+    def expires_at
+      Date.parse( @attributes[ :expiresAt ] ) rescue nil
+    end
+
+    def url 
+      @attributes[ :url ] 
+    end 
+    
+    def next_url 
+      @attributes[ :next ] || @attributes[ :url ] 
+    end 
+
+    def scrape_results 
+      success = @attributes[ :success ]
+      # note the &.compact is here because I've noted null entries in the data
+      ( @attributes[ :data ]&.compact || [] ).map do | attr |
+        ScrapeResult.new( success, attr )
+      end
+    end 
+
+    def merge( attributes )
+      self.class.new( attributes[ :success ], @attributes.merge( attributes ) )
+    end 
+  end
+end

data/lib/firecrawl/crawl_options.rb

@@ -0,0 +1,47 @@
+module Firecrawl
+  class CrawlOptions 
+    include DynamicSchema::Definable 
+    include DynamicSchema::Buildable
+
+    FORMATS = [ :markdown, :links, :html, :raw_html, :screenshot ]
+
+    ACTIONS = [ :wait, :click, :write, :press, :screenshot, :scrape ]
+
+    schema do 
+      exclude_paths         String, as: :excludePaths, array: true
+      include_paths         String, as: :includePaths, array: true
+      maximum_depth         Integer, as: :maxDepth 
+      ignore_sitemap        [ TrueClass, FalseClass ], as: :ignoreSitemap 
+      limit                 Integer 
+      allow_backward_links  [ TrueClass, FalseClass ], as: :allowBackwardLinks
+      allow_external_links  [ TrueClass, FalseClass ], as: :allowExternalLinks
+      webhook               String 
+      scrape_options        as: :scrapeOptions, &ScrapeOptions.schema
+    end
+
+    def self.build( options = nil, &block )
+      new( api_options: builder.build( options, &block ) )
+    end 
+
+    def self.build!( options = nil, &block )
+      new( api_options: builder.build!( options, &block ) )
+    end 
+
+    def initialize( options, api_options: nil )
+      @options = self.class.builder.build( options || {} )
+      @options = api_options.merge( @options ) if api_options 
+      
+      scrape_options = @options[ :scrapeOptions ]
+      if scrape_options 
+        scrape_options[ :formats ]&.map!( &method( :string_camelize ) )
+      end 
+    end
+
+    def to_h
+      @options.to_h 
+    end
+
+  end
+end
+
+

data/lib/firecrawl/error_result.rb

@@ -0,0 +1,45 @@
+module Firecrawl
+  class ErrorResult
+
+    attr_reader :error_type, :error_description  
+    
+    def initialize( status_code, attributes = nil )
+      @error_code, @error_description = status_code_to_error( status_code )
+      @error_description = attributes[ :error ] if @attributes&.respond_to?( :[] )
+    end 
+
+  private 
+    def status_code_to_error( status_code )
+      case status_code 
+      # this is here because I've noted invalid payloads being returned with a 200
+      when 200
+        [ :unexpected_error, 
+          "The response was successful but it did not include a valid payload." ]
+      when 400
+        [ :invalid_request_error, 
+          "There was an issue with the format or content of your request." ]
+      when 401
+        [ :authentication_error, 
+          "There's an issue with your API key." ]
+      when 402
+        [ :payment_required, 
+          "The request requires a paid account" ]
+      when 404
+        [ :not_found_error, 
+          "The requested resource was not found." ]
+      when 429
+        [ :rate_limit_error, 
+          "Your account has hit a rate limit." ]
+      when 500..505
+        [ :api_error, 
+          "An unexpected Firecrawl server error has occurred." ]
+      when 529
+        [ :overloaded_error, 
+          "The Firecrawl service is overloaded." ]
+      else
+        [ :unknown_error, 
+          "The Firecrawl service returned an unexpected status code: '#{status_code}'." ]
+      end
+    end
+  end
+end

data/lib/firecrawl/helpers.rb

@@ -0,0 +1,8 @@
+module Firecrawl 
+  module Helpers
+    def string_camelize( string )
+      words = string.split( /[\s_\-]/ )
+      words.map.with_index { |word, index| index.zero? ? word.downcase : word.capitalize }.join
+    end
+  end
+end

data/lib/firecrawl/map_options.rb

@@ -0,0 +1,32 @@
+module Firecrawl
+  class MapOptions
+    include DynamicSchema::Definable 
+
+    schema do 
+      search            String 
+      ignore_sitemap    [ TrueClass, FalseClass ]
+      ignore_subdomains [ TrueClass, FalseClass ]
+      limit             Integer
+    end
+
+    def self.build( options = nil, &block )
+      new( api_options: builder.build( options, &block ) )
+    end 
+
+    def self.build!( options = nil, &block )
+      new( api_options: builder.build!( options, &block ) )
+    end 
+
+    def initialize( options = {}, api_options: nil )
+      @options = self.class.builder.build( options || {} )
+      @options = api_options.merge( @options ) if api_options 
+    end
+
+    def to_h
+      @options.to_h 
+    end
+
+  end
+end
+
+

data/lib/firecrawl/map_request.rb

@@ -0,0 +1,58 @@
+module Firecrawl 
+  ##
+  # The +MapRequest+ class encapsulates a '/map' POST request to the Firecrawl API. After creating
+  # a new +MapRequest+ instance you can make the request by calling the +map+ method to crawl the 
+  # site and retrieve +links+
+  #
+  # === examples
+  # 
+  # require 'firecrawl'
+  #
+  # request = Firecrawl::MapRequest.new( api_key: ENV[ 'FIRECRAWL_API_KEY' )
+  #
+  # response = request.map( 'https://example.com', { limit: 100 } )
+  # if response.success?
+  #   result = response.result 
+  #   if result.success? 
+  #     result.links.each do | link |
+  #       puts link 
+  #     end 
+  #   end 
+  # else 
+  #   puts response.result.error_description 
+  # end 
+  # 
+  class MapRequest < Request 
+
+    ## 
+    # The +map+ method makes a Firecrawl '/map' POST request which will scrape the site with 
+    # given url. 
+    # 
+    # The response is always an instance of +Faraday::Response+. If +response.success?+ is true,
+    # then +response.result+ will be an instance +MapResult+. If the request is not successful 
+    # then +response.result+ will be an instance of +ErrorResult+.
+    #
+    def map( url, options = nil, &block )    
+      if options
+        options = options.is_a?( MapOptions ) ? options : MapOptions.build( options.to_h ) 
+        options = options.to_h
+      else 
+        options = {}
+      end
+      options[ :url ] = url.to_s     
+
+      response = post( "#{BASE_URI}/map", options, &block )
+      result = nil 
+      if response.success?  
+        attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
+        attributes ||= { success: false }
+        result = MapResult.new( attributes[ :success ], attributes )
+      else 
+        result = ErrorResult.new( response.status, attributes ) 
+      end
+      
+      ResponseMethods.install( response, result )     
+    end 
+
+  end 
+end

data/lib/firecrawl/map_result.rb

@@ -0,0 +1,29 @@
+module Firecrawl 
+  class MapResult 
+
+    def initialize( success, attributes )
+      @success = success
+      @attributes = attributes
+    end
+
+    ##
+    # The +success?+ method returns +true+ if the scraping was successful. 
+    #
+    # Note that the response +success?+ tells you if the call to the Firecrawl api was successful 
+    # while this +success?+ method tells you if the actual scraping operation was successful.
+    #
+    def success?  
+      @success || false 
+    end 
+
+    ##
+    # The +links+ method returns an array of the links that were scraped from the the page. 
+    # The +links+ are empty unless the request options +formats+ included +links+.
+    #
+    def links 
+      @attributes[ :links ] || [] 
+    end 
+
+  end
+end
+

data/lib/firecrawl/request.rb

@@ -0,0 +1,75 @@
+module Firecrawl 
+
+  ##
+  # The +Request+ class encapsulates a request to the Firecrawl API. After creating a new 
+  # +Request+ instance you can make the actual request by calling the +scrape+, +begin_crawl+ 
+  # +crawl+, +end_crawl+ or +map+ methods.
+  #
+  # === example
+  # 
+  # require 'firecrawl'
+  #
+  # request = Firecrawl::Request.new( api_key: ENV[ 'FIRECRAWL_API_KEY' )
+  #
+  # options = Firecrawl::ScrapeOptions.build do 
+  #   format                [ :markdown, :screenshot ]
+  #   only_main_content     true
+  # end
+  #
+  # response = request.scrape( 'https://cnn.com', criteria )
+  # if response.success?
+  #   result = response.result 
+  #   puts response.metadata[ :title ] 
+  #   puts '---'
+  #   puts response.markdown
+  # else 
+  #   puts response.result.error_description 
+  # end 
+  # 
+  class Request 
+
+    DEFAULT_CONNECTION = Faraday.new { | builder | builder.adapter Faraday.default_adapter }
+
+    BASE_URI = 'https://api.firecrawl.dev/v1'
+
+    ##
+    # The +initialize+ method initializes the +Request+ instance. You MUST pass an +api_key+ and
+    # and optionally a (Faraday) +connection+.
+    #
+    def initialize( connection: nil, api_key: nil )
+      @connection = connection || DEFAULT_CONNECTION
+      @api_key = api_key || Firecrawl.api_key
+      raise ArgumentError, "An 'api_key' is required unless configured using 'Firecrawl.api_key'." \
+        unless @api_key
+    end
+
+  protected
+
+    def post( uri, body, &block )
+      headers = { 
+        'Authorization' => "Bearer #{@api_key}", 
+        'Content-Type' => 'application/json'
+      }   
+
+      @connection.post( uri ) do | request |
+        headers.each { | key, value | request.headers[ key ] = value }
+        request.body = body.is_a?( String ) ? body : JSON.generate( body )  
+        block.call( request ) if block 
+      end
+    end 
+
+    def get( uri, &block )
+      headers = { 
+        'Authorization' => "Bearer #{@api_key}", 
+        'Content-Type' => 'application/json'
+      }   
+
+      @connection.get( uri ) do | request |
+        headers.each { | key, value | request.headers[ key ] = value }
+        block.call( request ) if block 
+      end
+    end 
+
+  end
+
+end

data/lib/firecrawl/response_methods.rb

@@ -0,0 +1,15 @@
+module Firecrawl
+  #
+  # The ResponseMethods module extends a Faraday reponse, adding the +result+ method. 
+  #
+  module ResponseMethods
+    def self.install( response, result )
+      response.instance_variable_set( "@_firecrawl_result", result )
+      response.extend( ResponseMethods )    
+    end 
+
+    def result
+      @_firecrawl_result
+    end 
+  end
+end 

data/lib/firecrawl/scrape_options.rb

@@ -0,0 +1,58 @@
+module Firecrawl
+  class ScrapeOptions 
+    include DynamicSchema::Definable 
+    include Helpers
+
+    FORMATS = [ :markdown, :links, :html, :raw_html, :screenshot, :"screenshot@full_page" ]
+
+    ACTIONS = [ :wait, :click, :write, :press, :screenshot, :scrape ]
+
+    schema do 
+      # note: both format and formats are defined as a semantic convenience
+      format            String, as: :formats, array: true, in: FORMATS 
+      formats           String, array: true, in: FORMATS 
+      only_main_content [ TrueClass, FalseClass ], as: :onlyMainContent 
+      include_tags      String, as: :includeTags, array: true 
+      exclude_tags      String, as: :excludeTags, array: true 
+      wait_for          Integer 
+      timeout           Integer 
+      extract           do 
+        #schema          Hash
+        system_prompt   String, as: :systemPrompt
+        prompt          String
+      end 
+      action            as: :actions, arguments: :type, array: true do 
+        type            Symbol, required: true, in: ACTIONS        
+        # wait  
+        milliseconds    Integer 
+        # click 
+        selector        String 
+        # write 
+        text            String 
+        # press
+        key             String 
+      end 
+    end
+
+    def self.build( options = nil, &block )
+      new( api_options: builder.build( options, &block ) )
+    end 
+
+    def self.build!( options = nil, &block )
+      new( api_options: builder.build!( options, &block ) )
+    end 
+
+    def initialize( options = {}, api_options: nil )
+      @options = self.class.builder.build( options || {} )
+      @options = api_options.merge( @options ) if api_options 
+      @options[ :formats ]&.map! { | format | string_camelize( format.to_s ) }
+    end
+
+    def to_h
+      @options.to_h 
+    end
+
+  end
+end
+
+

data/lib/firecrawl/scrape_request.rb

@@ -0,0 +1,60 @@
+module Firecrawl 
+  ##
+  # The +ScrapeRequest+ class encapsulates a '/scrape' POST request to the Firecrawl API. After 
+  # creating a new +ScrapeRequest+ instance you can initiate the request by calling the +scrape+ 
+  # method to perform synchronous scraping.  
+  #
+  # === examples
+  # 
+  # require 'firecrawl'
+  #
+  # request = Firecrawl::ScrapeRequest.new( api_key: ENV[ 'FIRECRAWL_API_KEY' )
+  #
+  # options = Firecrawl::ScrapeOptions.build do 
+  #   format                [ :markdown, 'screenshot@full_page' ]
+  #   only_main_content     true
+  # end
+  #
+  # response = request.scrape( 'https://example.com', options )
+  # if response.success?
+  #   result = response.result 
+  #   puts response.metadata[ 'title ] 
+  #   puts '---'
+  #   puts response.markdown
+  # else 
+  #   puts response.result.error_description 
+  # end 
+  # 
+  class ScrapeRequest < Request 
+
+    ## 
+    # The +scrape+ method makes a Firecrawl '/scrape' POST request which will scrape the given url. 
+    # 
+    # The response is always an instance of +Faraday::Response+. If +response.success?+ is true,
+    # then +response.result+ will be an instance +ScrapeResult+. If the request is not successful 
+    # then +response.result+ will be an instance of +ErrorResult+.
+    #
+    def scrape( url, options = nil, &block )    
+      if options
+        options = options.is_a?( ScrapeOptions ) ? options : ScrapeOptions.build( options.to_h ) 
+        options = options.to_h
+      else 
+        options = {}
+      end
+      options[ :url ] = url.to_s
+
+      response = post( "#{BASE_URI}/scrape", options, &block )
+      result = nil 
+      if response.success?  
+        attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
+        attributes ||= { success: false }
+        result = ScrapeResult.new( attributes[ :success ], attributes[ :data ] )
+      else 
+        result = ErrorResult.new( response.status, attributes )
+      end
+      
+      ResponseMethods.install( response, result )     
+    end 
+
+  end 
+end

data/lib/firecrawl/scrape_result.rb

@@ -0,0 +1,92 @@
+module Firecrawl 
+  class ScrapeResult 
+
+    def initialize( success, attributes )
+      @success = success 
+      @attributes = attributes || {}
+    end
+
+    ##
+    # The +success?+ method returns +true+ if the scraping was successful. 
+    #
+    # Note that the response +success?+ tells you if the call to the Firecrawl api was successful 
+    # while this +success?+ method tells you if the actual scraping operation was successful.
+    #
+    def success?  
+      @success || false 
+    end
+
+    ##
+    # The +markdown+ method returns scraped content that has been converted to markdown. The 
+    # markdown content is present only if the request options +formats+ included +markdown+.  
+    #
+    def markdown
+      @attributes[ :markdown ]
+    end 
+
+    ##
+    # The +html+ method returns scraped html content. The html content is present only if the 
+    # request options +formats+ included +html+.  
+    #
+    def html 
+      @attributes[ :html ] 
+    end 
+
+    ##
+    # The +raw_html+ method returns the full scraped html content of the page. The raw html 
+    # content is present only if the request options +formats+ included +raw_html+.  
+    #
+    def raw_html
+      @attributes[ :rawHtml ]
+    end 
+
+    ##
+    # The +screenshot_url+ method returns the url of the screenshot of the requested page. The 
+    # screenshot url is present only if the request options +formats+ included +screenshot+ or 
+    # +screenshot@full_page+.
+    #
+    def screenshot_url
+      @attributes[ :screenshot ]
+    end 
+
+    ##
+    # The +links+ method returns an array of the links that were scraped from the the page. 
+    # The +links+ are empty unless the request options +formats+ included +links+.
+    #
+    def links 
+      @attributes[ :links ] || [] 
+    end 
+
+    ##
+    # The +actions+ method returns an object of action results ( +scrapes+ or +screenshots+ ).  
+    # The +actions+ are empty unless the request options included +scrape+ or +scresshot+ 
+    # actions.
+    #
+    def actions 
+      @attributes[ :actions ] || {}
+    end
+
+    def metadata 
+      unless @metadata 
+        metadata = @attributes[ :metadata ] || {}
+        @metadata = metadata.transform_keys do | key |
+          key.to_s.gsub( /([a-z])([A-Z])/, '\1_\2' ).downcase
+        end
+        # remove the camelCase forms injected by Firecrawl
+        @metadata.delete_if do | key, _ | 
+          key.start_with?( 'og_' ) && @metadata.key?( key.sub( 'og_', 'og:' ) )
+        end 
+      end 
+      @metadata 
+    end
+
+    def llm_extraction 
+      @attributes[ :llm_extraction ] || {}
+    end
+
+    def warning 
+      @attributes[ :warning ] 
+    end
+
+  end
+end

data/lib/firecrawl.rb

@@ -0,0 +1,27 @@
+require 'json'
+require 'base64'
+require 'uri'
+
+require 'faraday'
+require 'dynamic_schema'
+
+require_relative 'firecrawl/helpers'
+require_relative 'firecrawl/error_result'
+require_relative 'firecrawl/request'
+require_relative 'firecrawl/response_methods'
+
+require_relative 'firecrawl/scrape_options'
+require_relative 'firecrawl/scrape_result'
+require_relative 'firecrawl/scrape_request'
+require_relative 'firecrawl/batch_scrape_result'
+require_relative 'firecrawl/batch_scrape_request'
+require_relative 'firecrawl/map_options'
+require_relative 'firecrawl/map_result'
+require_relative 'firecrawl/map_request'
+
+module Firecrawl
+  class << self 
+    attr_accessor :api_key 
+  end
+end
+

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: firecrawl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kristoph Cichocki-Romanov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: dynamicschema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta03
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta03
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+description: |-
+  The Firecrawl gem implements a lightweight interface to the Firecrawl.dev API. Firecrawl can take a URL, scrape the page contents and return the whole page or principal content as html, markdown, or structured data.
+
+  In addition, Firecrawl can crawl an entire site returning the pages it encounters or just the map of the pages, which can be used for subsequent scraping.
+email:
+- rubygems.org@kristoph.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- firecrawl.gemspec
+- lib/firecrawl.rb
+- lib/firecrawl/batch_scrape_request.rb
+- lib/firecrawl/batch_scrape_result.rb
+- lib/firecrawl/crawl_options.rb
+- lib/firecrawl/error_result.rb
+- lib/firecrawl/helpers.rb
+- lib/firecrawl/map_options.rb
+- lib/firecrawl/map_request.rb
+- lib/firecrawl/map_result.rb
+- lib/firecrawl/request.rb
+- lib/firecrawl/response_methods.rb
+- lib/firecrawl/scrape_options.rb
+- lib/firecrawl/scrape_request.rb
+- lib/firecrawl/scrape_result.rb
+homepage: https://github.com/EndlessInternational/firecrawl
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/EndlessInternational/firecrawl
+  bug_tracker_uri: https://github.com/EndlessInternational/firecrawl/issues
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.19
+signing_key:
+specification_version: 4
+summary: The Firecrawl gem implements a lightweight interface to the Firecrawl.dev
+  API which takes a URL, crawls it and returns html, markdown, or structured data.
+  It is of particular valuewhen used with LLM's for grounding.
+test_files: []