firecrawl 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e55dc5e433f0632ab0c11feda818bf82ddf77ae8ea2fdaa624c5a9af1dccf4c
-  data.tar.gz: 2781378e0a6b62c2e7befb0ebd0298b0b91b2cf249fe33115dc675606aed7bfe
+  metadata.gz: 5f895548e284fae55f2284c335f3e36d3d7dfb2c0c81fea525235968c90a2c8d
+  data.tar.gz: 91a8f19d1281a37c87b12cd37f1e312e1044808e3c7b14a976d250aea51fb989
 SHA512:
-  metadata.gz: 6fa88114c36df02f9cd261159132298e9a44b01464334daeb31ea2d8d5b11122321066ddf55207fddc3bef72704b353cf42d7aebaa46ac70298ea1efe19c6885
-  data.tar.gz: 622fd277c01854131b4a21742c915359c97fffa54b4dc41aff47d09b06d4d7c6c485361ff6409dc1361cf561f313d42096674a426987b5bbf696dcba1f52cd96
+  metadata.gz: 21420a482873a02c56a1caa7e7e149acd3f00543d6dc9e969f817a0abd42ff7ec442ea1cb0ebb18020f21c72af0da66927085c2595915388e5e169a3bea8a93f
+  data.tar.gz: 17b793284e7480b693f980d0b5f03e8ec1e4c606cabef3b8bd9728ad91029f7279b430bb4f6ea820347f45a7eb8cac02f82f9704fdde793ccd1028b7b86e3d18

data/firecrawl.gemspec

@@ -1,7 +1,9 @@
+require_relative 'lib/firecrawl/version'
+
 Gem::Specification.new do | spec |
 
   spec.name             = 'firecrawl'
-  spec.version          = '0.2.0'
+  spec.version          = Firecrawl::VERSION
   spec.authors          = [ 'Kristoph Cichocki-Romanov' ]
   spec.email            = [ 'rubygems.org@kristoph.net' ]
 
@@ -28,8 +30,8 @@ Gem::Specification.new do | spec |
   spec.files            = Dir[ "lib/**/*.rb", "LICENSE", "README.md", "firecrawl.gemspec" ]
   spec.require_paths    = [ "lib" ]
 
-  spec.add_runtime_dependency 'faraday', '~> 2.7'
-  spec.add_runtime_dependency 'dynamicschema', '~> 1.0.0.beta04'
+  spec.add_runtime_dependency 'faraday', '~> 2'
+  spec.add_runtime_dependency 'dynamicschema', '~> 2'
 
   spec.add_development_dependency 'rspec', '~> 3.13'
   spec.add_development_dependency 'debug', '~> 1.9'

data/lib/firecrawl/batch_scrape_options.rb

@@ -0,0 +1,67 @@
+module Firecrawl
+  class BatchScrapeOptions
+    include DynamicSchema::Definable 
+    include Helpers
+
+    schema do 
+      proxy                 Symbol, in: [ :basic, :stealth, :auto ] 
+      skip_tls_verification [ TrueClass, FalseClass ], as: :skipTlsVerification
+      mobile                [ TrueClass, FalseClass ]
+      location              arguments: :country do  
+        country             String, required: true # two digit country code
+        languages           String, array: true #en-US jp etc
+      end
+      max_age               Integer, as: :maxAge
+      headers               Hash
+
+      only_main_content     [ TrueClass, FalseClass ], as: :onlyMainContent 
+      include_tags          String, as: :includeTags, array: true 
+      exclude_tags          String, as: :excludeTags, array: true 
+      remove_base64_images  [ TrueClass, FalseClass ], as: :removeBase64Images
+      block_ads             [ TrueClass, FalseClass ], as: :blockAds
+
+      wait_for              Integer, as: :waitFor
+      timeout               Integer 
+      parsers               Symbol, array: true, in: [ :pdf ]
+
+      formats               Symbol, in: ScrapeOptions::FORMATS, array: true
+      screenshot            do 
+        type                String, default: 'screenshot'
+        full_page           [ TrueClass, FalseClass ], as: :fullPage 
+        quality             Integer, in: 0..100
+        viewport            do 
+          height            Integer, required: true
+          width             Integer, required: true 
+        end
+      end
+
+      webhook               do 
+        url                 URI, required: true 
+        headers             Hash
+        metadata            Hash 
+        events              Symbol, array: true, in: [ :completed, :page, :failed, :started ]
+      end 
+
+      cache                 [ TrueClass, FalseClass ], as: :storeInCache
+      zero_data_retention   [ TrueClass, FalseClass ], as: :zeroDataRetention
+    end
+
+    def self.build( options = nil, &block )
+      new( api_options: builder.build( options, &block ) )
+    end 
+
+    def self.build!( options = nil, &block )
+      new( api_options: builder.build!( options, &block ) )
+    end 
+
+    def initialize( options = nil, api_options: nil )
+      @options = self.class.builder.build( options || {} )
+      @options = api_options.merge( @options ) if api_options 
+    end
+
+    def to_h
+      @options.to_h 
+    end
+
+  end
+end

data/lib/firecrawl/batch_scrape_request.rb

@@ -54,7 +54,7 @@ module Firecrawl
     def submit( urls, options = nil, &block )        
       if options
         options = options.is_a?( ScrapeOptions ) ? options : ScrapeOptions.build( options.to_h ) 
-        options = options.to_h
+        options = ScrapeOptions.normalize_options( options.to_h )
       else 
         options = {}
       end
@@ -63,8 +63,7 @@ module Firecrawl
       result = nil 
       attributes = JSON.parse( response.body, symbolize_names: true ) rescue nil
       if response.success?
-        attributes ||= { success: false, status: :failed  }
-        result = BatchScrapeResult.new( attributes[ :success ], attributes )
+        result = BatchScrapeResult.new( attributes )
       else 
         result = ErrorResult.new( response.status, attributes || {} )
       end
@@ -95,7 +94,7 @@ module Firecrawl
       attributes = JSON.parse( response.body, symbolize_names: true ) rescue nil
       if response.success? 
         attributes ||= { success: false, status: :failed  }
-        result = batch_result.merge( attributes  )
+        result = batch_result.merge_attributes( attributes  )
       else 
         result = ErrorResult.new( response.status, attributes || {} )
       end 
@@ -131,7 +130,7 @@ module Firecrawl
         # the next url should not be set by this method so that retrieve and retrieve_all do 
         # not impact each other 
         attributes.delete( :next )
-        result = batch_result.merge( attributes  )
+        result = batch_result.merge_attributes( attributes  )
       else 
         result = ErrorResult.new( response.status, attributes || {} )
       end 

data/lib/firecrawl/batch_scrape_result.rb

@@ -1,63 +1,38 @@
 module Firecrawl 
-  class BatchScrapeResult 
 
-    def initialize( success, attributes )
-      @success = success
-      @attributes = attributes || {}
-    end
-
-    def success?
-      @success || false 
-    end
+  BatchScrapeResultSchema = DynamicSchema::Struct.define do
+    success             [ TrueClass, FalseClass ]
+    id                  String
+    invalid_urls        String, array: true
 
-    def status
-      # the initial Firecrawl response does not have a status so we synthesize a 'scraping'
-      # status if the operation was otherwise successful 
-      @attributes[ :status ]&.to_sym || ( @success ? :scraping : :failed )
-    end 
+    total               Integer
+    completed           Integer
+    credits_used        Integer, as: :creditsUsed
 
-    def status?( status )
-      self.status == status 
-    end 
-
-    def id
-      @attributes[ :id ]
-    end
+    url                 String
+    _value              :next, type: String
+    
+    expires_at          Date, as: :expiresAt
+    data                ScrapeResultData, array: true, default: []
+  end
 
-    def total 
-      @attributes[ :total ] || 0
-    end
+  class BatchScrapeResult < BatchScrapeResultSchema
+    extend Forwardable
 
-    def completed
-      @attributes[ :completed ] || 0
-    end 
+    def_delegators      :data,
+                        :[], :[]=, :<<, :push, :pop, :shift, :unshift,
+                        :length, :size, :empty?, :each
 
-    def credits_used 
-      @attributes[ :creditsUsed ] || 0
-    end 
+    def scraping?()     = !!self.next_url
+      
+    def next_url()      = self.next || self.url 
 
-    def expires_at
-      Date.parse( @attributes[ :expiresAt ] ) rescue nil
+    def merge_attributes( attributes )
+      new_attributes = self.to_h.merge( attributes )
+      data = attributes[ :data ]
+      new_attributes[ :data ] = self.data.concat( data ) if data 
+      self.class.new( attributes )
     end
-
-    def url 
-      @attributes[ :url ] 
-    end 
-    
-    def next_url 
-      @attributes[ :next ] || @attributes[ :url ] 
-    end 
-
-    def scrape_results 
-      success = @attributes[ :success ]
-      # note the &.compact is here because I've noted null entries in the data
-      ( @attributes[ :data ]&.compact || [] ).map do | attr |
-        ScrapeResult.new( success, attr )
-      end
-    end 
-
-    def merge( attributes )
-      self.class.new( attributes[ :success ], @attributes.merge( attributes ) )
-    end 
   end
+
 end

data/lib/firecrawl/crawl_options.rb

@@ -6,12 +6,23 @@ module Firecrawl
     schema do 
       exclude_paths         String, as: :excludePaths, array: true
       include_paths         String, as: :includePaths, array: true
-      maximum_depth         Integer, as: :maxDepth 
-      ignore_sitemap        [ TrueClass, FalseClass ], as: :ignoreSitemap 
+      maximum_depth         Integer, as: :maxDiscoveryDepth 
+      sitemap               Symbol, in: [ :skip, :include ] 
       limit                 Integer, in: (0..)
-      allow_backward_links  [ TrueClass, FalseClass ], as: :allowBackwardLinks
+      ignore_query_parameters [ TrueClass, FalseClass ], as: :ignoreQueryParameters
+      crawl_entire_domain   [ TrueClass, FalseClass ], as: :crawlEntireDomain
       allow_external_links  [ TrueClass, FalseClass ], as: :allowExternalLinks
-      webhook_uri           URI, as: :webhook 
+      allow_subdomains      String, as: :allowSubdomains
+      delay                 Integer, in: (0..)
+      max_concurency        Integer, in: (0..), as: :maxConcurrency
+
+      webhook               do 
+        url                 URI, required: true 
+        headers             Hash
+        metadata            Hash 
+        events              Symbol, array: true, in: [ :completed, :page, :failed, :started ]
+      end 
+
       scrape_options        as: :scrapeOptions, &ScrapeOptions.schema
     end
 
@@ -26,11 +37,6 @@ module Firecrawl
     def initialize( options = nil, api_options: nil )
       @options = self.class.builder.build( options || {} )
       @options = api_options.merge( @options ) if api_options 
-      
-      scrape_options = @options[ :scrapeOptions ]
-      if scrape_options 
-        scrape_options[ :formats ]&.map! { | format | string_camelize( format.to_s ) }
-      end 
     end
 
     def to_h

data/lib/firecrawl/crawl_request.rb

@@ -57,6 +57,11 @@ module Firecrawl
       if options
         options = options.is_a?( CrawlOptions ) ? options : CrawlOptions.build( options.to_h ) 
         options = options.to_h
+
+        scrape_options = options[ :scrapeOptions ]
+        if scrape_options 
+          options[ :scrapeOptions ] = ScrapeOptions.normalize_options( scrape_options )
+        end 
       else 
         options = {}
       end
@@ -65,8 +70,7 @@ module Firecrawl
       result = nil 
       attributes = JSON.parse( response.body, symbolize_names: true ) rescue nil
       if response.success?
-        attributes ||= { success: false, status: :failed  }
-        result = CrawlResult.new( attributes[ :success ], attributes )
+        result = CrawlResult.new( attributes )
       else 
         result = ErrorResult.new( response.status, attributes )
       end
@@ -96,7 +100,7 @@ module Firecrawl
       result = nil 
       attributes = JSON.parse( response.body, symbolize_names: true ) rescue nil
       if response.success? 
-        result = crawl_result.merge( attributes || { success: false, status: :failed  } )
+        result = crawl_result.merge_attributes( attributes || { success: false, status: :failed  } )
       else 
         result = ErrorResult.new( response.status, attributes || {} )
       end 
@@ -123,7 +127,7 @@ module Firecrawl
       result = nil 
       attributes = JSON.parse( response.body, symbolize_names: true ) rescue nil
       if response.success? 
-        result = crawl_result.merge( attributes || { success: false, status: :failed  } )
+        result = crawl_result.merge_attributes( attributes || { success: false, status: :failed  } )
       else 
         result = ErrorResult.new( response.status, attributes || {} )
       end 

data/lib/firecrawl/crawl_result.rb

@@ -1,63 +1,33 @@
 module Firecrawl 
-  class CrawlResult
-
-    def initialize( success, attributes )
-      @success = success
-      @attributes = attributes || {}
-    end
-
-    def success?
-      @success || false 
-    end
-
-    def status
-      # the initial Firecrawl response does not have a status so we synthesize a 'crawling'
-      # status if the operation was otherwise successful 
-      @attributes[ :status ]&.to_sym || ( @success ? :scraping : :failed )
-    end 
-
-    def status?( status )
-      self.status == status 
-    end 
+  CrawlResultSchema = DynamicSchema::Struct.define do
+    success             [ TrueClass, FalseClass ]
+    id                  String
+    total               Integer
+    completed           Integer
+    credits_used        Integer, as: :creditsUsed
+    url                 String
+    _value              :next, type: String
+    expires_at          Date, as: :expiresAt
+    data                ScrapeResultData, array: true, default: []
+  end
 
-    def id
-      @attributes[ :id ]
-    end
+  class CrawlResult < CrawlResultSchema
+    extend Forwardable
 
-    def total 
-      @attributes[ :total ] || 0
-    end
+    def_delegators      :data,
+                        :[], :[]=, :<<, :push, :pop, :shift, :unshift,
+                        :length, :size, :empty?, :each
 
-    def completed
-      @attributes[ :completed ] || 0
-    end 
+    def success?()      = self.success 
+    def crawling?()     = !!self.next_url
 
-    def credits_used 
-      @attributes[ :creditsUsed ] || 0
-    end 
+    def next_url()      = self.next || self.url 
 
-    def expires_at
-      Date.parse( @attributes[ :expiresAt ] ) rescue nil
+    def merge_attributes( attributes )
+      new_attributes = self.to_h.merge( attributes )
+      data = attributes[ :data ]
+      new_attributes[ :data ] = self.data.concat( data ) if data 
+      self.class.new( attributes )
     end
-
-    def url 
-      @attributes[ :url ] 
-    end 
-    
-    def next_url 
-      @attributes[ :next ] || @attributes[ :url ] 
-    end 
-
-    def scrape_results 
-      success = @attributes[ :success ]
-      # note the &.compact is here because I've noted null entries in the data
-      ( @attributes[ :data ]&.compact || [] ).map do | attr |
-        ScrapeResult.new( success, attr )
-      end
-    end 
-
-    def merge( attributes )
-      self.class.new( attributes[ :success ], @attributes.merge( attributes ) )
-    end 
   end
 end

data/lib/firecrawl/map_options.rb

@@ -1,30 +1,27 @@
 module Firecrawl
   class MapOptions
     include DynamicSchema::Definable 
+    include DynamicSchema::Buildable 
 
     schema do 
-      search            String 
-      ignore_sitemap    [ TrueClass, FalseClass ]
-      ignore_subdomains [ TrueClass, FalseClass ]
-      limit             Integer
+      search                      String
+      sitemap                     Symbol, in: [ :skip, :include ] 
+      include_subdomains          [ TrueClass, FalseClass ], as: :includeSubdomains
+      ignore_query_parameters     [ TrueClass, FalseClass ], as: :ignoreQueryParameters
+      limit                       Integer, in: (0..)
+      timeout                     Integer, in: (0..)
+      location                    arguments: :country do  
+        country                   String, required: true # two digit country code
+        languages                 String, array: true #en-US jp etc
+      end
     end
 
-    def self.build( options = nil, &block )
-      new( api_options: builder.build( options, &block ) )
-    end 
-
-    def self.build!( options = nil, &block )
-      new( api_options: builder.build!( options, &block ) )
-    end 
-
     def initialize( options = {}, api_options: nil )
       @options = self.class.builder.build( options || {} )
       @options = api_options.merge( @options ) if api_options 
     end
 
-    def to_h
-      @options.to_h 
-    end
+    def to_h()                    = @options.to_h
 
   end
 end

data/lib/firecrawl/map_request.rb

@@ -45,8 +45,7 @@ module Firecrawl
       result = nil 
       if response.success?  
         attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
-        attributes ||= { success: false }
-        result = MapResult.new( attributes[ :success ], attributes )
+        result = MapResult.new( attributes )
       else 
         result = ErrorResult.new( response.status, attributes ) 
       end

data/lib/firecrawl/map_result.rb

@@ -1,29 +1,21 @@
 module Firecrawl 
-  class MapResult 
 
-    def initialize( success, attributes )
-      @success = success
-      @attributes = attributes
-    end
-
-    ##
-    # The +success?+ method returns +true+ if the scraping was successful. 
-    #
-    # Note that the response +success?+ tells you if the call to the Firecrawl api was successful 
-    # while this +success?+ method tells you if the actual scraping operation was successful.
-    #
-    def success?  
-      @success || false 
-    end 
-
-    ##
-    # The +links+ method returns an array of the links that were scraped from the the page. 
-    # The +links+ are empty unless the request options +formats+ included +links+.
-    #
-    def links 
-      @attributes[ :links ] || [] 
-    end 
+  MapResultSchema = DynamicSchema::Struct.define do
+    success             [ TrueClass, FalseClass ]
+    links               array: true do 
+      url               String 
+      title             String 
+      description       String 
+    end              
+  end
 
+  class MapResult < MapResultSchema
+    extend Forwardable
+    def_delegators      :links,
+                        :[], :[]=, :<<, :push, :pop, :shift, :unshift,
+                        :length, :size, :empty?, :each
+    def success?()      = success
   end
+
 end
 

data/lib/firecrawl/request.rb

@@ -28,7 +28,7 @@ module Firecrawl
   # 
   class Request 
 
-    BASE_URI = 'https://api.firecrawl.dev/v1'
+    BASE_URI = 'https://api.firecrawl.dev/v2'
 
     ##
     # The +initialize+ method initializes the +Request+ instance. You MUST pass an +api_key+ and

data/lib/firecrawl/scrape_options.rb

@@ -3,24 +3,42 @@ module Firecrawl
     include DynamicSchema::Definable 
     include Helpers
 
-    FORMATS = [ :markdown, :links, :html, :raw_html, :screenshot, :"screenshot@full_page" ]
+    FORMATS = [ :summary, :markdown, :html, :raw_html, :links, :screenshot ]
 
     ACTIONS = [ :wait, :click, :write, :press, :screenshot, :scrape ]
 
     schema do 
-      # note: both format and formats are defined as a semantic convenience
-      format            String, as: :formats, array: true, in: FORMATS
-      formats           String, array: true, in: FORMATS 
+
+      proxy             Symbol, in: [ :basic, :stealth, :auto ] 
+      skip_tls_verification [ TrueClass, FalseClass ], as: :skipTlsVerification
+      mobile            [ TrueClass, FalseClass ]
+      location          arguments: :country do  
+        country         String, required: true # two digit country code
+        languages       String, array: true #en-US jp etc
+      end
+      max_age           Integer, as: :maxAge
+
+      headers           Hash
+
       only_main_content [ TrueClass, FalseClass ], as: :onlyMainContent 
       include_tags      String, as: :includeTags, array: true 
       exclude_tags      String, as: :excludeTags, array: true 
-      wait_for          Integer 
+      wait_for          Integer, as: :waitFor
       timeout           Integer 
-      extract           do 
-        schema          Hash
-        system_prompt   String, as: :systemPrompt
-        prompt          String
-      end 
+      parsers           Symbol, array: true, in: [ :pdf ]
+
+      formats           Symbol, in: FORMATS, array: true
+      screenshot        do 
+        type            String, default: 'screenshot'
+        full_page       [ TrueClass, FalseClass ], as: :fullPage 
+        quality         Integer, in: 0..100
+        viewport        do 
+          height        Integer, required: true
+          width         Integer, required: true 
+        end
+      end
+
+
       action            as: :actions, arguments: :type, array: true do 
         type            Symbol, required: true, in: ACTIONS        
         # wait  
@@ -32,6 +50,9 @@ module Firecrawl
         # press
         key             String 
       end 
+
+      cache             [ TrueClass, FalseClass ], as: :storeInCache
+      zero_retention    [ TrueClass, FalseClass ], as: :zeroDataRetention
     end
 
     def self.build( options = nil, &block )
@@ -42,10 +63,21 @@ module Firecrawl
       new( api_options: builder.build!( options, &block ) )
     end 
 
+    def self.normalize_options( options )
+      options = options&.dup || {}
+      screenshot = options.delete( :screenshot )
+      if screenshot
+        formats = options[ :formats ] || []
+        formats.delete( :screenshot )
+        formats << screenshot 
+        options[ :formats ] = formats
+      end
+      options 
+    end
+
     def initialize( options = {}, api_options: nil )
       @options = self.class.builder.build( options || {} )
       @options = api_options.merge( @options ) if api_options 
-      @options[ :formats ]&.map! { | format | string_camelize( format.to_s ) }
     end
 
     def to_h

data/lib/firecrawl/scrape_request.rb

@@ -36,8 +36,8 @@ module Firecrawl
     #
     def submit( url, options = nil, &block )    
       if options
-        options = options.is_a?( ScrapeOptions ) ? options : ScrapeOptions.build( options.to_h ) 
-        options = options.to_h
+        options = options.is_a?( ScrapeOptions ) ? options : ScrapeOptions.build!( options.to_h ) 
+        options = ScrapeOptions.normalize_options( options.to_h )
       else 
         options = {}
       end
@@ -47,8 +47,7 @@ module Firecrawl
       result = nil 
       if response.success?  
         attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
-        attributes ||= { success: false }
-        result = ScrapeResult.new( attributes[ :success ], attributes[ :data ] )
+        result = ScrapeResult.new( attributes )
       else 
         result = ErrorResult.new( response.status, attributes )
       end
@@ -56,5 +55,7 @@ module Firecrawl
       ResponseMethods.install( response, result )     
     end 
 
+  private 
+
   end 
 end

data/lib/firecrawl/scrape_result.rb

@@ -1,92 +1,47 @@
 module Firecrawl 
-  class ScrapeResult 
 
-    def initialize( success, attributes )
-      @success = success 
-      @attributes = attributes || {}
-    end
-
-    ##
-    # The +success?+ method returns +true+ if the scraping was successful. 
-    #
-    # Note that the response +success?+ tells you if the call to the Firecrawl api was successful 
-    # while this +success?+ method tells you if the actual scraping operation was successful.
-    #
-    def success?  
-      @success || false 
-    end
-
-    def metadata 
-      unless @metadata 
-        metadata = @attributes[ :metadata ] || {}
-        @metadata = metadata.transform_keys do | key |
-          key.to_s.gsub( /([a-z])([A-Z])/, '\1_\2' ).downcase
-        end
-        # remove the camelCase forms injected by Firecrawl
-        @metadata.delete_if do | key, _ | 
-          key.start_with?( 'og_' ) && @metadata.key?( key.sub( 'og_', 'og:' ) )
-        end 
-      end 
-      @metadata 
-    end
-
-    ##
-    # The +markdown+ method returns scraped content that has been converted to markdown. The 
-    # markdown content is present only if the request options +formats+ included +markdown+.  
-    #
-    def markdown
-      @attributes[ :markdown ]
-    end 
+  ScrapeResultDataSchema = DynamicSchema::Struct.define do
+    metadata            Hash
 
-    ##
-    # The +html+ method returns scraped html content. The html content is present only if the 
-    # request options +formats+ included +html+.  
-    #
-    def html 
-      @attributes[ :html ] 
-    end 
+    markdown            String
+    html                String 
+    raw_html            String,           as: :rawHtml
 
-    ##
-    # The +raw_html+ method returns the full scraped html content of the page. The raw html 
-    # content is present only if the request options +formats+ included +raw_html+.  
-    #
-    def raw_html
-      @attributes[ :rawHtml ]
-    end 
+    screenshot_url      String,           as: :screenshot
+    links               String,           array: true
+    actions             Hash,             default: {}
 
-    ##
-    # The +screenshot_url+ method returns the url of the screenshot of the requested page. The 
-    # screenshot url is present only if the request options +formats+ included +screenshot+ or 
-    # +screenshot@full_page+.
-    #
-    def screenshot_url
-      @attributes[ :screenshot ]
-    end 
-
-    ##
-    # The +links+ method returns an array of the links that were scraped from the the page. 
-    # The +links+ are empty unless the request options +formats+ included +links+.
-    #
-    def links 
-      @attributes[ :links ] || [] 
-    end 
+    warning             String
+  end
 
-    ##
-    # The +actions+ method returns an object of action results ( +scrapes+ or +screenshots+ ).  
-    # The +actions+ are empty unless the request options included +scrape+ or +scresshot+ 
-    # actions.
-    #
-    def actions 
-      @attributes[ :actions ] || {}
+  class ScrapeResultData < ScrapeResultDataSchema 
+    def metadata()        = @metadata ||= normalize_metadata( super )
+  private
+    def normalize_metadata( metadata )
+      return nil unless metadata
+      metadata = metadata.transform_keys do | key |
+        key.to_s.gsub( /([a-z])([A-Z])/, '\1_\2' ).downcase
+      end
+      # remove the camelCase forms injected by Firecrawl
+      metadata.delete_if do | key, _ | 
+        key.start_with?( 'og_' ) && metadata.key?( key.sub( 'og_', 'og:' ) )
+      end 
+      metadata
     end
+  end
 
-    def llm_extraction 
-      @attributes[ :llm_extraction ] || {}
-    end
+  ScrapeResultSchema = DynamicSchema::Struct.define do
+    success             [ TrueClass, FalseClass ]
+    data                ScrapeResultData
+  end
 
-    def warning 
-      @attributes[ :warning ] 
-    end
+  class ScrapeResult < ScrapeResultSchema
+    extend Forwardable
+    def success?()      = self.success 
+    def_delegators      :data,
+                        :metadata, :warning,
+                        :markdown, :html, :raw_html, 
+                        :screenshot_url, :links, :actions
 
   end
 end

data/lib/firecrawl/version.rb

@@ -0,0 +1,3 @@
+module Firecrawl 
+  VERSION = '0.3.0'
+end

data/lib/firecrawl.rb

@@ -5,6 +5,8 @@ require 'uri'
 require 'faraday'
 require 'dynamic_schema'
 
+require_relative 'firecrawl/version'
+
 require_relative 'firecrawl/helpers'
 require_relative 'firecrawl/error_result'
 require_relative 'firecrawl/request'
@@ -14,6 +16,7 @@ require_relative 'firecrawl/scrape_options'
 require_relative 'firecrawl/scrape_result'
 require_relative 'firecrawl/scrape_request'
 require_relative 'firecrawl/batch_scrape_result'
+require_relative 'firecrawl/batch_scrape_options'
 require_relative 'firecrawl/batch_scrape_request'
 require_relative 'firecrawl/map_options'
 require_relative 'firecrawl/map_result'
@@ -28,4 +31,3 @@ module Firecrawl
   extend ModuleMethods
 end
 
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: firecrawl
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Kristoph Cichocki-Romanov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-29 00:00:00.000000000 Z
+date: 2025-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: dynamicschema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta04
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta04
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,7 @@ files:
 - README.md
 - firecrawl.gemspec
 - lib/firecrawl.rb
+- lib/firecrawl/batch_scrape_options.rb
 - lib/firecrawl/batch_scrape_request.rb
 - lib/firecrawl/batch_scrape_result.rb
 - lib/firecrawl/crawl_options.rb
@@ -110,6 +111,7 @@ files:
 - lib/firecrawl/scrape_options.rb
 - lib/firecrawl/scrape_request.rb
 - lib/firecrawl/scrape_result.rb
+- lib/firecrawl/version.rb
 homepage: https://github.com/EndlessInternational/firecrawl
 licenses:
 - MIT
@@ -131,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.19
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: The Firecrawl gem implements a lightweight interface to the Firecrawl.dev