spidercloud 1.0.0

data/lib/spider_cloud/links_result.rb

@@ -0,0 +1,55 @@
+module SpiderCloud
+
+  LinksResultItemSchema = DynamicSchema::Struct.define do
+    url                   String
+    status                Integer
+    duration_elapsed_ms   Integer, as: :duration_elapsed_ms
+    error                 String
+  end
+
+  class LinksResultItem < LinksResultItemSchema
+    def success?
+      error.nil? && ( status.nil? || ( status >= 200 && status < 300 ) )
+    end
+  end
+
+  LinksResultSchema = DynamicSchema::Struct.define do
+    items                 LinksResultItem, array: true
+  end
+
+  class LinksResult < LinksResultSchema
+    extend Forwardable
+    include Enumerable
+
+    def_delegators :items, :each, :[], :count, :size, :length, :first, :last, :empty?
+
+    def self.from_array( array )
+      new( items: array )
+    end
+
+    def success?
+      items&.all?( &:success? ) || false
+    end
+
+    # get all discovered URLs
+    def urls
+      items&.map( &:url ) || []
+    end
+
+    # get failed URLs
+    def failed
+      items&.reject( &:success? ) || []
+    end
+
+    # get successful URLs
+    def succeeded
+      items&.select( &:success? ) || []
+    end
+
+    # get URLs by status code
+    def with_status( code )
+      items&.select { | item | item.status == code } || []
+    end
+  end
+
+end

data/lib/spider_cloud/module_methods.rb

@@ -0,0 +1,31 @@
+module SpiderCloud
+  module ModuleMethods
+
+    def connection( connection = nil )
+      @connection = connection if connection
+      @connection ||= Faraday.new { | builder | builder.adapter Faraday.default_adapter }
+    end
+
+    def api_key( api_key = nil )
+      @api_key = api_key || @api_key
+      @api_key
+    end
+
+    def scrape( url, options = nil, &block )
+      SpiderCloud::ScrapeRequest.new.submit( url, options, &block )
+    end
+
+    def crawl( url, options = nil, &block )
+      SpiderCloud::CrawlRequest.new.submit( url, options, &block )
+    end
+
+    def screenshot( url, options = nil, &block )
+      SpiderCloud::ScreenshotRequest.new.submit( url, options, &block )
+    end
+
+    def links( url, options = nil, &block )
+      SpiderCloud::LinksRequest.new.submit( url, options, &block )
+    end
+
+  end
+end

data/lib/spider_cloud/request.rb

@@ -0,0 +1,41 @@
+module SpiderCloud
+  class Request
+
+    BASE_URI = 'https://api.spider.cloud'
+
+    def initialize( connection: nil, api_key: nil )
+      @connection = connection || SpiderCloud.connection
+      @api_key = api_key || SpiderCloud.api_key
+      raise ArgumentError, "An 'api_key' is required unless configured using 'SpiderCloud.api_key'." \
+        unless @api_key
+    end
+
+  protected
+
+    def post( uri, body, &block )
+      headers = {
+        'Authorization' => "Bearer #{ @api_key }",
+        'Content-Type' => 'application/json'
+      }
+
+      @connection.post( uri ) do | request |
+        headers.each { | key, value | request.headers[ key ] = value }
+        request.body = body.is_a?( String ) ? body : JSON.generate( body )
+        block.call( request ) if block
+      end
+    end
+
+    def get( uri, &block )
+      headers = {
+        'Authorization' => "Bearer #{ @api_key }",
+        'Content-Type' => 'application/json'
+      }
+
+      @connection.get( uri ) do | request |
+        headers.each { | key, value | request.headers[ key ] = value }
+        block.call( request ) if block
+      end
+    end
+
+  end
+end

data/lib/spider_cloud/response_methods.rb

@@ -0,0 +1,15 @@
+module SpiderCloud
+  #
+  # The ResponseMethods module extends a Faraday response, adding the +result+ method.
+  #
+  module ResponseMethods
+    def self.install( response, result )
+      response.instance_variable_set( "@_spider_cloud_result", result )
+      response.extend( ResponseMethods )
+    end
+
+    def result
+      @_spider_cloud_result
+    end
+  end
+end

data/lib/spider_cloud/scrape_options.rb

@@ -0,0 +1,164 @@
+module SpiderCloud
+  class ScrapeOptions
+    include DynamicSchema::Definable
+    include Helpers
+
+    schema do
+      request                        Symbol, in: REQUEST_TYPES
+      return_format                  Symbol, in: RETURN_FORMATS
+      request_timeout                Integer, in: 5..255
+      lite_mode                      [ TrueClass, FalseClass ]
+
+      network_blacklist              String, array: true
+      network_whitelist              String, array: true
+      disable_hints                  [ TrueClass, FalseClass ]
+      disable_intercept              [ TrueClass, FalseClass ]
+      preserve_host                  [ TrueClass, FalseClass ]
+      anti_bot                       [ TrueClass, FalseClass ]
+
+      session                        [ TrueClass, FalseClass ]
+      cookies                        String
+      headers                        Hash
+      user_agent                     String
+
+      proxy                          Symbol, in: PROXY_TYPES
+      proxy_enabled                  [ TrueClass, FalseClass ]
+      remote_proxy                   String
+      country_code                   String
+      locale                         String
+
+      readability                    [ TrueClass, FalseClass ]
+      root_selector                  String
+      exclude_selector               String
+      css_extraction_map             Hash
+      clean_html                     [ TrueClass, FalseClass ]
+      filter_svg                     [ TrueClass, FalseClass ]
+      filter_images                  [ TrueClass, FalseClass ]
+      filter_main_only               [ TrueClass, FalseClass ]
+      filter_output_svg              [ TrueClass, FalseClass ]
+      filter_output_images           [ TrueClass, FalseClass ]
+      filter_output_main_only        [ TrueClass, FalseClass ]
+
+      return_json_data               [ TrueClass, FalseClass ]
+      return_headers                 [ TrueClass, FalseClass ]
+      return_cookies                 [ TrueClass, FalseClass ]
+      return_page_links              [ TrueClass, FalseClass ]
+      return_embeddings              [ TrueClass, FalseClass ]
+      metadata                       [ TrueClass, FalseClass ]
+      encoding                       String
+
+      gpt_config do
+        prompt                       String
+        model                        String
+        max_tokens                   Integer
+        temperature                  Float
+        top_p                        Float
+        api_key                      String
+        extra_ai_data                [ TrueClass, FalseClass ]
+        screenshot                   [ TrueClass, FalseClass ]
+      end
+      custom_prompt                  String
+      custom_function                String
+      model                          String
+
+      chunking_algorithm             as: :chunking_alg do
+        type                         Symbol, in: CHUNKING_TYPES
+        value                        Integer
+      end
+
+      wait_for do
+        idle_network do
+          timeout do
+            seconds                  Integer, as: :secs
+            nanoseconds              Integer, as: :nanos
+          end
+        end
+        selector                     String
+        dom do
+          timeout do
+            seconds                  Integer, as: :secs
+            nanoseconds              Integer, as: :nanos
+          end
+        end
+        delay do
+          timeout do
+            seconds                  Integer, as: :secs
+            nanoseconds              Integer, as: :nanos
+          end
+        end
+        page_navigations do
+          timeout do
+            seconds                  Integer, as: :secs
+            nanoseconds              Integer, as: :nanos
+          end
+        end
+      end
+
+      fingerprint                    [ TrueClass, FalseClass ]
+      stealth                        [ TrueClass, FalseClass ]
+      viewport do
+        width                        Integer
+        height                       Integer
+      end
+      device                         Symbol, in: DEVICE_TYPES
+      scroll                         Integer
+      block_ads                      [ TrueClass, FalseClass ]
+      block_analytics                [ TrueClass, FalseClass ]
+      block_stylesheets              [ TrueClass, FalseClass ]
+      block_images                   [ TrueClass, FalseClass ]
+      omit_background                [ TrueClass, FalseClass ]
+      service_worker_enabled         [ TrueClass, FalseClass ]
+      virtual_display                [ TrueClass, FalseClass ]
+
+      automation_scripts             Hash
+      execution_scripts              Hash
+      evaluate_on_new_document       String
+
+      blacklist                      String, array: true
+      whitelist                      String, array: true
+      external_domains               String, array: true
+      redirect_policy                Symbol, in: REDIRECT_POLICIES
+      link_rewrite                   Hash
+
+      sitemap                        [ TrueClass, FalseClass ]
+      sitemap_only                   [ TrueClass, FalseClass ]
+      sitemap_path                   String
+      subdomains                     [ TrueClass, FalseClass ]
+      tld                            [ TrueClass, FalseClass ]
+
+      cache                          [ TrueClass, FalseClass ]
+      skip_config_checks             [ TrueClass, FalseClass ]
+      storageless                    [ TrueClass, FalseClass ]
+      store_data                     [ TrueClass, FalseClass ]
+      respect_robots                 [ TrueClass, FalseClass ]
+      concurrency_limit              Integer
+      delay                          Integer
+
+      max_credits_per_page           Integer
+      max_credits_allowed            Integer
+      budget                         Hash
+
+      webhooks                       Hash
+      run_in_background              [ TrueClass, FalseClass ]
+      event_tracker                  Hash
+    end
+
+    def self.build( options = nil, &block )
+      new( api_options: builder.build( options, &block ) )
+    end
+
+    def self.build!( options = nil, &block )
+      new( api_options: builder.build!( options, &block ) )
+    end
+
+    def initialize( options = {}, api_options: nil )
+      @options = self.class.builder.build( options || {} )
+      @options = api_options.merge( @options ) if api_options
+    end
+
+    def to_h
+      @options.to_h
+    end
+
+  end
+end

data/lib/spider_cloud/scrape_request.rb

@@ -0,0 +1,29 @@
+module SpiderCloud
+  class ScrapeRequest < Request
+
+    def submit( url, options = nil, &block )
+      if options
+        options = options.is_a?( ScrapeOptions ) ? options : \
+          ScrapeOptions.build!( options.to_h )
+        options = options.to_h
+      else
+        options = {}
+      end
+      options[ :url ] = Helpers.normalize_url( url )
+
+      response = post( "#{ BASE_URI }/scrape", options, &block )
+      attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
+
+      result = if response.success? && attributes.is_a?( Array )
+        ScrapeResult.from_array( attributes )
+      elsif response.success?
+        ErrorResult.new( response.status, attributes )
+      else
+        ErrorResult.new( response.status, attributes )
+      end
+
+      ResponseMethods.install( response, result )
+    end
+
+  end
+end

data/lib/spider_cloud/scrape_result.rb

@@ -0,0 +1,62 @@
+module SpiderCloud
+
+  ScrapeResultItemSchema = DynamicSchema::Struct.define do
+    content               String
+    error                 String
+    status                Integer
+    duration_elapsed_ms   Integer, as: :duration_elapsed_ms
+    costs                 Costs
+    url                   String
+  end
+
+  class ScrapeResultItem < ScrapeResultItemSchema
+    def success?
+      error.nil? && ( status.nil? || ( status >= 200 && status < 300 ) )
+    end
+  end
+
+  ScrapeResultSchema = DynamicSchema::Struct.define do
+    items                 ScrapeResultItem, array: true
+  end
+
+  class ScrapeResult < ScrapeResultSchema
+    extend Forwardable
+    include Enumerable
+
+    def_delegators :items, :each, :[], :count, :size, :length, :first, :last, :empty?
+
+    def self.from_array( array )
+      new( items: array )
+    end
+
+    def success?
+      items&.all?( &:success? ) || false
+    end
+
+    # convenience method for single URL scrapes
+    def content
+      first&.content
+    end
+
+    def error
+      first&.error
+    end
+
+    def status
+      first&.status
+    end
+
+    def url
+      first&.url
+    end
+
+    def costs
+      first&.costs
+    end
+
+    def duration_elapsed_ms
+      first&.duration_elapsed_ms
+    end
+  end
+
+end

data/lib/spider_cloud/screenshot_options.rb

@@ -0,0 +1,84 @@
+module SpiderCloud
+  class ScreenshotOptions
+    include DynamicSchema::Definable
+    include Helpers
+
+    schema do
+      full_page                      [ TrueClass, FalseClass ]
+      binary                         [ TrueClass, FalseClass ]
+      omit_background                [ TrueClass, FalseClass ]
+      block_images                   [ TrueClass, FalseClass ]
+
+      cdp_params do
+        format                       Symbol, in: IMAGE_FORMATS
+        quality                      Integer, in: 0..100
+        clip do
+          x                          Integer
+          y                          Integer
+          width                      Integer
+          height                     Integer
+          scale                      [ Integer, Float ]
+        end
+        from_surface                 [ TrueClass, FalseClass ]
+        capture_beyond_viewport      [ TrueClass, FalseClass ]
+      end
+
+      limit                          Integer
+      request                        Symbol, in: REQUEST_TYPES
+
+      viewport do
+        width                        Integer
+        height                       Integer
+      end
+      device                         Symbol, in: DEVICE_TYPES
+
+      wait_for do
+        idle_network do
+          timeout do
+            seconds                  Integer, as: :secs
+            nanoseconds              Integer, as: :nanos
+          end
+        end
+        selector                     String
+        delay do
+          timeout do
+            seconds                  Integer, as: :secs
+            nanoseconds              Integer, as: :nanos
+          end
+        end
+      end
+
+      proxy                          Symbol, in: PROXY_TYPES
+      proxy_enabled                  [ TrueClass, FalseClass ]
+      country_code                   String
+
+      stealth                        [ TrueClass, FalseClass ]
+      fingerprint                    [ TrueClass, FalseClass ]
+      scroll                         Integer
+      block_ads                      [ TrueClass, FalseClass ]
+      virtual_display                [ TrueClass, FalseClass ]
+
+      cookies                        String
+      headers                        Hash
+      automation_scripts             Hash
+    end
+
+    def self.build( options = nil, &block )
+      new( api_options: builder.build( options, &block ) )
+    end
+
+    def self.build!( options = nil, &block )
+      new( api_options: builder.build!( options, &block ) )
+    end
+
+    def initialize( options = {}, api_options: nil )
+      @options = self.class.builder.build( options || {} )
+      @options = api_options.merge( @options ) if api_options
+    end
+
+    def to_h
+      @options.to_h
+    end
+
+  end
+end

data/lib/spider_cloud/screenshot_request.rb

@@ -0,0 +1,29 @@
+module SpiderCloud
+  class ScreenshotRequest < Request
+
+    def submit( url, options = nil, &block )
+      if options
+        options = options.is_a?( ScreenshotOptions ) ? options : \
+          ScreenshotOptions.build!( options.to_h )
+        options = options.to_h
+      else
+        options = {}
+      end
+      options[ :url ] = Helpers.normalize_url( url )
+
+      response = post( "#{ BASE_URI }/screenshot", options, &block )
+      attributes = ( JSON.parse( response.body, symbolize_names: true ) rescue nil )
+
+      result = if response.success? && attributes.is_a?( Array )
+        ScreenshotResult.from_array( attributes )
+      elsif response.success?
+        ErrorResult.new( response.status, attributes )
+      else
+        ErrorResult.new( response.status, attributes )
+      end
+
+      ResponseMethods.install( response, result )
+    end
+
+  end
+end

data/lib/spider_cloud/screenshot_result.rb

@@ -0,0 +1,69 @@
+module SpiderCloud
+
+  ScreenshotResultItemSchema = DynamicSchema::Struct.define do
+    content               String
+    error                 String
+    status                Integer
+    url                   String
+  end
+
+  class ScreenshotResultItem < ScreenshotResultItemSchema
+    def success?
+      error.nil? && ( status.nil? || ( status >= 200 && status < 300 ) )
+    end
+
+    # decode base64 content to binary
+    def image_data
+      return nil unless content
+      Base64.decode64( content )
+    end
+  end
+
+  ScreenshotResultSchema = DynamicSchema::Struct.define do
+    items                 ScreenshotResultItem, array: true
+  end
+
+  class ScreenshotResult < ScreenshotResultSchema
+    extend Forwardable
+    include Enumerable
+
+    def_delegators :items, :each, :[], :count, :size, :length, :first, :last, :empty?
+
+    def self.from_array( array )
+      new( items: array )
+    end
+
+    def success?
+      items&.all?( &:success? ) || false
+    end
+
+    # convenience method for single URL screenshots
+    def content
+      first&.content
+    end
+
+    def image_data
+      first&.image_data
+    end
+
+    def error
+      first&.error
+    end
+
+    def status
+      first&.status
+    end
+
+    def url
+      first&.url
+    end
+
+    # save screenshot to file
+    def save_to( path )
+      return false unless image_data
+      File.binwrite( path, image_data )
+      true
+    end
+  end
+
+end

data/lib/spider_cloud/shared_schemas.rb

@@ -0,0 +1,80 @@
+module SpiderCloud
+
+  RETURN_FORMATS                 = [ :markdown, :commonmark, :raw, :text, :html2text,
+                                     :xml, :bytes, :empty ]
+  REQUEST_TYPES                  = [ :http, :chrome, :smart ]
+  PROXY_TYPES                    = [ :residential, :mobile, :isp ]
+  DEVICE_TYPES                   = [ :mobile, :tablet, :desktop ]
+  REDIRECT_POLICIES              = [ :loose, :strict, :none ]
+  CHUNKING_TYPES                 = [ :no, :by_words, :by_lines, :by_character_length,
+                                     :by_sentence ]
+  IMAGE_FORMATS                  = [ :png, :jpeg ]
+
+  # reusable schema fragments for dynamic_schema definable classes
+  module SharedSchemas
+
+    # timeout schema fragment: seconds and nanoseconds
+    def timeout_schema
+      proc do
+        timeout do
+          seconds                  Integer, as: :secs
+          nanoseconds              Integer, as: :nanos
+        end
+      end
+    end
+
+    # viewport schema fragment: width and height
+    def viewport_schema
+      proc do
+        viewport do
+          width                    Integer
+          height                   Integer
+        end
+      end
+    end
+
+    # wait_for schema fragment with all wait conditions
+    def wait_for_schema( include_dom: true, include_page_navigations: true )
+      timeout_proc = timeout_schema
+      proc do
+        wait_for do
+          idle_network do
+            instance_eval( &timeout_proc )
+          end
+          selector                 String
+          if include_dom
+            dom do
+              instance_eval( &timeout_proc )
+            end
+          end
+          delay do
+            instance_eval( &timeout_proc )
+          end
+          if include_page_navigations
+            page_navigations do
+              instance_eval( &timeout_proc )
+            end
+          end
+        end
+      end
+    end
+
+    # gpt_config schema fragment for ai integration
+    def gpt_config_schema
+      proc do
+        gpt_config do
+          prompt                   String
+          model                    String
+          max_tokens               Integer
+          temperature              Float
+          top_p                    Float
+          api_key                  String
+          extra_ai_data            [ TrueClass, FalseClass ]
+          screenshot               [ TrueClass, FalseClass ]
+        end
+      end
+    end
+
+  end
+
+end

data/lib/spider_cloud/version.rb

@@ -0,0 +1,3 @@
+module SpiderCloud
+  VERSION = '1.0.0'
+end