spidy 0.3.12 → 1.0.0

data/lib/spidy/connector/lightpanda.rb

@@ -0,0 +1,161 @@
+#
+# Lightpanda connector for JavaScript-rendered pages via CDP
+# Using Ferrum for direct CDP connection
+#
+class Spidy::Connector::Lightpanda
+  include Spidy::Connector::StaticAccessor
+
+  attr_reader :user_agent, :host, :port
+
+  DEFAULT_HOST = '127.0.0.1'.freeze
+  DEFAULT_PORT = 9222
+
+  def initialize(user_agent:, host: nil, port: nil)
+    begin
+      require 'ferrum'
+    rescue LoadError
+      raise 'Ferrum gem is required. Please install with: gem install ferrum'
+    end
+
+    @user_agent = user_agent
+    @host = host || ENV['LIGHTPANDA_HOST'] || DEFAULT_HOST
+    @port = port || ENV['LIGHTPANDA_PORT'] || DEFAULT_PORT
+  end
+
+  def call(url)
+    fail 'url is not specified' if url.blank?
+
+    # Clean the URL by removing any whitespace or newlines
+    clean_url = url.to_s.strip
+
+    puts "Processing URL: #{clean_url}" if ENV['DEBUG']
+
+    # Create a page-like object similar to Mechanize
+    page = fetch_with_ferrum(clean_url)
+
+    # Apply yielder to the page
+    yield(page)
+  end
+
+  def refresh!
+    # No special refresh actions needed for Lightpanda
+  end
+
+  private
+
+  # Try to wait for network to be idle
+  def wait_for_network_idle(browser)
+    puts 'Waiting for network idle...' if ENV['DEBUG']
+
+    begin
+      # Try with timeout parameter
+      browser.network.wait_for_idle(timeout: 15)
+    rescue ArgumentError
+      begin
+        # Try without timeout parameter
+        browser.network.wait_for_idle
+      rescue StandardError => e
+        # If wait_for_idle fails, fall back to a simple sleep
+        puts "Warning: Could not wait for network idle: #{e.message}" if ENV['DEBUG']
+        sleep 5 # Simple fallback
+      end
+    end
+  end
+
+  # Navigate to URL and get page content
+  def navigate_and_get_content(browser, url)
+    # Navigate to the URL
+    puts "Navigating to: #{url}" if ENV['DEBUG']
+    browser.goto(url)
+
+    # Wait for network idle
+    wait_for_network_idle(browser)
+
+    # Get the content
+    puts 'Getting page content...' if ENV['DEBUG']
+    browser.body
+  end
+
+  # Create browser options
+  def create_browser_options
+    options = {
+      headless: true,         # Run in headless mode
+      timeout: 20,            # Increase timeout
+      process_timeout: 20,    # Process timeout
+      window_size: [1280, 800] # Set window size
+    }
+
+    # Add Chrome path if available
+    options[:browser_path] = ENV['CHROME_PATH'] if ENV['CHROME_PATH'] && File.exist?(ENV['CHROME_PATH'])
+
+    options
+  end
+
+  # Connect to CDP server with Ferrum and fetch the page
+  def fetch_with_ferrum(url) # rubocop:todo Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    puts 'Using direct Chrome/Chromium instead of Lightpanda' if ENV['DEBUG']
+    browser = nil
+    html_content = nil
+
+    begin
+      # Create Ferrum browser
+      browser = Ferrum::Browser.new(create_browser_options)
+
+      # Skip user agent setting - not supported in this Ferrum version
+      if @user_agent.present? && ENV.fetch('DEBUG', nil)
+        puts 'User agent setting will be skipped - not supported in your Ferrum version'
+      end
+
+      # Navigate and get content
+      html_content = navigate_and_get_content(browser, url)
+    rescue StandardError => e
+      puts "Error during page navigation: #{e.class} - #{e.message}" if ENV['DEBUG']
+      raise e
+    ensure
+      # Clean up - ensure browser is always closed, even if an error occurred
+      browser&.quit if defined?(browser) && browser
+    end
+
+    # Create a Mechanize-like page object
+    LightpandaPage.new(url, html_content)
+  end
+
+  # Page-like object that mimics the Mechanize::Page interface
+  class LightpandaPage
+    attr_reader :uri, :body, :title, :code, :response_code
+
+    def initialize(url, html_content)
+      @uri = url
+      @body = html_content
+      @doc = Nokogiri::HTML(html_content)
+      @title = @doc.title
+      @code = '200'
+      @response_code = '200'
+    end
+
+    # Common methods from Mechanize::Page that might be used in the application
+    def search(*)
+      @doc.search(*)
+    end
+
+    def at(*)
+      @doc.at(*)
+    end
+
+    def css(*)
+      @doc.css(*)
+    end
+
+    def xpath(*)
+      @doc.xpath(*)
+    end
+
+    def encoding
+      @doc.encoding
+    end
+
+    def try(*args)
+      send(*args) if respond_to?(args.first)
+    end
+  end
+end

data/lib/spidy/connector/xml.rb

@@ -1,20 +1,18 @@
-# frozen_string_literal: true
-
 #
 # xml
 #
 class Spidy::Connector::Xml
   include Spidy::Connector::StaticAccessor
 
-  def call(url, &block)
+  def call(url, &)
     fail 'URL is undefined' if url.blank?
 
-    connect(url, &block)
+    connect(url, &)
   end
 
-  def connect(url, &block)
+  def connect(url)
     OpenURI.open_uri(url, 'User-Agent' => @user_agent) do |body|
-      block.call Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
+      yield Nokogiri::XML(body.read.gsub(/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/, ''), url)
     end
   rescue OpenURI::HTTPError => e
     raise Spidy::Connector::Retry.new(error: e, response_code: e.io.status[0])

data/lib/spidy/connector.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #
 # This class is responsible for actually making a network connection and downloading hypertext
 #
@@ -9,6 +7,7 @@ module Spidy::Connector
   autoload :Html
   autoload :Json
   autoload :Xml
+  autoload :Lightpanda
 
   DEFAULT_WAIT_TIME = 5
 
@@ -35,9 +34,9 @@ module Spidy::Connector
   module StaticAccessor
     extend ActiveSupport::Concern
     class_methods do
-      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
+      def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &)
         ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(
-          url, &block
+          url, &
         )
       end
     end
@@ -75,9 +74,9 @@ module Spidy::Connector
       connect(url, &block)
     end
 
-    def connect(url, retry_attempt_count: @retry_attempt_count, &block)
+    def connect(url, retry_attempt_count: @retry_attempt_count, &)
       logger.call('connnector.get': url, 'connnector.accessed': Time.current)
-      origin_connector.call(url, &block)
+      origin_connector.call(url, &)
     rescue Spidy::Connector::Retry => e
       logger.call('retry.accessed': Time.current,
                   'retry.uri': url,
@@ -105,9 +104,9 @@ module Spidy::Connector
       @socks_proxy = socks_proxy
     end
 
-    def call(url, &block)
+    def call(url, &)
       Socksify.proxy(socks_proxy[:host], socks_proxy[:port]) do
-        connector.call(url, &block)
+        connector.call(url, &)
       end
     end
 

data/lib/spidy/console.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #
 # spidy console
 #

data/lib/spidy/definition.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #
 # Class representing a website defined by DSL
 #
@@ -42,12 +40,12 @@ module Spidy::Definition
     end
   end
 
-  def spider(name = :default, connector: nil, as: nil, &define_block)
+  def spider(name = :default, connector: nil, as: nil)
     @namespace ||= {}
     connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
                                                       socks_proxy: @socks_proxy)
     @namespace[:"#{name}_spider"] = proc do |source, &yielder|
-      define_block.call(yielder, connector, source)
+      yield(yielder, connector, source)
     end
   end
 

data/lib/spidy/definition_file.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #
 # spidy interface binding
 #

data/lib/spidy/definition_object.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #
 # An object that represents the scraper defined by define block.
 #

data/lib/spidy/shell.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #
 # spidy Shell
 #
@@ -9,7 +7,12 @@ class Spidy::Shell
   end
 
   def interactive
-    Pry.start(Spidy::Console.new(@definition_file))
+    console = Spidy::Console.new(@definition_file)
+    require 'irb'
+    IRB.setup(nil)
+    irb = IRB::Irb.new(IRB::WorkSpace.new(console))
+    IRB.conf[:MAIN_CONTEXT] = irb.context
+    irb.eval_input
   end
 
   def command_line

data/lib/spidy/spider.rb

@@ -1,11 +1,9 @@
-# frozen_string_literal: true
-
 #
 # Spider
 #
 class Spidy::Spider
-  def initialize(&block)
-    define_singleton_method(:bind, &block)
+  def initialize(&)
+    define_singleton_method(:bind, &)
   end
 
   def call(resource)

data/lib/spidy/version.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module Spidy
-  VERSION = '0.3.12'
+  VERSION = '1.0.0'.freeze
 end

data/lib/spidy.rb

@@ -1,6 +1,4 @@
-# frozen_string_literal: true
-
-require 'spidy/version'
+require_relative 'spidy/version'
 require 'active_support/all'
 require 'mechanize'
 require 'open-uri'
@@ -29,11 +27,11 @@ module Spidy
     Spidy::DefinitionFile.open(filepath).spidy
   end
 
-  def self.define(&block)
+  def self.define(&)
     spidy = Module.new do
       class_eval do
         extend ::Spidy::Definition
-        module_eval(&block)
+        module_eval(&)
       end
     end
     spidy.instance_eval do

data/spidy.gemspec

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'spidy/version'
@@ -23,21 +21,10 @@ Gem::Specification.new do |spec|
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
-
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'capybara_discoball'
-  spec.add_development_dependency 'ffaker'
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'rake', '~> 13.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rspec-command'
-  spec.add_development_dependency 'sinatra'
-
-  spec.add_runtime_dependency 'activesupport'
-  spec.add_runtime_dependency 'mechanize'
-  spec.add_runtime_dependency 'pry'
-  spec.add_runtime_dependency 'socksify'
-  spec.add_runtime_dependency 'tor'
+  spec.add_dependency 'activesupport', '~> 7.1'
+  spec.add_dependency 'mechanize'
+  spec.add_dependency 'socksify'
+  spec.add_dependency 'tor'
   spec.metadata = {
     'rubygems_mfa_required' => 'true'
   }

metadata

@@ -1,141 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: spidy
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 1.0.0
 platform: ruby
 authors:
 - aileron
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-09 00:00:00.000000000 Z
+date: 2025-04-15 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: capybara_discoball
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: ffaker
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: rspec-command
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -150,20 +37,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: socksify
   requirement: !ruby/object:Gem::Requirement
@@ -192,7 +65,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email:
 - aileron.cc@gmail.com
 executables:
@@ -203,10 +75,10 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".rubocop_todo.yml"
 - ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
+- CLAUDE.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -215,9 +87,16 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/check_ferrum.rb
+- example/check_lightpanda.rb
+- example/connect_test.rb
+- example/lightpanda_links.rb
 - example/master_detail.rb
 - example/proxy.rb
 - example/retry.rb
+- example/run_with_lightpanda.rb
+- example/simple_test.rb
+- example/test_lightpanda.rb
 - example/wikip.rb
 - exe/spidy
 - lib/spidy.rb
@@ -231,6 +110,7 @@ files:
 - lib/spidy/connector/direct.rb
 - lib/spidy/connector/html.rb
 - lib/spidy/connector/json.rb
+- lib/spidy/connector/lightpanda.rb
 - lib/spidy/connector/xml.rb
 - lib/spidy/console.rb
 - lib/spidy/definition.rb
@@ -246,7 +126,6 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -261,8 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key:
+rubygems_version: 3.6.6
 specification_version: 4
 summary: web spider dsl
 test_files: []

data/.rubocop_todo.yml

@@ -1,13 +0,0 @@
-# This configuration was generated by
-# `rubocop --auto-gen-config`
-# on 2019-03-29 18:00:03 +0900 using RuboCop version 0.66.0.
-# The point is for the user to remove these configuration records
-# one by one as the offenses are removed from the code base.
-# Note that changes in the inspected code, or installation of new
-# versions of RuboCop, may require this file to be generated again.
-
-# Offense count: 7
-# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
-# URISchemes: http, https
-Metrics/LineLength:
-  Max: 96