spidy 0.4.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6a7d70df09642e17d34cc85e1973914b8b7e151c34670526cb4d6b2d3589227
-  data.tar.gz: 012b7def5510c16d68676bada533d452315244fc5d47d7f26a9bd71068a3f9a3
+  metadata.gz: 743eadfde1aa8f5e9dbfde067b1c92e38014f274bd59502ca64d845d622c3e53
+  data.tar.gz: 3b89159ea679762e361214ecf9ece14642ff1aeb48b978084bfd35c71e3ad8ea
 SHA512:
-  metadata.gz: 8b6682cd3d1499b115cdfba3964cab62ae65f3a7943fb87f53d8613c2a6553b4f2a4a0728f2af07990dc90ae03c07df4d33914739ab4df462a37c2b04f5efdc5
-  data.tar.gz: a2680dd41fb1a6dead95ecd20742560d749f5c3f27367baac7d3f1294d4c6ee7de946cb173e1f2750169e6a341915197e375d1e56ca0faf7c5be4491f5a55ae9
+  metadata.gz: bad3fd94c682d94a2d92130759178ccea388701cbcbf2ef0e125db8d558af349e81e6c2f623d78277dfffcb89f82d60d7c06d32fef158990cdd3ec19118dc63f
+  data.tar.gz: 24217cbb1c12ebbc4fee9f18e3a3583f2194e33c7dfac99a270cde46f93367a6a397375b08905bb04de39a12f12226f13795c794355ca87ba6f155f8da45b5c6

data/.rubocop.yml

@@ -1,34 +1,79 @@
-inherit_from: .rubocop_todo.yml
+plugins:
+  - rubocop-performance
+  - rubocop-rspec
+
 AllCops:
-  TargetRubyVersion: 3.0.2
+  TargetRubyVersion: 3.4.2
   NewCops: enable
   DisplayCopNames: true
+  Exclude:
+    - 'vendor/**/*'
+    - 'bin/**/*'
+    - 'tmp/**/*'
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  EnforcedStyle: never
 
+# Style
 Style/ClassAndModuleChildren:
   Enabled: false
 
 Style/SignalException:
   EnforcedStyle: semantic
 
+Style/Documentation:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: single_quotes
+  ConsistentQuotesInMultiline: true
+
+# Naming
 Naming/MethodParameterName:
   AllowedNames:
     - as
+    - id
+    - io
+    - ip
+    - of
+    - on
+    - to
+    - up
 
+# Metrics
 Metrics/AbcSize:
-  Max: 21
+  Max: 25
   Exclude:
+    - 'lib/spidy/connector/lightpanda.rb'
 
 Metrics/MethodLength:
-  Max: 15
+  Max: 20
+  Exclude:
+    - 'lib/spidy/connector/lightpanda.rb'
+
+Metrics/ClassLength:
+  Max: 150
+  Exclude:
+    - 'lib/spidy/connector/lightpanda.rb'
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 130
 
 Metrics/BlockLength:
   Max: 120
-
-SignalException:
-  EnforcedStyle: semantic
+  Exclude:
+    - 'spec/**/*'
+    - 'example/**/*'
 
 Layout/EmptyLineAfterGuardClause:
   Enabled: false
+
+# RSpec
+RSpec/ExampleLength:
+  Max: 15
+
+RSpec/MultipleExpectations:
+  Max: 5

data/Gemfile

@@ -1,10 +1,24 @@
-# frozen_string_literal: true
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in crawler.gemspec
 gemspec
 
-gem 'webrick'
-gem 'rackup'
 gem 'irb'
+gem 'rackup'
+gem 'webrick'
+
+gem 'capybara_discoball'
+gem 'ffaker'
+gem 'rake', '~> 13.0'
+gem 'rspec', '~> 3.0'
+gem 'rspec-command'
+gem 'sinatra'
+
+group :development do
+  gem 'ferrum'
+
+  gem 'rubocop', require: false
+  gem 'rubocop-performance', require: false
+  gem 'rubocop-rake', require: false
+  gem 'rubocop-rspec', require: false
+end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.3.12)
+    spidy (1.0.0)
       activesupport (~> 7.1)
       mechanize
       socksify
@@ -24,6 +24,7 @@ GEM
       tzinfo (~> 2.0, >= 2.0.5)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
+    ast (2.4.3)
     base64 (0.2.0)
     benchmark (0.4.0)
     bigdecimal (3.1.9)
@@ -41,20 +42,29 @@ GEM
     concurrent-ruby (1.3.5)
     connection_pool (2.5.0)
     date (3.4.1)
-    diff-lcs (1.6.0)
+    diff-lcs (1.6.1)
     domain_name (0.6.20240107)
     drb (2.2.1)
+    ferrum (0.16)
+      addressable (~> 2.5)
+      base64 (~> 0.2)
+      concurrent-ruby (~> 1.1)
+      webrick (~> 1.7)
+      websocket-driver (~> 0.7)
     ffaker (2.24.0)
     http-cookie (1.0.8)
       domain_name (~> 0.5)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     io-console (0.8.0)
-    irb (1.15.1)
+    irb (1.15.2)
       pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
-    logger (1.6.6)
+    json (2.10.2)
+    language_server-protocol (3.17.0.4)
+    lint_roller (1.1.0)
+    logger (1.7.0)
     matrix (0.4.2)
     mechanize (2.14.0)
       addressable (~> 2.8)
@@ -69,10 +79,10 @@ GEM
       rubyntlm (~> 0.6, >= 0.6.3)
       webrick (~> 1.7)
       webrobots (~> 0.1.2)
-    mime-types (3.6.1)
+    mime-types (3.6.2)
       logger
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2025.0318)
+    mime-types-data (3.2025.0408)
     mini_mime (1.1.5)
     minitest (5.25.5)
     mixlib-shellout (2.4.4)
@@ -82,31 +92,36 @@ GEM
     net-http-persistent (4.0.5)
       connection_pool (~> 2.2)
     nkf (0.2.0)
-    nokogiri (1.18.5-aarch64-linux-gnu)
+    nokogiri (1.18.7-aarch64-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.5-aarch64-linux-musl)
+    nokogiri (1.18.7-aarch64-linux-musl)
       racc (~> 1.4)
-    nokogiri (1.18.5-arm-linux-gnu)
+    nokogiri (1.18.7-arm-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.5-arm-linux-musl)
+    nokogiri (1.18.7-arm-linux-musl)
       racc (~> 1.4)
-    nokogiri (1.18.5-arm64-darwin)
+    nokogiri (1.18.7-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.5-x86_64-darwin)
+    nokogiri (1.18.7-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.5-x86_64-linux-gnu)
+    nokogiri (1.18.7-x86_64-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.5-x86_64-linux-musl)
+    nokogiri (1.18.7-x86_64-linux-musl)
       racc (~> 1.4)
+    parallel (1.27.0)
+    parser (3.3.8.0)
+      ast (~> 2.4.1)
+      racc
     pp (0.6.2)
       prettyprint
     prettyprint (0.2.0)
+    prism (1.4.0)
     psych (5.2.3)
       date
       stringio
     public_suffix (6.0.1)
     racc (1.8.1)
-    rack (3.1.12)
+    rack (3.1.13)
     rack-protection (4.1.1)
       base64 (>= 0.1.0)
       logger (>= 1.6.0)
@@ -118,11 +133,12 @@ GEM
       rack (>= 1.3)
     rackup (2.2.1)
       rack (>= 3)
+    rainbow (3.1.1)
     rake (13.2.1)
-    rdoc (6.12.0)
+    rdoc (6.13.1)
       psych (>= 4.0.0)
     regexp_parser (2.10.0)
-    reline (0.6.0)
+    reline (0.6.1)
       io-console (~> 0.5)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -144,6 +160,31 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.2)
+    rubocop (1.75.2)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
+    rubocop-rake (0.7.1)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.72.1)
+    rubocop-rspec (3.5.0)
+      lint_roller (~> 1.1)
+      rubocop (~> 1.72, >= 1.72.1)
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyntlm (0.6.5)
       base64
@@ -156,13 +197,20 @@ GEM
       rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
     socksify (1.7.1)
-    stringio (3.1.5)
+    stringio (3.1.6)
     tilt (2.6.0)
     tor (0.1.7)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
     webrick (1.9.1)
     webrobots (0.1.2)
+    websocket-driver (0.7.7)
+      base64
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
 
@@ -177,14 +225,18 @@ PLATFORMS
   x86_64-linux-musl
 
 DEPENDENCIES
-  bundler (~> 2.0)
   capybara_discoball
+  ferrum
   ffaker
   irb
   rackup
   rake (~> 13.0)
   rspec (~> 3.0)
   rspec-command
+  rubocop
+  rubocop-performance
+  rubocop-rake
+  rubocop-rspec
   sinatra
   spidy!
   webrick

data/README.md

@@ -20,14 +20,73 @@ Or install it yourself as:
 
 ## Usage
 
-### When used from the command line
+### Connectors
 
-website.rb
+Spidy supports different connectors for fetching web pages:
+
+1. **HTML Connector (Mechanize)**: Default connector for regular HTTP requests and HTML parsing
+2. **JSON Connector**: For parsing JSON APIs
+3. **XML Connector**: For parsing XML responses
+4. **Lightpanda Connector**: For JavaScript-rendered websites (uses Playwright)
+
+#### Lightpanda Connector for JavaScript-Rendered Websites
+
+The Lightpanda connector allows you to process JavaScript-rendered websites by connecting to a running lightpanda CDP server.
+
+##### Prerequisites
+
+1. Install the Playwright Ruby client:
+
+```bash
+$ gem install playwright-ruby-client
+```
+
+2. Start a lightpanda CDP server in a separate terminal:
+
+```bash
+$ lightpanda serve --host 127.0.0.1 --port 9222
+```
+
+##### Usage
+
+```ruby
+# Define a scraper with lightpanda support
+scraper = Spidy.define do
+  # Use the :lightpanda connector for JavaScript-rendered sites
+  spider(as: :lightpanda) do |yielder, connector, url|
+    connector.call(url) do |page|
+      # Process the JavaScript-rendered page
+      # page is a Nokogiri-like object
+      yielder.call(page)
+    end
+  end
+
+  define(as: :html) do
+    let(:title, 'title')
+    # Extract content from JS-rendered page...
+  end
+end
+```
+
+##### Configuration
+
+You can customize the lightpanda CDP server connection using environment variables:
+
+```bash
+# Set custom host and port
+$ LIGHTPANDA_HOST=192.168.1.100 LIGHTPANDA_PORT=9333 ruby your_script.rb
+```
+
+Check `example/playwright_example.rb` for a complete example.
+
+### Command Line Usage
+
+Create a definition file (e.g., website.rb):
 ```rb
-Spidy.defin do
+Spidy.define do
   spider(as: :html) do |yielder, connector, url|
     connector.call(url) do |html|
-      # html as nokogiri object ( mechanize )
+      # html is a Nokogiri object (from Mechanize)
       yielder.call(url)
     end
   end
@@ -37,41 +96,61 @@ Spidy.defin do
   end
 end
 ```
+
+Use it from the command line:
 ```bash
 echo 'http://example.com' | spidy each website.rb > urls
 cat urls | spidy call website.rb > website.json
-# shorthands
+# shorthand
 echo 'http://example.com' | spidy each website.rb | spidy call website.rb | jq .
 ```
 
-### When development console
+### Development Console
+
+Start an interactive console with your definition:
 ```bash
 spidy console website.rb
 ```
 
-### reload source code
+Reload your source code during development:
 ```
 irb(#<Spidy::Console>)> reload!
 ```
 
+Example console usage:
 ```rb
 each('http://example.com') { |url| break url }
-call('http://example.com') { |html| break html } # html as nokogiri object ( mechanize )
+call('http://example.com') { |html| break html } # html is a Nokogiri object (from Mechanize)
 ```
 
-### When used from the ruby code
+### Ruby Code Usage
+
+Create and use a scraper in your Ruby code:
 ```rb
-a = Spidy.define do
-  # Implementing spiders and scrapers
+scraper = Spidy.define do
+  # Implement spiders and scrapers
+  spider(as: :html) do |yielder, connector, url|
+    connector.call(url) do |page|
+      yielder.call(page)
+    end
+  end
+  
+  define(as: :html) do
+    let(:title, 'title')
+    let(:links) { |doc| doc.css('a').map { |a| a['href'] } }
+  end
 end
 
-a.each(url) do |url|
-  # Loop for the number of retrieved URLs
+# Extract URLs from a site
+scraper.each(url) do |page_url|
+  # Process each URL found
+  puts page_url
 end
 
-a.call(url) do |object|
-  # The scrape result is passed as a defined object
-end
+# Extract structured data from a site
+result = scraper.call(url)
+puts "Title: #{result[:title]}"
+puts "Found #{result[:links].size} links"
 ```
 
 ## Development

data/Rakefile

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 

data/example/check_ferrum.rb

@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+
+# This script checks what Ferrum API methods are available
+begin
+  require 'ferrum'
+  puts 'Ferrum gem is loaded!'
+
+  # Check Ferrum version
+  puts "Ferrum version: #{begin
+    Ferrum::VERSION
+  rescue StandardError
+    'unknown'
+  end}"
+
+  # Try to create a browser instance
+  puts "\nTrying to create a browser instance..."
+
+  # Find Chrome executable path
+  def find_chrome_path
+    # Common locations on macOS
+    macos_paths = [
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+      '/Applications/Chromium.app/Contents/MacOS/Chromium',
+      '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'
+    ]
+
+    # Check macOS paths
+    macos_paths.each do |path|
+      return path if File.exist?(path)
+    end
+
+    # Try to locate Chrome using 'which' command
+    %w[google-chrome chromium chromium-browser].each do |browser|
+      path = `which #{browser} 2>/dev/null`.strip
+      return path if path != '' && File.exist?(path)
+    end
+
+    nil
+  end
+
+  # Get Chrome path
+  chrome_path = ENV['CHROME_PATH'] || find_chrome_path
+  if chrome_path
+    puts "Using Chrome executable: #{chrome_path}"
+  else
+    puts 'No Chrome executable found. Using default.'
+  end
+
+  # Create browser with options
+  options = {
+    headless: true,
+    window_size: [1280, 800]
+  }
+
+  # Add Chrome path if available
+  options[:browser_path] = chrome_path if chrome_path
+
+  browser = Ferrum::Browser.new(options)
+  puts 'Browser instance created successfully!'
+
+  # Check available methods on browser
+  puts "\nAvailable methods on browser object:"
+  browser_methods = (browser.methods - Object.methods).sort
+  puts browser_methods.join(', ')
+
+  # Check if headers method exists
+  puts "\nDoes browser respond to 'headers='? #{browser.respond_to?(:headers=)}"
+
+  # Check available methods on browser.network
+  if browser.respond_to?(:network)
+    puts "\nAvailable methods on browser.network object:"
+    network_methods = (browser.network.methods - Object.methods).sort
+    puts network_methods.join(', ')
+
+    # Check if wait_for_idle method exists and what parameters it accepts
+    if browser.network.respond_to?(:wait_for_idle)
+      puts "\nExamine wait_for_idle method:"
+      begin
+        # Try with timeout parameter
+        browser.network.wait_for_idle(timeout: 1)
+        puts 'wait_for_idle accepts timeout parameter'
+      rescue ArgumentError => e
+        puts "wait_for_idle does not accept timeout parameter: #{e.message}"
+      rescue StandardError => e
+        puts "Error calling wait_for_idle with timeout: #{e.message}"
+      end
+    else
+      puts "\nwait_for_idle method not available on network object"
+    end
+  else
+    puts "\nnetwork method not available on browser object"
+  end
+
+  # Test goto method
+  puts "\nTesting navigation with goto method:"
+  begin
+    browser.goto('https://example.com')
+    puts 'Navigation successful!'
+    puts "Page title: #{browser.title}"
+  rescue StandardError => e
+    puts "Error during navigation: #{e.message}"
+  end
+
+  # Clean up
+  browser.quit
+  puts "\nBrowser closed successfully"
+rescue LoadError => e
+  puts "Error: Ferrum gem is not installed: #{e.message}"
+  puts 'Install it with: gem install ferrum'
+rescue StandardError => e
+  puts "Error: #{e.message}"
+  puts e.backtrace.join("\n")
+end

data/example/check_lightpanda.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+# Script to check if Lightpanda is running and start it if needed
+require 'net/http'
+
+def lightpanda_running?(host = '127.0.0.1', port = 9222)
+  uri = URI("http://#{host}:#{port}/json/version")
+  response = Net::HTTP.get_response(uri)
+  response.is_a?(Net::HTTPSuccess)
+rescue StandardError
+  false
+end
+
+def start_lightpanda(host = '127.0.0.1', port = 9222)
+  puts 'Starting Lightpanda...'
+
+  # Build the command to start Lightpanda in the background
+  cmd = "/Users/aileron/bin/lightpanda serve --host #{host} --port #{port} > /tmp/lightpanda.log 2>&1 &"
+
+  # Execute the command
+  result = system(cmd)
+
+  if result
+    puts "Lightpanda started! Service should be available at http://#{host}:#{port}"
+
+    # Wait for it to be ready
+    10.times do
+      if lightpanda_running?(host, port)
+        puts 'Lightpanda is now running and accepting connections!'
+        return true
+      end
+      puts 'Waiting for Lightpanda to start...'
+      sleep 1
+    end
+
+    puts "Lightpanda might have started but isn't responding yet."
+    puts 'Check /tmp/lightpanda.log for details.'
+  else
+    puts 'Failed to start Lightpanda. Make sure the path is correct: /Users/aileron/bin/lightpanda'
+  end
+  false
+end
+
+# Main script
+host = '127.0.0.1'
+port = 9222
+
+if lightpanda_running?(host, port)
+  puts "✅ Lightpanda is already running at http://#{host}:#{port}"
+else
+  puts "❌ Lightpanda is not running at http://#{host}:#{port}"
+
+  if ARGV.include?('--start') || ARGV.include?('-s')
+    start_lightpanda(host, port)
+  else
+    puts 'Run this script with --start or -s option to start Lightpanda automatically:'
+    puts "  #{$PROGRAM_NAME} --start"
+  end
+end

data/example/connect_test.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+# Test connecting to existing Chrome instance at 127.0.0.1:9222
+
+begin
+  require 'ferrum'
+  puts 'Successfully loaded Ferrum'
+rescue LoadError => e
+  puts "Ferrum is not installed: #{e.message}"
+  puts "Run 'gem install ferrum' to install it"
+  exit 1
+end
+
+puts 'Testing connection to Chrome at 127.0.0.1:9222'
+puts '=============================================='
+
+begin
+  # Connect to the remote Chrome instance
+  # Note: We're setting process: false to prevent launching a new browser
+  browser = Ferrum::Browser.new(
+    url: 'http://127.0.0.1:9222',
+    process: false
+  )
+
+  # Access a test URL
+  url = 'https://example.com'
+  puts "Accessing: #{url}"
+  browser.goto(url)
+
+  # Get page title
+  title = browser.title
+  puts "Page title: #{title}"
+
+  # Clean up browser connection (but don't close Chrome)
+  browser.quit
+
+  puts "\nSuccess! Connected to Chrome at 127.0.0.1:9222"
+rescue StandardError => e
+  puts "Error: #{e.message}"
+  puts e.backtrace.join("\n")
+
+  puts "\nTroubleshooting tips:"
+  puts '1. Make sure Chrome is running with remote debugging enabled'
+  puts '2. Verify the command: /Users/aileron/bin/lightpanda serve --host 127.0.0.1 --port 9222'
+  puts '3. Check if you can access http://127.0.0.1:9222/json/version in your browser'
+end
+
+puts "\nTest completed"