scraper_utils 0.5.1 → 0.7.0

data/lib/scraper_utils/spec_support.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "scraperwiki"
+
+module ScraperUtils
+  # Methods to support specs
+  module SpecSupport
+    AUSTRALIAN_STATES = %w[ACT NSW NT QLD SA TAS VIC WA].freeze
+    COMMON_STREET_TYPES =
+      %w[
+        Avenue Ave Boulevard Court Crt Circle Chase Circuit Close Crescent
+        Drive Drv Lane Loop Parkway Place Parade Road Rd Street St Square Terrace Way
+      ].freeze
+    AUSTRALIAN_POSTCODES = /\b\d{4}\b/.freeze
+
+    # Check if an address is likely to be geocodable by analyzing its format.
+    # This is a bit stricter than needed - typically assert >= 75% match
+    # @param address [String] The address to check
+    # @return [Boolean] True if the address appears to be geocodable.
+    def self.geocodable?(address, ignore_case: false)
+      return false if address.nil? || address.empty?
+      check_address = ignore_case ? address.upcase : address
+
+      # Basic structure check - must have a street name, suburb, state and postcode
+      has_state = AUSTRALIAN_STATES.any? { |state| check_address.end_with?(" #{state}") || check_address.include?(" #{state} ") }
+      has_postcode = address.match?(AUSTRALIAN_POSTCODES)
+
+      has_street_type = COMMON_STREET_TYPES.any? { |type| check_address.include?(" #{type}") || check_address.include?(" #{type.upcase}") }
+
+      has_unit_or_lot = address.match?(/\b(Unit|Lot:?)\s+\d+/i)
+
+      has_suburb_stats = check_address.match?(/\b[A-Z]{2,}(\s+[A-Z]+)*,?\s+(#{AUSTRALIAN_STATES.join('|')})\b/)
+
+      if ENV["DEBUG"]
+        missing = []
+        unless has_street_type || has_unit_or_lot
+          missing << "street type / unit / lot"
+        end
+        missing << "state" unless has_state
+        missing << "postcode" unless has_postcode
+        missing << "#{ignore_case ? '' : 'uppercase '}suburb state" unless has_suburb_stats
+        puts "  address: #{address} is not geocodable, missing #{missing.join(', ')}" if missing.any?
+      end
+
+      (has_street_type || has_unit_or_lot) && has_state && has_postcode && has_suburb_stats
+    end
+
+    PLACEHOLDERS = [
+      /no description/i,
+      /not available/i,
+      /to be confirmed/i,
+      /\btbc\b/i,
+      %r{\bn/a\b}i
+    ].freeze
+
+    def self.placeholder?(text)
+      PLACEHOLDERS.any? { |placeholder| text.to_s.match?(placeholder) }
+    end
+
+    # Check if this looks like a "reasonable" description
+    # This is a bit stricter than needed - typically assert >= 75% match
+    def self.reasonable_description?(text)
+      !placeholder?(text) && text.to_s.split.size >= 3
+    end
+  end
+end
+

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.5.1"
+  VERSION = "0.7.0"
 end

data/lib/scraper_utils.rb

@@ -1,20 +1,22 @@
 # frozen_string_literal: true
 
-require "scraper_utils/adaptive_delay"
+require "scraper_utils/version"
+
+# Public Apis (responsible for requiring their own dependencies)
 require "scraper_utils/authority_utils"
 require "scraper_utils/cycle_utils"
 require "scraper_utils/data_quality_monitor"
 require "scraper_utils/date_range_utils"
 require "scraper_utils/db_utils"
 require "scraper_utils/debug_utils"
-require "scraper_utils/fiber_scheduler"
 require "scraper_utils/log_utils"
+require "scraper_utils/randomize_utils"
+require "scraper_utils/scheduler"
+require "scraper_utils/spec_support"
+
+# Mechanize utilities
 require "scraper_utils/mechanize_actions"
-require "scraper_utils/mechanize_utils/agent_config"
 require "scraper_utils/mechanize_utils"
-require "scraper_utils/randomize_utils"
-require "scraper_utils/robots_checker"
-require "scraper_utils/version"
 
 # Utilities for planningalerts scrapers
 module ScraperUtils
@@ -22,17 +24,13 @@ module ScraperUtils
   AUSTRALIAN_PROXY_ENV_VAR = "MORPH_AUSTRALIAN_PROXY"
 
   # Fatal Error
-  class Error < StandardError
-  end
+  class Error < StandardError; end
 
   # Fatal error with the site - retrying won't help
-  class UnprocessableSite < Error
-  end
+  class UnprocessableSite < Error; end
 
-  # Content validation errors that should not be retried for that record,
-  # but other records may be processable
-  class UnprocessableRecord < Error
-  end
+  # Fatal Error for a record - other records may be processable
+  class UnprocessableRecord < Error; end
 
   def self.australian_proxy
     ap = ENV[AUSTRALIAN_PROXY_ENV_VAR].to_s

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-05 00:00:00.000000000 Z
+date: 2025-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -64,6 +64,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- ".yardopts"
 - CHANGELOG.md
 - GUIDELINES.md
 - Gemfile
@@ -73,30 +74,41 @@ files:
 - Rakefile
 - SPECS.md
 - bin/console
+- bin/rspec
 - bin/setup
 - docs/debugging.md
+- docs/enhancing_specs.md
 - docs/example_scrape_with_fibers.rb
 - docs/example_scraper.rb
+- docs/fibers_and_threads.md
 - docs/getting_started.md
 - docs/interleaving_requests.md
 - docs/mechanize_utilities.md
+- docs/parallel_requests.md
 - docs/randomizing_requests.md
 - docs/reducing_server_load.md
 - lib/scraper_utils.rb
-- lib/scraper_utils/adaptive_delay.rb
 - lib/scraper_utils/authority_utils.rb
 - lib/scraper_utils/cycle_utils.rb
 - lib/scraper_utils/data_quality_monitor.rb
 - lib/scraper_utils/date_range_utils.rb
 - lib/scraper_utils/db_utils.rb
 - lib/scraper_utils/debug_utils.rb
-- lib/scraper_utils/fiber_scheduler.rb
 - lib/scraper_utils/log_utils.rb
 - lib/scraper_utils/mechanize_actions.rb
 - lib/scraper_utils/mechanize_utils.rb
+- lib/scraper_utils/mechanize_utils/adaptive_delay.rb
 - lib/scraper_utils/mechanize_utils/agent_config.rb
+- lib/scraper_utils/mechanize_utils/robots_checker.rb
 - lib/scraper_utils/randomize_utils.rb
-- lib/scraper_utils/robots_checker.rb
+- lib/scraper_utils/scheduler.rb
+- lib/scraper_utils/scheduler/constants.rb
+- lib/scraper_utils/scheduler/operation_registry.rb
+- lib/scraper_utils/scheduler/operation_worker.rb
+- lib/scraper_utils/scheduler/process_request.rb
+- lib/scraper_utils/scheduler/thread_request.rb
+- lib/scraper_utils/scheduler/thread_response.rb
+- lib/scraper_utils/spec_support.rb
 - lib/scraper_utils/version.rb
 - scraper_utils.gemspec
 homepage: https://github.com/ianheggie-oaf/scraper_utils
@@ -106,7 +118,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.5.1
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.7.0
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message:

data/lib/scraper_utils/adaptive_delay.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module ScraperUtils
-  # Adapts delays between requests based on server response times.
-  # Target delay is proportional to response time based on max_load setting.
-  # Uses an exponential moving average to smooth variations in response times.
-  class AdaptiveDelay
-    DEFAULT_MIN_DELAY = 0.0
-    DEFAULT_MAX_DELAY = 30.0 # Presumed default timeout for Mechanize
-
-    attr_reader :min_delay, :max_delay, :max_load
-
-    # Creates a new adaptive delay calculator
-    #
-    # @param min_delay [Float] Minimum delay between requests in seconds
-    # @param max_delay [Float] Maximum delay between requests in seconds
-    # @param max_load [Float] Maximum load percentage (1-99) we aim to place on the server
-    #                         Lower values are more conservative (e.g., 20% = 4x response time delay)
-    def initialize(min_delay: DEFAULT_MIN_DELAY, max_delay: DEFAULT_MAX_DELAY, max_load: 20.0)
-      @delays = {} # domain -> last delay used
-      @min_delay = min_delay.to_f
-      @max_delay = max_delay.to_f
-      @max_load = max_load.to_f.clamp(1.0, 99.0)
-      @response_multiplier = (100.0 - @max_load) / @max_load
-
-      return unless DebugUtils.basic?
-
-      ScraperUtils::FiberScheduler.log(
-        "AdaptiveDelay initialized with delays between #{@min_delay} and #{@max_delay} seconds, " \
-          "Max_load #{@max_load}% thus response multiplier: #{@response_multiplier.round(2)}x"
-      )
-    end
-
-    # @param uri [URI::Generic, String] The URL to extract the domain from
-    # @return [String] The domain in the format "scheme://host"
-    def domain(uri)
-      uri = URI(uri) unless uri.is_a?(URI)
-      "#{uri.scheme}://#{uri.host}".downcase
-    end
-
-    # @param uri [URI::Generic, String] URL to get delay for
-    # @return [Float] Current delay for the domain, or min_delay if no delay set
-    def delay(uri)
-      @delays[domain(uri)] || @min_delay
-    end
-
-    # @param uri [URI::Generic, String] URL the response came from
-    # @param response_time [Float] Time in seconds the server took to respond
-    # @return [Float] The calculated delay to use with the next request
-    def next_delay(uri, response_time)
-      uris_domain = domain(uri)
-      target_delay = (response_time * @response_multiplier).clamp(0.0, @max_delay)
-      current_delay = @delays[uris_domain] || target_delay
-      delay = ((9.0 * current_delay) + target_delay) / 10.0
-      delay = delay.clamp(@min_delay, @max_delay)
-
-      if DebugUtils.basic?
-        ScraperUtils::FiberScheduler.log(
-          "Adaptive delay for #{uris_domain} updated to #{delay.round(2)}s (target: " \
-            "#{@response_multiplier.round(1)}x response_time of #{response_time.round(2)}s)"
-        )
-      end
-
-      @delays[uris_domain] = delay
-      delay
-    end
-  end
-end

data/lib/scraper_utils/fiber_scheduler.rb

@@ -1,229 +0,0 @@
-# frozen_string_literal: true
-
-require "fiber"
-
-module ScraperUtils
-  # A utility module for interleaving multiple scraping operations
-  # using fibers during connection delay periods. This allows efficient
-  # use of wait time by switching between operations.
-  module FiberScheduler
-    # @return [Array<Fiber>] List of active fibers managed by the scheduler
-    def self.registry
-      @registry ||= []
-    end
-
-    # Checks if the current code is running within a registered fiber
-    #
-    # @return [Boolean] true if running in a registered fiber, false otherwise
-    def self.in_fiber?
-      !Fiber.current.nil? && registry.include?(Fiber.current)
-    end
-
-    # Gets the authority associated with the current fiber
-    #
-    # @return [String, nil] the authority name or nil if not in a fiber
-    def self.current_authority
-      return nil unless in_fiber?
-
-      Fiber.current.instance_variable_get(:@authority)
-    end
-
-    # Logs a message, automatically prefixing with authority name if in a fiber
-    #
-    # @param message [String] the message to log
-    # @return [void]
-    def self.log(message)
-      authority = current_authority
-      $stderr.flush
-      if authority
-        puts "[#{authority}] #{message}"
-      else
-        puts message
-      end
-      $stdout.flush
-    end
-
-    # Returns a hash of exceptions encountered during processing, indexed by authority
-    #
-    # @return [Hash{Symbol => Exception}] exceptions by authority
-    def self.exceptions
-      @exceptions ||= {}
-    end
-
-    # Returns a hash of the yielded / block values
-    #
-    # @return [Hash{Symbol => Any}] values by authority
-    def self.values
-      @values ||= {}
-    end
-
-    # Checks if fiber scheduling is currently enabled
-    #
-    # @return [Boolean] true if enabled, false otherwise
-    def self.enabled?
-      @enabled ||= false
-    end
-
-    # Enables fiber scheduling
-    #
-    # @return [void]
-    def self.enable!
-      reset! unless enabled?
-      @enabled = true
-    end
-
-    # Disables fiber scheduling
-    #
-    # @return [void]
-    def self.disable!
-      @enabled = false
-    end
-
-    # Resets the scheduler state, and disables. Use before retrying failed authorities.
-    #
-    # @return [void]
-    def self.reset!
-      @registry = []
-      @exceptions = {}
-      @values = {}
-      @enabled = false
-      @delay_requested = 0.0
-      @time_slept = 0.0
-      @resume_count = 0
-      @initial_resume_at = Time.now - 60.0 # one minute ago
-    end
-
-    # Registers a block to scrape for a specific authority
-    #
-    # @param authority [String] the name of the authority being processed
-    # @yield to the block containing the scraping operation to be run in the fiber
-    # @return [Fiber] a fiber that calls the block. With @authority and @resume_at instance vars
-    def self.register_operation(authority, &block)
-      # Automatically enable fiber scheduling when operations are registered
-      enable!
-
-      fiber = Fiber.new do
-        values[authority] = block.call
-      rescue StandardError => e
-        # Store exception against the authority
-        exceptions[authority] = e
-      ensure
-        # Remove itself when done regardless of success/failure
-        registry.delete(Fiber.current)
-      end
-
-      # Start fibres in registration order
-      @initial_resume_at += 0.1
-      fiber.instance_variable_set(:@resume_at, @initial_resume_at)
-      fiber.instance_variable_set(:@authority, authority)
-      registry << fiber
-
-      if DebugUtils.basic?
-        FiberScheduler.log "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving"
-      end
-      # Process immediately when testing
-      fiber.resume if ScraperUtils::RandomizeUtils.sequential?
-      fiber
-    end
-
-    # Run all registered fibers until completion
-    #
-    # @return [Hash] Exceptions that occurred during execution
-    def self.run_all
-      count = registry.size
-      while (fiber = find_earliest_fiber)
-        if fiber.alive?
-          authority = begin
-            fiber.instance_variable_get(:@authority)
-          rescue StandardError
-            nil
-          end
-          @resume_count ||= 0
-          @resume_count += 1
-          values[authority] = fiber.resume
-        else
-          FiberScheduler.log "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
-          registry.delete(fiber)
-        end
-      end
-
-      if @time_slept&.positive? && @delay_requested&.positive?
-        percent_slept = (100.0 * @time_slept / @delay_requested).round(1)
-      end
-      puts
-      FiberScheduler.log "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, " \
-           "sleeping #{percent_slept}% (#{@time_slept&.round(1)}) of the " \
-           "#{@delay_requested&.round(1)} seconds requested."
-      puts
-
-      exceptions
-    end
-
-    # Delays the current fiber and potentially runs another one
-    # Falls back to regular sleep if fiber scheduling is not enabled
-    #
-    # @param seconds [Numeric] the number of seconds to delay
-    # @return [Integer] return from sleep operation or 0
-    def self.delay(seconds)
-      seconds = 0.0 unless seconds&.positive?
-      @delay_requested ||= 0.0
-      @delay_requested += seconds
-
-      current_fiber = Fiber.current
-
-      if !enabled? || !current_fiber || registry.size <= 1
-        @time_slept ||= 0.0
-        @time_slept += seconds
-        log("Sleeping #{seconds.round(3)} seconds") if DebugUtils.basic?
-        return sleep(seconds)
-      end
-
-      now = Time.now
-      resume_at = now + seconds
-
-      # Don't resume at the same time as someone else,
-      # FIFO queue if seconds == 0
-      @other_resumes ||= []
-      @other_resumes = @other_resumes.delete_if { |t| t < now }
-      while @other_resumes.include?(resume_at) && resume_at
-        resume_at += 0.01
-      end
-
-      # Used to compare when other fibers need to be resumed
-      current_fiber.instance_variable_set(:@resume_at, resume_at)
-
-      # Yield control back to the scheduler so another fiber can run
-      Fiber.yield
-
-      # When we get control back, check if we need to sleep more
-      remaining = resume_at - Time.now
-      if remaining.positive?
-        @time_slept ||= 0.0
-        @time_slept += remaining
-        log("Sleeping remaining #{remaining.round(3)} seconds") if DebugUtils.basic?
-        sleep(remaining)
-      end || 0
-    end
-
-    # Finds the fiber with the earliest wake-up time
-    #
-    # @return [Fiber, nil] the fiber with the earliest wake-up time or nil if none found
-    def self.find_earliest_fiber
-      earliest_time = nil
-      earliest_fiber = nil
-
-      registry.each do |fiber|
-        resume_at = fiber.instance_variable_get(:@resume_at)
-        if earliest_time.nil? || resume_at < earliest_time
-          earliest_time = resume_at
-          earliest_fiber = fiber
-        end
-      end
-
-      earliest_fiber
-    end
-
-    # Mark methods as private
-    private_class_method :find_earliest_fiber
-  end
-end

data/lib/scraper_utils/robots_checker.rb

@@ -1,149 +0,0 @@
-# frozen_string_literal: true
-
-module ScraperUtils
-  # robots.txt checker with deliberately simplistic rules
-  class RobotsChecker
-    # @return [String] Lowercased user_agent for matching
-    attr_reader :user_agent
-
-    # Initialize with full user agent string like:
-    # "Mozilla/5.0 (compatible; ScraperUtils/0.1.0 2025-02-22; +https://github.com/ianheggie-oaf/scraper_utils)"
-    # Extracts the bot name (e.g. "ScraperUtils") to check against robots.txt
-    # Checks for
-    # * Disallow for User-agent: bot_name and
-    # * Crawl-delay from either User-agent: bot name or * (default)
-    def initialize(user_agent)
-      @user_agent = extract_user_agent(user_agent).downcase
-      if DebugUtils.basic?
-        ScraperUtils::FiberScheduler.log(
-          "Checking robots.txt for user agent prefix: #{@user_agent} (case insensitive)"
-        )
-      end
-      @rules = {} # domain -> {rules: [], delay: int}
-      @delay = nil # Delay from last robots.txt check
-    end
-
-    # Check if a URL is disallowed based on robots.txt rules specific to our user agent
-    # @param url [String] The full URL to check
-    # @return [Boolean] true if specifically blocked for our user agent, otherwise false
-    def disallowed?(url)
-      return false unless url
-
-      uri = URI(url)
-      domain = "#{uri.scheme}://#{uri.host}"
-      path = uri.path || "/"
-
-      # Get or fetch robots.txt rules
-      rules = get_rules(domain)
-      return false unless rules # If we can't get robots.txt, assume allowed
-
-      # Store any delay found for this domain
-      @delay = rules[:our_delay]
-
-      # Check rules specific to our user agent
-      matches_any_rule?(path, rules[:our_rules])
-    end
-
-    # Returns the crawl delay (if any) that applied to the last URL checked
-    # Should be called after disallowed? to get relevant delay
-    # @return [Integer, nil] The delay in seconds, or nil if no delay specified
-    def crawl_delay
-      @delay
-    end
-
-    private
-
-    def extract_user_agent(user_agent)
-      if user_agent =~ /^(.*compatible;\s*)?([-_a-z0-9]+)/i
-        user_agent = ::Regexp.last_match(2)&.strip
-      end
-      user_agent&.strip
-    end
-
-    def matches_any_rule?(path, rules)
-      rules&.any? { |rule| path.start_with?(rule) }
-    end
-
-    def get_rules(domain)
-      return @rules[domain] if @rules.key?(domain)
-
-      begin
-        response = Net::HTTP.get_response(URI("#{domain}/robots.txt"))
-        return nil unless response.code.start_with?("2") # 2xx response
-
-        rules = parse_robots_txt(response.body)
-        @rules[domain] = rules
-        rules
-      rescue StandardError => e
-        if DebugUtils.basic?
-          ScraperUtils::FiberScheduler.log(
-            "WARNING: Failed to fetch robots.txt for #{domain}: #{e.message}"
-          )
-        end
-        nil
-      end
-    end
-
-    # Parse robots.txt content into structured rules
-    # Only collects rules for our specific user agent and generic crawl-delay
-    # @param content [String] The robots.txt content
-    # @return [Hash] Hash containing :our_rules and :our_delay
-    def parse_robots_txt(content)
-      sections = [] # Array of {agent:, rules:[], delay:} hashes
-      current_section = nil
-
-      content.each_line do |line|
-        line = line.strip.downcase
-        next if line.empty? || line.start_with?("#")
-
-        if line.start_with?("user-agent:")
-          agent = line.split(":", 2).last.strip
-          # Check if this is a continuation of the previous section
-          if current_section && current_section[:rules].empty? && current_section[:delay].nil?
-            current_section[:agents] << agent
-          else
-            current_section = { agents: [agent], rules: [], delay: nil }
-            sections << current_section
-          end
-          next
-        end
-
-        next unless current_section # Skip rules before first user-agent
-
-        if line.start_with?("disallow:")
-          path = line.split(":", 2).last.strip
-          current_section[:rules] << path unless path.empty?
-        elsif line.start_with?("crawl-delay:")
-          delay = line.split(":", 2).last.strip.to_i
-          current_section[:delay] = delay if delay.positive?
-        end
-      end
-
-      # Sort sections by most specific agent match first
-      matched_section = sections.find do |section|
-        section[:agents].any? do |agent|
-          # Our user agent starts with the agent from robots.txt
-          @user_agent.start_with?(agent) ||
-            # Or the agent from robots.txt starts with our user agent
-            # (handles ScraperUtils matching ScraperUtils/1.0)
-            agent.start_with?(@user_agent)
-        end
-      end
-
-      # Use matched section or fall back to wildcard
-      if matched_section
-        {
-          our_rules: matched_section[:rules],
-          our_delay: matched_section[:delay]
-        }
-      else
-        # Find default section
-        default_section = sections.find { |s| s[:agents].include?("*") }
-        {
-          our_rules: [],
-          our_delay: default_section&.dig(:delay)
-        }
-      end
-    end
-  end
-end