ligamagic-scraper 0.6.0

data/lib/ligamagic_scraper/loggable.rb

@@ -0,0 +1,43 @@
+module LigaMagicScraper
+  module Loggable
+    attr_reader :logs
+
+    def initialize_logs
+      @logs = []
+    end
+
+    def log(message, level: :info)
+      @logs << {
+        timestamp: Time.now,
+        level: level,
+        message: message,
+        source: self.class.name
+      }
+    end
+
+    def log_info(message)
+      log(message, level: :info)
+    end
+
+    def log_debug(message)
+      log(message, level: :debug)
+    end
+
+    def log_warning(message)
+      log(message, level: :warning)
+    end
+
+    def log_error(message)
+      log(message, level: :error)
+    end
+
+    def formatted_logs
+      @logs.map { |entry| entry[:message] }
+    end
+
+    def clear_logs
+      @logs = []
+    end
+  end
+end
+

data/lib/ligamagic_scraper/scrapers/base_scraper.rb

@@ -0,0 +1,126 @@
+module LigaMagicScraper
+  class BaseScraper
+    include Capybara::DSL
+    include Loggable
+
+    attr_reader :browser_mode, :alert_system
+
+    def initialize(browser_mode: 'headed', alert_config: nil)
+      @browser_mode = browser_mode
+      @alert_system = AlertSystem.new(alert_config || {}) if alert_config
+      initialize_logs
+      configure_browser
+    end
+
+    def configure_browser
+      case browser_mode.downcase
+      when 'headless'
+        Capybara.register_driver :selenium_chrome_headless do |app|
+          options = Selenium::WebDriver::Chrome::Options.new
+          options.add_argument('--headless')
+          options.add_argument('--disable-gpu')
+          options.add_argument('--no-sandbox')
+          options.add_argument('--disable-dev-shm-usage')
+          
+          Capybara::Selenium::Driver.new(app, browser: :chrome, options: options)
+        end
+        Capybara.default_driver = :selenium_chrome_headless
+      when 'headed'
+        Capybara.default_driver = :selenium_chrome
+      else
+        log_warning("⚠️  Unknown browser mode '#{browser_mode}', using default (headed)")
+        Capybara.default_driver = :selenium_chrome
+      end
+    end
+
+    def generate_slug(name)
+      return nil if name.nil? || name.empty?
+      
+      # Transliterate accented characters to ASCII
+      slug = name.downcase
+                 .tr('áàãâäåāăąǎǟǡǻȁȃȧ', 'a')
+                 .tr('éèêëēĕėęěȅȇȩ', 'e')
+                 .tr('íìîïĩīĭįıȉȋ', 'i')
+                 .tr('óòôõöōŏőơǒǿȍȏȫȭȯȱ', 'o')
+                 .tr('úùûüũūŭůűųưȕȗ', 'u')
+                 .tr('çćĉċč', 'c')
+                 .tr('ñńņňʼn', 'n')
+                 .gsub(/[^a-z0-9]+/, '_')
+                 .gsub(/^_+|_+$/, '')
+      
+      slug
+    end
+
+    def parse_price(price_text)
+      return nil if price_text.nil? || price_text.empty?
+      
+      price_text.gsub(/R\$\s*/, '')
+                .gsub(/\./, '')
+                .gsub(/,/, '.')
+                .to_f
+    rescue
+      nil
+    end
+
+    def save_to_json(products)
+      filename = generate_filename
+      
+      dir = File.dirname(filename)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      
+      data = build_json_data(products)
+      
+      if @alert_system
+        previous_file = find_previous_scrape(filename)
+        @alert_system.process(current_data: data, previous_file:)
+      end
+      
+      File.write(filename, JSON.pretty_generate(data))
+      log_info("💾 Results saved to: #{filename}")
+      filename
+    end
+
+    def find_previous_scrape(current_filename)
+      dir = File.dirname(current_filename)
+      
+      # Extract the slug pattern from current filename (e.g., "__booster_box.json")
+      basename = File.basename(current_filename)
+      slug_pattern = basename.match(/__(.+)\.json$/)
+      
+      return nil unless slug_pattern
+      
+      slug = slug_pattern[1]
+      
+      # Get all JSON files with the same slug pattern, sorted by name (chronological)
+      matching_files = Dir.glob(File.join(dir, "*__#{slug}.json")).sort.reverse
+      
+      # Return the first file that's not the current one (most recent previous)
+      matching_files.find { |f| f != current_filename }
+    end
+
+    # Override in subclasses
+    def generate_filename
+      raise NotImplementedError, "Subclasses must implement generate_filename"
+    end
+
+    # Override in subclasses
+    def build_json_data(products)
+      raise NotImplementedError, "Subclasses must implement build_json_data"
+    end
+
+    # Override in subclasses
+    def scrape
+      raise NotImplementedError, "Subclasses must implement scrape"
+    end
+
+    def close_browser
+      log_info("🔒 Closing browser...")
+      begin
+        Capybara.current_session.driver.quit
+      rescue => e
+        log_error("⚠️  Error closing browser: #{e.message}")
+      end
+    end
+  end
+end
+

data/lib/ligamagic_scraper/scrapers/global_scraper.rb

@@ -0,0 +1,240 @@
+require_relative 'base_scraper'
+require 'nokogiri'
+require 'benchmark'
+require 'set'
+
+module LigaMagicScraper
+  class GlobalScraper < BaseScraper
+    BASE_URL = 'https://www.ligamagic.com.br/?view=cards%2Fsearch&tipo=1'
+    MAX_CLICKS = 50
+
+    attr_reader :search_term
+
+    def initialize(search_term:, browser_mode: 'headed', alert_config: nil)
+      @search_term = search_term
+      @product_html_snapshots = []
+      @timings = {}
+      super(browser_mode:, alert_config:)
+      
+      log_info("🚀 Starting Liga Magic global search scraper...")
+      log_info("🔍 Search term: #{search_term}")
+      log_info("🖥️  Browser mode: #{browser_mode}")
+    end
+
+    def url
+      "#{BASE_URL}&card=#{CGI.escape(search_term)}"
+    end
+
+    def scrape
+      start_time = Time.now
+      
+      visit url
+      log_info("📄 Loaded initial page")
+      
+      # Phase 1: Load all pages (browser open, no capturing)
+      load_time = Benchmark.measure do
+        load_all_products
+      end
+      @timings[:loading] = load_time.real
+      log_info("⏱️  Phase 1 (Loading): #{format('%.2f', load_time.real)}s")
+      
+      # Phase 2: Capture all products at once (browser still open)
+      capture_time = Benchmark.measure do
+        capture_all_products
+      end
+      @timings[:capture] = capture_time.real
+      log_info("⏱️  Phase 2 (Capture): #{format('%.2f', capture_time.real)}s")
+      
+      # Close browser ASAP
+      close_time = Benchmark.measure do
+        close_browser
+      end
+      @timings[:browser_close] = close_time.real
+      log_info("✅ Browser closed in #{format('%.2f', close_time.real)}s")
+      
+      # Phase 3: Extract products from memory (browser closed)
+      log_info("🔍 Extracting products from memory...")
+      products = nil
+      extraction_time = Benchmark.measure do
+        products = extract_products
+      end
+      @timings[:extraction] = extraction_time.real
+      log_info("⏱️  Phase 3 (Extraction): #{format('%.2f', extraction_time.real)}s")
+      
+      total_time = Time.now - start_time
+      @timings[:total] = total_time
+      
+      log_info("=" * 60)
+      log_info("📊 PERFORMANCE SUMMARY")
+      log_info("=" * 60)
+      log_info("⏱️  Loading (browser open):           #{format('%.2f', @timings[:loading])}s")
+      log_info("⏱️  Capture (browser open):           #{format('%.2f', @timings[:capture])}s")
+      log_info("⏱️  Browser Close:                    #{format('%.2f', @timings[:browser_close])}s")
+      log_info("⏱️  Extraction (browser closed):      #{format('%.2f', @timings[:extraction])}s")
+      log_info("⏱️  Total Time:                       #{format('%.2f', @timings[:total])}s")
+      log_info("📦 Products captured:                 #{@product_html_snapshots.count}")
+      log_info("✅ Valid products extracted:          #{products.count}")
+      log_info("=" * 60)
+      
+      products
+    rescue => e
+      log_error("❌ Error during scraping: #{e.message}")
+      log_debug(e.backtrace.first(5).join("\n"))
+      []
+    ensure
+      close_browser if @driver
+    end
+
+    def generate_filename
+      datetime_str = Time.now.strftime('%Y%m%d_%H%M%S')
+      slug = search_term.downcase.gsub(/[^a-z0-9]+/, '_').gsub(/^_|_$/, '')
+      "scrapped/global/#{datetime_str}__#{slug}.json"
+    end
+
+    def build_json_data(products)
+      {
+        search_term:,
+        search_type: 'global',
+        scraped_at: Time.now.iso8601,
+        total_products: products.count,
+        products:
+      }
+    end
+
+    private
+
+    def load_all_products
+      log_info("🔄 Starting pagination...")
+      click_count = 0
+      
+      loop do
+        if has_unavailable_products?
+          log_warning("⚠️  Found unavailable products, stopping pagination")
+          break
+        end
+
+        load_more_button = find_load_more_button
+        
+        if load_more_button.nil?
+          log_info("✅ No more 'Load More' button found")
+          break
+        end
+
+        if click_count >= MAX_CLICKS
+          log_warning("⚠️  Reached maximum click limit (#{MAX_CLICKS})")
+          break
+        end
+
+        begin
+          load_more_button.click
+          click_count += 1
+          log_debug("🔄 Clicked 'Load More' button (#{click_count}x), waiting for new products...")
+          sleep 1
+        rescue => e
+          log_error("❌ Error clicking 'Load More': #{e.message}")
+          break
+        end
+      end
+      
+      log_info("✅ Pagination complete. Total clicks: #{click_count}")
+    end
+
+    def has_unavailable_products?
+      page.has_text?(/indispon[ií]vel|esgotado/i, wait: 1)
+    end
+
+    def find_load_more_button
+      begin
+        element = page.find('input.exibir-mais[value="Exibir mais"]', visible: true, wait: 1)
+        return element if element
+      rescue Capybara::ElementNotFound
+        nil
+      end
+    end
+
+    def capture_all_products
+      log_info("📸 Capturing all products...")
+      # Find all product elements in one go and capture their outer HTML
+      product_elements = page.all('div.box.p25 div.mtg-single', wait: 5)
+      product_elements.each do |element|
+        @product_html_snapshots << element[:outerHTML]
+      end
+      log_info("📸 Captured #{@product_html_snapshots.count} product HTML snapshots")
+    end
+
+    def extract_products
+      log_info("🔍 Extracting product data from #{@product_html_snapshots.count} HTML snapshots...")
+      products = []
+      seen_ids = Set.new
+      
+      @product_html_snapshots.each_with_index do |html, index|
+        log_debug(".") if (index + 1) % 10 == 0
+        
+        begin
+          doc = Nokogiri::HTML(html)
+          
+          # Check availability first
+          text_content = doc.text.downcase
+          next if text_content.match?(/indispon[ií]vel|esgotado|sem estoque/)
+          
+          name_data = extract_product_name_and_id_from_doc(doc)
+          next unless name_data
+          
+          # Skip duplicates (website shows same products on multiple pages)
+          next if seen_ids.include?(name_data[:id])
+          seen_ids.add(name_data[:id])
+          
+          prices = extract_product_prices_from_doc(doc)
+          next unless prices
+          
+          slug = generate_slug(name_data[:name])
+          
+          products << {
+            id: name_data[:id],
+            slug:,
+            name: name_data[:name],
+            min_price: prices[:min_price],
+            avg_price: prices[:avg_price],
+            max_price: prices[:max_price]
+          }
+        rescue => e
+          log_warning("⚠️  Error extracting product #{index + 1}: #{e.message}")
+        end
+      end
+      
+      log_info("✅ Extraction complete. Valid products: #{products.count} (from #{@product_html_snapshots.count} snapshots)")
+      products
+    end
+
+    def extract_product_name_and_id_from_doc(doc)
+      link = doc.at_css('.mtg-names .mtg-name-prod a')
+      return nil unless link
+      
+      href = link['href']
+      match = href&.match(/pcode=(\d+)/)
+      
+      name = link.text.strip
+      id = match ? match[1] : nil
+      
+      return nil if name.empty?
+      
+      { name:, id: }
+    end
+
+    def extract_product_prices_from_doc(doc)
+      price_unit = doc.at_css('.mtg-prices .mtg-price-unit')
+      return nil unless price_unit
+      
+      min_price_el = price_unit.at_css('.price-min')
+      avg_price_el = price_unit.at_css('.price-avg')
+      max_price_el = price_unit.at_css('.price-max')
+      
+      {
+        min_price: parse_price(min_price_el&.text&.strip),
+        avg_price: parse_price(avg_price_el&.text&.strip),
+        max_price: parse_price(max_price_el&.text&.strip)
+      }
+    end
+  end
+end
+