ligamagic-scraper 0.6.0

data/lib/ligamagic_scraper/scrapers/store_scraper.rb

@@ -0,0 +1,392 @@
+require_relative 'base_scraper'
+require 'nokogiri'
+require 'benchmark'
+require 'set'
+
+module LigaMagicScraper
+  class StoreScraper < BaseScraper
+    attr_reader :store_domain, :store_url, :search_term, :max_pages
+
+    def initialize(store_domain:, search_term: nil, max_pages: nil, browser_mode: 'headed', alert_config: nil)
+      @store_domain = store_domain
+      @search_term = search_term
+      @max_pages = max_pages
+      @store_url = build_store_url(store_domain, search_term)
+      @product_html_snapshots = []
+      @timings = {}
+      
+      # Validate: max_pages required when no search term
+      if search_term.nil? && max_pages.nil?
+        raise ArgumentError, "max_pages is required when listing store products without a search term"
+      end
+      
+      super(browser_mode:, alert_config:)
+      
+      log_info("🚀 Starting Liga Magic Store scraper...")
+      log_info("🏪 Store domain: #{store_domain}")
+      log_info("🔍 Search term: #{search_term}") if search_term
+      log_info("📄 Max pages: #{max_pages}") if max_pages
+      log_info("🔗 Store URL: #{@store_url}")
+      log_info("🖥️  Browser mode: #{browser_mode}")
+    end
+
+    def scrape
+      start_time = Time.now
+      log_info("📄 Starting pagination...")
+      
+      # Phase 1: Load all pages and capture HTML (browser open)
+      load_time = Benchmark.measure do
+        load_all_pages
+      end
+      @timings[:loading_and_capture] = load_time.real
+      log_info("⏱️  Phase 1 (Loading + Capture): #{format('%.2f', load_time.real)}s")
+      
+      # Close browser ASAP
+      close_time = Benchmark.measure do
+        close_browser
+      end
+      @timings[:browser_close] = close_time.real
+      log_info("✅ Browser closed in #{format('%.2f', close_time.real)}s")
+      
+      # Phase 2: Extract products from memory (browser closed)
+      log_info("🔍 Extracting products from memory...")
+      all_products = nil
+      extraction_time = Benchmark.measure do
+        all_products = extract_products_from_html
+      end
+      @timings[:extraction] = extraction_time.real
+      log_info("⏱️  Phase 2 (Extraction): #{format('%.2f', extraction_time.real)}s")
+      
+      total_time = Time.now - start_time
+      @timings[:total] = total_time
+      
+      log_info("=" * 60)
+      log_info("📊 PERFORMANCE SUMMARY")
+      log_info("=" * 60)
+      log_info("⏱️  Loading + Capture (browser open): #{format('%.2f', @timings[:loading_and_capture])}s")
+      log_info("⏱️  Browser Close:                    #{format('%.2f', @timings[:browser_close])}s")
+      log_info("⏱️  Extraction (browser closed):      #{format('%.2f', @timings[:extraction])}s")
+      log_info("⏱️  Total Time:                       #{format('%.2f', @timings[:total])}s")
+      log_info("📦 Products captured:                 #{@product_html_snapshots.count}")
+      log_info("✅ Valid products extracted:          #{all_products.count}")
+      log_info("=" * 60)
+      
+      all_products
+    rescue => e
+      log_error("❌ Error during scraping: #{e.message}")
+      log_debug(e.backtrace.first(5).join("\n"))
+      []
+    ensure
+      close_browser if @driver
+    end
+
+    def generate_filename
+      datetime_str = Time.now.strftime('%Y%m%d_%H%M%S')
+      store_slug = generate_slug(@store_domain)
+      
+      if @search_term && !@search_term.empty?
+        search_slug = generate_slug(@search_term)
+        "scrapped/stores/#{store_slug}/#{datetime_str}__#{search_slug}.json"
+      else
+        "scrapped/stores/#{store_slug}/#{datetime_str}.json"
+      end
+    end
+
+    def build_json_data(products)
+      data = {
+        store_domain:,
+        store_url:,
+        search_type: 'store',
+        scraped_at: Time.now.iso8601,
+        total_products: products.count,
+        products:
+      }
+      
+      data[:search_term] = search_term if search_term && !search_term.empty?
+      data[:max_pages] = max_pages if max_pages
+      data
+    end
+
+    private
+    
+    # ============================================================================
+    # OBFUSCATED PRICE/QUANTITY EXTRACTION (Currently Not Implemented)
+    # ============================================================================
+    #
+    # When using store search with a search term (-u STORE -s TERM), Liga Magic
+    # employs sophisticated anti-scraping protection for prices and quantities:
+    #
+    # 1. CSS CLASS OBFUSCATION:
+    #    - Digit values are encoded using randomized CSS class names
+    #    - Example: <div class="qYlMh mImKn lJcCw">&nbsp;</div>
+    #    - Classes change with each page load/session
+    #
+    # 2. SPRITE-BASED RENDERING:
+    #    - Digits are rendered using CSS background-position from a sprite image
+    #    - Example CSS: .lJcCw{background-position:-488px -2px;}
+    #    - The sprite image URL also rotates per session
+    #
+    # 3. DYNAMIC MAPPING:
+    #    - Both class names AND background positions change between sessions
+    #    - Example: Session 1: .bPzEo{-216px -2px} → '0'
+    #              Session 2: .nLwKv{-392px -65px} → '0'
+    #    - No text content in DOM (innerText/textContent return empty strings)
+    #
+    # ATTEMPTED SOLUTIONS:
+    # - Static CSS mapping: Failed (classes/positions rotate)
+    # - JavaScript DOM extraction: Failed (no text content, purely visual)
+    # - CSS parsing: Partial (can extract mapping structure but positions change)
+    #
+    # POTENTIAL SOLUTIONS (Not Implemented):
+    # - Download sprite image and use OCR/image analysis (requires external gems)
+    # - Use image template matching at background-position coordinates
+    # - Analyze sprite pixel data to identify digits
+    #
+    # CURRENT BEHAVIOR:
+    # - Store listings (no search term): Price/qty extracted normally ✓
+    # - Store searches (with search term): Price/qty set to nil (cards extracted without pricing)
+    #
+    # ============================================================================
+    
+    def extract_obfuscated_price(product)
+      # Placeholder for future implementation
+      # Would require sprite image download and analysis
+      nil
+    end
+    
+    def extract_obfuscated_quantity(product)
+      # Placeholder for future implementation  
+      # Would require sprite image download and analysis
+      nil
+    end
+    
+
+    def build_store_url(domain, search_term = nil)
+      # Build URL: https://www.<domain>.com.br/?view=ecom/itens&tcg=1
+      base_domain = domain.include?('.') ? domain : "#{domain}.com.br"
+      base_domain = "www.#{base_domain}" unless base_domain.start_with?('www.')
+      
+      url = "https://#{base_domain}/?view=ecom/itens"
+      
+      if search_term && !search_term.empty?
+        # Add search parameter
+        url += "&busca=#{CGI.escape(search_term)}"
+      else
+        # Keep tcg=1 for non-search listing
+        url += "&tcg=1"
+      end
+      
+      # Add ordering by price (most expensive to cheapest)
+      url += "&txt_order=6"
+      
+      # Add filter for only in-stock items
+      url += "&txt_estoque=1"
+      
+      url
+    end
+    
+    def build_page_url(page_number)
+      return @store_url if page_number == 1
+      
+      # Add page parameter
+      separator = @store_url.include?('?') ? '&' : '?'
+      "#{@store_url}#{separator}page=#{page_number}"
+    end
+    
+    def should_continue_pagination?(current_page)
+      # If max_pages is set (no search term), check limit
+      if @max_pages
+        if current_page >= @max_pages
+          log_info("📄 Reached max pages limit (#{@max_pages})")
+          return false
+        end
+      end
+      
+      # Check if there's a next page button
+      has_next_page?
+    end
+    
+    def has_next_page?
+      # Look for next page link or button with &gt; (>)
+      # Use wait: 0 to avoid long waits when pagination doesn't exist
+      pagination_links = page.all('a.ecomresp-paginacao', wait: 0)
+      
+      if pagination_links.empty?
+        log_info("   ℹ️  No pagination found (single page)")
+        return false
+      end
+      
+      next_button = pagination_links.find do |link|
+        link.text.strip == '>' || link.text.include?('&gt;')
+      end
+      
+      if next_button
+        log_debug("   ✓ Next page button found")
+        true
+      else
+        log_info("   ℹ️  No more pages available")
+        false
+      end
+    rescue => e
+      log_debug("   ⚠️  Error checking for next page: #{e.message}")
+      false
+    end
+
+    def load_all_pages
+      log_info("🔄 Loading all pages and capturing HTML...")
+      current_page = 1
+      
+      loop do
+        log_info("📄 Loading page #{current_page}...")
+        page_url = build_page_url(current_page)
+        visit page_url
+        
+        # Capture products from this page
+        product_elements = page.all('.card-item', wait: 3)
+        if product_elements.empty?
+          log_warning("⚠️  No products found on page #{current_page}")
+          break
+        end
+        
+        # Capture HTML from this page
+        product_elements.each do |element|
+          @product_html_snapshots << element[:outerHTML]
+        end
+        log_info("   Captured #{product_elements.count} products from page #{current_page} (total: #{@product_html_snapshots.count})")
+        
+        # Check if we should continue to next page
+        if should_continue_pagination?(current_page)
+          current_page += 1
+          sleep 1  # Be nice to the server
+        else
+          break
+        end
+      end
+      
+      log_info("✅ Loading complete. Total pages: #{current_page}, Total products captured: #{@product_html_snapshots.count}")
+    end
+
+    def capture_all_products
+      # This method is now called from load_all_pages incrementally
+      # Keeping it here for compatibility but it won't be used in the optimized flow
+      log_info("📸 Capturing remaining products (if any)...")
+      product_elements = page.all('.card-item', wait: 1)
+      if product_elements.any?
+        product_elements.each do |element|
+          @product_html_snapshots << element[:outerHTML]
+        end
+        log_info("📸 Captured #{product_elements.count} additional products")
+      end
+      log_info("📸 Total snapshots: #{@product_html_snapshots.count}")
+    end
+
+    def extract_products_from_html
+      log_info("🔍 Extracting product data from #{@product_html_snapshots.count} HTML snapshots...")
+      products = []
+      seen_ids = Set.new
+      
+      # Warn if using search term (price/qty won't be extracted)
+      if @search_term && !@search_term.empty?
+        log_warning("⚠️  Search term detected - price/qty extraction disabled (CSS obfuscation)")
+      end
+      
+      @product_html_snapshots.each_with_index do |html, index|
+        log_debug(".") if (index + 1) % 10 == 0
+        
+        begin
+          doc = Nokogiri::HTML(html)
+          product_data = extract_product_data_from_doc(doc)
+          
+          if product_data
+            # Skip duplicates
+            next if seen_ids.include?(product_data[:card_id])
+            seen_ids.add(product_data[:card_id]) if product_data[:card_id]
+            
+            products << product_data
+          end
+        rescue => e
+          log_warning("⚠️  Error extracting product #{index + 1}: #{e.message}")
+        end
+      end
+      
+      log_info("✅ Extraction complete. Valid products: #{products.count} (from #{@product_html_snapshots.count} snapshots)")
+      products
+    end
+
+    def extract_product_data_from_doc(doc)
+      name = extract_product_name_from_doc(doc)
+      
+      if name.nil? || name.empty?
+        log_debug("   ⚠️  Skipping product: no name found")
+        return nil
+      end
+      
+      card_id = extract_card_id_from_doc(doc)
+      
+      # Price and quantity extraction depends on search mode
+      if @search_term && !@search_term.empty?
+        # When search term is provided, Liga Magic uses CSS obfuscation for prices/quantities
+        price = nil
+        qtd = nil
+        available = nil
+      else
+        # Store listings (without search) have normal HTML structure
+        price = extract_product_price_from_doc(doc)
+        qtd = extract_quantity_from_doc(doc)
+        available = qtd && qtd > 0  # Available if has quantity
+      end
+      
+      slug = generate_slug(name)
+      
+      {card_id:, name:, slug:, price:, qtd:, available:}
+    end
+
+    def extract_card_id_from_doc(doc)
+      link = doc.at_css('.card-desc .title a')
+      return nil unless link
+      
+      href = link['href']
+      return nil unless href
+      
+      match = href.match(/[?&]card=(\d+)/)
+      match ? match[1] : nil
+    rescue => e
+      log_debug("   ⚠️  Error extracting card ID: #{e.message}")
+      nil
+    end
+
+    def extract_product_name_from_doc(doc)
+      link = doc.at_css('.card-desc .title a')
+      return nil unless link
+      
+      link.text.strip
+    rescue => e
+      log_debug("   ⚠️  Error extracting name: #{e.message}")
+      nil
+    end
+
+    def extract_product_price_from_doc(doc)
+      # For store listings (no search term), prices are in plain text
+      price_element = doc.at_css('.card-desc .price .align-price')
+      return nil unless price_element
+      
+      price_text = price_element.text.strip
+      parse_price(price_text)
+    rescue => e
+      log_debug("   ⚠️  Error extracting price: #{e.message}")
+      nil
+    end
+
+    def extract_quantity_from_doc(doc)
+      # For store listings (no search term), quantities are in plain text spans
+      qty_element = doc.at_css('.card-desc .qty span')
+      return nil unless qty_element
+      
+      qty_text = qty_element.text.strip
+      qty_text.to_i
+    rescue => e
+      log_debug("   ⚠️  Error extracting quantity: #{e.message}")
+      nil
+    end
+  end
+end

data/lib/ligamagic_scraper/version.rb

@@ -0,0 +1,4 @@
+module LigaMagicScraper
+  VERSION = "0.6.0"
+end
+

data/lib/ligamagic_scraper.rb

@@ -0,0 +1,18 @@
+require 'capybara'
+require 'capybara/dsl'
+require 'selenium-webdriver'
+require 'json'
+require 'date'
+require 'cgi'
+require 'uri'
+require 'fileutils'
+
+require_relative 'ligamagic_scraper/version'
+require_relative 'ligamagic_scraper/loggable'
+require_relative 'ligamagic_scraper/scrapers/base_scraper'
+require_relative 'ligamagic_scraper/scrapers/global_scraper'
+require_relative 'ligamagic_scraper/scrapers/store_scraper'
+require_relative 'ligamagic_scraper/alerts/alert_system'
+require_relative 'ligamagic_scraper/cli'
+
+Capybara.default_max_wait_time = 5

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: ligamagic-scraper
+version: !ruby/object:Gem::Version
+  version: 0.6.0
+platform: ruby
+authors:
+- Vinicius Kammradt
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-11-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.40'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.40'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.15'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+description: A Ruby gem to scrape card prices and information from ligamagic.com.br
+email:
+- vinicius.kammradt1@gmail.com
+executables:
+- ligamagic-scraper
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/ligamagic-scraper
+- lib/ligamagic_scraper.rb
+- lib/ligamagic_scraper/alerts/alert_system.rb
+- lib/ligamagic_scraper/alerts/base_alert.rb
+- lib/ligamagic_scraper/alerts/file_alert.rb
+- lib/ligamagic_scraper/alerts/telegram_alert.rb
+- lib/ligamagic_scraper/cli.rb
+- lib/ligamagic_scraper/loggable.rb
+- lib/ligamagic_scraper/scrapers/base_scraper.rb
+- lib/ligamagic_scraper/scrapers/global_scraper.rb
+- lib/ligamagic_scraper/scrapers/store_scraper.rb
+- lib/ligamagic_scraper/version.rb
+homepage: https://github.com/kammradt/ligamagic-scrapper
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/kammradt/ligamagic-scrapper
+  source_code_uri: https://github.com/kammradt/ligamagic-scrapper
+  changelog_uri: https://github.com/kammradt/ligamagic-scrapper/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: A web scraper for Liga Magic product prices
+test_files: []