ligamagic-scraper 0.6.0

data/Rakefile

@@ -0,0 +1,121 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require_relative "lib/ligamagic_scraper/version"
+
+RSpec::Core::RakeTask.new(:spec)
+
+desc "Show current version"
+task :version do
+  puts "Current version: #{LigaMagicScraper::VERSION}"
+end
+
+desc "Bump patch version (0.1.0 -> 0.1.1)"
+task :bump_patch do
+  bump_version(:patch)
+end
+
+desc "Bump minor version (0.1.0 -> 0.2.0)"
+task :bump_minor do
+  bump_version(:minor)
+end
+
+desc "Bump major version (0.1.0 -> 1.0.0)"
+task :bump_major do
+  bump_version(:major)
+end
+
+desc "Build and install gem locally"
+task :install_local do
+  Rake::Task["clean"].invoke
+  
+  puts "🔨 Building gem..."
+  system("gem build ligamagic-scraper.gemspec")
+  
+  gem_file = Dir["ligamagic-scraper-*.gem"].first
+  
+  if gem_file
+    puts "📦 Installing #{gem_file}..."
+    system("gem install #{gem_file}")
+    puts "✅ Gem installed successfully!"
+    puts "   You can now use: ligamagic-scraper -s 'search term'"
+  else
+    puts "❌ Failed to build gem"
+    exit 1
+  end
+end
+
+desc "Clean built gems"
+task :clean do
+  puts "🧹 Cleaning old gem files..."
+  Dir["*.gem"].each do |gem_file|
+    File.delete(gem_file)
+    puts "   Deleted: #{gem_file}"
+  end
+end
+
+desc "Build gem without installing"
+task :build do
+  Rake::Task["clean"].invoke
+  puts "🔨 Building gem..."
+  system("gem build ligamagic-scraper.gemspec")
+  gem_file = Dir["ligamagic-scraper-*.gem"].first
+  puts "✅ Built: #{gem_file}" if gem_file
+end
+
+desc "Release new version (bump patch, build, install)"
+task :release_patch do
+  bump_version(:patch)
+  Rake::Task["install_local"].invoke
+end
+
+desc "Release new minor version (bump minor, build, install)"
+task :release_minor do
+  bump_version(:minor)
+  Rake::Task["install_local"].invoke
+end
+
+desc "Release new major version (bump major, build, install)"
+task :release_major do
+  bump_version(:major)
+  Rake::Task["install_local"].invoke
+end
+
+desc "Uninstall the gem"
+task :uninstall do
+  puts "🗑️  Uninstalling ligamagic-scraper..."
+  system("gem uninstall ligamagic-scraper -x")
+  puts "✅ Gem uninstalled"
+end
+
+def bump_version(type)
+  version_file = "lib/ligamagic_scraper/version.rb"
+  content = File.read(version_file)
+  
+  current_version = LigaMagicScraper::VERSION
+  parts = current_version.split('.').map(&:to_i)
+  
+  case type
+  when :patch
+    parts[2] += 1
+  when :minor
+    parts[1] += 1
+    parts[2] = 0
+  when :major
+    parts[0] += 1
+    parts[1] = 0
+    parts[2] = 0
+  end
+  
+  new_version = parts.join('.')
+  
+  new_content = content.gsub(/VERSION = "#{Regexp.escape(current_version)}"/, 
+                              "VERSION = \"#{new_version}\"")
+  
+  File.write(version_file, new_content)
+  
+  puts "📈 Version bumped: #{current_version} -> #{new_version}"
+  new_version
+end
+
+task default: :spec
+

data/bin/ligamagic-scraper

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/ligamagic_scraper'
+
+begin
+  cli = LigaMagicScraper::CLI.new.parse
+  scraper = cli.create_scraper
+  
+  results = scraper.scrape
+  scraper.save_to_json(results)
+  
+  scraper.formatted_logs.each { |msg| puts msg }
+  
+  if scraper.alert_system
+    scraper.alert_system.formatted_logs.each { |msg| puts msg }
+  end
+  
+  puts "✅ Scraping completed successfully!"
+  puts "👋 Exiting..."
+rescue Interrupt
+  puts "⚠️  Interrupted by user (Ctrl+C)"
+  puts "🔒 Cleaning up..."
+ensure
+  Capybara.reset_sessions!
+end
+
+exit(0)
+

data/lib/ligamagic_scraper/alerts/alert_system.rb

@@ -0,0 +1,218 @@
+require 'json'
+require_relative 'base_alert'
+require_relative 'file_alert'
+require_relative 'telegram_alert'
+
+module LigaMagicScraper
+  class AlertSystem
+    include Loggable
+    
+    attr_reader :config, :alerts
+
+    def initialize(config = {})
+      initialize_logs
+      @config = {
+        enabled: config[:enabled] || false,
+        alert_types: config[:alert_types] || [:file],
+        compare_previous: config[:compare_previous] != false
+      }
+      @alerts = []
+      
+      setup_alerts if @config[:enabled]
+    end
+
+    def process(current_data:, previous_file: nil)
+      return unless @config[:enabled]
+      
+      log_info("🔔 Alert system enabled")
+      
+      previous_data = load_previous_data(previous_file) if @config[:compare_previous]
+      
+      if previous_data
+        log_info("📊 Comparing with previous scrape...")
+        changes = detect_changes(previous_data, current_data)
+        
+        if changes[:has_changes]
+          log_info("✨ Changes detected!")
+          notify_all(changes)
+        else
+          log_info("✅ No changes detected")
+        end
+      else
+        log_info("ℹ️  No previous data to compare")
+      end
+    end
+
+    def detect_changes(previous_data, current_data)
+      changes = {
+        has_changes: false,
+        timestamp: Time.now.iso8601,
+        search_type: current_data[:search_type],
+        search_term: current_data[:search_term] || current_data[:store_domain],
+        new_products: [],
+        removed_products: [],
+        price_changes: [],
+        quantity_changes: [],
+        availability_changes: []
+      }
+
+      prev_products = index_products(previous_data[:products])
+      curr_products = index_products(current_data[:products])
+
+      # Detect new products
+      curr_products.each do |id, product|
+        unless prev_products.key?(id)
+          changes[:new_products] << product
+          changes[:has_changes] = true
+        end
+      end
+
+      # Detect removed products
+      prev_products.each do |id, product|
+        unless curr_products.key?(id)
+          changes[:removed_products] << product
+          changes[:has_changes] = true
+        end
+      end
+
+      # Detect price changes
+      curr_products.each do |id, curr_product|
+        if prev_products.key?(id)
+          prev_product = prev_products[id]
+          
+          # Check price changes (works for both global and store scrapers)
+          price_change = detect_price_change(prev_product, curr_product)
+          if price_change
+            changes[:price_changes] << price_change
+            changes[:has_changes] = true
+          end
+
+          # Check quantity changes (store scraper)
+          if curr_product.key?(:qtd) && prev_product.key?(:qtd)
+            prev_qtd = prev_product[:qtd]
+            curr_qtd = curr_product[:qtd]
+            
+            if prev_qtd != curr_qtd
+              changes[:quantity_changes] << {
+                id:,
+                name: curr_product[:name],
+                previous_qtd: prev_qtd,
+                current_qtd: curr_qtd,
+                change: curr_qtd.to_i - prev_qtd.to_i
+              }
+              changes[:has_changes] = true
+            end
+          end
+
+          # Check availability changes (store scraper)
+          if curr_product.key?(:available) && prev_product.key?(:available)
+            if curr_product[:available] != prev_product[:available]
+              changes[:availability_changes] << {
+                id:,
+                name: curr_product[:name],
+                previous: prev_product[:available],
+                current: curr_product[:available]
+              }
+              changes[:has_changes] = true
+            end
+          end
+        end
+      end
+
+      changes
+    end
+
+    private
+
+    def setup_alerts
+      @config[:alert_types].each do |type|
+        alert = create_alert(type)
+        @alerts << alert if alert
+      end
+      
+      log_info("📢 Configured alerts: #{@alerts.map(&:class).map(&:name).join(', ')}")
+    end
+
+    def create_alert(type)
+      case type
+      when :file
+        Alerts::FileAlert.new
+      when :telegram
+        Alerts::TelegramAlert.new
+      else
+        log_warning("⚠️  Unknown alert type: #{type}")
+        nil
+      end
+    end
+
+    def load_previous_data(filename)
+      return nil unless filename && File.exist?(filename)
+      
+      JSON.parse(File.read(filename), symbolize_names: true)
+    rescue => e
+      log_error("⚠️  Error loading previous data: #{e.message}")
+      nil
+    end
+
+    def index_products(products)
+      indexed = {}
+      products.each do |product|
+        # Use card_id if available (store scraper), otherwise use id (global scraper)
+        id = product[:card_id] || product[:id] || product[:slug]
+        indexed[id] = product if id
+      end
+      indexed
+    end
+
+    def detect_price_change(prev_product, curr_product)
+      # For global scraper (has min/avg/max prices)
+      if curr_product.key?(:min_price)
+        prev_min = prev_product[:min_price]
+        curr_min = curr_product[:min_price]
+        
+        if prev_min && curr_min && prev_min != curr_min
+          return {
+            id: curr_product[:id],
+            name: curr_product[:name],
+            previous_min: prev_min,
+            current_min: curr_min,
+            change: curr_min - prev_min,
+            change_percent: ((curr_min - prev_min) / prev_min * 100).round(2)
+          }
+        end
+      end
+
+      # For store scraper (has single price)
+      if curr_product.key?(:price)
+        prev_price = prev_product[:price]
+        curr_price = curr_product[:price]
+        
+        if prev_price && curr_price && prev_price != curr_price
+          return {
+            id: curr_product[:card_id],
+            name: curr_product[:name],
+            previous_price: prev_price,
+            current_price: curr_price,
+            change: curr_price - prev_price,
+            change_percent: ((curr_price - prev_price) / prev_price * 100).round(2)
+          }
+        end
+      end
+
+      nil
+    end
+
+    def notify_all(changes)
+      @alerts.each do |alert|
+        begin
+          alert.notify(changes)
+          # Merge alert logs into system logs
+          @logs.concat(alert.logs) if alert.respond_to?(:logs)
+        rescue => e
+          log_error("❌ Error sending alert via #{alert.class.name}: #{e.message}")
+        end
+      end
+    end
+  end
+end
+

data/lib/ligamagic_scraper/alerts/base_alert.rb

@@ -0,0 +1,75 @@
+module LigaMagicScraper
+  module Alerts
+    class BaseAlert
+      def notify(changes)
+        raise NotImplementedError, "Subclasses must implement the notify method"
+      end
+
+      protected
+
+      def format_changes(changes)
+        summary = []
+        
+        summary << "📊 Changes detected at #{changes[:timestamp]}"
+        summary << "🔍 Search: #{changes[:search_term]} (#{changes[:search_type]})"
+        summary << ""
+        
+        if changes[:new_products].any?
+          summary << "🆕 New products (#{changes[:new_products].count}):"
+          changes[:new_products].first(5).each do |product|
+            summary << "  - #{product[:name]}"
+          end
+          summary << "  ... and #{changes[:new_products].count - 5} more" if changes[:new_products].count > 5
+          summary << ""
+        end
+        
+        if changes[:removed_products].any?
+          summary << "❌ Removed products (#{changes[:removed_products].count}):"
+          changes[:removed_products].first(5).each do |product|
+            summary << "  - #{product[:name]}"
+          end
+          summary << "  ... and #{changes[:removed_products].count - 5} more" if changes[:removed_products].count > 5
+          summary << ""
+        end
+        
+        if changes[:price_changes].any?
+          summary << "💰 Price changes (#{changes[:price_changes].count}):"
+          changes[:price_changes].first(5).each do |change|
+            arrow = change[:change] > 0 ? "📈" : "📉"
+            summary << "  #{arrow} #{change[:name]}: R$ #{format_price(change[:previous_price] || change[:previous_min])} → R$ #{format_price(change[:current_price] || change[:current_min])} (#{change[:change_percent]}%)"
+          end
+          summary << "  ... and #{changes[:price_changes].count - 5} more" if changes[:price_changes].count > 5
+          summary << ""
+        end
+        
+        if changes[:quantity_changes].any?
+          summary << "📊 Quantity changes (#{changes[:quantity_changes].count}):"
+          changes[:quantity_changes].first(5).each do |change|
+            arrow = change[:change] > 0 ? "📈" : "📉"
+            summary << "  #{arrow} #{change[:name]}: #{change[:previous_qtd]} → #{change[:current_qtd]} (#{change[:change] > 0 ? '+' : ''}#{change[:change]})"
+          end
+          summary << "  ... and #{changes[:quantity_changes].count - 5} more" if changes[:quantity_changes].count > 5
+          summary << ""
+        end
+        
+        if changes[:availability_changes].any?
+          summary << "📦 Availability changes (#{changes[:availability_changes].count}):"
+          changes[:availability_changes].first(5).each do |change|
+            status = change[:current] ? "✅ Available" : "❌ Unavailable"
+            summary << "  #{status}: #{change[:name]}"
+          end
+          summary << "  ... and #{changes[:availability_changes].count - 5} more" if changes[:availability_changes].count > 5
+          summary << ""
+        end
+        
+        summary.join("\n")
+      end
+
+      def format_price(price)
+        return "N/A" unless price
+        "%.2f" % price
+      end
+    end
+  end
+end
+

data/lib/ligamagic_scraper/alerts/file_alert.rb

@@ -0,0 +1,56 @@
+require_relative 'base_alert'
+require 'fileutils'
+require 'json'
+
+module LigaMagicScraper
+  module Alerts
+    class FileAlert < BaseAlert
+      include Loggable
+      
+      ALERTS_BASE_DIR = 'alerts_json'
+
+      def initialize
+        super
+        initialize_logs
+      end
+
+      def notify(changes)
+        log_info("📁 FileAlert: Saving changes to JSON file...")
+        
+        subdir = if changes[:search_type] == 'store'
+          search_identifier = sanitize_filename(changes[:search_term] || 'unknown')
+          File.join(ALERTS_BASE_DIR, 'stores', search_identifier)
+        else
+          File.join(ALERTS_BASE_DIR, 'global')
+        end
+        
+        FileUtils.mkdir_p(subdir) unless Dir.exist?(subdir)
+        
+        filename = generate_alert_filename(changes)
+        filepath = File.join(subdir, filename)
+        
+        File.write(filepath, JSON.pretty_generate(changes))
+        
+        log_info("   ✅ Changes saved to: #{filepath}")
+        log_info("   📊 Summary:")
+        log_info("      🆕 New products: #{changes[:new_products].count}")
+        log_info("      ❌ Removed products: #{changes[:removed_products].count}")
+        log_info("      💰 Price changes: #{changes[:price_changes].count}")
+        log_info("      📊 Quantity changes: #{changes[:quantity_changes].count}") if changes[:quantity_changes]
+        log_info("      📦 Availability changes: #{changes[:availability_changes].count}")
+      end
+
+      private
+
+      def generate_alert_filename(changes)
+        datetime_str = Time.now.strftime('%Y%m%d_%H%M%S')
+        "#{datetime_str}.json"
+      end
+
+      def sanitize_filename(name)
+        name.to_s.downcase.gsub(/[^a-z0-9]+/, '_').gsub(/^_+|_+$/, '')
+      end
+    end
+  end
+end
+

data/lib/ligamagic_scraper/alerts/telegram_alert.rb

@@ -0,0 +1,36 @@
+require_relative 'base_alert'
+
+module LigaMagicScraper
+  module Alerts
+    class TelegramAlert < BaseAlert
+      include Loggable
+
+      def initialize
+        super
+        initialize_logs
+      end
+
+      def notify(changes)
+        log_info("📱 TelegramAlert: Preparing to send Telegram message...")
+        
+        # TODO: Implement Telegram alert logic
+        # Ideas:
+        # - Use Telegram Bot API
+        # - Configure bot token and chat ID
+        # - Send formatted messages with emojis
+        # - Support for inline keyboards
+        # - Send photos of cards (if available)
+        # - Rate limiting for multiple changes
+        
+        log_debug("   Would send to Telegram:")
+        log_debug("   Bot Token: [Configure bot token]")
+        log_debug("   Chat ID: [Configure chat ID]")
+        log_debug("   Message preview:")
+        format_changes(changes).split("\n").each { |line| log_debug("   #{line}") }
+        
+        log_warning("   ⚠️  Telegram alert action not implemented yet")
+      end
+    end
+  end
+end
+

data/lib/ligamagic_scraper/cli.rb

@@ -0,0 +1,152 @@
+require 'optparse'
+
+module LigaMagicScraper
+  class CLI
+    attr_reader :options
+
+    def initialize(argv = ARGV)
+      @argv = argv
+      @options = default_options
+    end
+
+    def parse
+      option_parser.parse!(@argv)
+      validate_options
+      self
+    end
+
+    def create_scraper
+      alert_config = build_alert_config
+      
+      if store_search?
+        StoreScraper.new(
+          store_domain: options[:store_domain],
+          search_term: options[:search],
+          max_pages: options[:max_pages],
+          browser_mode: options[:browser_mode],
+          alert_config: alert_config
+        )
+      else
+        GlobalScraper.new(
+          search_term: options[:search],
+          browser_mode: options[:browser_mode],
+          alert_config: alert_config
+        )
+      end
+    end
+
+    private
+
+    def default_options
+      {
+        search: nil,
+        store_domain: nil,
+        max_pages: nil,
+        browser_mode: 'headed',
+        search_type: 'global',
+        alerts_enabled: false,
+        alert_types: [:file]
+      }
+    end
+
+    def option_parser
+      @option_parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: ligamagic-scraper [options]"
+        opts.separator ""
+        opts.separator "Search Options (choose one):"
+        
+        opts.on("-s", "--search SEARCH", "Search term (global search or store search with -u)") do |search|
+          options[:search] = search
+          # Don't set search_type here - will be determined by presence of -u flag
+        end
+        
+        opts.on("-u", "--store DOMAIN", "Store domain (e.g., 'test-store')") do |domain|
+          options[:store_domain] = domain
+          options[:search_type] = 'store'
+        end
+        
+        opts.on("-p", "--pages N", Integer, "Pages to scrape (max: 5, required with -u if no -s)") do |pages|
+          # Cap at 5 pages maximum
+          options[:max_pages] = [pages, 5].min
+        end
+        
+        opts.separator ""
+        opts.separator "Options:"
+        
+        opts.on("-g", "--global", "Use global Liga Magic search (default)") do
+          options[:search_type] = 'global'
+        end
+        
+        opts.on("-b", "--browser-mode MODE", "Browser mode: 'headed' (default) or 'headless'") do |mode|
+          options[:browser_mode] = mode
+        end
+        
+        opts.on("-a", "--alerts", "Enable alert system (detects changes from previous scrapes)") do
+          options[:alerts_enabled] = true
+        end
+        
+        opts.on("--alert-types TYPES", Array, "Alert types: file, email, telegram, webhook (comma-separated)") do |types|
+          options[:alert_types] = types.map(&:to_sym)
+        end
+        
+        opts.on("-h", "--help", "Show this help message") do
+          puts opts
+          exit
+        end
+        
+        opts.on("--version", "Show version") do
+          puts "LigaMagic Scraper v#{VERSION}"
+          exit
+        end
+      end
+    end
+
+    def validate_options
+      # At least one search parameter required
+      if options[:search].nil? && options[:store_domain].nil?
+        puts "❌ Error: Either search term (-s) or store domain (-u) is required"
+        puts ""
+        puts "Usage examples:"
+        puts "  ligamagic-scraper -s SEARCH               # Global search"
+        puts "  ligamagic-scraper -u test-store -p 3     # List store products (max 5 pages)"
+        puts "  ligamagic-scraper -u test-store -s TERM  # Search within store"
+        puts ""
+        puts "Try 'ligamagic-scraper -h' for more information"
+        exit(1)
+      end
+      
+      # Validate: max_pages required when using -u without -s
+      if options[:store_domain] && options[:search].nil? && options[:max_pages].nil?
+        puts "❌ Error: -p (--pages) is required when listing store products without search term"
+        puts ""
+        puts "Example: ligamagic-scraper -u test-store -p 3"
+        puts ""
+        puts "Note: Maximum 5 pages allowed. Use -s flag for unlimited pagination."
+        exit(1)
+      end
+      
+      # Validate: -p only makes sense with -u
+      if options[:max_pages] && options[:store_domain].nil?
+        puts "❌ Error: -p (--pages) can only be used with -u (--store)"
+        puts ""
+        puts "Usage: ligamagic-scraper -u STORE -p N"
+        exit(1)
+      end
+    end
+
+    def build_alert_config
+      return nil unless options[:alerts_enabled]
+      
+      {
+        enabled: true,
+        alert_types: options[:alert_types],
+        compare_previous: true
+      }
+    end
+
+    def store_search?
+      options[:search_type] == 'store'
+    end
+  end
+end
+