UrlCategorise 0.0.2 → 0.1.0

data/lib/url_categorise/active_record_client.rb

@@ -0,0 +1,118 @@
+require_relative 'models'
+
+module UrlCategorise
+  class ActiveRecordClient < Client
+    def initialize(**kwargs)
+      raise "ActiveRecord not available" unless UrlCategorise::Models.available?
+      
+      @use_database = kwargs.delete(:use_database) { true }
+      super(**kwargs)
+      
+      populate_database if @use_database
+    end
+
+    def categorise(url)
+      return super(url) unless @use_database && UrlCategorise::Models.available?
+      
+      host = (URI.parse(url).host || url).downcase.gsub("www.", "")
+      
+      # Try database first
+      categories = UrlCategorise::Models::Domain.categorise(host)
+      return categories unless categories.empty?
+      
+      # Fallback to memory-based categorization
+      super(url)
+    end
+
+    def categorise_ip(ip_address)
+      return super(ip_address) unless @use_database && UrlCategorise::Models.available?
+      
+      # Try database first
+      categories = UrlCategorise::Models::IpAddress.categorise(ip_address)
+      return categories unless categories.empty?
+      
+      # Fallback to memory-based categorization
+      super(ip_address)
+    end
+
+    def update_database
+      return unless @use_database && UrlCategorise::Models.available?
+      
+      populate_database
+    end
+
+    def database_stats
+      return {} unless @use_database && UrlCategorise::Models.available?
+      
+      {
+        domains: UrlCategorise::Models::Domain.count,
+        ip_addresses: UrlCategorise::Models::IpAddress.count,
+        list_metadata: UrlCategorise::Models::ListMetadata.count,
+        categories: UrlCategorise::Models::Domain.distinct.pluck(:categories).flatten.uniq.size
+      }
+    end
+
+    private
+
+    def populate_database
+      return unless UrlCategorise::Models.available?
+      
+      # Store list metadata
+      @host_urls.each do |category, urls|
+        urls.each do |url|
+          next unless url.is_a?(String)
+          
+          metadata = @metadata[url] || {}
+          UrlCategorise::Models::ListMetadata.find_or_create_by(url: url) do |record|
+            record.name = category.to_s
+            record.categories = [category.to_s]
+            record.file_hash = metadata[:content_hash]
+            record.fetched_at = metadata[:last_updated]
+          end
+        end
+      end
+
+      # Store domain data
+      @hosts.each do |category, domains|
+        domains.each do |domain|
+          next if domain.nil? || domain.empty?
+          
+          existing = UrlCategorise::Models::Domain.find_by(domain: domain)
+          if existing
+            # Add category if not already present
+            categories = existing.categories | [category.to_s]
+            existing.update(categories: categories) if categories != existing.categories
+          else
+            UrlCategorise::Models::Domain.create!(
+              domain: domain,
+              categories: [category.to_s]
+            )
+          end
+        end
+      end
+
+      # Store IP data (for IP-based lists)
+      ip_categories = [:sanctions_ips, :compromised_ips, :tor_exit_nodes, :open_proxy_ips, 
+                       :banking_trojans, :malicious_ssl_certificates, :top_attack_sources]
+      
+      ip_categories.each do |category|
+        next unless @hosts[category]
+        
+        @hosts[category].each do |ip|
+          next if ip.nil? || ip.empty? || !ip.match(/^\d+\.\d+\.\d+\.\d+$/)
+          
+          existing = UrlCategorise::Models::IpAddress.find_by(ip_address: ip)
+          if existing
+            categories = existing.categories | [category.to_s]
+            existing.update(categories: categories) if categories != existing.categories
+          else
+            UrlCategorise::Models::IpAddress.create!(
+              ip_address: ip,
+              categories: [category.to_s]
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/url_categorise/client.rb

@@ -2,15 +2,23 @@ module UrlCategorise
   class Client < ApiPattern::Client
     include ::UrlCategorise::Constants
 
-    attr_reader :host_urls, :hosts
-
-    # TODO: Save to folder
-    # TODO: Read from disk the database
-    # TODO: Sanctioned IPs
-    # TODO: ActiveRecord support
-    # TODO: List of abuse IPs
-    def initialize(host_urls: DEFAULT_HOST_URLS)
+    def self.compatible_api_version
+      'v2'
+    end
+
+    def self.api_version
+      'v2 2023-04-12'
+    end
+
+    attr_reader :host_urls, :hosts, :cache_dir, :force_download, :dns_servers, :metadata, :request_timeout
+
+    def initialize(host_urls: DEFAULT_HOST_URLS, cache_dir: nil, force_download: false, dns_servers: ['1.1.1.1', '1.0.0.1'], request_timeout: 10)
       @host_urls = host_urls
+      @cache_dir = cache_dir
+      @force_download = force_download
+      @dns_servers = dns_servers
+      @request_timeout = request_timeout
+      @metadata = {}
       @hosts = fetch_and_build_host_lists
     end
 
@@ -19,10 +27,35 @@ module UrlCategorise
       host = host.gsub("www.", "")
 
       @hosts.keys.select do |category|
-        @hosts[category].include?(host)
+        @hosts[category].any? do |blocked_host|
+          host == blocked_host || host.end_with?(".#{blocked_host}")
+        end
+      end
+    end
+
+    def categorise_ip(ip_address)
+      @hosts.keys.select do |category|
+        @hosts[category].include?(ip_address)
       end
     end
 
+    def resolve_and_categorise(domain)
+      categories = categorise(domain)
+      
+      begin
+        resolver = Resolv::DNS.new(nameserver: @dns_servers)
+        ip_addresses = resolver.getaddresses(domain).map(&:to_s)
+        
+        ip_addresses.each do |ip|
+          categories.concat(categorise_ip(ip))
+        end
+      rescue
+        # DNS resolution failed, return domain categories only
+      end
+      
+      categories.uniq
+    end
+
     def count_of_hosts
       @hosts.keys.map do |category|
         @hosts[category].size
@@ -41,10 +74,12 @@ module UrlCategorise
 
     def hash_size_in_mb(hash)
       size = 0
+
       hash.each do |key, value|
         size += value.join.length
       end
-      (size / 1.megabyte).round(2)
+
+      (size / ONE_MEGABYTE).round(2)
     end
 
     def fetch_and_build_host_lists
@@ -63,25 +98,160 @@ module UrlCategorise
         end
 
         original_value << extra_category_values
-        @hosts[category] = original_value
+        @hosts[category] = original_value.uniq.compact
       end
 
       @hosts
     end
 
     def build_host_data(urls)
-      urls.map do |url|
+      all_hosts = []
+      
+      urls.each do |url|
         next unless url_valid?(url)
-
-        raw_data = HTTParty.get(url)
-        raw_data.split("\n").reject do |line|
-          line[0] == "#"
-        end.map do |line|
-          line.split(' ')[1] # Select the domain name # gsub("0.0.0.0 ", "")
+        
+        hosts_data = nil
+        
+        if @cache_dir && !@force_download
+          hosts_data = read_from_cache(url)
+        end
+        
+        if hosts_data.nil?
+          hosts_data = download_and_parse_list(url)
+          save_to_cache(url, hosts_data) if @cache_dir
         end
-      end.flatten.compact.sort
+        
+        all_hosts.concat(hosts_data) if hosts_data
+      end
+      
+      all_hosts.compact.sort.uniq
+    end
+
+    def download_and_parse_list(url)
+      begin
+        raw_data = HTTParty.get(url, timeout: @request_timeout)
+        return [] if raw_data.body.nil? || raw_data.body.empty?
+        
+        # Store metadata
+        etag = raw_data.headers['etag']
+        last_modified = raw_data.headers['last-modified']
+        @metadata[url] = {
+          last_updated: Time.now,
+          etag: etag,
+          last_modified: last_modified,
+          content_hash: Digest::SHA256.hexdigest(raw_data.body),
+          status: 'success'
+        }
+        
+        parse_list_content(raw_data.body, detect_list_format(raw_data.body))
+      rescue HTTParty::Error, Net::HTTPError, SocketError, Timeout::Error, URI::InvalidURIError, StandardError => e
+        # Log the error but continue with other lists
+        @metadata[url] = {
+          last_updated: Time.now,
+          error: e.message,
+          status: 'failed'
+        }
+        return []
+      end
+    end
+
+    def parse_list_content(content, format)
+      lines = content.split("\n").reject { |line| line.empty? || line.strip.start_with?('#') }
+      
+      case format
+      when :hosts
+        lines.map { |line| line.split(' ')[1] }.compact
+      when :plain
+        lines.map(&:strip)
+      when :dnsmasq
+        lines.map { |line| 
+          match = line.match(/address=\/(.+?)\//)
+          match ? match[1] : nil
+        }.compact
+      when :ublock
+        lines.map { |line| line.gsub(/^\|\|/, '').gsub(/[\$\^].*$/, '').strip }.reject(&:empty?)
+      else
+        lines.map(&:strip)
+      end
+    end
+
+    def detect_list_format(content)
+      sample_lines = content.split("\n").first(10)
+      
+      return :hosts if sample_lines.any? { |line| line.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\s+/) }
+      return :dnsmasq if sample_lines.any? { |line| line.include?('address=/') }
+      return :ublock if sample_lines.any? { |line| line.match(/^\|\|/) }
+      
+      :plain
+    end
+
+    def cache_file_path(url)
+      return nil unless @cache_dir
+      
+      FileUtils.mkdir_p(@cache_dir) unless Dir.exist?(@cache_dir)
+      filename = Digest::MD5.hexdigest(url) + '.cache'
+      File.join(@cache_dir, filename)
+    end
+
+    def read_from_cache(url)
+      cache_file = cache_file_path(url)
+      return nil unless cache_file && File.exist?(cache_file)
+      
+      cache_data = Marshal.load(File.read(cache_file))
+      
+      # Check if we should update based on hash or time
+      if should_update_cache?(url, cache_data)
+        return nil
+      end
+      
+      cache_data[:hosts]
+    rescue
+      nil
+    end
+
+    def save_to_cache(url, hosts_data)
+      cache_file = cache_file_path(url)
+      return unless cache_file
+      
+      cache_data = {
+        hosts: hosts_data,
+        metadata: @metadata[url],
+        cached_at: Time.now
+      }
+      
+      File.write(cache_file, Marshal.dump(cache_data))
+    rescue
+      # Cache save failed, continue without caching
+    end
+
+    def should_update_cache?(url, cache_data)
+      return true if @force_download
+      return true unless cache_data[:metadata]
+      
+      # Update if cache is older than 24 hours
+      cache_age = Time.now - cache_data[:cached_at]
+      return true if cache_age > 24 * 60 * 60
+      
+      # Check if remote content has changed
+      begin
+        head_response = HTTParty.head(url, timeout: @request_timeout)
+        remote_etag = head_response.headers['etag']
+        remote_last_modified = head_response.headers['last-modified']
+        
+        cached_metadata = cache_data[:metadata]
+        
+        return true if remote_etag && cached_metadata[:etag] && remote_etag != cached_metadata[:etag]
+        return true if remote_last_modified && cached_metadata[:last_modified] && remote_last_modified != cached_metadata[:last_modified]
+      rescue HTTParty::Error, Net::HTTPError, SocketError, Timeout::Error, URI::InvalidURIError, StandardError
+        # If HEAD request fails, assume we should update
+        return true
+      end
+      
+      false
     end
 
+    private
+
     def categories_with_keys
       keyed_categories = {}
 

data/lib/url_categorise/constants.rb

@@ -1,5 +1,6 @@
 module UrlCategorise
   module Constants
+    ONE_MEGABYTE = 1048576
     DEFAULT_HOST_URLS = {
       abuse: ["https://github.com/blocklistproject/Lists/raw/master/abuse.txt"],
       adobe: ["https://github.com/blocklistproject/Lists/raw/master/adobe.txt"],
@@ -13,7 +14,7 @@ module UrlCategorise
       drugs: ["https://github.com/blocklistproject/Lists/raw/master/drugs.txt"],
       facebook: ["https://github.com/blocklistproject/Lists/raw/master/facebook.txt", "https://www.github.developerdan.com/hosts/lists/facebook-extended.txt", "https://raw.githubusercontent.com/blocklistproject/Lists/master/facebook.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/all", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/facebook.com"],
       fraud: ["https://blocklistproject.github.io/Lists/fraud.txt"],
-      gambling: ["https://blocklistproject.github.io/Lists/gambling.txt"],
+      gambling: ["https://blocklistproject.github.io/Lists/gambling.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/gambling.txt"],
       gaming: ["https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-ubisoft.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-steam.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-activision.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-blizzard.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-ea.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-epicgames.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-nintendo.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-rockstargames.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-roblox.txt"],
       google: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/google/all"],
       hate_and_junk: ["https://www.github.developerdan.com/hosts/lists/hate-and-junk-extended.txt"],
@@ -25,13 +26,13 @@ module UrlCategorise
       nsa: ["https://raw.githubusercontent.com/tigthor/NSA-CIA-Blocklist/main/HOSTS/HOSTS"],
       phishing: ["https://blocklistproject.github.io/Lists/phishing.txt"],
       pinterest: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/pinterest/all"],
-      piracy: ["https://github.com/blocklistproject/Lists/raw/master/piracy.txt"],
+      piracy: ["https://github.com/blocklistproject/Lists/raw/master/piracy.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/anti.piracy.txt"],
       pornography: ["https://blocklistproject.github.io/Lists/porn.txt"],
       reddit: ["https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-reddit.txt"],
       redirect: ["https://github.com/blocklistproject/Lists/raw/master/redirect.txt"],
       scam: ["https://blocklistproject.github.io/Lists/scam.txt"],
       smart_tv: ["https://github.com/blocklistproject/Lists/raw/master/smart-tv.txt"],
-      social_media: [:facebook, :instagram, :linkedin, :pinterest, :reddit,:tiktok, :twitter, :whatsapp, :youtube],
+      social_media: [:facebook, :instagram, :linkedin, :pinterest, :reddit, :tiktok, :twitter, :whatsapp, :youtube],
       tiktok: ["https://blocklistproject.github.io/Lists/tiktok.txt"],
       torrent: ["https://github.com/blocklistproject/Lists/raw/master/torrent.txt"],
       tracking: ["https://blocklistproject.github.io/Lists/tracking.txt"],
@@ -39,6 +40,67 @@ module UrlCategorise
       vaping: ["https://github.com/blocklistproject/Lists/raw/master/vaping.txt"],
       whatsapp: ["https://github.com/blocklistproject/Lists/raw/master/whatsapp.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/whatsapp"],
       youtube: ["https://github.com/blocklistproject/Lists/raw/master/youtube.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/google/youtube"],
+      
+      # Hagezi DNS Blocklists - specialized categories only
+      threat_intelligence: ["https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/tif.txt"],
+      dyndns: ["https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/dyndns.txt"],
+      badware_hoster: ["https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/hoster.txt"],
+      most_abused_tlds: ["https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/tlds.txt"],
+      newly_registered_domains: ["https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/nrd.txt"],
+      dns_over_https_bypass: ["https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@release/adblock/doh-vpn-proxy-bypass.txt"],
+      
+      # StevenBlack hosts lists - specific categories only
+      fakenews: ["https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts"],
+      
+      # Security threat lists
+      banking_trojans: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+      malware_domains: ["https://bazaar.abuse.ch/downloads/domain_blocklist.txt"],
+      malicious_ssl_certificates: ["https://sslbl.abuse.ch/blacklist/sslipblacklist.txt"],
+      threat_indicators: ["https://threatfox.abuse.ch/downloads/hostfile.txt"],
+      
+      # Additional IP-based sanctions and abuse lists
+      sanctions_ips: ["https://lists.blocklist.de/lists/all.txt"],
+      compromised_ips: ["https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"],
+      tor_exit_nodes: ["https://www.dan.me.uk/torlist/"],
+      open_proxy_ips: ["https://raw.githubusercontent.com/stamparm/ipsum/master/ipsum.txt"],
+      
+      # Network security feeds
+      top_attack_sources: ["https://www.dshield.org/feeds/suspiciousdomains_High.txt"],
+      suspicious_domains: ["https://www.dshield.org/feeds/suspiciousdomains_Medium.txt"],
+      
+      # Extended categories for better organization
+      cryptojacking: ["https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt"],
+      ransomware: ["https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt"],
+      botnet_command_control: ["https://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt"],
+      phishing_extended: ["https://openphish.com/feed.txt"],
+      
+      # Regional and specialized lists
+      chinese_ad_hosts: ["https://raw.githubusercontent.com/jdlingyu/ad-wars/master/hosts"],
+      korean_ad_hosts: ["https://raw.githubusercontent.com/yous/YousList/master/hosts.txt"],
+      
+      # Mobile and app-specific
+      mobile_ads: ["https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/MobileFilter/sections/adservers.txt"],
+      smart_tv_ads: ["https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt"],
+      
+      # Content and informational categories
+      news: ["https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-only/hosts"],
+      legitimate_news: ["https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/.dev-tools/_domains_fake_news/domains.txt"],
+      blogs: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/blogs-nl.txt"],
+      forums: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/forums-nl.txt"],
+      educational: ["https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/education/hosts"],
+      government: ["https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/.dev-tools/_domains_government/domains.txt"],
+      health: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/health-nl.txt"],
+      finance: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/finance-nl.txt"],
+      streaming: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/streaming-nl.txt"],
+      shopping: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/shopping-nl.txt"],
+      
+      # Professional and business
+      business: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/business-nl.txt"],
+      technology: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/tech-nl.txt"],
+      
+      # Regional content
+      local_news: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/local-news-nl.txt"],
+      international_news: ["https://raw.githubusercontent.com/blocklistproject/Lists/master/alt-version/international-news-nl.txt"],
     }
   end
 end

data/lib/url_categorise/models.rb

@@ -0,0 +1,105 @@
+begin
+  require 'active_record'
+rescue LoadError
+  # ActiveRecord not available, skip model definitions
+  module UrlCategorise
+    module Models
+      def self.available?
+        false
+      end
+    end
+  end
+else
+  module UrlCategorise
+    module Models
+      def self.available?
+        true
+      end
+
+      class ListMetadata < ActiveRecord::Base
+        self.table_name = 'url_categorise_list_metadata'
+        
+        validates :name, presence: true, uniqueness: true
+        validates :url, presence: true
+        validates :categories, presence: true
+        
+        serialize :categories, coder: JSON
+        
+        scope :by_category, ->(category) { where('categories LIKE ?', "%#{category}%") }
+        scope :updated_since, ->(time) { where('updated_at > ?', time) }
+      end
+
+      class Domain < ActiveRecord::Base
+        self.table_name = 'url_categorise_domains'
+        
+        validates :domain, presence: true, uniqueness: true
+        validates :categories, presence: true
+        
+        serialize :categories, coder: JSON
+        
+        scope :by_category, ->(category) { where('categories LIKE ?', "%#{category}%") }
+        scope :search, ->(term) { where('domain LIKE ?', "%#{term}%") }
+        
+        def self.categorise(domain_name)
+          record = find_by(domain: domain_name.downcase.gsub('www.', ''))
+          record ? record.categories : []
+        end
+      end
+
+      class IpAddress < ActiveRecord::Base
+        self.table_name = 'url_categorise_ip_addresses'
+        
+        validates :ip_address, presence: true, uniqueness: true
+        validates :categories, presence: true
+        
+        serialize :categories, coder: JSON
+        
+        scope :by_category, ->(category) { where('categories LIKE ?', "%#{category}%") }
+        scope :in_subnet, ->(subnet) { where('ip_address LIKE ?', "#{subnet}%") }
+        
+        def self.categorise(ip)
+          record = find_by(ip_address: ip)
+          record ? record.categories : []
+        end
+      end
+
+      # Generator for Rails integration
+      def self.generate_migration
+        <<~MIGRATION
+          class CreateUrlCategoriseTables < ActiveRecord::Migration[8.0]
+            def change
+              create_table :url_categorise_list_metadata do |t|
+                t.string :name, null: false, index: { unique: true }
+                t.string :url, null: false
+                t.text :categories, null: false
+                t.string :file_path
+                t.datetime :fetched_at
+                t.string :file_hash
+                t.datetime :file_updated_at
+                t.timestamps
+              end
+
+              create_table :url_categorise_domains do |t|
+                t.string :domain, null: false, index: { unique: true }
+                t.text :categories, null: false
+                t.timestamps
+              end
+              
+              add_index :url_categorise_domains, :domain
+              add_index :url_categorise_domains, :categories
+
+              create_table :url_categorise_ip_addresses do |t|
+                t.string :ip_address, null: false, index: { unique: true }
+                t.text :categories, null: false
+                t.timestamps
+              end
+              
+              add_index :url_categorise_ip_addresses, :ip_address
+              add_index :url_categorise_ip_addresses, :categories
+            end
+          end
+        MIGRATION
+      end
+    end
+  end
+end

data/lib/url_categorise/version.rb

@@ -1,3 +1,3 @@
 module UrlCategorise
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/lib/url_categorise.rb

@@ -1,5 +1,8 @@
 require 'httparty'
 require 'nokogiri'
+require 'digest'
+require 'fileutils'
+require 'resolv'
 
 require 'api-pattern'
 
@@ -8,6 +11,14 @@ require 'url_categorise/constants'
 
 require 'url_categorise/client'
 
+# Optional ActiveRecord integration
+begin
+  require 'url_categorise/models'
+  require 'url_categorise/active_record_client'
+rescue LoadError
+  # ActiveRecord not available, skip
+end
+
 module UrlCategorise
   class Error < StandardError; end
 end

data/url_categorise.gemspec

@@ -21,16 +21,24 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.required_ruby_version = ">= 3.0.0"
 
-  spec.add_dependency "api_pattern", "~> 0.0.4"
+  spec.add_dependency "api_pattern", ">= 0.0.5", "< 1.0"
+  spec.add_dependency "httparty", ">= 0.22.0", "< 1.0"
+  spec.add_dependency "nokogiri", ">= 1.16.0", "< 2.0"
+  spec.add_dependency "csv", ">= 3.3.0", "< 4.0"
+  spec.add_dependency "digest", ">= 3.1.0", "< 4.0"
+  spec.add_dependency "fileutils", ">= 1.7.0", "< 2.0"
+  spec.add_dependency "resolv", ">= 0.4.0", "< 1.0"
 
   # Development dependancies
-  spec.add_development_dependency "rake", "~> 13.0.6"
-  spec.add_development_dependency "minitest", "~> 5.18.0"
-  spec.add_development_dependency "minitest-focus", "~> 1.3.1"
-  spec.add_development_dependency "minitest-reporters", "~> 1.6.0"
-  spec.add_development_dependency "timecop", "~> 0.9.6"
-  spec.add_development_dependency "mocha", "~> 2.0.2"
-  spec.add_development_dependency "pry", "~> 0.14.2"
-  spec.add_development_dependency "webmock", "~> 3.18.1"
+  spec.add_development_dependency "rake", "~> 13.3.0"
+  spec.add_development_dependency "minitest", "~> 5.25.5"
+  spec.add_development_dependency "minitest-focus", "~> 1.4.0"
+  spec.add_development_dependency "minitest-reporters", "~> 1.7.1"
+  spec.add_development_dependency "timecop", "~> 0.9.10"
+  spec.add_development_dependency "mocha", "~> 2.4.5"
+  spec.add_development_dependency "pry", "~> 0.15.2"
+  spec.add_development_dependency "webmock", "~> 3.24.0"
+  spec.add_development_dependency "simplecov", "~> 0.22.0"
 end