UrlCategorise 0.1.2 → 0.1.6

data/bin/export_csv

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'optparse'
+require_relative '../lib/url_categorise'
+
+options = {
+  output_path: nil,
+  cache_dir: nil,
+  verbose: false,
+  iab_compliance: false,
+  smart_categorization: false
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Export all categorized domains and metadata as a single CSV file for AI training"
+  opts.separator ""
+  
+  opts.on("-o", "--output PATH", "Output directory path (default: cache_dir/exports/csv or ./exports/csv)") do |path|
+    options[:output_path] = path
+  end
+  
+  opts.on("-c", "--cache-dir PATH", "Cache directory path for client initialization") do |path|
+    options[:cache_dir] = path
+  end
+  
+  opts.on("--iab-compliance", "Enable IAB compliance for category mapping") do
+    options[:iab_compliance] = true
+  end
+  
+  opts.on("--smart-categorization", "Enable smart categorization") do
+    options[:smart_categorization] = true
+  end
+  
+  opts.on("-v", "--verbose", "Verbose output") do
+    options[:verbose] = true
+  end
+  
+  opts.on("-h", "--help", "Show this help message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+puts "=== UrlCategorise CSV Data Export ===" if options[:verbose]
+puts "Initializing client..." if options[:verbose]
+
+begin
+  client = UrlCategorise::Client.new(
+    cache_dir: options[:cache_dir],
+    iab_compliance: options[:iab_compliance],
+    smart_categorization: options[:smart_categorization]
+  )
+  
+  puts "Exporting CSV data..." if options[:verbose]
+  
+  result = client.export_csv_data(options[:output_path])
+  
+  puts "\n✅ Export completed successfully!"
+  puts "📁 Export directory: #{result[:export_directory]}"
+  puts "📄 CSV file: #{result[:csv_file]}"
+  puts "📄 Metadata file: #{result[:metadata_file]}"
+  
+  puts "\n📊 Data Summary:"
+  puts "  Total domains: #{result[:summary][:total_domains]}"
+  puts "  Total categories: #{result[:summary][:total_categories]}"
+  puts "  Dataset categories: #{result[:summary][:dataset_categories_count]}"
+  puts "  Blocklist categories: #{result[:summary][:blocklist_categories_count]}"
+  
+  if options[:verbose]
+    puts "\n🏷️  Categories included:"
+    result[:summary][:categories].each do |category|
+      puts "    - #{category}"
+    end
+  end
+  
+rescue StandardError => e
+  puts "❌ Error: #{e.message}"
+  puts e.backtrace if options[:verbose]
+  exit 1
+end

data/bin/export_hosts

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'optparse'
+require_relative '../lib/url_categorise'
+
+options = {
+  output_path: nil,
+  cache_dir: nil,
+  verbose: false
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Export all categorized domains as separate hosts files per category"
+  opts.separator ""
+  
+  opts.on("-o", "--output PATH", "Output directory path (default: cache_dir/exports/hosts or ./exports/hosts)") do |path|
+    options[:output_path] = path
+  end
+  
+  opts.on("-c", "--cache-dir PATH", "Cache directory path for client initialization") do |path|
+    options[:cache_dir] = path
+  end
+  
+  opts.on("-v", "--verbose", "Verbose output") do
+    options[:verbose] = true
+  end
+  
+  opts.on("-h", "--help", "Show this help message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+puts "=== UrlCategorise Hosts Export ===" if options[:verbose]
+puts "Initializing client..." if options[:verbose]
+
+begin
+  client = UrlCategorise::Client.new(
+    cache_dir: options[:cache_dir]
+  )
+  
+  puts "Exporting hosts files..." if options[:verbose]
+  
+  result = client.export_hosts_files(options[:output_path])
+  
+  summary = result.delete(:_summary)
+  
+  puts "\n✅ Export completed successfully!"
+  puts "📁 Export directory: #{summary[:export_directory]}"
+  puts "📊 Total categories exported: #{summary[:total_categories]}"
+  puts "🌐 Total domains exported: #{summary[:total_domains]}"
+  puts "📄 Summary file: #{summary[:path]}"
+  
+  if options[:verbose]
+    puts "\n📋 Files created:"
+    result.each do |category, info|
+      puts "  #{info[:filename]} - #{info[:count]} domains"
+    end
+  end
+  
+rescue StandardError => e
+  puts "❌ Error: #{e.message}"
+  puts e.backtrace if options[:verbose]
+  exit 1
+end

data/bin/rake

@@ -0,0 +1,2 @@
+#!/bin/bash
+rake

data/correct_usage_example.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require './lib/url_categorise'
+
+puts "=== Large Dataset Loading Example ==="
+
+# Configuration for handling large datasets (300+ MB)
+# First test with cache-only mode
+puts "Creating client with cached datasets only..."
+client = UrlCategorise::Client.new(
+  cache_dir: './url_cache',
+  auto_load_datasets: true,
+  smart_categorization: true,
+  dataset_config: {
+    cache_path: './url_cache/datasets',
+    download_path: './url_cache/downloads',
+    kaggle: { credentials_file: '~/kaggle.json' }
+  }
+)
+
+puts "Client created successfully!"
+puts ""
+puts "Dataset Statistics:"
+puts "  Total categories: #{client.count_of_categories}"
+puts "  Dataset categories: #{client.count_of_dataset_categories}"
+puts "  Blocklist categories: #{client.count_of_categories - client.count_of_dataset_categories}"
+puts ""
+puts "  Total hosts: #{client.count_of_hosts.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse}"
+puts "  Dataset hosts: #{client.count_of_dataset_hosts.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse}"
+puts ""
+puts "  Total data size: #{client.size_of_data.round(1)} MB"
+puts "  Dataset data size: #{client.size_of_dataset_data.round(1)} MB"  
+puts "  Blocklist data size: #{client.size_of_blocklist_data.round(1)} MB"
+
+puts ""
+puts "Dataset-specific Statistics:"
+# Get dataset metadata if available
+metadata = client.dataset_metadata
+if metadata && !metadata.empty?
+  puts "  Datasets loaded: #{metadata.size}"
+  
+  # Calculate size for each dataset by finding its categories and domains
+  dataset_categories = client.instance_variable_get(:@dataset_categories)
+  total_dataset_size = 0
+  
+  metadata.each_with_index do |(hash, data), index|
+    # Estimate size contribution of this dataset
+    dataset_portion = data[:total_entries].to_f / metadata.values.sum { |d| d[:total_entries] }
+    dataset_size_mb = (client.size_of_dataset_data * dataset_portion).round(2)
+    total_dataset_size += dataset_size_mb
+    
+    puts "  Dataset #{index + 1}:"
+    puts "    Processed at: #{data[:processed_at]}"
+    puts "    Total entries: #{data[:total_entries].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse}"
+    puts "    Estimated size: #{dataset_size_mb} MB"
+    puts "    Data hash: #{hash[0..12]}..."
+  end
+  
+  puts ""
+  puts "  Total dataset size: #{total_dataset_size.round(2)} MB (#{client.size_of_dataset_data.round(1)} MB actual)"
+else
+  puts "  No dataset metadata available"
+end

data/docs/v0.1.4-features.md

@@ -0,0 +1,215 @@
+# UrlCategorise v0.1.4 Release Notes
+
+## New Features Added
+
+### 🆕 Dataset-Specific Helper Methods
+
+Added dedicated helper methods for tracking dataset-only statistics separately from DNS blocklists:
+
+```ruby
+client = UrlCategorise::Client.new(dataset_config: { kaggle: {} })
+client.load_kaggle_dataset('owner', 'dataset-name')
+
+# New dataset-specific methods
+client.count_of_dataset_hosts      # Returns hosts from datasets only
+client.count_of_dataset_categories # Returns categories from datasets only
+
+# Existing methods still work and count both
+client.count_of_hosts     # All hosts (DNS lists + datasets)
+client.count_of_categories # All categories (DNS lists + datasets)
+```
+
+**Implementation Details:**
+- `count_of_dataset_hosts` - Sums hosts from categories tracked in `@dataset_categories` Set
+- `count_of_dataset_categories` - Returns size of `@dataset_categories` Set
+- Gracefully handles nil categories with safe navigation (`&.size || 0`)
+- Both methods return 0 when no datasets are loaded
+
+### 🆕 IAB Content Taxonomy Compliance
+
+Full support for IAB (Interactive Advertising Bureau) Content Taxonomy standards:
+
+```ruby
+# Enable IAB v3.0 compliance (recommended)
+client = UrlCategorise::Client.new(
+  iab_compliance: true,
+  iab_version: :v3
+)
+
+# Enable IAB v2.0 compliance
+client = UrlCategorise::Client.new(
+  iab_compliance: true,
+  iab_version: :v2
+)
+
+# Categorization returns IAB codes instead of custom categories
+categories = client.categorise("badsite.com")
+puts categories # => ["626"] (IAB v3 code for illegal content)
+
+# Helper methods
+client.iab_compliant?            # => true/false
+client.get_iab_mapping(:malware) # => "626" (v3) or "IAB25" (v2)
+```
+
+**IAB Category Mappings:**
+
+**IAB Content Taxonomy v3.0:**
+- Security threats (`malware`, `phishing`, `illegal`) → `626` (Illegal Content)
+- Advertising (`advertising`, `mobile_ads`) → `3` (Advertising)
+- Gambling → `7-39` (Gambling subcategory)
+- Adult content (`pornography`) → `626` (Adult Content)
+- Social platforms → `14` (Society)
+- Technology → `19` (Technology & Computing)
+
+**IAB Content Taxonomy v2.0:**
+- Security threats → `IAB25` (Non-Standard Content)
+- Advertising → `IAB3` (Advertising)
+- Gambling → `IAB7-39` (Gambling subcategory)
+- Adult content → `IAB25-3` (Pornography)
+
+**Implementation Details:**
+- New `IabCompliance` module with comprehensive mappings
+- Support for both v2.0 and v3.0 standards
+- IAB compliance affects all categorization methods (`categorise`, `categorise_ip`, `resolve_and_categorise`)
+- Automatic deduplication of IAB codes
+- Graceful handling of unknown categories (returns 'Unknown')
+
+### 🧪 Comprehensive Test Coverage
+
+Added extensive test suites for new features:
+
+**Dataset Methods Tests:**
+- `client_dataset_methods_test.rb` (9 tests)
+- Tests for empty, populated, and mixed dataset scenarios
+- Integration tests with existing methods
+- Edge case handling (nil categories, empty arrays)
+
+**IAB Compliance Tests:**
+- `iab_compliance_test.rb` (14 tests) - Module functionality
+- `client_iab_compliance_test.rb` (19 tests) - Client integration
+- Comprehensive mapping validation for v2 and v3
+- Integration with all categorization methods
+- DNS resolution with IAB compliance
+- Error handling and edge cases
+
+**Total Test Stats:**
+- **316 tests, 2455 assertions, 0 failures, 0 errors**
+- **94.69% line coverage** (660/697 lines)
+- All new features fully tested with edge cases
+
+### 🔧 Technical Implementation
+
+**New Files:**
+- `lib/url_categorise/iab_compliance.rb` - IAB mapping module
+- `test/url_categorise/client_dataset_methods_test.rb` - Dataset helper tests
+- `test/url_categorise/iab_compliance_test.rb` - IAB module tests
+- `test/url_categorise/client_iab_compliance_test.rb` - IAB client integration tests
+
+**Updated Files:**
+- `lib/url_categorise/client.rb` - Added dataset helpers and IAB support
+- `lib/url_categorise.rb` - Required new IAB module
+- `lib/url_categorise/version.rb` - Bumped to 0.1.4
+
+**New Client Attributes:**
+- `iab_compliance_enabled` - Boolean flag for IAB compliance
+- `iab_version` - IAB taxonomy version (:v2 or :v3)
+
+**New Client Methods:**
+- `count_of_dataset_hosts` - Dataset-specific host count
+- `count_of_dataset_categories` - Dataset-specific category count
+- `iab_compliant?` - Check IAB compliance status
+- `get_iab_mapping(category)` - Get IAB code for category
+
+### 🚀 Usage Examples
+
+**Dataset Statistics:**
+```ruby
+client = UrlCategorise::Client.new(dataset_config: { kaggle: {} })
+client.load_kaggle_dataset('owner', 'dataset')
+
+puts "Total hosts: #{client.count_of_hosts}"           # All sources
+puts "Dataset hosts: #{client.count_of_dataset_hosts}" # Datasets only
+puts "DNS list hosts: #{client.count_of_hosts - client.count_of_dataset_hosts}"
+```
+
+**IAB Compliance:**
+```ruby
+# Production environment with IAB compliance
+client = UrlCategorise::Client.new(
+  iab_compliance: true,
+  iab_version: :v3,
+  dataset_config: { kaggle: { username: 'user', api_key: 'key' } }
+)
+
+# All methods return IAB codes
+domain_cats = client.categorise("example.com")        # => ["3", "626"]
+ip_cats = client.categorise_ip("192.168.1.100")      # => ["626"]
+resolved_cats = client.resolve_and_categorise("site.com") # => ["3"]
+
+# Check compliance
+puts "IAB compliant: #{client.iab_compliant?}"       # => true
+puts "Using version: #{client.iab_version}"          # => :v3
+```
+
+**Rails Service Integration:**
+```ruby
+class UrlCategorizerService
+  def initialize
+    @client = UrlCategorise::ActiveRecordClient.new(
+      iab_compliance: Rails.env.production?,
+      iab_version: :v3,
+      dataset_config: {
+        kaggle: {
+          username: ENV['KAGGLE_USERNAME'],
+          api_key: ENV['KAGGLE_API_KEY']
+        }
+      }
+    )
+  end
+
+  def stats
+    {
+      total_hosts: @client.count_of_hosts,
+      dataset_hosts: @client.count_of_dataset_hosts,
+      dns_list_hosts: @client.count_of_hosts - @client.count_of_dataset_hosts,
+      iab_compliant: @client.iab_compliant?,
+      iab_version: @client.iab_version
+    }
+  end
+end
+```
+
+### 🔄 Migration Guide
+
+**From v0.1.3 to v0.1.4:**
+
+1. **No Breaking Changes** - All existing code continues to work
+2. **Optional New Features** - IAB compliance and dataset helpers are opt-in
+3. **Enhanced Statistics** - Use new helper methods for better insights
+
+**Recommended Updates:**
+```ruby
+# Before (still works)
+client = UrlCategorise::Client.new
+puts "Total: #{client.count_of_hosts}"
+
+# After (enhanced)
+client = UrlCategorise::Client.new(
+  iab_compliance: true,  # Optional IAB compliance
+  iab_version: :v3       # Optional version selection
+)
+puts "Total hosts: #{client.count_of_hosts}"
+puts "Dataset hosts: #{client.count_of_dataset_hosts}"
+puts "IAB compliant: #{client.iab_compliant?}"
+```
+
+### 📊 Quality Assurance
+
+- ✅ All 316 tests pass with 0 failures/errors
+- ✅ 94.69% line coverage maintained
+- ✅ Code style enforced with rubocop
+- ✅ Comprehensive edge case testing
+- ✅ Memory-efficient implementation
+- ✅ Backward compatibility preserved
+
+This release adds powerful new features while maintaining the reliability and performance standards established in previous versions.

data/lib/url_categorise/active_record_client.rb

@@ -3,65 +3,125 @@ require_relative 'models'
 module UrlCategorise
   class ActiveRecordClient < Client
     def initialize(**kwargs)
-      raise "ActiveRecord not available" unless UrlCategorise::Models.available?
-      
+      raise 'ActiveRecord not available' unless UrlCategorise::Models.available?
+
       @use_database = kwargs.delete(:use_database) { true }
       super(**kwargs)
-      
+
       populate_database if @use_database
     end
 
     def categorise(url)
       return super(url) unless @use_database && UrlCategorise::Models.available?
-      
-      host = (URI.parse(url).host || url).downcase.gsub("www.", "")
-      
+
+      host = (URI.parse(url).host || url).downcase.gsub('www.', '')
+
       # Try database first
       categories = UrlCategorise::Models::Domain.categorise(host)
       return categories unless categories.empty?
-      
+
       # Fallback to memory-based categorization
       super(url)
     end
 
     def categorise_ip(ip_address)
       return super(ip_address) unless @use_database && UrlCategorise::Models.available?
-      
+
       # Try database first
       categories = UrlCategorise::Models::IpAddress.categorise(ip_address)
       return categories unless categories.empty?
-      
+
       # Fallback to memory-based categorization
       super(ip_address)
     end
 
     def update_database
       return unless @use_database && UrlCategorise::Models.available?
-      
+
       populate_database
     end
 
     def database_stats
       return {} unless @use_database && UrlCategorise::Models.available?
-      
+
       {
         domains: UrlCategorise::Models::Domain.count,
         ip_addresses: UrlCategorise::Models::IpAddress.count,
         list_metadata: UrlCategorise::Models::ListMetadata.count,
+        dataset_metadata: UrlCategorise::Models::DatasetMetadata.count,
         categories: UrlCategorise::Models::Domain.distinct.pluck(:categories).flatten.uniq.size
       }
     end
 
+    def load_kaggle_dataset(dataset_owner, dataset_name, options = {})
+      result = super(dataset_owner, dataset_name, options)
+
+      # Store dataset metadata in database if enabled
+      if @use_database && UrlCategorise::Models.available? && @dataset_metadata
+        store_dataset_metadata_in_db(
+          source_type: 'kaggle',
+          identifier: "#{dataset_owner}/#{dataset_name}",
+          metadata: @dataset_metadata.values.last,
+          category_mappings: options[:category_mappings],
+          processing_options: options
+        )
+      end
+
+      # Repopulate database with integrated dataset domains
+      populate_database if @use_database
+
+      result
+    end
+
+    def load_csv_dataset(url, options = {})
+      result = super(url, options)
+
+      # Store dataset metadata in database if enabled
+      if @use_database && UrlCategorise::Models.available? && @dataset_metadata
+        store_dataset_metadata_in_db(
+          source_type: 'csv',
+          identifier: url,
+          metadata: @dataset_metadata.values.last,
+          category_mappings: options[:category_mappings],
+          processing_options: options
+        )
+      end
+
+      # Repopulate database with integrated dataset domains
+      populate_database if @use_database
+
+      result
+    end
+
+    def dataset_history(source_type: nil, limit: 10)
+      return [] unless @use_database && UrlCategorise::Models.available?
+
+      query = UrlCategorise::Models::DatasetMetadata.order(processed_at: :desc).limit(limit)
+      query = query.by_source(source_type) if source_type
+
+      query.map do |record|
+        {
+          source_type: record.source_type,
+          identifier: record.identifier,
+          data_hash: record.data_hash,
+          total_entries: record.total_entries,
+          processed_at: record.processed_at,
+          category_mappings: record.category_mappings,
+          processing_options: record.processing_options
+        }
+      end
+    end
+
     private
 
     def populate_database
       return unless UrlCategorise::Models.available?
-      
+
       # Store list metadata
-      @host_urls.each do |category, urls|
+      (host_urls || {}).each do |category, urls|
         urls.each do |url|
           next unless url.is_a?(String)
-          
+
           metadata = @metadata[url] || {}
           UrlCategorise::Models::ListMetadata.find_or_create_by(url: url) do |record|
             record.name = category.to_s
@@ -76,7 +136,7 @@ module UrlCategorise
       @hosts.each do |category, domains|
         domains.each do |domain|
           next if domain.nil? || domain.empty?
-          
+
           existing = UrlCategorise::Models::Domain.find_by(domain: domain)
           if existing
             # Add category if not already present
@@ -92,15 +152,15 @@ module UrlCategorise
       end
 
       # Store IP data (for IP-based lists)
-      ip_categories = [:sanctions_ips, :compromised_ips, :tor_exit_nodes, :open_proxy_ips, 
-                       :banking_trojans, :malicious_ssl_certificates, :top_attack_sources]
-      
+      ip_categories = %i[sanctions_ips compromised_ips tor_exit_nodes open_proxy_ips
+                         banking_trojans malicious_ssl_certificates top_attack_sources]
+
       ip_categories.each do |category|
         next unless @hosts[category]
-        
+
         @hosts[category].each do |ip|
           next if ip.nil? || ip.empty? || !ip.match(/^\d+\.\d+\.\d+\.\d+$/)
-          
+
           existing = UrlCategorise::Models::IpAddress.find_by(ip_address: ip)
           if existing
             categories = existing.categories | [category.to_s]
@@ -114,5 +174,22 @@ module UrlCategorise
         end
       end
     end
+
+    def store_dataset_metadata_in_db(source_type:, identifier:, metadata:, category_mappings: nil,
+                                     processing_options: nil)
+      return unless UrlCategorise::Models.available?
+
+      UrlCategorise::Models::DatasetMetadata.find_or_create_by(data_hash: metadata[:data_hash]) do |record|
+        record.source_type = source_type
+        record.identifier = identifier
+        record.total_entries = metadata[:total_entries]
+        record.category_mappings = category_mappings || {}
+        record.processing_options = processing_options || {}
+        record.processed_at = metadata[:processed_at] || Time.now
+      end
+    rescue ActiveRecord::RecordInvalid => e
+      # Dataset metadata already exists or validation failed
+      puts "Warning: Failed to store dataset metadata: #{e.message}" if ENV['DEBUG']
+    end
   end
-end
+end