UrlCategorise 0.1.2 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74ab2a721a954a91958dc1c184fd76a4a2c3acd250730bb7db58c30286579fcd
-  data.tar.gz: 3b01bf42b654266dddcd85d684c2f3e25d455d0fe9b82094f07de2b95c026bd3
+  metadata.gz: c1271c16022cb3fb5e9efe9af22deca404204ee54e1b23243c4ebd862565c76b
+  data.tar.gz: '082814ad46484b87027e0a1d0042f3f37ab20a3d491f4a8e39a585e14fb40ac8'
 SHA512:
-  metadata.gz: 2d175a1f72e2fda10771c770a694f07235cd7f5a3f93d1b487d82c98b4f7b490f7aaf665834dcf226bc1a6ae2cc833038731e8d9581af0a570302a7c17dad1c1
-  data.tar.gz: 2bd46865492bea3411b8506f294cf96b507c5c5beafad2584b2438977043fbfd3850e79c0a0342b11079829ef5bf3fd91c18c3f45ad5e0537fae8c78c9645245
+  metadata.gz: 8f74b667ba1d993237e224d3db7fcb0323816716961280e5fd27fbf162ecb162f83fd126af89ce812e98fbf8fe48a15c47697b656e345d23465a23ab9913581e
+  data.tar.gz: 57ed94a3e9a9be6db60c3d48dd9d4ac4da52836951b06a9e6937c44968e5509cda43193491c9ec20039649e01d5bb4a6214f769aab2d6c27c8d047d7b9bd5273

data/.claude/settings.local.json

@@ -6,7 +6,16 @@
       "Bash(ruby:*)",
       "Bash(bundle exec ruby:*)",
       "Bash(find:*)",
-      "Bash(grep:*)"
+      "Bash(grep:*)",
+      "Bash(for file in test/url_categorise/*dataset*test.rb)",
+      "Bash(do echo \"Checking $file...\")",
+      "Bash(done)",
+      "Bash(bundle exec rubocop:*)",
+      "Bash(rubocop:*)",
+      "Bash(timeout 30 ruby dataset_loading_example.rb)",
+      "Bash(timeout:*)",
+      "Bash(DEBUG=1 timeout 300 ruby correct_usage_example.rb)",
+      "Bash(chmod:*)"
     ],
     "deny": []
   }

data/.gitignore

@@ -54,3 +54,4 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+url_cache/*

data/CLAUDE.md

@@ -78,12 +78,95 @@ The gem includes automatic monitoring and cleanup of broken URLs:
 - ActiveRecord/Rails integration (optional)
 - URL health monitoring and reporting
 - Automatic cleanup of broken blocklist sources
+- **Dataset Processing**: Kaggle and CSV dataset integration with three auth methods
+- **Optional Kaggle**: Can disable Kaggle functionality entirely while keeping CSV processing
+- **Smart Caching**: Cached datasets work without credentials, avoiding unnecessary authentication
+- **Data Hashing**: SHA256 content hashing for dataset change detection
+- **Category Mapping**: Flexible column detection and category mapping for datasets
+- **Credential Warnings**: Helpful warnings when Kaggle credentials are missing but functionality continues
+- **IAB Compliance**: Full support for IAB Content Taxonomy v2.0 and v3.0 standards
+- **Dataset-Specific Metrics**: Separate counting methods for dataset vs DNS list categorization
+- **Enhanced Statistics**: Extended helper methods for comprehensive data insights
+- **ActiveAttr Settings**: In-memory modification of client settings using attribute setters
+- **Data Export**: Multiple export formats including hosts files per category and CSV data exports
+- **CLI Commands**: Command-line utilities for data export and list checking
 
 ### Architecture
-- `Client` class: Main interface for categorization
+- `Client` class: Main interface for categorization with IAB compliance support and ActiveAttr attributes
+- `DatasetProcessor` class: Handles Kaggle and CSV dataset processing
+- `IabCompliance` module: Maps categories to IAB Content Taxonomy v2.0/v3.0 standards
 - `Constants` module: Contains default list URLs and categories
-- Modular design allows extending with new list sources
-- Support for custom list directories and caching
+- `ActiveRecordClient` class: Database-backed client with dataset history
+- Modular design allows extending with new list sources and datasets
+- Support for custom list directories, caching, dataset integration, IAB compliance, and data export
+- ActiveAttr integration for dynamic setting modification and attribute validation
+
+### New Features (Latest Version)
+
+#### Dynamic Settings with ActiveAttr
+The Client class now uses ActiveAttr to provide dynamic attribute modification:
+
+```ruby
+client = UrlCategorise::Client.new
+
+# Modify settings in-memory
+client.smart_categorization_enabled = true
+client.iab_compliance_enabled = true
+client.iab_version = :v2
+client.request_timeout = 30
+client.dns_servers = ['8.8.8.8', '8.8.4.4']
+
+# Settings take effect immediately
+categories = client.categorise('reddit.com') # Uses new smart categorization rules
+```
+
+#### Data Export Features
+
+##### Hosts File Export
+Export all categorized domains as separate hosts files per category:
+
+```ruby
+# Export to default location (cache_dir/exports/hosts or ./exports/hosts)
+result = client.export_hosts_files
+
+# Export to custom location
+result = client.export_hosts_files('/custom/export/path')
+
+# Returns hash with file information:
+# {
+#   malware: { path: '/path/malware.hosts', filename: 'malware.hosts', count: 1500 },
+#   advertising: { path: '/path/advertising.hosts', filename: 'advertising.hosts', count: 25000 },
+#   _summary: { total_categories: 15, total_domains: 50000, export_directory: '/path' }
+# }
+```
+
+##### CSV Data Export
+Export all data as a single CSV file for AI training and analysis:
+
+```ruby
+# Export to default location (cache_dir/exports/csv or ./exports/csv)
+result = client.export_csv_data
+
+# Export to custom location  
+result = client.export_csv_data('/custom/export/path')
+
+# CSV includes: domain, category, source_type, is_dataset_category, iab_category_v2, iab_category_v3, export_timestamp
+# Metadata file includes: export info, client settings, data summary, dataset metadata
+```
+
+#### CLI Commands
+New command-line utilities for data export:
+
+```bash
+# Export hosts files
+$ bundle exec export_hosts --output /tmp/hosts --verbose
+
+# Export CSV data with IAB compliance
+$ bundle exec export_csv --output /tmp/csv --iab-compliance --verbose
+
+# Check URL health (existing)
+$ bundle exec check_lists
+```
 
 ### List Sources
 Primary sources include:
@@ -91,6 +174,8 @@ Primary sources include:
 - hagezi/dns-blocklists  
 - StevenBlack/hosts
 - Various specialized security lists
+- **Kaggle datasets**: Public URL classification datasets
+- **Custom CSV files**: Direct CSV dataset URLs with flexible column mapping
 
 ### Testing Guidelines
 - Mock all HTTP requests using WebMock

data/Gemfile

@@ -1,6 +1,6 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/TRex22/url_categorise" }
+git_source(:github) { |_repo_name| 'https://github.com/TRex22/url_categorise' }
 
 # Specify your gem's dependencies in url_categorise.gemspec
 gemspec

data/Gemfile.lock

@@ -1,14 +1,18 @@
 PATH
   remote: .
   specs:
-    UrlCategorise (0.1.2)
+    UrlCategorise (0.1.6)
+      active_attr (>= 0.17.1, < 1.0)
       api_pattern (>= 0.0.6, < 1.0)
       csv (>= 3.3.0, < 4.0)
       digest (>= 3.1.0, < 4.0)
       fileutils (>= 1.7.0, < 2.0)
       httparty (>= 0.22.0, < 1.0)
+      json (>= 2.7.0, < 3.0)
+      kaggle (>= 0.0.3, < 1.0)
       nokogiri (>= 1.18.9, < 2.0)
       resolv (>= 0.4.0, < 1.0)
+      rubyzip (>= 2.3.0, < 3.0)
 
 GEM
   remote: https://rubygems.org/
@@ -78,19 +82,25 @@ GEM
     erubi (1.13.1)
     fileutils (1.7.3)
     hashdiff (1.2.0)
-    httparty (0.22.0)
+    httparty (0.23.1)
       csv
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
+    json (2.13.2)
+    kaggle (0.0.3)
+      csv (>= 3.3)
+      fileutils (>= 1.7)
+      httparty (>= 0.23)
+      oj (= 3.16.11)
+      rubyzip (>= 2.0)
     logger (1.7.0)
     loofah (2.24.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     method_source (1.1.0)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.9)
     minitest (5.25.5)
     minitest-focus (1.4.0)
       minitest (>= 4, < 6)
@@ -103,11 +113,12 @@ GEM
       ruby2_keywords (>= 0.0.5)
     multi_xml (0.7.2)
       bigdecimal (~> 3.1)
-    nokogiri (1.18.9)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
     nokogiri (1.18.9-arm64-darwin)
       racc (~> 1.4)
+    oj (3.16.11)
+      bigdecimal (>= 3.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.3)
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -130,6 +141,7 @@ GEM
     rexml (3.4.1)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
+    rubyzip (2.4.1)
     securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
@@ -137,8 +149,6 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    sqlite3 (2.7.3)
-      mini_portile2 (~> 2.8.0)
     sqlite3 (2.7.3-arm64-darwin)
     timecop (0.9.10)
     timeout (0.4.3)
@@ -153,7 +163,6 @@ GEM
 
 PLATFORMS
   arm64-darwin-24
-  ruby
 
 DEPENDENCIES
   UrlCategorise!