UrlCategorise 0.0.3 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 926043e28097f20035b4dbc943534c63a5c3c0a429745e4bb42e7dd0701295c1
-  data.tar.gz: 3ef850d4c43266a6ec15a7653c64b97720efc2e46298ed36be07d48d70ada772
+  metadata.gz: f9e0158a598c4ce31320e56da6cfa74eaf795b6961cf432dc36bfc806b291a80
+  data.tar.gz: cfc035a4f344ef9d70f3336259fec31e76c6f2f9367934dd79a3fff932872040
 SHA512:
-  metadata.gz: c6504dd8ec0a5f284bc78dfcbb7e45b9e1752c50f6f32a380c52128c19364b976232bd4af93c943d73f0b7cbb6e2ac3e44574e90a0fde38da87f5585b92fc3c0
-  data.tar.gz: c2de16323a1cfa085ac15590f7601b5307cc7c660243f48b6a5db10df25809041b119eb3f48da382effbe120150c2967ae4224213508fca896c45889bad6bce1
+  metadata.gz: c4f140c5b7eaafe8d556c5db8d8e8ed17925c9c34999205bdecb521283b9b9f8f7124e43f421000dd78a441ea57516fe31226710018e32a1db1c038f103f5465
+  data.tar.gz: b033b13f7143399ff908449ee0f5c932f8fed1dc892295256802b3aca5b5114e07b20d81403484299b22dd664b6b6691e68b6a344fed0ef91571b68ca39cefe1

data/.claude/settings.local.json

@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(bundle exec rake:*)",
+      "Bash(bundle install:*)",
+      "Bash(ruby:*)",
+      "Bash(bundle exec ruby:*)",
+      "Bash(find:*)",
+      "Bash(grep:*)"
+    ],
+    "deny": []
+  }
+}

data/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['3.0', '3.1', '3.2', '3.3', '3.4']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Ruby ${{ matrix.ruby-version }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Run tests
+      run: bundle exec rake test
+
+    - name: Run linter (if available)
+      run: bundle exec rubocop || true
+      continue-on-error: true
+
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4'
+        bundler-cache: true
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Run tests with coverage
+      run: bundle exec rake test
+      env:
+        COVERAGE: true
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        fail_ci_if_error: false

data/.ruby-version

@@ -0,0 +1 @@
+3.4.5

data/CLAUDE.md

@@ -0,0 +1,134 @@
+# UrlCategorise Development Guidelines
+
+## Overview
+UrlCategorise is a Ruby gem for categorizing URLs and domains based on various security and content blocklists. It downloads and processes multiple types of lists to provide comprehensive domain categorization.
+
+## Development Requirements
+
+### Testing Standards
+- **ALL new changes MUST include new tests**
+- **Test coverage MUST be 90% or higher**
+- **NEVER delete, skip, or environment-check tests to make them pass**
+- **Tests MUST pass because the underlying code works correctly**
+- Use minitest for all testing
+- Use WebMock for HTTP request stubbing in tests
+- Run tests with: `bundle exec rake test`
+- SimpleCov integration is mandatory for coverage tracking
+
+### Dependencies and Rails Support
+- **MUST use the latest stable versions of gems**
+- Ruby >= 3.0.0 (currently using 3.4+)
+- **MUST use minitest and rake** for testing and build automation
+- **Rails compatibility MUST support Rails 8.x** and current stable versions
+- Dependencies are managed via Gemfile and gemspec
+- ActiveRecord integration must be optional and backward compatible
+
+#### Rails 8 Integration
+- ActiveRecord models use `coder: JSON` for serialization (Rails 8 compatible)
+- Migration version set to `ActiveRecord::Migration[8.0]` 
+- Optional database integration with automatic fallback to memory-based categorization
+- Installation: Generate migration with `UrlCategorise::Models.generate_migration`
+- Usage: Use `UrlCategorise::ActiveRecordClient` instead of `UrlCategorise::Client`
+
+### Code Quality
+- Follow Ruby best practices and conventions
+- Use meaningful variable and method names
+- Add appropriate error handling
+- Ensure thread safety where applicable
+
+### Supported List Formats
+The gem supports multiple blocklist formats:
+- Standard hosts files (0.0.0.0 domain.com)
+- pfSense format
+- AdSense lists  
+- uBlock Origin files
+- dnsmasq format
+- Plain text domain lists
+
+### Category Management Guidelines
+- **Category names MUST be human-readable and intuitive**
+- **NEVER add combined/meta lists as categories** (e.g., hagezi_light, stevenblack_all)
+- **First try to add new lists to existing categories** before creating new ones
+- **Use descriptive names instead of provider prefixes**:
+  - ❌ Bad: `abuse_ch_feodo`, `dshield_block_list`, `botnet_c2`, `doh_vpn_proxy_bypass`
+  - ✅ Good: `banking_trojans`, `suspicious_domains`, `botnet_command_control`, `dns_over_https_bypass`
+- **Logical category organization**:
+  - Security threats: `malware`, `phishing`, `threat_indicators`, `cryptojacking`, `phishing_extended`
+  - Content filtering: `advertising`, `gambling`, `pornography`, `social_media`
+  - Network security: `suspicious_domains`, `threat_intelligence`, `dns_over_https_bypass`
+  - Geographic/specialized: `sanctions_ips`, `newly_registered_domains`, `chinese_ad_hosts`, `korean_ad_hosts`
+  - IP-based security: `compromised_ips`, `tor_exit_nodes`, `open_proxy_ips`, `top_attack_sources`
+  - Content categories: `news`, `fakenews` (remaining active categories)
+  - Mobile/TV: `mobile_ads`, `smart_tv_ads`
+
+### URL Health Monitoring and Cleanup
+The gem includes automatic monitoring and cleanup of broken URLs:
+- **Automatic removal of broken URLs**: Categories with URLs returning 403, 404, or persistent errors are commented out
+- **Health checking tools**: Use `bin/check_lists` to verify all URLs in constants
+- **Programmatic checking**: The `Client#check_all_lists` method provides detailed health reports
+- **Recently removed categories**: Categories like `botnet_command_control` (403 Forbidden), `blogs`, `forums`, `educational`, `health`, `finance`, `streaming`, `shopping`, `business`, `technology`, `government` (404 Not Found) have been commented out until working URLs are found
+
+### Core Features
+- Domain/URL categorization
+- Multiple list format parsing
+- Hash-based file update detection
+- Optional local file caching
+- IP sanctions list checking
+- DNS resolution for domain-to-IP mapping
+- ActiveRecord/Rails integration (optional)
+- URL health monitoring and reporting
+- Automatic cleanup of broken blocklist sources
+
+### Architecture
+- `Client` class: Main interface for categorization
+- `Constants` module: Contains default list URLs and categories
+- Modular design allows extending with new list sources
+- Support for custom list directories and caching
+
+### List Sources
+Primary sources include:
+- The Block List Project
+- hagezi/dns-blocklists  
+- StevenBlack/hosts
+- Various specialized security lists
+
+### Testing Guidelines
+- Mock all HTTP requests using WebMock
+- Test both success and failure scenarios
+- Verify proper parsing of different list formats
+- Test edge cases (empty responses, malformed data)
+- Include integration tests for the full categorization flow
+
+### Performance Considerations
+- Implement efficient parsing for large lists
+- Use appropriate data structures for fast lookups
+- Consider memory usage with large datasets
+- Provide options for selective list loading
+
+### Configuration
+- Allow custom list URLs
+- Support for local file directories
+- Configurable DNS servers for IP resolution
+- Optional caching parameters
+
+## Build and Release Process
+1. Update version number in `lib/url_categorise/version.rb`
+2. Update CHANGELOG.md with new features
+3. Run full test suite: `bundle exec rake test`
+4. Update documentation as needed
+5. Build gem: `gem build url_categorise.gemspec`
+6. Release: `gem push url_categorise-x.x.x.gem`
+
+## Contributing
+- Fork the repository
+- Create a feature branch
+- Add comprehensive tests for new functionality
+- Ensure all tests pass
+- Update documentation
+- Submit a pull request
+
+## CI/CD
+- GitHub Actions workflow runs tests on multiple Ruby versions
+- All tests must pass before merging
+- Coverage reporting with Codecov integration
+- Automated dependency updates where appropriate

data/Gemfile.lock

@@ -1,112 +1,172 @@
 PATH
   remote: .
   specs:
-    UrlCategorise (0.0.3)
-      api_pattern (~> 0.0.4)
+    UrlCategorise (0.1.1)
+      api_pattern (>= 0.0.6, < 1.0)
+      csv (>= 3.3.0, < 4.0)
+      digest (>= 3.1.0, < 4.0)
+      fileutils (>= 1.7.0, < 2.0)
+      httparty (>= 0.22.0, < 1.0)
+      nokogiri (>= 1.16.0, < 2.0)
+      resolv (>= 0.4.0, < 1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
-      actionview (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      rack (~> 2.0, >= 2.2.0)
+    actionpack (8.0.2.1)
+      actionview (= 8.0.2.1)
+      activesupport (= 8.0.2.1)
+      nokogiri (>= 1.8.5)
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.4.3)
-      activesupport (= 7.0.4.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+      useragent (~> 0.16)
+    actionview (8.0.2.1)
+      activesupport (= 8.0.2.1)
       builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    active_attr (0.15.4)
-      actionpack (>= 3.0.2, < 7.1)
-      activemodel (>= 3.0.2, < 7.1)
-      activesupport (>= 3.0.2, < 7.1)
-    activemodel (7.0.4.3)
-      activesupport (= 7.0.4.3)
-    activesupport (7.0.4.3)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    active_attr (0.17.1)
+      actionpack (>= 3.0.2, < 8.1)
+      activemodel (>= 3.0.2, < 8.1)
+      activesupport (>= 3.0.2, < 8.1)
+    activemodel (8.0.2.1)
+      activesupport (= 8.0.2.1)
+    activerecord (8.0.2.1)
+      activemodel (= 8.0.2.1)
+      activesupport (= 8.0.2.1)
+      timeout (>= 0.4.0)
+    activesupport (8.0.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      tzinfo (~> 2.0)
-    addressable (2.8.4)
-      public_suffix (>= 2.0.2, < 6.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     ansi (1.5.0)
-    api_pattern (0.0.4)
-      active_attr (~> 0.15.4)
-      httparty (~> 0.21.0)
-      nokogiri (~> 1.14.3)
-    builder (3.2.4)
+    api_pattern (0.0.6)
+      active_attr (>= 0.15.4)
+      csv (>= 3.3.0)
+      httparty (>= 0.22.0)
+      nokogiri (>= 1.16.0)
+    base64 (0.3.0)
+    benchmark (0.4.1)
+    bigdecimal (3.2.2)
+    builder (3.3.0)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
-    crack (0.4.5)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    crack (1.0.0)
+      bigdecimal
       rexml
     crass (1.0.6)
-    erubi (1.12.0)
-    hashdiff (1.0.1)
-    httparty (0.21.0)
+    csv (3.3.5)
+    digest (3.2.0)
+    docile (1.4.1)
+    drb (2.2.3)
+    erubi (1.13.1)
+    fileutils (1.7.3)
+    hashdiff (1.2.0)
+    httparty (0.22.0)
+      csv
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
-    i18n (1.13.0)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    loofah (2.21.3)
+    logger (1.7.0)
+    loofah (2.24.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
-    method_source (1.0.0)
-    mini_mime (1.1.2)
-    minitest (5.18.0)
-    minitest-focus (1.3.1)
+    method_source (1.1.0)
+    mini_mime (1.1.5)
+    mini_portile2 (2.8.9)
+    minitest (5.25.5)
+    minitest-focus (1.4.0)
       minitest (>= 4, < 6)
-    minitest-reporters (1.6.0)
+    minitest-reporters (1.7.1)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    mocha (2.0.2)
+    mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    multi_xml (0.6.0)
-    nokogiri (1.14.4-arm64-darwin)
+    multi_xml (0.7.2)
+      bigdecimal (~> 3.1)
+    nokogiri (1.16.8)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    pry (0.14.2)
+    pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.1)
-    racc (1.6.2)
-    rack (2.2.7)
-    rack-test (2.1.0)
+    public_suffix (6.0.2)
+    racc (1.8.1)
+    rack (2.2.17)
+    rack-session (1.0.2)
+      rack (< 3)
+    rack-test (2.2.0)
       rack (>= 1.3)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
+    rails-dom-testing (2.3.0)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    rake (13.0.6)
-    rexml (3.2.5)
+    rails-html-sanitizer (1.6.2)
+      loofah (~> 2.21)
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    rake (13.3.0)
+    resolv (0.6.2)
+    rexml (3.4.1)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    timecop (0.9.6)
+    securerandom (0.4.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.2)
+    simplecov_json_formatter (0.1.4)
+    sqlite3 (2.7.3)
+      mini_portile2 (~> 2.8.0)
+    sqlite3 (2.7.3-arm64-darwin)
+    timecop (0.9.10)
+    timeout (0.4.3)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    webmock (3.18.1)
+    uri (1.0.3)
+    useragent (0.16.11)
+    webmock (3.24.0)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
-  arm64-darwin-20
+  arm64-darwin-24
+  ruby
 
 DEPENDENCIES
   UrlCategorise!
-  minitest (~> 5.18.0)
-  minitest-focus (~> 1.3.1)
-  minitest-reporters (~> 1.6.0)
-  mocha (~> 2.0.2)
-  pry (~> 0.14.2)
-  rake (~> 13.0.6)
-  timecop (~> 0.9.6)
-  webmock (~> 3.18.1)
+  activerecord (>= 8.0)
+  logger
+  minitest (~> 5.25.5)
+  minitest-focus (~> 1.4.0)
+  minitest-reporters (~> 1.7.1)
+  mocha (~> 2.4.5)
+  pry (~> 0.15.2)
+  rake (~> 13.3.0)
+  simplecov (~> 0.22.0)
+  sqlite3 (>= 2.7)
+  timecop (~> 0.9.10)
+  webmock (~> 3.24.0)
 
 BUNDLED WITH
-   2.3.16
+   2.7.1