UrlCategorise 0.1.6 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1271c16022cb3fb5e9efe9af22deca404204ee54e1b23243c4ebd862565c76b
-  data.tar.gz: '082814ad46484b87027e0a1d0042f3f37ab20a3d491f4a8e39a585e14fb40ac8'
+  metadata.gz: c22ada722efc33979930ece5d8737625943f1d815d2c7acdbfbc0f5b0fede15f
+  data.tar.gz: 15747c8a4b23c4c805cd74118627dfa8d102d4d813ae9f08c8c49011ff2ebad8
 SHA512:
-  metadata.gz: 8f74b667ba1d993237e224d3db7fcb0323816716961280e5fd27fbf162ecb162f83fd126af89ce812e98fbf8fe48a15c47697b656e345d23465a23ab9913581e
-  data.tar.gz: 57ed94a3e9a9be6db60c3d48dd9d4ac4da52836951b06a9e6937c44968e5509cda43193491c9ec20039649e01d5bb4a6214f769aab2d6c27c8d047d7b9bd5273
+  metadata.gz: b1c21b59b8a7631c4c39f206fb37bb9c51c1925a544ddd5f8a627b595dafa92d366d880a33c6303eafc477ae9be4130055af060953e27617e5ce198618e5f9bd
+  data.tar.gz: 1621c52578daf8957546c778125df120f9480dbe2f2d365feb6024890ddfe5affac3aae77100047ea46b15c96939db1fc228b876806197c63dda954199536bc9

data/.claude/settings.local.json

@@ -15,8 +15,16 @@
       "Bash(timeout 30 ruby dataset_loading_example.rb)",
       "Bash(timeout:*)",
       "Bash(DEBUG=1 timeout 300 ruby correct_usage_example.rb)",
-      "Bash(chmod:*)"
+      "Bash(chmod:*)",
+      "Bash(bundle exec bin/export_csv:*)",
+      "Bash(bundle exec rg:*)",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:api.github.com)",
+      "WebFetch(domain:raw.githubusercontent.com)",
+      "WebSearch",
+      "WebFetch(domain:firebog.net)",
+      "Bash(curl:*)"
     ],
     "deny": []
   }
-}
+}

data/.gitignore

@@ -55,3 +55,5 @@ build-iPhoneSimulator/
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
 url_cache/*
+exports/*
+.DS_Store

data/.rubocop.yml

@@ -0,0 +1,56 @@
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 3.0
+  NewCops: enable
+  Exclude:
+    - 'bin/*'
+    - 'vendor/**/*'
+    - 'tmp/**/*'
+    - 'db/migrate/*'
+
+# Allow longer lines for URL constants
+Layout/LineLength:
+  Max: 200
+  Exclude:
+    - 'lib/url_categorise/constants.rb'
+    - 'test/**/*'
+
+# Allow complex methods in client due to categorization logic
+Metrics/MethodLength:
+  Max: 30
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+    - 'test/**/*'
+
+# Allow complex classes for main client
+Metrics/ClassLength:
+  Max: 500
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+    - 'test/**/*'
+
+# Allow higher complexity for categorization methods
+Metrics/CyclomaticComplexity:
+  Max: 15
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+
+# Allow higher ABC size for complex categorization logic
+Metrics/AbcSize:
+  Max: 25
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+    - 'test/**/*'
+
+# Allow higher parameter count for client initialization
+Metrics/ParameterLists:
+  Max: 8
+
+# Allow block length for tests and constants
+Metrics/BlockLength:
+  Exclude:
+    - 'test/**/*'
+    - 'lib/url_categorise/constants.rb'
+    - 'url_categorise.gemspec'

data/.sublime-project

@@ -0,0 +1,14 @@
+{
+  "folders": [
+    {
+      "path": ".",
+      "folder_exclude_patterns": [
+        "node_modules", ".git", "tmp", "log",
+        "vendor/bundle", "coverage", ".bundle"
+      ],
+      "file_exclude_patterns": [
+        ".log", ".min.js", "*.min.css"
+      ]
+    }
+  ]
+}

data/CLAUDE.md

@@ -103,6 +103,34 @@ The gem includes automatic monitoring and cleanup of broken URLs:
 
 ### New Features (Latest Version)
 
+#### Video Hosting Detection and Regex Categorization
+Advanced video content detection system with:
+
+- **Comprehensive Video Hosting Lists**: Generate PiHole-compatible hosts files from yt-dlp extractors  
+- **Regex-Based Content Detection**: Distinguish between video content URLs vs homepages/profiles/playlists
+- **Direct Video URL Detection**: `video_url?` method to check if URLs are direct video content links
+- **Automatic List Generation**: `bin/generate_video_lists` script fetches and processes yt-dlp data
+- **Video Hosting Category**: Separate `video_hosting` category with 3,500+ domains
+- **Smart Content Categorization**: URLs matching video patterns get `*_content` suffix categories
+- **Remote Pattern Files**: Automatically downloads video patterns from GitHub repository
+
+```ruby
+# Enable regex categorization for video content detection (uses remote patterns by default)
+client = UrlCategorise::Client.new(regex_categorization: true)
+
+# Basic domain categorization
+client.categorise('https://youtube.com') # => [:video_hosting]
+
+# Enhanced content detection  
+client.categorise('https://youtube.com/watch?v=abc123') # => [:video_hosting, :video_hosting_content]
+
+# Direct video URL detection
+client.video_url?('https://youtube.com/watch?v=abc123')  # => true
+client.video_url?('https://youtube.com')                # => false
+client.video_url?('https://vimeo.com/123456789')       # => true
+client.video_url?('https://tiktok.com/@user/video/123') # => true
+```
+
 #### Dynamic Settings with ActiveAttr
 The Client class now uses ActiveAttr to provide dynamic attribute modification:
 
@@ -141,7 +169,7 @@ result = client.export_hosts_files('/custom/export/path')
 ```
 
 ##### CSV Data Export
-Export all data as a single CSV file for AI training and analysis:
+Export all data as a single comprehensive CSV file for AI training and analysis:
 
 ```ruby
 # Export to default location (cache_dir/exports/csv or ./exports/csv)
@@ -150,33 +178,68 @@ result = client.export_csv_data
 # Export to custom location  
 result = client.export_csv_data('/custom/export/path')
 
-# CSV includes: domain, category, source_type, is_dataset_category, iab_category_v2, iab_category_v3, export_timestamp
-# Metadata file includes: export info, client settings, data summary, dataset metadata
+# Returns:
+# {
+#   csv_file: 'url_categorise_comprehensive_export_TIMESTAMP.csv',
+#   summary_file: 'export_summary_TIMESTAMP.json',
+#   total_entries: 75000,
+#   summary: { domain_categorization_entries: 50000, dataset_content_entries: 25000 },
+#   export_directory: '/export/path'
+# }
 ```
 
+**Comprehensive Export Features:**
+- **Everything in One File**: Combined domains, categories, and raw dataset content
+- **Rich Dataset Content**: Original titles, descriptions, summaries, and text from datasets
+- **Dynamic Headers**: Automatically includes all available fields from any dataset
+- **Data Type Tracking**: Distinguishes between processed domains and raw dataset entries
+- **Perfect for AI/ML**: Single file with both structured categorization and rich textual features
+
 #### CLI Commands
-New command-line utilities for data export:
+Command-line utilities for comprehensive data export:
 
 ```bash
-# Export hosts files
+# Export hosts files per category
 $ bundle exec export_hosts --output /tmp/hosts --verbose
 
-# Export CSV data with IAB compliance
-$ bundle exec export_csv --output /tmp/csv --iab-compliance --verbose
+# Full CSV export with datasets and all features
+$ bundle exec export_csv --auto-load-datasets --iab-compliance --smart-categorization --verbose
+
+# Custom configuration export
+$ bundle exec export_csv --cache-dir ./custom_cache --kaggle-credentials ~/kaggle.json --output /tmp/export
 
-# Check URL health (existing)
+# Basic domain categorization only
+$ bundle exec export_csv --output /tmp/basic
+
+# Health check for all blocklist URLs
 $ bundle exec check_lists
+
+# Generate updated video hosting lists
+$ ruby bin/generate_video_lists
 ```
 
+**Enhanced CLI Features:**
+- `--auto-load-datasets`: Automatically load datasets from constants for rich content export
+- `--kaggle-credentials FILE`: Custom Kaggle API credentials file path
+- Full integration with all client features (IAB compliance, smart categorization, etc.)
+- Verbose output shows dataset statistics and loading progress
+- Video list generation from yt-dlp extractors with manual curation
+
 ### List Sources
 Primary sources include:
 - The Block List Project
 - hagezi/dns-blocklists  
 - StevenBlack/hosts
 - Various specialized security lists
+- **yt-dlp video extractors**: Comprehensive video hosting domain detection (3,500+ domains)
+- **GitHub-hosted video patterns**: Remote video URL detection patterns with manual curation
 - **Kaggle datasets**: Public URL classification datasets
 - **Custom CSV files**: Direct CSV dataset URLs with flexible column mapping
 
+**Video hosting lists are now automatically fetched from:**
+- Video domains: `https://raw.githubusercontent.com/TRex22/url_categorise/refs/heads/main/lists/video_hosting_domains.hosts`
+- URL patterns: `https://raw.githubusercontent.com/TRex22/url_categorise/refs/heads/main/lists/video_url_patterns.txt`
+
 ### Testing Guidelines
 - Mock all HTTP requests using WebMock
 - Test both success and failure scenarios

data/Gemfile

@@ -1,6 +1,6 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-git_source(:github) { |_repo_name| 'https://github.com/TRex22/url_categorise' }
+git_source(:github) { |_repo_name| "https://github.com/TRex22/url_categorise" }
 
 # Specify your gem's dependencies in url_categorise.gemspec
 gemspec

data/Gemfile.lock

@@ -1,16 +1,17 @@
 PATH
   remote: .
   specs:
-    UrlCategorise (0.1.6)
+    UrlCategorise (0.1.9)
       active_attr (>= 0.17.1, < 1.0)
       api_pattern (>= 0.0.6, < 1.0)
       csv (>= 3.3.0, < 4.0)
       digest (>= 3.1.0, < 4.0)
       fileutils (>= 1.7.0, < 2.0)
-      httparty (>= 0.22.0, < 1.0)
+      httparty (>= 0.24.0, < 1.0)
       json (>= 2.7.0, < 3.0)
       kaggle (>= 0.0.3, < 1.0)
-      nokogiri (>= 1.18.9, < 2.0)
+      nokogiri (>= 1.19.1, < 2.0)
+      reline (>= 0.6.2, < 2.0)
       resolv (>= 0.4.0, < 1.0)
       rubyzip (>= 2.3.0, < 3.0)
 
@@ -64,10 +65,14 @@ GEM
       csv (>= 3.3.0)
       httparty (>= 0.22.0)
       nokogiri (>= 1.16.0)
+    ast (2.4.3)
     base64 (0.3.0)
     benchmark (0.4.1)
-    bigdecimal (3.2.2)
+    bigdecimal (4.1.0)
     builder (3.3.0)
+    bundler-audit (0.9.3)
+      bundler (>= 1.2.0)
+      thor (~> 1.0)
     coderay (1.1.3)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
@@ -82,12 +87,13 @@ GEM
     erubi (1.13.1)
     fileutils (1.7.3)
     hashdiff (1.2.0)
-    httparty (0.23.1)
+    httparty (0.24.2)
       csv
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
+    io-console (0.8.1)
     json (2.13.2)
     kaggle (0.0.3)
       csv (>= 3.3)
@@ -95,6 +101,8 @@ GEM
       httparty (>= 0.23)
       oj (= 3.16.11)
       rubyzip (>= 2.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
     logger (1.7.0)
     loofah (2.24.1)
       crass (~> 1.0.2)
@@ -111,22 +119,28 @@ GEM
       ruby-progressbar
     mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    multi_xml (0.7.2)
-      bigdecimal (~> 3.1)
-    nokogiri (1.18.9-arm64-darwin)
+    multi_xml (0.8.1)
+      bigdecimal (>= 3.1, < 5)
+    nokogiri (1.19.2-arm64-darwin)
       racc (~> 1.4)
     oj (3.16.11)
       bigdecimal (>= 3.0)
       ostruct (>= 0.2)
     ostruct (0.6.3)
+    parallel (1.27.0)
+    parser (3.3.9.0)
+      ast (~> 2.4.1)
+      racc
+    prism (1.4.0)
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (6.0.2)
     racc (1.8.1)
-    rack (2.2.17)
-    rack-session (1.0.2)
-      rack (< 3)
+    rack (3.2.5)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0)
     rack-test (2.2.0)
       rack (>= 1.3)
     rails-dom-testing (2.3.0)
@@ -136,9 +150,41 @@ GEM
     rails-html-sanitizer (1.6.2)
       loofah (~> 2.21)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    rainbow (3.1.1)
     rake (13.3.0)
+    regexp_parser (2.11.2)
+    reline (0.6.2)
+      io-console (~> 0.5)
     resolv (0.6.2)
-    rexml (3.4.1)
+    rexml (3.4.4)
+    rubocop (1.80.1)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.46.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.46.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
+    rubocop-rails (2.33.3)
+      activesupport (>= 4.2.0)
+      lint_roller (~> 1.1)
+      rack (>= 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
+    rubocop-rails-omakase (1.1.0)
+      rubocop (>= 1.72)
+      rubocop-performance (>= 1.24)
+      rubocop-rails (>= 2.30)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.4.1)
@@ -150,11 +196,15 @@ GEM
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
     sqlite3 (2.7.3-arm64-darwin)
+    thor (1.5.0)
     timecop (0.9.10)
     timeout (0.4.3)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    uri (1.0.3)
+    unicode-display_width (3.1.5)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    uri (1.1.1)
     useragent (0.16.11)
     webmock (3.24.0)
       addressable (>= 2.8.0)
@@ -163,10 +213,12 @@ GEM
 
 PLATFORMS
   arm64-darwin-24
+  arm64-darwin-25
 
 DEPENDENCIES
   UrlCategorise!
   activerecord (>= 8.0)
+  bundler-audit (~> 0.9)
   logger
   minitest (~> 5.25.5)
   minitest-focus (~> 1.4.0)
@@ -174,6 +226,7 @@ DEPENDENCIES
   mocha (~> 2.4.5)
   pry (~> 0.15.2)
   rake (~> 13.3.0)
+  rubocop-rails-omakase (~> 1.0)
   simplecov (~> 0.22.0)
   sqlite3 (>= 2.7)
   timecop (~> 0.9.10)