UrlCategorise 0.1.7 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4725733a26240e3bc23bfe9292d3ed0222db4e0308c5228da12a9ede1347e4b
-  data.tar.gz: a315a03357b48260c543459fab91ac8ec0601a0d7001b120bdc6f72e543e3671
+  metadata.gz: c22ada722efc33979930ece5d8737625943f1d815d2c7acdbfbc0f5b0fede15f
+  data.tar.gz: 15747c8a4b23c4c805cd74118627dfa8d102d4d813ae9f08c8c49011ff2ebad8
 SHA512:
-  metadata.gz: 74c8be653a2ae97ad74300105a95c326685d9669defd5c980cf16636e96965080a587453c21d5af6014874145f8f93d66c444574142f1568588f82b9577a7440
-  data.tar.gz: 450ae86237aa6a743a289e9b1165b9bae49f78068af8d727114763629b863567dc37de1b2ba38522e93a0ae08dd1683b79a062df2e2d18e76884fd4f8415c1db
+  metadata.gz: b1c21b59b8a7631c4c39f206fb37bb9c51c1925a544ddd5f8a627b595dafa92d366d880a33c6303eafc477ae9be4130055af060953e27617e5ce198618e5f9bd
+  data.tar.gz: 1621c52578daf8957546c778125df120f9480dbe2f2d365feb6024890ddfe5affac3aae77100047ea46b15c96939db1fc228b876806197c63dda954199536bc9

data/.claude/settings.local.json

@@ -16,8 +16,15 @@
       "Bash(timeout:*)",
       "Bash(DEBUG=1 timeout 300 ruby correct_usage_example.rb)",
       "Bash(chmod:*)",
-      "Bash(bundle exec bin/export_csv:*)"
+      "Bash(bundle exec bin/export_csv:*)",
+      "Bash(bundle exec rg:*)",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:api.github.com)",
+      "WebFetch(domain:raw.githubusercontent.com)",
+      "WebSearch",
+      "WebFetch(domain:firebog.net)",
+      "Bash(curl:*)"
     ],
     "deny": []
   }
-}
+}

data/.gitignore

@@ -56,3 +56,4 @@ build-iPhoneSimulator/
 # .rubocop-https?--*
 url_cache/*
 exports/*
+.DS_Store

data/.rubocop.yml

@@ -0,0 +1,56 @@
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 3.0
+  NewCops: enable
+  Exclude:
+    - 'bin/*'
+    - 'vendor/**/*'
+    - 'tmp/**/*'
+    - 'db/migrate/*'
+
+# Allow longer lines for URL constants
+Layout/LineLength:
+  Max: 200
+  Exclude:
+    - 'lib/url_categorise/constants.rb'
+    - 'test/**/*'
+
+# Allow complex methods in client due to categorization logic
+Metrics/MethodLength:
+  Max: 30
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+    - 'test/**/*'
+
+# Allow complex classes for main client
+Metrics/ClassLength:
+  Max: 500
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+    - 'test/**/*'
+
+# Allow higher complexity for categorization methods
+Metrics/CyclomaticComplexity:
+  Max: 15
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+
+# Allow higher ABC size for complex categorization logic
+Metrics/AbcSize:
+  Max: 25
+  Exclude:
+    - 'lib/url_categorise/client.rb'
+    - 'test/**/*'
+
+# Allow higher parameter count for client initialization
+Metrics/ParameterLists:
+  Max: 8
+
+# Allow block length for tests and constants
+Metrics/BlockLength:
+  Exclude:
+    - 'test/**/*'
+    - 'lib/url_categorise/constants.rb'
+    - 'url_categorise.gemspec'

data/.sublime-project

@@ -0,0 +1,14 @@
+{
+  "folders": [
+    {
+      "path": ".",
+      "folder_exclude_patterns": [
+        "node_modules", ".git", "tmp", "log",
+        "vendor/bundle", "coverage", ".bundle"
+      ],
+      "file_exclude_patterns": [
+        ".log", ".min.js", "*.min.css"
+      ]
+    }
+  ]
+}

data/Gemfile

@@ -1,6 +1,6 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-git_source(:github) { |_repo_name| 'https://github.com/TRex22/url_categorise' }
+git_source(:github) { |_repo_name| "https://github.com/TRex22/url_categorise" }
 
 # Specify your gem's dependencies in url_categorise.gemspec
 gemspec

data/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    UrlCategorise (0.1.7)
+    UrlCategorise (0.1.9)
       active_attr (>= 0.17.1, < 1.0)
       api_pattern (>= 0.0.6, < 1.0)
       csv (>= 3.3.0, < 4.0)
       digest (>= 3.1.0, < 4.0)
       fileutils (>= 1.7.0, < 2.0)
-      httparty (>= 0.22.0, < 1.0)
+      httparty (>= 0.24.0, < 1.0)
       json (>= 2.7.0, < 3.0)
       kaggle (>= 0.0.3, < 1.0)
-      nokogiri (>= 1.18.9, < 2.0)
-      reline (>= 0.6.2)
+      nokogiri (>= 1.19.1, < 2.0)
+      reline (>= 0.6.2, < 2.0)
       resolv (>= 0.4.0, < 1.0)
       rubyzip (>= 2.3.0, < 3.0)
 
@@ -65,10 +65,14 @@ GEM
       csv (>= 3.3.0)
       httparty (>= 0.22.0)
       nokogiri (>= 1.16.0)
+    ast (2.4.3)
     base64 (0.3.0)
     benchmark (0.4.1)
-    bigdecimal (3.2.2)
+    bigdecimal (4.1.0)
     builder (3.3.0)
+    bundler-audit (0.9.3)
+      bundler (>= 1.2.0)
+      thor (~> 1.0)
     coderay (1.1.3)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
@@ -83,7 +87,7 @@ GEM
     erubi (1.13.1)
     fileutils (1.7.3)
     hashdiff (1.2.0)
-    httparty (0.23.1)
+    httparty (0.24.2)
       csv
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
@@ -97,6 +101,8 @@ GEM
       httparty (>= 0.23)
       oj (= 3.16.11)
       rubyzip (>= 2.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
     logger (1.7.0)
     loofah (2.24.1)
       crass (~> 1.0.2)
@@ -113,22 +119,28 @@ GEM
       ruby-progressbar
     mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    multi_xml (0.7.2)
-      bigdecimal (~> 3.1)
-    nokogiri (1.18.9-arm64-darwin)
+    multi_xml (0.8.1)
+      bigdecimal (>= 3.1, < 5)
+    nokogiri (1.19.2-arm64-darwin)
       racc (~> 1.4)
     oj (3.16.11)
       bigdecimal (>= 3.0)
       ostruct (>= 0.2)
     ostruct (0.6.3)
+    parallel (1.27.0)
+    parser (3.3.9.0)
+      ast (~> 2.4.1)
+      racc
+    prism (1.4.0)
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (6.0.2)
     racc (1.8.1)
-    rack (2.2.17)
-    rack-session (1.0.2)
-      rack (< 3)
+    rack (3.2.5)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0)
     rack-test (2.2.0)
       rack (>= 1.3)
     rails-dom-testing (2.3.0)
@@ -138,11 +150,41 @@ GEM
     rails-html-sanitizer (1.6.2)
       loofah (~> 2.21)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    rainbow (3.1.1)
     rake (13.3.0)
+    regexp_parser (2.11.2)
     reline (0.6.2)
       io-console (~> 0.5)
     resolv (0.6.2)
-    rexml (3.4.1)
+    rexml (3.4.4)
+    rubocop (1.80.1)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.46.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.46.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
+    rubocop-rails (2.33.3)
+      activesupport (>= 4.2.0)
+      lint_roller (~> 1.1)
+      rack (>= 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
+    rubocop-rails-omakase (1.1.0)
+      rubocop (>= 1.72)
+      rubocop-performance (>= 1.24)
+      rubocop-rails (>= 2.30)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.4.1)
@@ -154,11 +196,15 @@ GEM
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
     sqlite3 (2.7.3-arm64-darwin)
+    thor (1.5.0)
     timecop (0.9.10)
     timeout (0.4.3)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    uri (1.0.3)
+    unicode-display_width (3.1.5)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    uri (1.1.1)
     useragent (0.16.11)
     webmock (3.24.0)
       addressable (>= 2.8.0)
@@ -167,10 +213,12 @@ GEM
 
 PLATFORMS
   arm64-darwin-24
+  arm64-darwin-25
 
 DEPENDENCIES
   UrlCategorise!
   activerecord (>= 8.0)
+  bundler-audit (~> 0.9)
   logger
   minitest (~> 5.25.5)
   minitest-focus (~> 1.4.0)
@@ -178,6 +226,7 @@ DEPENDENCIES
   mocha (~> 2.4.5)
   pry (~> 0.15.2)
   rake (~> 13.3.0)
+  rubocop-rails-omakase (~> 1.0)
   simplecov (~> 0.22.0)
   sqlite3 (>= 2.7)
   timecop (~> 0.9.10)

data/README.md

@@ -6,6 +6,8 @@ A comprehensive Ruby gem for categorizing URLs and domains based on various secu
 
 - **Comprehensive Coverage**: 60+ high-quality categories including security, content, and specialized lists
 - **Video Content Detection**: Advanced regex-based categorization with `video_url?` method to distinguish video content from other website resources
+- **Blog Content Detection**: Simple string-based `blog_url?` method to identify blog-related URLs and content
+- **Debug Mode**: Comprehensive debug logging with timing information for initialization and operations
 - **Custom Video Lists**: Generate and maintain comprehensive video hosting domain lists using yt-dlp extractors
 - **Kaggle Dataset Integration**: Automatic loading and processing of machine learning datasets from Kaggle
 - **Multiple Data Sources**: Supports blocklists, CSV datasets, and Kaggle ML datasets  
@@ -162,11 +164,11 @@ $ bundle exec export_hosts --output /tmp/hosts --verbose
 # Export CSV data with all features enabled
 $ bundle exec export_csv --output /tmp/csv --iab-compliance --smart-categorization --auto-load-datasets --verbose
 
-# Generate updated video hosting lists
-$ ruby bin/generate_video_lists
+# Generate updated video hosting lists (with custom worker configuration)
+$ ruby bin/generate_video_lists --workers 8 --batch-size 50
 
-# Check health of all blocklist URLs
-$ bundle exec check_lists
+# Check health of all blocklist URLs (with parallel processing)
+$ bundle exec check_lists --parallel --threads 16
 
 # Export with custom Kaggle credentials
 $ bundle exec export_csv --auto-load-datasets --kaggle-credentials ~/my-kaggle.json --verbose
@@ -183,6 +185,8 @@ $ bundle exec check_lists
 - `--kaggle-credentials FILE`: Specify custom Kaggle credentials file
 - `--iab-compliance`: Enable IAB Content Taxonomy mapping
 - `--smart-categorization`: Enable intelligent category filtering
+- `--parallel` / `--threads NUM`: Enable parallel URL checking with configurable thread count
+- `--workers NUM`: Configure number of Ractor workers for video list generation
 
 ## Advanced Configuration
 
@@ -244,10 +248,32 @@ client = UrlCategorise::Client.new(
   iab_compliance: true,                                    # Enable IAB compliance
   iab_version: :v3,                                        # Use IAB Content Taxonomy v3.0
   auto_load_datasets: false,                               # Disable automatic dataset loading (default)
-  smart_categorization: false                              # Disable smart post-processing (default)
+  smart_categorization: false,                             # Disable smart post-processing (default)
+  max_threads: 8,                                          # Max threads for parallel processing (default: 8)
+  max_ractor_workers: 4,                                   # Max Ractor workers for parallel processing (default: 4 or CPU count)
+  parallel_loading: true                                   # Enable parallel loading when available (default: true)
 )
 ```
 
+### Parallel Processing Configuration
+
+The gem uses Ruby Ractors (when available) or Threads for parallel processing to significantly improve performance:
+
+```ruby
+# Configure parallel processing parameters
+client = UrlCategorise::Client.new(
+  parallel_loading: true,        # Enable/disable parallel loading (default: true if Ractors available)
+  max_threads: 16,              # Maximum threads for thread-based processing (default: 8)
+  max_ractor_workers: 8         # Maximum Ractor workers for Ractor-based processing (default: 4 or CPU count)
+)
+```
+
+**Performance Notes:**
+- **Ractors**: Used by default on Ruby 3.0+ when available, providing better isolation and performance
+- **Threads**: Fallback method when Ractors are unavailable, still provides significant speedup
+- **Test Environment**: Automatically switches to sequential processing during tests to avoid issues
+- **Worker Pools**: Both Ractors and Threads use worker pool patterns to prevent resource exhaustion
+
 ### Custom Lists
 
 Use your own curated lists or subset of categories:
@@ -273,7 +299,7 @@ The gem maintains a comprehensive list of video hosting domains extracted from y
 
 ```ruby
 # Generate/update video hosting lists
-system("ruby bin/generate_video_lists")
+system("ruby bin/generate_video_lists --workers 4 --batch-size 25")
 
 # Use video hosting categorization
 client = UrlCategorise::Client.new
@@ -331,13 +357,56 @@ client.video_url?("https://google.com/search?q=cats")       # => false
 3. Returns `true` only if both conditions are met
 4. Handles invalid URLs gracefully (returns `false`)
 
+#### Additional Video URL Helper Methods
+
+The gem provides specialized helper methods for different types of video content:
+
+```ruby
+client = UrlCategorise::Client.new(regex_categorization: true)
+
+# Detect short-form video content
+client.shorts_url?("https://youtube.com/shorts/abc123defgh")     # => true
+client.shorts_url?("https://tiktok.com/@user/video/123456789")  # => true
+client.shorts_url?("https://youtube.com/watch?v=test123")       # => false
+
+# Detect playlist URLs
+client.playlist_url?("https://youtube.com/playlist?list=PLtest123")        # => true
+client.playlist_url?("https://youtube.com/watch?v=abc123&list=PLtest123")  # => true
+client.playlist_url?("https://vimeo.com/album/123456")                     # => true
+client.playlist_url?("https://youtube.com/watch?v=test123")                # => false
+
+# Detect music content (works with video platforms hosting music)
+client.music_url?("https://music.youtube.com/watch?v=abc123")              # => true
+client.music_url?("https://youtube.com/watch?v=abc123defgh&list=PLmusic")  # => true
+client.music_url?("https://youtube.com/c/musicchannel")                    # => true
+client.music_url?("https://youtube.com/watch?v=regularvideo")              # => false
+
+# Detect channel/profile URLs
+client.channel_url?("https://youtube.com/@channelname")       # => true
+client.channel_url?("https://tiktok.com/@username")           # => true
+client.channel_url?("https://twitch.tv/streamername")         # => true
+client.channel_url?("https://youtube.com/watch?v=test123")    # => false
+
+# Detect live stream URLs
+client.live_stream_url?("https://youtube.com/live/streamid")      # => true
+client.live_stream_url?("https://twitch.tv/streamername")         # => true
+client.live_stream_url?("https://youtube.com/watch?v=test123")    # => false
+```
+
+**All helper methods:**
+- Require `regex_categorization: true` to be enabled
+- First verify the URL is from a video hosting domain
+- Use specific regex patterns for accurate detection
+- Handle invalid URLs gracefully (return `false`)
+- Work across multiple video platforms (YouTube, TikTok, Vimeo, Twitch, etc.)
+
 #### Maintaining Video Lists
 
 The gem includes a script to generate and maintain comprehensive video hosting lists:
 
 ```bash
-# Generate updated video hosting lists
-ruby bin/generate_video_lists
+# Generate updated video hosting lists (configurable parallel processing)
+ruby bin/generate_video_lists --workers 8 --batch-size 50 --threshold 20
 
 # This creates:
 # - lists/video_hosting_domains.hosts (PiHole compatible)  
@@ -346,6 +415,92 @@ ruby bin/generate_video_lists
 
 The script fetches data from yt-dlp extractors and combines it with manually curated major platforms to ensure comprehensive coverage.
 
+### Blog Content Detection
+
+The gem provides a `blog_url?` method to identify blog-related URLs using simple string matching patterns:
+
+```ruby
+client = UrlCategorise::Client.new
+
+# Basic blog path detection
+client.blog_url?("https://example.com/blog/")           # => true
+client.blog_url?("https://example.com/blogs/tech")      # => true
+client.blog_url?("https://example.com/blog?page=1")     # => true
+
+# Blog subdomains
+client.blog_url?("https://blog.example.com/")          # => true
+client.blog_url?("https://blog.company.org/article")   # => true
+
+# Blog platforms
+client.blog_url?("https://example.wordpress.com/")     # => true
+client.blog_url?("https://example.blogspot.com/post")  # => true
+client.blog_url?("https://medium.com/@user/article")   # => true
+client.blog_url?("https://user.substack.com/p/post")   # => true
+
+# Blog-like content paths
+client.blog_url?("https://example.com/post/123")       # => true
+client.blog_url?("https://example.com/articles/tech")  # => true
+client.blog_url?("https://example.com/diary/entry")    # => true
+
+# Blog keywords in URLs
+client.blog_url?("https://example.com/corporate-blog") # => true
+client.blog_url?("https://example-blog.com/")          # => true
+
+# Non-blog URLs return false
+client.blog_url?("https://example.com/")               # => false
+client.blog_url?("https://example.com/products")       # => false
+```
+
+**Detection patterns include:**
+- `/blog/` or `/blogs/` in URL paths
+- `blog.` subdomains
+- `blog-` or `-blog` in domain names
+- Common blog platforms (WordPress, Blogspot, Medium, Substack)
+- Blog-like content paths (`/post/`, `/articles/`, `/diary/`, `/journal/`)
+- The word "blog" anywhere in the URL
+- Case-insensitive matching
+- Graceful handling of invalid URLs
+
+### Debug Mode
+
+The gem includes comprehensive debug functionality to help you understand what's happening during initialization and operation:
+
+```ruby
+# Enable debug mode during initialization
+client = UrlCategorise::Client.new(debug: true)
+
+# Or enable it dynamically using ActiveAttr
+client = UrlCategorise::Client.new
+client.debug_enabled = true
+
+# Debug output shows:
+# [UrlCategorise DEBUG] Initializing UrlCategorise Client with debug enabled
+# [UrlCategorise DEBUG] Loading host lists from 15 categories
+# [UrlCategorise DEBUG] Processing host list: https://example.com/malware.txt
+# [UrlCategorise DEBUG] Cache miss for https://example.com/malware.txt
+# [UrlCategorise DEBUG] Downloading and parsing https://example.com/malware.txt completed in 234.56ms
+# [UrlCategorise DEBUG] Downloaded 1500 hosts from https://example.com/malware.txt
+# [UrlCategorise DEBUG] Total unique hosts collected: 45000
+# [UrlCategorise DEBUG] Host lists loading completed in 2543.21ms
+# [UrlCategorise DEBUG] Client initialization completed
+```
+
+**Debug features include:**
+- Initialization timing and progress
+- Host list loading with individual URL timing
+- Cache hit/miss information
+- Download progress and host counts
+- Dataset loading progress (when datasets are enabled)
+- Regex pattern loading information
+- Total timing for major operations
+- Off by default, easily enabled via constructor or ActiveAttr
+
+**Timing accuracy:**
+- Millisecond precision timing
+- Individual operation timing
+- Cumulative timing for complex operations
+- Helpful for performance optimization and debugging
+
 ### Smart Categorization (Post-Processing)
 
 Smart categorization solves the problem of overly broad domain-level categorization. For example, `reddit.com` might appear in health & fitness blocklists, but not all Reddit content is health-related.
@@ -559,8 +714,11 @@ end
 
 Use the included script to check all URLs:
 ```bash
-# Check all URLs in constants
+# Check all URLs in constants (sequential)
 ruby bin/check_lists
+
+# Check URLs in parallel with custom thread count
+ruby bin/check_lists --parallel --threads 12 --verbose
 ```
 
 [View all 60+ categories in constants.rb](lib/url_categorise/constants.rb)

data/Rakefile

@@ -6,7 +6,7 @@ Rake::TestTask.new(:test) do |t|
   t.libs << 'test'
   t.libs << 'lib'
   t.test_files = FileList['test/**/*_test.rb']
-  t.ruby_opts = ['-rbundler/setup']
+  t.ruby_opts = [ '-rbundler/setup' ]
 end
 
 task default: :test