UrlCategorise 0.1.9 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c22ada722efc33979930ece5d8737625943f1d815d2c7acdbfbc0f5b0fede15f
-  data.tar.gz: 15747c8a4b23c4c805cd74118627dfa8d102d4d813ae9f08c8c49011ff2ebad8
+  metadata.gz: 6ed2176041a871f6fae72dc1b7c1f266910f6233879e1359d02eead164f8f6ff
+  data.tar.gz: 5ff46c4dd9faaa22218a86d7b40f54bedaa6f91e9914b8e9671b14ba55ec6495
 SHA512:
-  metadata.gz: b1c21b59b8a7631c4c39f206fb37bb9c51c1925a544ddd5f8a627b595dafa92d366d880a33c6303eafc477ae9be4130055af060953e27617e5ce198618e5f9bd
-  data.tar.gz: 1621c52578daf8957546c778125df120f9480dbe2f2d365feb6024890ddfe5affac3aae77100047ea46b15c96939db1fc228b876806197c63dda954199536bc9
+  metadata.gz: 71aa116bb3de0c269ccaf18a3f159f78c722ab010589a2118c411f826f1ff4de48b9e5ed8c9936a67a6eac3cb17b3f52d71e41bac552e73a91c67edee804f9c4
+  data.tar.gz: 4c2d8f50a9cad1b1f32c56ddc51b44e8ca28f49af5718970bb0bea02a12e934cb2c8075589fa9ca8db56437e3a377a5e0cc1afe7a30e44aab5b393cb94a70f3a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    UrlCategorise (0.1.9)
+    UrlCategorise (0.1.10)
       active_attr (>= 0.17.1, < 1.0)
       api_pattern (>= 0.0.6, < 1.0)
       csv (>= 3.3.0, < 4.0)
@@ -18,9 +18,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (8.0.2.1)
-      actionview (= 8.0.2.1)
-      activesupport (= 8.0.2.1)
+    actionpack (8.1.3)
+      actionview (= 8.1.3)
+      activesupport (= 8.1.3)
       nokogiri (>= 1.8.5)
       rack (>= 2.2.4)
       rack-session (>= 1.0.1)
@@ -28,37 +28,37 @@ GEM
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
       useragent (~> 0.16)
-    actionview (8.0.2.1)
-      activesupport (= 8.0.2.1)
+    actionview (8.1.3)
+      activesupport (= 8.1.3)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    active_attr (0.17.1)
-      actionpack (>= 3.0.2, < 8.1)
-      activemodel (>= 3.0.2, < 8.1)
-      activesupport (>= 3.0.2, < 8.1)
-    activemodel (8.0.2.1)
-      activesupport (= 8.0.2.1)
-    activerecord (8.0.2.1)
-      activemodel (= 8.0.2.1)
-      activesupport (= 8.0.2.1)
+    active_attr (0.17.2)
+      actionpack (>= 3.0.2, < 8.2)
+      activemodel (>= 3.0.2, < 8.2)
+      activesupport (>= 3.0.2, < 8.2)
+    activemodel (8.1.3)
+      activesupport (= 8.1.3)
+    activerecord (8.1.3)
+      activemodel (= 8.1.3)
+      activesupport (= 8.1.3)
       timeout (>= 0.4.0)
-    activesupport (8.0.2.1)
+    activesupport (8.1.3)
       base64
-      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      json
       logger (>= 1.4.2)
       minitest (>= 5.1)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
       uri (>= 0.13.1)
-    addressable (2.8.7)
-      public_suffix (>= 2.0.2, < 7.0)
+    addressable (2.9.0)
+      public_suffix (>= 2.0.2, < 8.0)
     ansi (1.5.0)
     api_pattern (0.0.6)
       active_attr (>= 0.15.4)
@@ -67,15 +67,14 @@ GEM
       nokogiri (>= 1.16.0)
     ast (2.4.3)
     base64 (0.3.0)
-    benchmark (0.4.1)
-    bigdecimal (4.1.0)
+    bigdecimal (4.1.2)
     builder (3.3.0)
     bundler-audit (0.9.3)
       bundler (>= 1.2.0)
       thor (~> 1.0)
     coderay (1.1.3)
-    concurrent-ruby (1.3.5)
-    connection_pool (2.5.3)
+    concurrent-ruby (1.3.6)
+    connection_pool (3.0.2)
     crack (1.0.0)
       bigdecimal
       rexml
@@ -91,10 +90,10 @@ GEM
       csv
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
-    i18n (1.14.7)
+    i18n (1.14.8)
       concurrent-ruby (~> 1.0)
     io-console (0.8.1)
-    json (2.13.2)
+    json (2.19.5)
     kaggle (0.0.3)
       csv (>= 3.3)
       fileutils (>= 1.7)
@@ -104,7 +103,7 @@ GEM
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     logger (1.7.0)
-    loofah (2.24.1)
+    loofah (2.25.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     method_source (1.1.0)
@@ -121,7 +120,7 @@ GEM
       ruby2_keywords (>= 0.0.5)
     multi_xml (0.8.1)
       bigdecimal (>= 3.1, < 5)
-    nokogiri (1.19.2-arm64-darwin)
+    nokogiri (1.19.3-arm64-darwin)
       racc (~> 1.4)
     oj (3.16.11)
       bigdecimal (>= 3.0)
@@ -135,10 +134,10 @@ GEM
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (6.0.2)
+    public_suffix (7.0.5)
     racc (1.8.1)
-    rack (3.2.5)
-    rack-session (2.1.1)
+    rack (3.2.6)
+    rack-session (2.1.2)
       base64 (>= 0.1.0)
       rack (>= 3.0.0)
     rack-test (2.2.0)
@@ -147,8 +146,8 @@ GEM
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.2)
-      loofah (~> 2.21)
+    rails-html-sanitizer (1.7.0)
+      loofah (~> 2.25)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
     rainbow (3.1.1)
     rake (13.3.0)

data/bin/generate_video_lists_focused

@@ -0,0 +1,381 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'fileutils'
+require 'time'
+require 'set'
+
+module VideoListGenerator
+  class FocusedExtractorParser
+    # Focus on the most popular video hosting platforms first
+    PRIORITY_EXTRACTORS = [
+      'youtube.py',
+      'vimeo.py', 
+      'dailymotion.py',
+      'twitch.py',
+      'tiktok.py',
+      'instagram.py',
+      'facebook.py',
+      'twitter.py',
+      'reddit.py',
+      'streamable.py',
+      'wistia.py',
+      'brightcove.py',
+      'jwplatform.py',
+      'kaltura.py',
+      'vidyard.py',
+      'vzaar.py',
+      'soundcloud.py',
+      'mixcloud.py',
+      'bandcamp.py',
+      'peertube.py',
+      'rumble.py',
+      'odysee.py',
+      'bitchute.py'
+    ].freeze
+
+    YT_DLP_API_URL = 'https://api.github.com/repos/yt-dlp/yt-dlp/contents/yt_dlp/extractor'
+    
+    def initialize
+      @video_domains = Set.new
+      @video_regexes = []
+      @manual_domains = get_manual_video_domains
+    end
+
+    def generate_lists
+      puts "🎬 Generating focused video hosting lists..."
+      
+      puts "📝 Adding manually curated video hosting domains..."
+      @manual_domains.each { |domain| @video_domains.add(domain) }
+      
+      puts "📡 Fetching priority yt-dlp extractors..."
+      fetch_priority_extractors
+      
+      create_lists_directory
+      generate_hosts_file
+      generate_regex_file
+      
+      puts "✅ Video hosting lists generated successfully!"
+      puts "📁 Files created in lists/ directory:"
+      puts "   - video_hosting_domains.hosts (PiHole compatible)"
+      puts "   - video_url_patterns.txt (Regex patterns for video detection)"
+    end
+
+    private
+
+    def get_manual_video_domains
+      # Manually curated list of major video hosting domains
+      [
+        'youtube.com', 'youtu.be', 'youtube-nocookie.com',
+        'vimeo.com', 'player.vimeo.com',
+        'dailymotion.com', 'dai.ly',
+        'twitch.tv', 'clips.twitch.tv',
+        'tiktok.com', 'vm.tiktok.com',
+        'instagram.com', 'instagr.am',
+        'facebook.com', 'fb.com', 'fb.watch',
+        'twitter.com', 'x.com', 't.co',
+        'reddit.com', 'redd.it', 'v.redd.it',
+        'streamable.com',
+        'wistia.com', 'fast.wistia.com', 'fast.wistia.net',
+        'brightcove.com', 'bcove.video',
+        'jwplatform.com', 'jwpcdn.com',
+        'kaltura.com',
+        'vidyard.com',
+        'vzaar.com',
+        'soundcloud.com', 'snd.sc',
+        'mixcloud.com',
+        'bandcamp.com',
+        'rumble.com',
+        'odysee.com', 'lbry.tv',
+        'bitchute.com',
+        'peertube.tv',
+        'archive.org',
+        'vevo.com',
+        'liveleak.com',
+        'metacafe.com',
+        'veoh.com',
+        'break.com',
+        'crunchyroll.com',
+        'funimation.com',
+        'vrv.co',
+        'roosterteeth.com',
+        'newgrounds.com',
+        'coub.com',
+        'vidme.com',
+        'vid.me',
+        'gfycat.com',
+        'imgur.com',
+        'giphy.com',
+        'tenor.com'
+      ]
+    end
+
+    def fetch_priority_extractors
+      uri = URI(YT_DLP_API_URL)
+      response = Net::HTTP.get_response(uri)
+      
+      unless response.code == '200'
+        puts "⚠️  Failed to fetch extractor list from GitHub API, using manual domains only"
+        return
+      end
+      
+      files = JSON.parse(response.body)
+      
+      # Filter to priority extractor files
+      priority_files = files.select { |file| PRIORITY_EXTRACTORS.include?(file['name']) }
+      
+      puts "📝 Processing #{priority_files.length} priority extractor files..."
+      
+      priority_files.each_with_index do |file, index|
+        puts "Processing #{index + 1}/#{priority_files.length}: #{file['name']}"
+        process_extractor_file(file)
+      end
+      
+      puts "🎯 Found #{@video_domains.size} unique video hosting domains"
+      puts "🔍 Extracted #{@video_regexes.size} URL pattern regexes"
+    end
+
+    def process_extractor_file(file_info)
+      begin
+        response = Net::HTTP.get_response(URI(file_info['download_url']))
+        return unless response.code == '200'
+        
+        content = response.body
+        
+        extract_domains_from_content(content, file_info['name'])
+        extract_regexes_from_content(content, file_info['name'])
+      rescue StandardError => e
+        puts "⚠️  Error processing #{file_info['name']}: #{e.message}"
+      end
+    end
+
+    def extract_domains_from_content(content, filename)
+      # Look for _VALID_URL patterns
+      valid_url_patterns = content.scan(/_VALID_URL\s*=\s*r?['"]([^'"]+)['"]/m)
+      
+      valid_url_patterns.each do |pattern|
+        pattern = pattern[0] if pattern.is_a?(Array)
+        
+        domains = extract_domains_from_regex(pattern)
+        domains.each { |domain| @video_domains.add(domain) }
+      end
+      
+      # Also look for explicit domain lists or IE_NAME patterns
+      ie_name_patterns = content.scan(/IE_NAME\s*=\s*['"]([^'"]+)['"]/m)
+      ie_name_patterns.each do |name|
+        name = name[0] if name.is_a?(Array)
+        next unless name
+        
+        # Many IE_NAME values correspond to domain names
+        if name.include?('.') && valid_domain?(name)
+          @video_domains.add(name.downcase)
+        end
+      end
+    end
+
+    def extract_regexes_from_content(content, filename)
+      # Extract _VALID_URL patterns for video detection
+      valid_url_patterns = content.scan(/_VALID_URL\s*=\s*r?['"]([^'"]+)['"]/m)
+      
+      valid_url_patterns.each do |pattern|
+        pattern = pattern[0] if pattern.is_a?(Array)
+        
+        cleaned_pattern = clean_regex_pattern(pattern)
+        if cleaned_pattern && !cleaned_pattern.empty?
+          @video_regexes << {
+            source: filename.gsub('.py', ''),
+            pattern: cleaned_pattern,
+            original: pattern,
+            description: generate_pattern_description(pattern, filename)
+          }
+        end
+      end
+    end
+
+    def extract_domains_from_regex(pattern)
+      domains = Set.new
+      
+      # Various patterns to extract domains from regex
+      # Pattern: https?://(?:www\.)?domain\.com
+      domain_matches = pattern.scan(/https?:\/\/[^\/]*?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i)
+      domain_matches.each do |match|
+        domain = match[0] if match.is_a?(Array)
+        domain = clean_domain(domain) if domain
+        domains.add(domain.downcase) if domain && valid_domain?(domain)
+      end
+      
+      # Pattern: (?:www\.)?domain\.com
+      simple_matches = pattern.scan(/(?:\(\?\:www\\\.\)\?)?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i)
+      simple_matches.each do |match|
+        domain = match[0] if match.is_a?(Array)
+        domain = clean_domain(domain) if domain
+        domains.add(domain.downcase) if domain && valid_domain?(domain)
+      end
+      
+      domains.to_a
+    end
+
+    def clean_domain(domain)
+      return nil unless domain
+      
+      # Remove common regex escapes
+      cleaned = domain.gsub(/\\\./, '.').gsub(/[\(\)\?\:\|\\]/, '')
+      cleaned = cleaned.strip
+      
+      # Remove any trailing regex syntax
+      cleaned = cleaned.split(/[\(\)\[\]]/)[0] if cleaned.include?('(') || cleaned.include?('[')
+      
+      cleaned
+    end
+
+    def clean_regex_pattern(pattern)
+      return nil unless pattern
+      
+      # Simplify common patterns for practical use
+      cleaned = pattern.dup
+      
+      # Remove complex lookaheads/lookbehinds
+      cleaned = cleaned.gsub(/\(\?\![^)]*\)/, '')
+      cleaned = cleaned.gsub(/\(\?\<\![^)]*\)/, '')
+      
+      # Simplify common patterns
+      cleaned = cleaned.gsub(/\(\?\:www\\\.\)\?/, '(?:www\.)?')
+      cleaned = cleaned.gsub(/\\\./, '\.')
+      
+      # Validate that it's a reasonable regex
+      begin
+        Regexp.new(cleaned)
+        cleaned
+      rescue RegexpError
+        # Return a simplified version if the complex regex fails
+        extract_simple_pattern(pattern)
+      end
+    end
+
+    def extract_simple_pattern(pattern)
+      # Extract a simple domain pattern from complex regex
+      if pattern.include?('https?://')
+        # Extract the basic domain part
+        match = pattern.match(/https?:\/\/[^\/]*?([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i)
+        if match
+          return "https?://[^/]*#{Regexp.escape(match[1])}"
+        end
+      end
+      
+      nil
+    end
+
+    def generate_pattern_description(pattern, filename)
+      service = filename.gsub('.py', '').capitalize
+      
+      case filename
+      when 'youtube.py'
+        'YouTube video, playlist, and channel URLs'
+      when 'vimeo.py'
+        'Vimeo video and album URLs'
+      when 'twitch.py'
+        'Twitch streams, clips, and VODs'
+      when 'tiktok.py'
+        'TikTok video URLs'
+      else
+        "#{service} video content URLs"
+      end
+    end
+
+    def valid_domain?(domain)
+      return false unless domain
+      return false if domain.length < 4
+      return false unless domain.match?(/^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/)
+      return false if domain.include?('example.') || domain.include?('test.')
+      return false if domain.include?('localhost')
+      true
+    end
+
+    def create_lists_directory
+      FileUtils.mkdir_p('lists')
+    end
+
+    def generate_hosts_file
+      hosts_file_path = 'lists/video_hosting_domains.hosts'
+      
+      File.open(hosts_file_path, 'w') do |file|
+        write_hosts_header(file)
+        
+        @video_domains.sort.each do |domain|
+          file.puts "0.0.0.0 #{domain}"
+          file.puts "0.0.0.0 www.#{domain}" unless domain.start_with?('www.')
+        end
+      end
+      
+      puts "📝 Generated hosts file: #{hosts_file_path} (#{@video_domains.size} domains)"
+    end
+
+    def generate_regex_file
+      regex_file_path = 'lists/video_url_patterns.txt'
+      
+      File.open(regex_file_path, 'w') do |file|
+        write_regex_header(file)
+        
+        @video_regexes.each do |regex_info|
+          file.puts "# Source: #{regex_info[:source]}"
+          file.puts "# Description: #{regex_info[:description]}"
+          file.puts "# Pattern: #{regex_info[:pattern]}"
+          file.puts "# Original: #{regex_info[:original]}"
+          file.puts regex_info[:pattern]
+          file.puts ""
+        end
+      end
+      
+      puts "📝 Generated regex file: #{regex_file_path} (#{@video_regexes.size} patterns)"
+    end
+
+    def write_hosts_header(file)
+      file.puts "# Video Hosting Domains - PiHole Compatible Hosts File"
+      file.puts "# Generated on: #{Time.now.strftime('%Y-%m-%d %H:%M:%S UTC')}"
+      file.puts "# Source: yt-dlp extractors + manual curation"
+      file.puts "# Purpose: Block or categorize video hosting websites"
+      file.puts "# Format: 0.0.0.0 domain.com"
+      file.puts "#"
+      file.puts "# This file contains domains from popular video hosting platforms"
+      file.puts "# extracted from yt-dlp video extractors and manually curated lists."
+      file.puts "# Use this for URL categorization or content filtering."
+      file.puts "#"
+      file.puts "# Generated by UrlCategorise gem: https://github.com/trex22/url_categorise"
+      file.puts "#"
+      file.puts ""
+    end
+
+    def write_regex_header(file)
+      file.puts "# Video URL Detection Patterns"
+      file.puts "# Generated on: #{Time.now.strftime('%Y-%m-%d %H:%M:%S UTC')}"
+      file.puts "# Source: yt-dlp extractors + manual curation"
+      file.puts "# Purpose: Regex patterns to detect video URLs vs other content"
+      file.puts "#"
+      file.puts "# These patterns help distinguish between:"
+      file.puts "# - Direct video content URLs (matches these patterns)"
+      file.puts "# - Homepage, playlist, user profile, community content URLs"
+      file.puts "#"
+      file.puts "# Usage: Use these patterns to categorize URLs from video hosting domains"
+      file.puts "#        to determine if they contain actual video content or other resources"
+      file.puts "#"
+      file.puts "# Each pattern includes:"
+      file.puts "# - Source extractor name"
+      file.puts "# - Human-readable description"
+      file.puts "# - Simplified regex pattern"
+      file.puts "# - Original complex pattern (for reference)"
+      file.puts "#"
+      file.puts "# Generated by UrlCategorise gem: https://github.com/trex22/url_categorise"
+      file.puts "#"
+      file.puts ""
+    end
+  end
+end
+
+# Run the generator if this file is executed directly
+if __FILE__ == $0
+  generator = VideoListGenerator::FocusedExtractorParser.new
+  generator.generate_lists
+end

data/lib/url_categorise/client.rb

@@ -101,10 +101,16 @@ module UrlCategorise
     end
 
     def categorise(url)
+      return [] if url.nil? || (url.respond_to?(:strip) && url.strip.empty?)
+
       debug_time("Categorizing '#{url}'") do
         debug_log("🔍 Starting categorization for URL: #{url}")
 
-        host = (URI.parse(url).host || url).downcase
+        host = begin
+          (URI.parse(url).host || url).downcase
+        rescue URI::InvalidURIError
+          url.to_s.downcase
+        end
         host = host.gsub("www.", "")
         debug_log("📌 Extracted host: #{host}")
 
@@ -1318,7 +1324,7 @@ module UrlCategorise
       # Phase 2: Process content in parallel using Ractors (if available) or threads
       debug_log("⚡ Phase 2: Processing content in parallel")
 
-      if self.class.ractor_available?
+      if self.class.ractor_available? && !test_environment?
         process_content_with_ractors(downloaded_content)
       else
         process_content_with_threads(downloaded_content)
@@ -1332,9 +1338,10 @@ module UrlCategorise
         original_verbose = $VERBOSE
         $VERBOSE = nil
         work_queue = Ractor.new do
+          # :nocov:
           tasks = []
           no_more_tasks = false
-          
+
           while (task = Ractor.receive) != :done
             if task == :get_task
               if tasks.empty? && no_more_tasks
@@ -1348,6 +1355,7 @@ module UrlCategorise
               tasks << task
             end
           end
+          # :nocov:
         end
 
         # Add all tasks to queue
@@ -1364,6 +1372,7 @@ module UrlCategorise
 
         workers = num_workers.times.map do
           Ractor.new(work_queue, cache_dir) do |queue, cache_path|
+            # :nocov:
             require "digest"
             require "fileutils"
 
@@ -1433,6 +1442,7 @@ module UrlCategorise
             end
 
             results
+            # :nocov:
           end
         end
 
@@ -1585,9 +1595,11 @@ module UrlCategorise
     end
 
     def extract_host(url)
+      return nil if url.nil? || (url.respond_to?(:strip) && url.strip.empty?)
+
       (URI.parse(url).host || url).downcase.gsub("www.", "")
     rescue URI::InvalidURIError
-      url.downcase.gsub("www.", "")
+      url.to_s.downcase.gsub("www.", "")
     end
 
     def build_host_data(urls)
@@ -1803,7 +1815,7 @@ module UrlCategorise
       return true if url.start_with?("file://")
 
       uri = URI.parse(url)
-      uri.is_a?(URI::HTTP) && !uri.host.nil?
+      uri.is_a?(URI::HTTP) && !uri.host.nil? && !uri.host.empty?
     rescue URI::InvalidURIError
       false
     end

data/lib/url_categorise/dataset_processor.rb

@@ -262,10 +262,16 @@ module UrlCategorise
 
     def extract_zip_file(zip_file_path, extract_to_dir)
       FileUtils.mkdir_p(extract_to_dir)
+      base_dir = File.expand_path(extract_to_dir)
 
       Zip::File.open(zip_file_path) do |zip_file|
         zip_file.each do |entry|
-          extract_path = File.join(extract_to_dir, entry.name)
+          extract_path = File.expand_path(File.join(base_dir, entry.name))
+
+          # Reject entries that would be written outside the extraction directory (zip-slip)
+          unless extract_path.start_with?("#{base_dir}#{File::SEPARATOR}")
+            raise Error, "Unsafe path in zip file: #{entry.name}"
+          end
 
           if entry.directory?
             FileUtils.mkdir_p(extract_path)

data/lib/url_categorise/version.rb

@@ -1,3 +1,3 @@
 module UrlCategorise
-  VERSION = "0.1.9"
+  VERSION = "0.1.11"
 end

data/lists/blogging_domains.hosts

@@ -1,5 +1,5 @@
 # Blogging Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/crypto_domains.hosts

@@ -1,5 +1,5 @@
 # Cryptocurrency Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/developer_domains.hosts

@@ -1,5 +1,5 @@
 # Developer Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/forum_domains.hosts

@@ -1,5 +1,5 @@
 # Forum and Q&A Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/gaming_domains.hosts

@@ -1,5 +1,5 @@
 # Gaming Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/messaging_domains.hosts

@@ -1,5 +1,5 @@
 # Messaging Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/music_domains.hosts

@@ -1,5 +1,5 @@
 # Music Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/security_domains.hosts

@@ -1,5 +1,5 @@
 # Security Community Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/social_media_domains.hosts

@@ -1,5 +1,5 @@
 # Social Media Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:27 UTC
+# Generated on: 2026-05-22 11:06:06 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated social media platform domains

data/lists/streaming_domains.hosts

@@ -1,5 +1,5 @@
 # Streaming Platform Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:10:15 UTC
+# Generated on: 2026-05-22 11:06:15 UTC
 # Sources:
 #   - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
 #   - Manually curated domain lists

data/lists/video_hosting_domains.hosts

@@ -1,5 +1,5 @@
 # Video Hosting Domains - PiHole Compatible Hosts File
-# Generated on: 2026-02-15 21:17:19 UTC
+# Generated on: 2026-05-22 11:05:43 UTC
 # Source: yt-dlp extractors (https://github.com/yt-dlp/yt-dlp)
 # Purpose: Block access to video hosting websites
 # Format: 0.0.0.0 domain.com
@@ -618,6 +618,8 @@
 0.0.0.0 www.api.vkplay
 0.0.0.0 api.vmh
 0.0.0.0 www.api.vmh
+0.0.0.0 api.watch
+0.0.0.0 www.api.watch
 0.0.0.0 api.whowatch
 0.0.0.0 www.api.whowatch
 0.0.0.0 api.zype
@@ -704,6 +706,8 @@
 0.0.0.0 www.ardaudiothek.de
 0.0.0.0 ardmediathek.de
 0.0.0.0 www.ardmediathek.de
+0.0.0.0 ardsounds.de
+0.0.0.0 www.ardsounds.de
 0.0.0.0 arhiiv.err
 0.0.0.0 www.arhiiv.err
 0.0.0.0 armed-f.akamaihd
@@ -1168,6 +1172,8 @@
 0.0.0.0 www.capitaltv.cy
 0.0.0.0 caracoltv.com
 0.0.0.0 www.caracoltv.com
+0.0.0.0 cas.global
+0.0.0.0 www.cas.global
 0.0.0.0 catalogue.pr
 0.0.0.0 www.catalogue.pr
 0.0.0.0 cbc.ca
@@ -1258,6 +1264,8 @@
 0.0.0.0 www.cdn.syndication
 0.0.0.0 cdn.teamcococdn
 0.0.0.0 www.cdn.teamcococdn
+0.0.0.0 cdn.thechosen
+0.0.0.0 www.cdn.thechosen
 0.0.0.0 cdn.vidyard
 0.0.0.0 www.cdn.vidyard
 0.0.0.0 cdn.viqeo
@@ -1304,8 +1312,6 @@
 0.0.0.0 www.cgtn.com
 0.0.0.0 ch.nicovideo
 0.0.0.0 www.ch.nicovideo
-0.0.0.0 ch.sooplive
-0.0.0.0 www.ch.sooplive
 0.0.0.0 channel.microcms
 0.0.0.0 www.channel.microcms
 0.0.0.0 channel4.com
@@ -2204,8 +2210,6 @@
 0.0.0.0 www.feeds.cbsn
 0.0.0.0 feeds.rasset
 0.0.0.0 www.feeds.rasset
-0.0.0.0 feeds.video
-0.0.0.0 www.feeds.video
 0.0.0.0 felixcosm.tumblr
 0.0.0.0 www.felixcosm.tumblr
 0.0.0.0 fem.com
@@ -3754,8 +3758,6 @@
 0.0.0.0 www.music.amazon
 0.0.0.0 music.apple
 0.0.0.0 www.music.apple
-0.0.0.0 music.yandex
-0.0.0.0 www.music.yandex
 0.0.0.0 music.youtube
 0.0.0.0 www.music.youtube
 0.0.0.0 musicdex.com
@@ -4730,6 +4732,8 @@
 0.0.0.0 www.public-api.wordpress
 0.0.0.0 public.api
 0.0.0.0 www.public.api
+0.0.0.0 public.aurora
+0.0.0.0 www.public.aurora
 0.0.0.0 puhutv.com
 0.0.0.0 www.puhutv.com
 0.0.0.0 puls4.com
@@ -5318,6 +5322,8 @@
 0.0.0.0 www.solidarum.org
 0.0.0.0 sonyliv.com
 0.0.0.0 www.sonyliv.com
+0.0.0.0 sooplive.com
+0.0.0.0 www.sooplive.com
 0.0.0.0 sound.orf
 0.0.0.0 www.sound.orf
 0.0.0.0 soundcloud.com
@@ -5776,6 +5782,8 @@
 0.0.0.0 www.the-voice-of-germany.de
 0.0.0.0 theater-complex.town
 0.0.0.0 www.theater-complex.town
+0.0.0.0 thechosen.com
+0.0.0.0 www.thechosen.com
 0.0.0.0 theepochtimes.com
 0.0.0.0 www.theepochtimes.com
 0.0.0.0 theguardian.com
@@ -7041,8 +7049,6 @@
 0.0.0.0 www.yahoo.com
 0.0.0.0 yandexdisk.com
 0.0.0.0 www.yandexdisk.com
-0.0.0.0 yandexmusic.com
-0.0.0.0 www.yandexmusic.com
 0.0.0.0 yinyue.kuwo
 0.0.0.0 www.yinyue.kuwo
 0.0.0.0 yoga.appsvcs

data/lists/video_url_patterns.txt

@@ -1,5 +1,5 @@
 # Video URL Detection Patterns
-# Generated on: 2026-02-15 21:17:19 UTC
+# Generated on: 2026-05-22 11:05:44 UTC
 # Source: yt-dlp extractors (https://github.com/yt-dlp/yt-dlp) + manual patterns
 # Purpose: Regex patterns to detect video URLs vs other content
 #
@@ -190,26 +190,26 @@ https?://(?:www\.)?gronkh\.tv(?:/feed)?/?(?:#|$)
 # Original: https?://(?:www\.)?gronkh\.tv/vods/streams/?(?:#|$)
 https?://(?:www\.)?gronkh\.tv/vods/streams/?(?:#|$)
 
-# Source: internetvideoarchive
-# Pattern: https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?
-# Original: https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?
-https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?
-
 # Source: iqiyi
 # Pattern: https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html
 # Original: https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html
 https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html
 
-# Source: jeuxvideo
-# Pattern: https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm
-# Original: https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm
-https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm
+# Source: internetvideoarchive
+# Pattern: https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?
+# Original: https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?
+https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?
 
 # Source: japandiet
 # Pattern: https?://www\.webtv\.sangiin\.go\.jp/webtv/index\.php
 # Original: https?://www\.webtv\.sangiin\.go\.jp/webtv/index\.php
 https?://www\.webtv\.sangiin\.go\.jp/webtv/index\.php
 
+# Source: jeuxvideo
+# Pattern: https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm
+# Original: https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm
+https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm
+
 # Source: leeco
 # Pattern: https?://yuntv\.letv\.com/bcloud.html\?.+
 # Original: https?://yuntv\.letv\.com/bcloud.html\?.+

data/url_categorise.gemspec

@@ -30,11 +30,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'fileutils', '>= 1.7.0', '< 2.0'
   spec.add_dependency 'httparty', '>= 0.24.0', '< 1.0'
   spec.add_dependency 'json', '>= 2.7.0', '< 3.0'
-  spec.add_dependency 'kaggle', '>= 0.0.3', '< 1.0'
+  spec.add_dependency 'kaggle', '>= 0.0.6', '< 1.0'
   spec.add_dependency 'nokogiri', '>= 1.19.1', '< 2.0'
   spec.add_dependency 'reline', '>= 0.6.2', '< 2.0'
   spec.add_dependency 'resolv', '>= 0.4.0', '< 1.0'
-  spec.add_dependency 'rubyzip', '>= 2.3.0', '< 3.0'
+  spec.add_dependency 'rubyzip', '>= 3.4.0', '< 4.0'
 
   # Development dependancies
   spec.add_development_dependency 'minitest', '~> 5.25.5'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: UrlCategorise
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.11
 platform: ruby
 authors:
 - trex22
@@ -155,7 +155,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.0.6
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
@@ -165,7 +165,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.0.6
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
@@ -235,20 +235,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 3.4.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 3.4.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -476,6 +476,7 @@ files:
 - bin/generate_categorised_lists
 - bin/generate_social_media_lists
 - bin/generate_video_lists
+- bin/generate_video_lists_focused
 - bin/rake
 - bin/setup
 - correct_usage_example.rb