UrlCategorise 0.1.3 → 0.1.6

data/lib/url_categorise/iab_compliance.rb

@@ -0,0 +1,147 @@
+module UrlCategorise
+  module IabCompliance
+    IAB_V2_MAPPINGS = {
+      # Content Categories
+      advertising: 'IAB3', # Advertising
+      automotive: 'IAB2', # Automotive
+      books_literature: 'IAB20', # Books & Literature
+      business: 'IAB3', # Business
+      careers: 'IAB4', # Careers
+      education: 'IAB5', # Education
+      entertainment: 'IAB1', # Arts & Entertainment
+      finance: 'IAB13', # Personal Finance
+      food_drink: 'IAB8', # Food & Drink
+      health: 'IAB7', # Health & Fitness
+      hobbies_interests: 'IAB9', # Hobbies & Interests
+      home_garden: 'IAB10', # Home & Garden
+      law_government: 'IAB11', # Law, Government & Politics
+      news: 'IAB12', # News
+      parenting: 'IAB6', # Family & Parenting
+      pets: 'IAB16', # Pets
+      philosophy: 'IAB21', # Philosophy/Religion
+      real_estate: 'IAB21', # Real Estate
+      science: 'IAB15', # Science
+      shopping: 'IAB22', # Shopping
+      sports: 'IAB17', # Sports
+      style_fashion: 'IAB18', # Style & Fashion
+      technology: 'IAB19', # Technology & Computing
+      travel: 'IAB20', # Travel
+
+      # Security & Malware Categories
+      malware: 'IAB25', # Non-Standard Content (custom extension)
+      phishing: 'IAB25', # Non-Standard Content (custom extension)
+      gambling: 'IAB7-39', # Gambling
+      pornography: 'IAB25-3', # Pornography
+      violence: 'IAB25', # Non-Standard Content (custom extension)
+      illegal: 'IAB25', # Non-Standard Content (custom extension)
+
+      # Network & Security
+      botnet_command_control: 'IAB25', # Non-Standard Content (custom extension)
+      threat_intelligence: 'IAB25', # Non-Standard Content (custom extension)
+      suspicious_domains: 'IAB25', # Non-Standard Content (custom extension)
+      compromised_ips: 'IAB25', # Non-Standard Content (custom extension)
+      tor_exit_nodes: 'IAB25', # Non-Standard Content (custom extension)
+
+      # Social & Media
+      social_media: 'IAB14', # Society
+      streaming: 'IAB1-2', # Music
+      blogs: 'IAB14', # Society
+      forums: 'IAB19', # Technology & Computing
+
+      # Geographic/Language Specific
+      chinese_ad_hosts: 'IAB3', # Advertising
+      korean_ad_hosts: 'IAB3', # Advertising
+      mobile_ads: 'IAB3', # Advertising
+      smart_tv_ads: 'IAB3', # Advertising
+
+      # Specialized
+      newly_registered_domains: 'IAB25', # Non-Standard Content (custom extension)
+      dns_over_https_bypass: 'IAB25', # Non-Standard Content (custom extension)
+      sanctions_ips: 'IAB25', # Non-Standard Content (custom extension)
+      cryptojacking: 'IAB25', # Non-Standard Content (custom extension)
+      phishing_extended: 'IAB25' # Non-Standard Content (custom extension)
+    }.freeze
+
+    IAB_V3_MAPPINGS = {
+      # Tier-1 Categories (IAB Content Taxonomy 3.0)
+      advertising: '3', # Advertising
+      automotive: '2', # Automotive
+      books_literature: '20', # Books & Literature
+      business: '3', # Business
+      careers: '4', # Careers
+      education: '5', # Education
+      entertainment: '1', # Arts & Entertainment
+      finance: '13', # Personal Finance
+      food_drink: '8', # Food & Drink
+      health: '7', # Health & Fitness & Wellness
+      hobbies_interests: '9', # Hobbies & Interests
+      home_garden: '10', # Home & Garden
+      law_government: '11', # Law, Government & Politics
+      news: '12', # News & Politics
+      parenting: '6', # Family & Parenting
+      pets: '16', # Pets
+      philosophy: '21', # Philosophy/Religion & Spirituality
+      real_estate: '21', # Real Estate
+      science: '15', # Science
+      shopping: '22', # Shopping
+      sports: '17', # Sports
+      style_fashion: '18', # Style & Fashion
+      technology: '19', # Technology & Computing
+      travel: '20', # Travel
+
+      # Security & Malware Categories (Custom extensions)
+      malware: '626', # Illegal Content (custom mapping)
+      phishing: '626', # Illegal Content (custom mapping)
+      gambling: '7-39', # Gambling (subcategory)
+      pornography: '626', # Adult Content
+      violence: '626', # Illegal Content (custom mapping)
+      illegal: '626', # Illegal Content
+
+      # Network & Security (Custom extensions)
+      botnet_command_control: '626', # Illegal Content (custom mapping)
+      threat_intelligence: '626', # Illegal Content (custom mapping)
+      suspicious_domains: '626', # Illegal Content (custom mapping)
+      compromised_ips: '626', # Illegal Content (custom mapping)
+      tor_exit_nodes: '626', # Illegal Content (custom mapping)
+
+      # Social & Media
+      social_media: '14', # Society
+      streaming: '1-2', # Music & Audio
+      blogs: '14', # Society
+      forums: '19', # Technology & Computing
+
+      # Geographic/Language Specific
+      chinese_ad_hosts: '3', # Advertising
+      korean_ad_hosts: '3', # Advertising
+      mobile_ads: '3', # Advertising
+      smart_tv_ads: '3', # Advertising
+
+      # Specialized
+      newly_registered_domains: '626', # Illegal Content (custom mapping)
+      dns_over_https_bypass: '626', # Illegal Content (custom mapping)
+      sanctions_ips: '626', # Illegal Content (custom mapping)
+      cryptojacking: '626', # Illegal Content (custom mapping)
+      phishing_extended: '626' # Illegal Content (custom mapping)
+    }.freeze
+
+    def self.map_category_to_iab(category, version = :v3)
+      category_sym = category.to_sym
+      mapping = version == :v2 ? IAB_V2_MAPPINGS : IAB_V3_MAPPINGS
+      mapping[category_sym] || 'Unknown'
+    end
+
+    def self.get_iab_categories(categories, version = :v3)
+      categories.map { |cat| map_category_to_iab(cat, version) }.uniq
+    end
+
+    def self.supported_versions
+      %i[v2 v3]
+    end
+
+    def self.category_exists?(category, version = :v3)
+      category_sym = category.to_sym
+      mapping = version == :v2 ? IAB_V2_MAPPINGS : IAB_V3_MAPPINGS
+      mapping.key?(category_sym)
+    end
+  end
+end

data/lib/url_categorise/version.rb

@@ -1,3 +1,3 @@
 module UrlCategorise
-  VERSION = '0.1.3'
+  VERSION = '0.1.6'
 end

data/lib/url_categorise.rb

@@ -3,12 +3,14 @@ require 'nokogiri'
 require 'digest'
 require 'fileutils'
 require 'resolv'
+require 'active_attr'
 
 require 'api-pattern'
 
 require 'url_categorise/version'
 require 'url_categorise/constants'
 require 'url_categorise/dataset_processor'
+require 'url_categorise/iab_compliance'
 
 require 'url_categorise/client'
 

data/url_categorise.gemspec

@@ -18,17 +18,19 @@ Gem::Specification.new do |spec|
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = 'bin'
+  spec.executables   = ['export_hosts', 'export_csv', 'check_lists']
   spec.require_paths = ['lib']
   spec.required_ruby_version = '>= 3.0.0'
 
+  spec.add_dependency 'active_attr', '>= 0.17.1', '< 1.0'
   spec.add_dependency 'api_pattern', '>= 0.0.6', '< 1.0'
   spec.add_dependency 'csv', '>= 3.3.0', '< 4.0'
   spec.add_dependency 'digest', '>= 3.1.0', '< 4.0'
   spec.add_dependency 'fileutils', '>= 1.7.0', '< 2.0'
   spec.add_dependency 'httparty', '>= 0.22.0', '< 1.0'
   spec.add_dependency 'json', '>= 2.7.0', '< 3.0'
+  spec.add_dependency 'kaggle', '>= 0.0.3', '< 1.0'
   spec.add_dependency 'nokogiri', '>= 1.18.9', '< 2.0'
   spec.add_dependency 'resolv', '>= 0.4.0', '< 1.0'
   spec.add_dependency 'rubyzip', '>= 2.3.0', '< 3.0'

metadata

@@ -1,14 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: UrlCategorise
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.6
 platform: ruby
 authors:
 - trex22
-bindir: exe
+bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: active_attr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: api_pattern
   requirement: !ruby/object:Gem::Requirement
@@ -129,6 +149,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: kaggle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -361,7 +401,10 @@ description: A client for using the UrlCategorise API in Ruby. Built from their
   documentation. This is an unofficial project.
 email:
 - contact@jasonchalom.com
-executables: []
+executables:
+- check_lists
+- export_csv
+- export_hosts
 extensions: []
 extra_rdoc_files: []
 files:
@@ -378,14 +421,20 @@ files:
 - Rakefile
 - bin/check_lists
 - bin/console
+- bin/export_csv
+- bin/export_hosts
+- bin/rake
 - bin/setup
+- correct_usage_example.rb
 - docs/.keep
 - docs/v0.1-context.md
+- docs/v0.1.4-features.md
 - lib/url_categorise.rb
 - lib/url_categorise/active_record_client.rb
 - lib/url_categorise/client.rb
 - lib/url_categorise/constants.rb
 - lib/url_categorise/dataset_processor.rb
+- lib/url_categorise/iab_compliance.rb
 - lib/url_categorise/models.rb
 - lib/url_categorise/version.rb
 - url_categorise.gemspec