UrlCategorise 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ed8fbd044a6405977b3e720454c6eb3bfebdc6ee38e6b00f860ac2e3d49d3da
-  data.tar.gz: c795f83ee5ce691af751f19681eaec865c2a01aadc27ef120c87ed7bf9ce339f
+  metadata.gz: 926043e28097f20035b4dbc943534c63a5c3c0a429745e4bb42e7dd0701295c1
+  data.tar.gz: 3ef850d4c43266a6ec15a7653c64b97720efc2e46298ed36be07d48d70ada772
 SHA512:
-  metadata.gz: e4aa69b7edecd520db0a2df0ea48498f730f6ddd49d6f0d3d116954aaa7b486ae87c395b655aa81523e43a4189536f7b3d5033443e5f5ecae649c445ff2b178f
-  data.tar.gz: c00b7d76d298538dcb55314cbda83dde5daa5bc78c2661bf44d818a62d663050957a6ff2d9063ec46c4cece3b92a18c14a471fdf67534b073f8a20fa560ad01d
+  metadata.gz: c6504dd8ec0a5f284bc78dfcbb7e45b9e1752c50f6f32a380c52128c19364b976232bd4af93c943d73f0b7cbb6e2ac3e44574e90a0fde38da87f5585b92fc3c0
+  data.tar.gz: c2de16323a1cfa085ac15590f7601b5307cc7c660243f48b6a5db10df25809041b119eb3f48da382effbe120150c2967ae4224213508fca896c45889bad6bce1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    UrlCategorise (0.0.1)
+    UrlCategorise (0.0.3)
       api_pattern (~> 0.0.4)
 
 GEM

data/README.md

@@ -15,15 +15,40 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install url_categorise
+    $ gem install UrlCategorise
 
 ## Usage
+The default host lists I picked for their separated categories.
+I didn't select them for the quality of data
+Use at your own risk!
 
 ```ruby
   require 'url_categorise'
   client = UrlCategorise::Client.new
 
+  client.count_of_hosts
+  client.count_of_categories
+  client.size_of_data # In megabytes
 
+  url = "www.google.com"
+  client.categorise(url)
+
+  # Can also initialise the client using a custom dataset
+  host_urls = {
+    abuse: ["https://github.com/blocklistproject/Lists/raw/master/abuse.txt"]
+  }
+
+  require 'url_categorise'
+  client = UrlCategorise::Client.new(host_urls: host_urls)
+
+  # You can also define symbols to combine other categories
+  host_urls = {
+    abuse: ["https://github.com/blocklistproject/Lists/raw/master/abuse.txt"],
+    bad_links: [:abuse]
+  }
+
+  require 'url_categorise'
+  client = UrlCategorise::Client.new(host_urls: host_urls)
 ```
 
 ## Development

data/lib/url_categorise/client.rb

@@ -4,27 +4,19 @@ module UrlCategorise
 
     attr_reader :host_urls, :hosts
 
+    # TODO: Save to folder
+    # TODO: Read from disk the database
     # TODO: Sanctioned IPs
-    # TODO: More default lists
     # TODO: ActiveRecord support
     # TODO: List of abuse IPs
-    # TODO: https://github.com/blocklistproject/Lists
-    # TODO: https://github.com/nickoppen/pihole-blocklists
     def initialize(host_urls: DEFAULT_HOST_URLS)
       @host_urls = host_urls
-      # @hosts = fetch_and_build_host_lists
-    end
-
-    def self.compatible_api_version
-      'v1'
-    end
-
-    def self.api_version
-      'v2 2023-05-19'
+      @hosts = fetch_and_build_host_lists
     end
 
     def categorise(url)
       host = (URI.parse(url).host || url).downcase
+      host = host.gsub("www.", "")
 
       @hosts.keys.select do |category|
         @hosts[category].include?(host)
@@ -42,7 +34,19 @@ module UrlCategorise
     end
 
     def size_of_data
+      hash_size_in_mb(@hosts)
+    end
+
+    private
+
+    def hash_size_in_mb(hash)
+      size = 0
 
+      hash.each do |key, value|
+        size += value.join.length
+      end
+
+      (size / ONE_MEGABYTE).round(2)
     end
 
     def fetch_and_build_host_lists
@@ -52,18 +56,57 @@ module UrlCategorise
         @hosts[category] = build_host_data(host_urls[category])
       end
 
+      sub_category_values = categories_with_keys
+      sub_category_values.keys.each do |category|
+        original_value = @hosts[category] || []
+
+        extra_category_values = sub_category_values[category].each do |sub_category|
+          @hosts[sub_category]
+        end
+
+        original_value << extra_category_values
+        @hosts[category] = original_value.uniq.compact
+      end
+
       @hosts
     end
 
     def build_host_data(urls)
       urls.map do |url|
+        next unless url_valid?(url)
+
         raw_data = HTTParty.get(url)
         raw_data.split("\n").reject do |line|
-          line.include?("#")
+          line[0] == "#"
         end.map do |line|
-          line.gsub("0.0.0.0 ", "")
+          line.split(' ')[1] # Select the domain name # gsub("0.0.0.0 ", "")
         end
       end.flatten.compact.sort
     end
+
+    def categories_with_keys
+      keyed_categories = {}
+
+      host_urls.keys.each do |category|
+        category_values = host_urls[category].select do |url|
+          url_not_valid?(url) && url.is_a?(Symbol)
+        end
+
+        keyed_categories[category] = category_values
+      end
+
+      keyed_categories
+    end
+
+    def url_not_valid?(url)
+      url_valid?(url)
+    end
+
+    def url_valid?(url)
+      uri = URI.parse(url)
+      uri.is_a?(URI::HTTP) && !uri.host.nil?
+    rescue URI::InvalidURIError
+      false
+    end
   end
 end

data/lib/url_categorise/constants.rb

@@ -1,36 +1,45 @@
 module UrlCategorise
   module Constants
-    # Resources used:
-    # https://blocklist.site/#
-    # https://github.com/lightswitch05/hosts
-    #
+    ONE_MEGABYTE = 1048576
     DEFAULT_HOST_URLS = {
-      advertising: ["https://blocklistproject.github.io/Lists/ads.txt"],
+      abuse: ["https://github.com/blocklistproject/Lists/raw/master/abuse.txt"],
+      adobe: ["https://github.com/blocklistproject/Lists/raw/master/adobe.txt"],
+      advertising: ["https://blocklistproject.github.io/Lists/ads.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-advert_01.txt"],
+      amazon: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/amazon/all"],
       amp_hosts: ["https://www.github.developerdan.com/hosts/lists/amp-hosts-extended.txt"],
+      apple: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/apple/all"],
+      cloudflare: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/cloudflare/all"],
+      crypto: ["https://github.com/blocklistproject/Lists/raw/master/crypto.txt"],
       dating_services: ["https://www.github.developerdan.com/hosts/lists/dating-services-extended.txt"],
-      facebook: ["https://github.com/blocklistproject/Lists/raw/master/facebook.txt", "https://www.github.developerdan.com/hosts/lists/facebook-extended.txt", "https://raw.githubusercontent.com/blocklistproject/Lists/master/facebook.txt"],
+      drugs: ["https://github.com/blocklistproject/Lists/raw/master/drugs.txt"],
+      facebook: ["https://github.com/blocklistproject/Lists/raw/master/facebook.txt", "https://www.github.developerdan.com/hosts/lists/facebook-extended.txt", "https://raw.githubusercontent.com/blocklistproject/Lists/master/facebook.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/all", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/facebook.com"],
       fraud: ["https://blocklistproject.github.io/Lists/fraud.txt"],
       gambling: ["https://blocklistproject.github.io/Lists/gambling.txt"],
-      hate: [],
-      junk: [],
-      malware: ["https://blocklistproject.github.io/Lists/malware.txt"],
+      gaming: ["https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-ubisoft.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-steam.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-activision.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-blizzard.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-ea.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-epicgames.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-nintendo.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-rockstargames.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-roblox.txt"],
+      google: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/google/all"],
+      hate_and_junk: ["https://www.github.developerdan.com/hosts/lists/hate-and-junk-extended.txt"],
+      instagram: ["https://github.com/jmdugan/blocklists/raw/master/corporations/facebook/instagram"],
+      linkedin: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/microsoft/linkedin"],
+      malware: ["https://blocklistproject.github.io/Lists/malware.txt", "http://www.malwaredomainlist.com/hostslist/hosts.txt"],
+      microsoft: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/microsoft/all"],
+      mozilla: ["https://github.com/jmdugan/blocklists/raw/master/corporations/mozilla/all"],
+      nsa: ["https://raw.githubusercontent.com/tigthor/NSA-CIA-Blocklist/main/HOSTS/HOSTS"],
       phishing: ["https://blocklistproject.github.io/Lists/phishing.txt"],
+      pinterest: ["https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/pinterest/all"],
+      piracy: ["https://github.com/blocklistproject/Lists/raw/master/piracy.txt"],
       pornography: ["https://blocklistproject.github.io/Lists/porn.txt"],
+      reddit: ["https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-reddit.txt"],
+      redirect: ["https://github.com/blocklistproject/Lists/raw/master/redirect.txt"],
       scam: ["https://blocklistproject.github.io/Lists/scam.txt"],
+      smart_tv: ["https://github.com/blocklistproject/Lists/raw/master/smart-tv.txt"],
+      social_media: [:facebook, :instagram, :linkedin, :pinterest, :reddit,:tiktok, :twitter, :whatsapp, :youtube],
       tiktok: ["https://blocklistproject.github.io/Lists/tiktok.txt"],
+      torrent: ["https://github.com/blocklistproject/Lists/raw/master/torrent.txt"],
       tracking: ["https://blocklistproject.github.io/Lists/tracking.txt"],
-      twitter: ["https://github.com/blocklistproject/Lists/raw/master/twitter.txt"],
+      twitter: ["https://github.com/blocklistproject/Lists/raw/master/twitter.txt", "https://github.com/jmdugan/blocklists/raw/master/corporations/twitter/all"],
       vaping: ["https://github.com/blocklistproject/Lists/raw/master/vaping.txt"],
-      whatsapp: ["https://github.com/blocklistproject/Lists/raw/master/whatsapp.txt"],
-      youtube: ["https://github.com/blocklistproject/Lists/raw/master/youtube.txt"],
-      torrent: ["https://github.com/blocklistproject/Lists/raw/master/torrent.txt"],
-      smart_tv: ["https://github.com/blocklistproject/Lists/raw/master/smart-tv.txt"],
-      redirect: ["https://github.com/blocklistproject/Lists/raw/master/redirect.txt"],
-      piracy: ["https://github.com/blocklistproject/Lists/raw/master/piracy.txt"],
-      drugs: ["https://github.com/blocklistproject/Lists/raw/master/drugs.txt"],
-      crypto: ["https://github.com/blocklistproject/Lists/raw/master/crypto.txt"],
-      adobe: ["https://github.com/blocklistproject/Lists/raw/master/adobe.txt"],
-      abuse: ["https://github.com/blocklistproject/Lists/raw/master/abuse.txt"],
+      whatsapp: ["https://github.com/blocklistproject/Lists/raw/master/whatsapp.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/whatsapp"],
+      youtube: ["https://github.com/blocklistproject/Lists/raw/master/youtube.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/google/youtube"],
     }
   end
 end

data/lib/url_categorise/version.rb

@@ -1,3 +1,3 @@
 module UrlCategorise
-  VERSION = "0.0.1"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: UrlCategorise
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - trex22
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-19 00:00:00.000000000 Z
+date: 2023-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: api_pattern