UrlCategorise 0.1.7 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/settings.local.json +9 -2
- data/.gitignore +1 -0
- data/.rubocop.yml +56 -0
- data/.sublime-project +14 -0
- data/Gemfile +2 -2
- data/Gemfile.lock +89 -41
- data/README.md +167 -9
- data/Rakefile +1 -1
- data/bin/check_lists +124 -22
- data/bin/export_csv +45 -36
- data/bin/export_hosts +34 -24
- data/bin/generate_categorised_lists +533 -0
- data/bin/generate_social_media_lists +398 -0
- data/bin/generate_video_lists +303 -124
- data/bin/generate_video_lists_focused +381 -0
- data/correct_usage_example.rb +18 -18
- data/lib/url_categorise/active_record_client.rb +11 -11
- data/lib/url_categorise/client.rb +856 -226
- data/lib/url_categorise/constants.rb +148 -70
- data/lib/url_categorise/dataset_processor.rb +68 -64
- data/lib/url_categorise/iab_compliance.rb +111 -99
- data/lib/url_categorise/models.rb +15 -15
- data/lib/url_categorise/version.rb +1 -1
- data/lib/url_categorise.rb +14 -14
- data/lists/blogging_domains.hosts +62 -0
- data/lists/crypto_domains.hosts +46 -0
- data/lists/developer_domains.hosts +108 -0
- data/lists/forum_domains.hosts +65 -0
- data/lists/gaming_domains.hosts +99 -0
- data/lists/messaging_domains.hosts +96 -0
- data/lists/music_domains.hosts +62 -0
- data/lists/security_domains.hosts +48 -0
- data/lists/social_media_domains.hosts +443 -0
- data/lists/streaming_domains.hosts +41 -0
- data/lists/video_hosting_domains.hosts +203 -175
- data/lists/video_url_patterns.txt +15 -15
- data/url_categorise.gemspec +11 -9
- metadata +69 -20
|
@@ -3,105 +3,183 @@ module UrlCategorise
|
|
|
3
3
|
ONE_MEGABYTE = 1_048_576
|
|
4
4
|
|
|
5
5
|
# Video URL patterns for detecting video content
|
|
6
|
-
VIDEO_URL_PATTERNS_FILE =
|
|
6
|
+
VIDEO_URL_PATTERNS_FILE = "https://raw.githubusercontent.com/TRex22/url_categorise/refs/heads/main/lists/video_url_patterns.txt".freeze
|
|
7
|
+
|
|
8
|
+
# Generated categorised domain lists from Mr.Holmes OSINT tool and curated sources
|
|
9
|
+
GENERATED_LISTS_BASE_URL = "https://raw.githubusercontent.com/TRex22/url_categorise/refs/heads/main/lists".freeze
|
|
10
|
+
SOCIAL_MEDIA_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/social_media_domains.hosts".freeze
|
|
11
|
+
GAMING_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/gaming_domains.hosts".freeze
|
|
12
|
+
DEVELOPER_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/developer_domains.hosts".freeze
|
|
13
|
+
MUSIC_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/music_domains.hosts".freeze
|
|
14
|
+
STREAMING_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/streaming_domains.hosts".freeze
|
|
15
|
+
FORUM_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/forum_domains.hosts".freeze
|
|
16
|
+
MESSAGING_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/messaging_domains.hosts".freeze
|
|
17
|
+
CRYPTO_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/crypto_domains.hosts".freeze
|
|
18
|
+
BLOGGING_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/blogging_domains.hosts".freeze
|
|
19
|
+
SECURITY_DOMAINS_FILE = "#{GENERATED_LISTS_BASE_URL}/security_domains.hosts".freeze
|
|
7
20
|
|
|
8
21
|
# crawler data
|
|
9
22
|
# https://commoncrawl.org/
|
|
10
23
|
|
|
11
24
|
# Usually used to train deep models. Using directly here
|
|
12
25
|
CATEGORIY_DATABASES = [
|
|
13
|
-
{ type: :kaggle, path:
|
|
14
|
-
{ type: :kaggle, path:
|
|
15
|
-
{ type: :kaggle, path:
|
|
16
|
-
{ type: :csv, path:
|
|
26
|
+
{ type: :kaggle, path: "shaurov/website-classification-using-url" },
|
|
27
|
+
{ type: :kaggle, path: "hetulmehta/website-classification" },
|
|
28
|
+
{ type: :kaggle, path: "shawon10/url-classification-dataset-dmoz" },
|
|
29
|
+
{ type: :csv, path: "https://query.data.world/s/zackomeddpgotrp3yel66aphvvlcuq?dws=00000" }
|
|
17
30
|
]
|
|
18
31
|
|
|
19
32
|
DEFAULT_HOST_URLS = {
|
|
20
|
-
abuse: [
|
|
21
|
-
adobe: [
|
|
33
|
+
abuse: [ "https://github.com/blocklistproject/Lists/raw/master/abuse.txt" ],
|
|
34
|
+
adobe: [ "https://github.com/blocklistproject/Lists/raw/master/adobe.txt" ],
|
|
22
35
|
adult: %i[pornography dating_services drugs gambling],
|
|
23
|
-
advertising: [
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
36
|
+
advertising: [ "https://blocklistproject.github.io/Lists/ads.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-advert_01.txt",
|
|
37
|
+
"https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt",
|
|
38
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Ads",
|
|
39
|
+
"https://adaway.org/hosts.txt",
|
|
40
|
+
"https://v.firebog.net/hosts/AdguardDNS.txt",
|
|
41
|
+
"https://v.firebog.net/hosts/Admiral.txt",
|
|
42
|
+
"https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext" ],
|
|
43
|
+
amazon: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/amazon/all" ],
|
|
44
|
+
amp_hosts: [ "https://www.github.developerdan.com/hosts/lists/amp-hosts-extended.txt" ],
|
|
45
|
+
apple: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/apple/all",
|
|
46
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Apple" ],
|
|
47
|
+
cloudflare: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/cloudflare/all" ],
|
|
48
|
+
crypto: [ "https://github.com/blocklistproject/Lists/raw/master/crypto.txt", "https://v.firebog.net/hosts/Prigent-Crypto.txt", CRYPTO_DOMAINS_FILE,
|
|
49
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Cryptocurrency" ],
|
|
50
|
+
dating_services: [ "https://www.github.developerdan.com/hosts/lists/dating-services-extended.txt",
|
|
51
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Dating" ],
|
|
52
|
+
drugs: [ "https://github.com/blocklistproject/Lists/raw/master/drugs.txt" ],
|
|
53
|
+
facebook: [ "https://github.com/blocklistproject/Lists/raw/master/facebook.txt",
|
|
54
|
+
"https://www.github.developerdan.com/hosts/lists/facebook-extended.txt", "https://raw.githubusercontent.com/blocklistproject/Lists/master/facebook.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/all", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/facebook.com" ],
|
|
55
|
+
fraud: [ "https://blocklistproject.github.io/Lists/fraud.txt",
|
|
56
|
+
"https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt" ],
|
|
57
|
+
gambling: [ "https://blocklistproject.github.io/Lists/gambling.txt", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/gambling.txt",
|
|
58
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Gambling" ],
|
|
59
|
+
gaming: [ "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-ubisoft.txt",
|
|
60
|
+
"https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-steam.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-activision.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-blizzard.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-ea.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-epicgames.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-nintendo.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-rockstargames.txt", "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-roblox.txt", GAMING_DOMAINS_FILE ],
|
|
61
|
+
google: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/google/all" ],
|
|
62
|
+
hate_and_junk: [ "https://www.github.developerdan.com/hosts/lists/hate-and-junk-extended.txt" ],
|
|
63
|
+
instagram: [ "https://github.com/jmdugan/blocklists/raw/master/corporations/facebook/instagram" ],
|
|
64
|
+
linkedin: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/microsoft/linkedin" ],
|
|
65
|
+
malware: [ "https://blocklistproject.github.io/Lists/malware.txt",
|
|
66
|
+
"https://feodotracker.abuse.ch/downloads/ipblocklist.txt", "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt",
|
|
67
|
+
"https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt",
|
|
68
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Malware",
|
|
69
|
+
"https://urlhaus.abuse.ch/downloads/hostfile/",
|
|
70
|
+
"https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt" ],
|
|
71
|
+
microsoft: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/microsoft/all",
|
|
72
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Microsoft" ],
|
|
73
|
+
mozilla: [ "https://github.com/jmdugan/blocklists/raw/master/corporations/mozilla/all" ],
|
|
74
|
+
nsa: [ "https://raw.githubusercontent.com/tigthor/NSA-CIA-Blocklist/main/HOSTS/HOSTS" ],
|
|
75
|
+
phishing: [ "https://blocklistproject.github.io/Lists/phishing.txt", "https://openphish.com/feed.txt",
|
|
76
|
+
"https://phishing.army/download/phishing_army_blocklist_extended.txt",
|
|
77
|
+
"https://malware-filter.gitlab.io/malware-filter/phishing-filter-hosts.txt" ],
|
|
78
|
+
pinterest: [ "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/pinterest/all" ],
|
|
79
|
+
piracy: [ "https://github.com/blocklistproject/Lists/raw/master/piracy.txt", "https://github.com/hagezi/dns-blocklists/raw/refs/heads/main/adblock/anti.piracy.txt" ],
|
|
80
|
+
pornography: [ "https://blocklistproject.github.io/Lists/porn.txt", "https://v.firebog.net/hosts/Prigent-Adult.txt",
|
|
81
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Adult",
|
|
82
|
+
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/nsfw.txt" ],
|
|
83
|
+
reddit: [ "https://raw.githubusercontent.com/nickoppen/pihole-blocklists/master/blocklist-reddit.txt" ],
|
|
84
|
+
redirect: [ "https://github.com/blocklistproject/Lists/raw/master/redirect.txt" ],
|
|
85
|
+
scam: [ "https://blocklistproject.github.io/Lists/scam.txt",
|
|
86
|
+
"https://raw.githubusercontent.com/jarelllama/Scam-Blocklist/main/lists/wildcard_domains/scams.txt",
|
|
87
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Scam" ],
|
|
88
|
+
smart_tv: [ "https://github.com/blocklistproject/Lists/raw/master/smart-tv.txt" ],
|
|
89
|
+
social_media: [ SOCIAL_MEDIA_DOMAINS_FILE, :facebook, :instagram, :linkedin, :pinterest, :reddit, :tiktok, :twitter, :whatsapp, :youtube ],
|
|
90
|
+
tiktok: [ "https://blocklistproject.github.io/Lists/tiktok.txt" ],
|
|
91
|
+
torrent: [ "https://github.com/blocklistproject/Lists/raw/master/torrent.txt" ],
|
|
92
|
+
tracking: [ "https://blocklistproject.github.io/Lists/tracking.txt",
|
|
93
|
+
"https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Tracking",
|
|
94
|
+
"https://raw.githubusercontent.com/tigthor/NSA-CIA-Blocklist/main/HOSTS/Trackers/trackers.txt",
|
|
95
|
+
"https://raw.githubusercontent.com/TRex22/general_pihole_lists/refs/heads/main/blocklists/whatsapp-tracking.txt",
|
|
96
|
+
"https://media.githubusercontent.com/media/zachlagden/Pi-hole-Optimized-Blocklists/main/lists/tracking.txt",
|
|
97
|
+
"https://v.firebog.net/hosts/Easyprivacy.txt",
|
|
98
|
+
"https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt",
|
|
99
|
+
"https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt",
|
|
100
|
+
"https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt",
|
|
101
|
+
"https://hostfiles.frogeye.fr/multiparty-trackers-hosts.txt" ],
|
|
102
|
+
twitter: [ "https://github.com/blocklistproject/Lists/raw/master/twitter.txt", "https://github.com/jmdugan/blocklists/raw/master/corporations/twitter/all" ],
|
|
103
|
+
vaping: [ "https://github.com/blocklistproject/Lists/raw/master/vaping.txt" ],
|
|
104
|
+
video_hosting: [ "https://raw.githubusercontent.com/TRex22/url_categorise/refs/heads/main/lists/video_hosting_domains.hosts",
|
|
105
|
+
"https://raw.githubusercontent.com/wilwade/pihole-block-video/master/hosts.txt" ],
|
|
106
|
+
streaming: [ STREAMING_DOMAINS_FILE ],
|
|
107
|
+
messaging: [ MESSAGING_DOMAINS_FILE ],
|
|
108
|
+
music: [ MUSIC_DOMAINS_FILE ],
|
|
109
|
+
developer_platforms: [ DEVELOPER_DOMAINS_FILE ],
|
|
110
|
+
blogging: [ BLOGGING_DOMAINS_FILE ],
|
|
111
|
+
forum: [ FORUM_DOMAINS_FILE ],
|
|
112
|
+
security_communities: [ SECURITY_DOMAINS_FILE ],
|
|
113
|
+
whatsapp: [ "https://github.com/blocklistproject/Lists/raw/master/whatsapp.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/facebook/whatsapp" ],
|
|
114
|
+
youtube: [ "https://github.com/blocklistproject/Lists/raw/master/youtube.txt", "https://raw.githubusercontent.com/jmdugan/blocklists/master/corporations/google/youtube" ],
|
|
64
115
|
|
|
65
116
|
# Hagezi DNS Blocklists - specialized categories only
|
|
66
|
-
threat_intelligence: [
|
|
67
|
-
dyndns: [
|
|
68
|
-
badware_hoster: [
|
|
69
|
-
most_abused_tlds: [
|
|
70
|
-
newly_registered_domains: [
|
|
71
|
-
dns_over_https_bypass: [
|
|
117
|
+
threat_intelligence: [ "https://github.com/hagezi/dns-blocklists/raw/refs/heads/main/ips/tif.txt" ],
|
|
118
|
+
dyndns: [ "https://github.com/hagezi/dns-blocklists/raw/refs/heads/main/adblock/dyndns.txt" ],
|
|
119
|
+
badware_hoster: [ "https://github.com/hagezi/dns-blocklists/raw/refs/heads/main/adblock/hoster.txt" ],
|
|
120
|
+
most_abused_tlds: [ "https://github.com/hagezi/dns-blocklists/raw/refs/heads/main/adblock/spam-tlds.txt" ],
|
|
121
|
+
newly_registered_domains: [ "https://github.com/xRuffKez/NRD/raw/refs/heads/main/lists/14-day/adblock/nrd-14day_adblock.txt" ],
|
|
122
|
+
dns_over_https_bypass: [ "https://github.com/hagezi/dns-blocklists/raw/refs/heads/main/adblock/doh-vpn-proxy-bypass.txt" ],
|
|
72
123
|
|
|
73
124
|
# StevenBlack hosts lists - specific categories only
|
|
74
|
-
fakenews: [
|
|
125
|
+
fakenews: [ "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts", "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-only/hosts" ],
|
|
75
126
|
|
|
76
127
|
# Security threat lists
|
|
77
|
-
threat_indicators: [
|
|
128
|
+
threat_indicators: [ "https://threatfox.abuse.ch/downloads/hostfile.txt" ],
|
|
78
129
|
|
|
79
130
|
# Additional IP-based sanctions and abuse lists
|
|
80
|
-
sanctions_ips: [
|
|
81
|
-
compromised_ips: [
|
|
82
|
-
tor_exit_nodes: [
|
|
83
|
-
open_proxy_ips: [
|
|
131
|
+
sanctions_ips: [ "https://lists.blocklist.de/lists/all.txt" ],
|
|
132
|
+
compromised_ips: [ "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt" ],
|
|
133
|
+
tor_exit_nodes: [ "https://www.dan.me.uk/torlist/" ],
|
|
134
|
+
open_proxy_ips: [ "https://raw.githubusercontent.com/stamparm/ipsum/master/ipsum.txt" ],
|
|
84
135
|
|
|
85
136
|
# Network security feeds
|
|
86
|
-
top_attack_sources: [
|
|
87
|
-
suspicious_domains: [
|
|
137
|
+
top_attack_sources: [ "https://www.dshield.org/feeds/suspiciousdomains_High.txt" ],
|
|
138
|
+
suspicious_domains: [ "https://www.dshield.org/feeds/suspiciousdomains_Medium.txt" ],
|
|
88
139
|
|
|
89
|
-
# Extended categories for better
|
|
90
|
-
cryptojacking: [
|
|
140
|
+
# Extended categories for better organisation
|
|
141
|
+
cryptojacking: [ "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt" ],
|
|
91
142
|
|
|
92
143
|
# Regional and specialized lists
|
|
93
|
-
chinese_ad_hosts: [
|
|
94
|
-
korean_ad_hosts: [
|
|
144
|
+
chinese_ad_hosts: [ "https://raw.githubusercontent.com/jdlingyu/ad-wars/master/hosts" ],
|
|
145
|
+
korean_ad_hosts: [ "https://raw.githubusercontent.com/yous/YousList/master/hosts.txt" ],
|
|
95
146
|
|
|
96
147
|
# Mobile and app-specific
|
|
97
|
-
mobile_ads: [
|
|
98
|
-
smart_tv_ads: [
|
|
148
|
+
mobile_ads: [ "https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/MobileFilter/sections/adservers.txt" ],
|
|
149
|
+
smart_tv_ads: [ "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt",
|
|
150
|
+
"https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt" ],
|
|
151
|
+
|
|
152
|
+
# Additional security categories
|
|
153
|
+
stalkerware: [ "https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/hosts" ],
|
|
154
|
+
ransomware: [ "https://blocklistproject.github.io/Lists/ransomware.txt" ],
|
|
155
|
+
|
|
156
|
+
# Telemetry and spam
|
|
157
|
+
telemetry: [ "https://raw.githubusercontent.com/MoralCode/pihole-antitelemetry/main/telemetry-domains.txt" ],
|
|
158
|
+
spam: [ "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts" ],
|
|
159
|
+
|
|
160
|
+
# Marketing and URL services
|
|
161
|
+
marketing: [ "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Marketing" ],
|
|
162
|
+
url_shorteners: [ "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/UrlShortener" ],
|
|
163
|
+
|
|
164
|
+
# Network tools and services
|
|
165
|
+
remote_access: [ "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Remote" ],
|
|
166
|
+
vpn_proxy: [ "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Tunnels" ],
|
|
167
|
+
|
|
168
|
+
# Risk and typosquatting
|
|
169
|
+
risky_domains: [ "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Risk" ],
|
|
170
|
+
typosquatting: [ "https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Typo" ],
|
|
171
|
+
|
|
172
|
+
# Game-specific
|
|
173
|
+
fortnite: [ "https://blocklistproject.github.io/Lists/fortnite.txt" ],
|
|
174
|
+
|
|
175
|
+
# Generated categorised lists from Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
|
|
176
|
+
# Run bin/generate_social_media_lists and bin/generate_categorised_lists to regenerate
|
|
99
177
|
|
|
100
178
|
# Content and informational categories
|
|
101
|
-
news: [
|
|
179
|
+
# news: [ "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-only/hosts" ],
|
|
102
180
|
# NOTE: The following categories had broken URLs and have been commented out:
|
|
103
181
|
# legitimate_news: URLs from mitchellkrogza repository return 404
|
|
104
|
-
# blogs,
|
|
182
|
+
# blogs, health, finance, shopping: blocklistproject alt-version URLs return 404
|
|
105
183
|
# educational: StevenBlack educational hosts URL returns 404
|
|
106
184
|
# government: mitchellkrogza government domains URL returns 404
|
|
107
185
|
# business, technology: blocklistproject alt-version URLs return 404
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
6
|
-
require
|
|
7
|
-
require
|
|
8
|
-
require
|
|
1
|
+
require "httparty"
|
|
2
|
+
require "csv"
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "net/http"
|
|
6
|
+
require "timeout"
|
|
7
|
+
require "zip"
|
|
8
|
+
require "json"
|
|
9
9
|
|
|
10
10
|
module UrlCategorise
|
|
11
11
|
class DatasetProcessor
|
|
12
12
|
include HTTParty
|
|
13
13
|
|
|
14
|
-
KAGGLE_BASE_URL =
|
|
15
|
-
DEFAULT_DOWNLOAD_PATH =
|
|
16
|
-
DEFAULT_CACHE_PATH =
|
|
14
|
+
KAGGLE_BASE_URL = "https://www.kaggle.com/api/v1"
|
|
15
|
+
DEFAULT_DOWNLOAD_PATH = "./downloads"
|
|
16
|
+
DEFAULT_CACHE_PATH = "./cache"
|
|
17
17
|
DEFAULT_TIMEOUT = 30
|
|
18
|
-
DEFAULT_CREDENTIALS_FILE = File.expand_path(
|
|
18
|
+
DEFAULT_CREDENTIALS_FILE = File.expand_path("~/.kaggle/kaggle.json")
|
|
19
19
|
|
|
20
20
|
attr_reader :username, :api_key, :download_path, :cache_path, :timeout, :kaggle_enabled
|
|
21
21
|
|
|
@@ -41,7 +41,7 @@ module UrlCategorise
|
|
|
41
41
|
|
|
42
42
|
def process_kaggle_dataset(dataset_owner, dataset_name, options = {})
|
|
43
43
|
unless @kaggle_enabled
|
|
44
|
-
raise Error,
|
|
44
|
+
raise Error, "Kaggle functionality is disabled. Set enable_kaggle: true to use Kaggle datasets."
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
dataset_path = "#{dataset_owner}/#{dataset_name}"
|
|
@@ -61,12 +61,12 @@ module UrlCategorise
|
|
|
61
61
|
# If credentials not available, return nil gracefully for cache mode
|
|
62
62
|
unless kaggle_credentials_available?
|
|
63
63
|
if options[:use_cache]
|
|
64
|
-
puts "Warning: Kaggle dataset '#{dataset_path}' not cached and no credentials available" if ENV[
|
|
64
|
+
puts "Warning: Kaggle dataset '#{dataset_path}' not cached and no credentials available" if ENV["DEBUG"]
|
|
65
65
|
return nil
|
|
66
66
|
else
|
|
67
|
-
raise Error,
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
raise Error, "Kaggle credentials required for downloading new datasets. " \
|
|
68
|
+
"Set KAGGLE_USERNAME/KAGGLE_KEY environment variables, provide credentials explicitly, " \
|
|
69
|
+
"or place kaggle.json file in ~/.kaggle/ directory."
|
|
70
70
|
end
|
|
71
71
|
end
|
|
72
72
|
|
|
@@ -109,28 +109,28 @@ module UrlCategorise
|
|
|
109
109
|
|
|
110
110
|
def generate_dataset_hash(data)
|
|
111
111
|
content = case data
|
|
112
|
-
|
|
112
|
+
when Hash
|
|
113
113
|
data.to_json
|
|
114
|
-
|
|
114
|
+
when Array
|
|
115
115
|
data.to_json
|
|
116
|
-
|
|
116
|
+
when String
|
|
117
117
|
data
|
|
118
|
-
|
|
118
|
+
else
|
|
119
119
|
data.to_s
|
|
120
|
-
|
|
120
|
+
end
|
|
121
121
|
|
|
122
122
|
Digest::SHA256.hexdigest(content)
|
|
123
123
|
end
|
|
124
124
|
|
|
125
125
|
def integrate_dataset_into_categorization(dataset, category_mappings = {})
|
|
126
|
-
|
|
126
|
+
categorised_data = {}
|
|
127
127
|
raw_content = []
|
|
128
128
|
|
|
129
129
|
case dataset
|
|
130
130
|
when Hash
|
|
131
131
|
# Single dataset with multiple files
|
|
132
132
|
dataset.each do |file_name, data|
|
|
133
|
-
process_dataset_file(data, file_name, category_mappings,
|
|
133
|
+
process_dataset_file(data, file_name, category_mappings, categorised_data)
|
|
134
134
|
# Collect raw content
|
|
135
135
|
if data.is_a?(Array)
|
|
136
136
|
raw_content.concat(data.map { |row| row.is_a?(Hash) ? row : {} })
|
|
@@ -138,26 +138,24 @@ module UrlCategorise
|
|
|
138
138
|
end
|
|
139
139
|
when Array
|
|
140
140
|
# Single file dataset
|
|
141
|
-
process_dataset_file(dataset,
|
|
141
|
+
process_dataset_file(dataset, "default", category_mappings, categorised_data)
|
|
142
142
|
# Collect raw content
|
|
143
143
|
raw_content.concat(dataset.map { |row| row.is_a?(Hash) ? row : {} })
|
|
144
144
|
else
|
|
145
145
|
raise Error, "Unsupported dataset format: #{dataset.class}"
|
|
146
146
|
end
|
|
147
147
|
|
|
148
|
-
# Store both processed
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
148
|
+
# Store both processed categorisation data and raw content
|
|
149
|
+
{
|
|
150
|
+
"categories" => categorised_data,
|
|
151
|
+
"raw_content" => raw_content,
|
|
152
|
+
"_metadata" => {
|
|
153
153
|
processed_at: Time.now,
|
|
154
154
|
data_hash: generate_dataset_hash(dataset),
|
|
155
155
|
total_entries: count_total_entries(dataset),
|
|
156
156
|
raw_content_entries: raw_content.length
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
|
-
|
|
160
|
-
result
|
|
161
159
|
end
|
|
162
160
|
|
|
163
161
|
private
|
|
@@ -169,9 +167,9 @@ module UrlCategorise
|
|
|
169
167
|
def warn_if_kaggle_credentials_missing
|
|
170
168
|
return if kaggle_credentials_available?
|
|
171
169
|
|
|
172
|
-
warn
|
|
173
|
-
|
|
174
|
-
|
|
170
|
+
warn "Warning: Kaggle credentials not found. Kaggle datasets will only work if they are already cached. " \
|
|
171
|
+
"To download new Kaggle datasets, set KAGGLE_USERNAME/KAGGLE_KEY environment variables, " \
|
|
172
|
+
"provide credentials explicitly, or place kaggle.json file in ~/.kaggle/ directory."
|
|
175
173
|
end
|
|
176
174
|
|
|
177
175
|
def valid_credential?(credential)
|
|
@@ -182,17 +180,17 @@ module UrlCategorise
|
|
|
182
180
|
# Try provided credentials file first
|
|
183
181
|
if credentials_file && File.exist?(credentials_file)
|
|
184
182
|
credentials = load_credentials_from_file(credentials_file)
|
|
185
|
-
@username = username || credentials[
|
|
186
|
-
@api_key = api_key || credentials[
|
|
183
|
+
@username = username || credentials["username"]
|
|
184
|
+
@api_key = api_key || credentials["key"]
|
|
187
185
|
# Try default kaggle.json file if no explicit credentials
|
|
188
186
|
elsif !username && !api_key && File.exist?(DEFAULT_CREDENTIALS_FILE)
|
|
189
187
|
credentials = load_credentials_from_file(DEFAULT_CREDENTIALS_FILE)
|
|
190
|
-
@username = credentials[
|
|
191
|
-
@api_key = credentials[
|
|
188
|
+
@username = credentials["username"]
|
|
189
|
+
@api_key = credentials["key"]
|
|
192
190
|
else
|
|
193
191
|
# Fall back to environment variables
|
|
194
|
-
@username = username || ENV[
|
|
195
|
-
@api_key = api_key || ENV[
|
|
192
|
+
@username = username || ENV["KAGGLE_USERNAME"]
|
|
193
|
+
@api_key = api_key || ENV["KAGGLE_KEY"]
|
|
196
194
|
end
|
|
197
195
|
end
|
|
198
196
|
|
|
@@ -214,7 +212,7 @@ module UrlCategorise
|
|
|
214
212
|
self.class.base_uri KAGGLE_BASE_URL
|
|
215
213
|
self.class.default_options.merge!({
|
|
216
214
|
headers: {
|
|
217
|
-
|
|
215
|
+
"User-Agent" => "url_categorise-ruby-client"
|
|
218
216
|
},
|
|
219
217
|
timeout: @timeout,
|
|
220
218
|
basic_auth: {
|
|
@@ -227,7 +225,7 @@ module UrlCategorise
|
|
|
227
225
|
def authenticated_request(method, endpoint, options = {})
|
|
228
226
|
self.class.send(method, endpoint, options)
|
|
229
227
|
rescue Timeout::Error, Net::ReadTimeout, Net::OpenTimeout
|
|
230
|
-
raise Error,
|
|
228
|
+
raise Error, "Request timed out"
|
|
231
229
|
rescue StandardError => e
|
|
232
230
|
raise Error, "Request failed: #{e.message}"
|
|
233
231
|
end
|
|
@@ -247,7 +245,7 @@ module UrlCategorise
|
|
|
247
245
|
end
|
|
248
246
|
|
|
249
247
|
def get_extracted_dir(dataset_path)
|
|
250
|
-
dir_name = dataset_path.gsub(
|
|
248
|
+
dir_name = dataset_path.gsub("/", "_").gsub(/[^a-zA-Z0-9_-]/, "_")
|
|
251
249
|
File.join(@download_path, dir_name)
|
|
252
250
|
end
|
|
253
251
|
|
|
@@ -255,7 +253,7 @@ module UrlCategorise
|
|
|
255
253
|
filename = "#{dataset_path.gsub('/', '_')}_#{Time.now.to_i}.zip"
|
|
256
254
|
file_path = File.join(@download_path, filename)
|
|
257
255
|
|
|
258
|
-
File.open(file_path,
|
|
256
|
+
File.open(file_path, "wb") do |file|
|
|
259
257
|
file.write(content)
|
|
260
258
|
end
|
|
261
259
|
|
|
@@ -264,10 +262,16 @@ module UrlCategorise
|
|
|
264
262
|
|
|
265
263
|
def extract_zip_file(zip_file_path, extract_to_dir)
|
|
266
264
|
FileUtils.mkdir_p(extract_to_dir)
|
|
265
|
+
base_dir = File.expand_path(extract_to_dir)
|
|
267
266
|
|
|
268
267
|
Zip::File.open(zip_file_path) do |zip_file|
|
|
269
268
|
zip_file.each do |entry|
|
|
270
|
-
extract_path = File.join(
|
|
269
|
+
extract_path = File.expand_path(File.join(base_dir, entry.name))
|
|
270
|
+
|
|
271
|
+
# Reject entries that would be written outside the extraction directory (zip-slip)
|
|
272
|
+
unless extract_path.start_with?("#{base_dir}#{File::SEPARATOR}")
|
|
273
|
+
raise Error, "Unsafe path in zip file: #{entry.name}"
|
|
274
|
+
end
|
|
271
275
|
|
|
272
276
|
if entry.directory?
|
|
273
277
|
FileUtils.mkdir_p(extract_path)
|
|
@@ -275,7 +279,7 @@ module UrlCategorise
|
|
|
275
279
|
parent_dir = File.dirname(extract_path)
|
|
276
280
|
FileUtils.mkdir_p(parent_dir) unless Dir.exist?(parent_dir)
|
|
277
281
|
|
|
278
|
-
File.open(extract_path,
|
|
282
|
+
File.open(extract_path, "wb") do |f|
|
|
279
283
|
f.write entry.get_input_stream.read
|
|
280
284
|
end
|
|
281
285
|
end
|
|
@@ -300,14 +304,14 @@ module UrlCategorise
|
|
|
300
304
|
end
|
|
301
305
|
|
|
302
306
|
def find_csv_files(directory)
|
|
303
|
-
Dir.glob(File.join(directory,
|
|
307
|
+
Dir.glob(File.join(directory, "**", "*.csv"))
|
|
304
308
|
end
|
|
305
309
|
|
|
306
310
|
def parse_csv_files_to_hash(csv_files)
|
|
307
311
|
result = {}
|
|
308
312
|
|
|
309
313
|
csv_files.each do |csv_file|
|
|
310
|
-
file_name = File.basename(csv_file,
|
|
314
|
+
file_name = File.basename(csv_file, ".csv")
|
|
311
315
|
result[file_name] = parse_csv_file(csv_file)
|
|
312
316
|
end
|
|
313
317
|
|
|
@@ -340,7 +344,7 @@ module UrlCategorise
|
|
|
340
344
|
end
|
|
341
345
|
|
|
342
346
|
def generate_cache_key(identifier, source_type)
|
|
343
|
-
sanitized = identifier.gsub(/[^a-zA-Z0-9_-]/,
|
|
347
|
+
sanitized = identifier.gsub(/[^a-zA-Z0-9_-]/, "_")
|
|
344
348
|
"#{source_type}_#{sanitized}_processed.json"
|
|
345
349
|
end
|
|
346
350
|
|
|
@@ -363,7 +367,7 @@ module UrlCategorise
|
|
|
363
367
|
# Cache write failed, continue without caching
|
|
364
368
|
end
|
|
365
369
|
|
|
366
|
-
def process_dataset_file(data, file_name, category_mappings,
|
|
370
|
+
def process_dataset_file(data, file_name, category_mappings, categorised_data)
|
|
367
371
|
return unless data.is_a?(Array) && !data.empty?
|
|
368
372
|
|
|
369
373
|
# If explicit column mappings are provided, use them for all rows
|
|
@@ -382,9 +386,9 @@ module UrlCategorise
|
|
|
382
386
|
# Determine category
|
|
383
387
|
category = determine_category(row, category_col, category_mappings, file_name)
|
|
384
388
|
|
|
385
|
-
# Add to
|
|
386
|
-
|
|
387
|
-
|
|
389
|
+
# Add to categorised data
|
|
390
|
+
categorised_data[category] ||= []
|
|
391
|
+
categorised_data[category] << domain unless categorised_data[category].include?(domain)
|
|
388
392
|
end
|
|
389
393
|
else
|
|
390
394
|
# Auto-detect columns for each row (handles mixed column structures)
|
|
@@ -408,9 +412,9 @@ module UrlCategorise
|
|
|
408
412
|
# Determine category
|
|
409
413
|
category = determine_category(row, category_col, category_mappings, file_name)
|
|
410
414
|
|
|
411
|
-
# Add to
|
|
412
|
-
|
|
413
|
-
|
|
415
|
+
# Add to categorised data
|
|
416
|
+
categorised_data[category] ||= []
|
|
417
|
+
categorised_data[category] << domain unless categorised_data[category].include?(domain)
|
|
414
418
|
end
|
|
415
419
|
end
|
|
416
420
|
end
|
|
@@ -440,12 +444,12 @@ module UrlCategorise
|
|
|
440
444
|
|
|
441
445
|
uri = URI.parse(url)
|
|
442
446
|
domain = uri.host&.downcase
|
|
443
|
-
domain = domain.gsub(/\Awww\./,
|
|
447
|
+
domain = domain.gsub(/\Awww\./, "") if domain # Remove www prefix
|
|
444
448
|
domain
|
|
445
449
|
rescue URI::InvalidURIError
|
|
446
450
|
# If URI parsing fails, try to extract domain manually
|
|
447
|
-
cleaned = url.gsub(%r{\A\w+://},
|
|
448
|
-
cleaned = cleaned.gsub(/\Awww\./,
|
|
451
|
+
cleaned = url.gsub(%r{\A\w+://}, "").gsub(%r{/.*\z}, "").downcase
|
|
452
|
+
cleaned = cleaned.gsub(/\Awww\./, "")
|
|
449
453
|
cleaned.empty? ? nil : cleaned
|
|
450
454
|
end
|
|
451
455
|
|
|
@@ -467,11 +471,11 @@ module UrlCategorise
|
|
|
467
471
|
|
|
468
472
|
# Sanitize and format category name
|
|
469
473
|
sanitized = original_name.to_s.downcase
|
|
470
|
-
.gsub(/[^a-z0-9_]/,
|
|
471
|
-
.gsub(/_+/,
|
|
472
|
-
.gsub(/\A_|_\z/,
|
|
474
|
+
.gsub(/[^a-z0-9_]/, "_")
|
|
475
|
+
.gsub(/_+/, "_")
|
|
476
|
+
.gsub(/\A_|_\z/, "")
|
|
473
477
|
|
|
474
|
-
sanitized.empty? ?
|
|
478
|
+
sanitized.empty? ? "dataset_category" : sanitized
|
|
475
479
|
end
|
|
476
480
|
|
|
477
481
|
def count_total_entries(dataset)
|