UrlCategorise 0.1.7 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/settings.local.json +9 -2
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +56 -0
  5. data/.sublime-project +14 -0
  6. data/Gemfile +2 -2
  7. data/Gemfile.lock +89 -41
  8. data/README.md +167 -9
  9. data/Rakefile +1 -1
  10. data/bin/check_lists +124 -22
  11. data/bin/export_csv +45 -36
  12. data/bin/export_hosts +34 -24
  13. data/bin/generate_categorised_lists +533 -0
  14. data/bin/generate_social_media_lists +398 -0
  15. data/bin/generate_video_lists +303 -124
  16. data/bin/generate_video_lists_focused +381 -0
  17. data/correct_usage_example.rb +18 -18
  18. data/lib/url_categorise/active_record_client.rb +11 -11
  19. data/lib/url_categorise/client.rb +856 -226
  20. data/lib/url_categorise/constants.rb +148 -70
  21. data/lib/url_categorise/dataset_processor.rb +68 -64
  22. data/lib/url_categorise/iab_compliance.rb +111 -99
  23. data/lib/url_categorise/models.rb +15 -15
  24. data/lib/url_categorise/version.rb +1 -1
  25. data/lib/url_categorise.rb +14 -14
  26. data/lists/blogging_domains.hosts +62 -0
  27. data/lists/crypto_domains.hosts +46 -0
  28. data/lists/developer_domains.hosts +108 -0
  29. data/lists/forum_domains.hosts +65 -0
  30. data/lists/gaming_domains.hosts +99 -0
  31. data/lists/messaging_domains.hosts +96 -0
  32. data/lists/music_domains.hosts +62 -0
  33. data/lists/security_domains.hosts +48 -0
  34. data/lists/social_media_domains.hosts +443 -0
  35. data/lists/streaming_domains.hosts +41 -0
  36. data/lists/video_hosting_domains.hosts +203 -175
  37. data/lists/video_url_patterns.txt +15 -15
  38. data/url_categorise.gemspec +11 -9
  39. metadata +69 -20
data/lists/gaming_domains.hosts ADDED
@@ -0,0 +1,99 @@
1
+ # Gaming Platform Domains - PiHole Compatible Hosts File
2
+ # Generated on: 2026-05-22 11:06:15 UTC
3
+ # Sources:
4
+ # - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
5
+ # - Manually curated domain lists
6
+ # Purpose: Identify and categorize gaming platforms and communities
7
+ # Format: 0.0.0.0 domain.com
8
+ #
9
+
10
+ 0.0.0.0 api.mojang.com
11
+ 0.0.0.0 www.api.mojang.com
12
+ 0.0.0.0 battle.net
13
+ 0.0.0.0 www.battle.net
14
+ 0.0.0.0 blizzard.com
15
+ 0.0.0.0 www.blizzard.com
16
+ 0.0.0.0 chess.com
17
+ 0.0.0.0 www.chess.com
18
+ 0.0.0.0 clips.twitch.tv
19
+ 0.0.0.0 www.clips.twitch.tv
20
+ 0.0.0.0 crunchyroll.com
21
+ 0.0.0.0 www.crunchyroll.com
22
+ 0.0.0.0 ea.com
23
+ 0.0.0.0 www.ea.com
24
+ 0.0.0.0 epicgames.com
25
+ 0.0.0.0 www.epicgames.com
26
+ 0.0.0.0 euw.op.gg
27
+ 0.0.0.0 www.euw.op.gg
28
+ 0.0.0.0 fandom.com
29
+ 0.0.0.0 www.fandom.com
30
+ 0.0.0.0 fortnitetracker.com
31
+ 0.0.0.0 www.fortnitetracker.com
32
+ 0.0.0.0 gamespot.com
33
+ 0.0.0.0 www.gamespot.com
34
+ 0.0.0.0 gog.com
35
+ 0.0.0.0 www.gog.com
36
+ 0.0.0.0 humblebundle.com
37
+ 0.0.0.0 www.humblebundle.com
38
+ 0.0.0.0 ign.com
39
+ 0.0.0.0 www.ign.com
40
+ 0.0.0.0 itch.io
41
+ 0.0.0.0 www.itch.io
42
+ 0.0.0.0 lichess.org
43
+ 0.0.0.0 www.lichess.org
44
+ 0.0.0.0 lolprofile.net
45
+ 0.0.0.0 www.lolprofile.net
46
+ 0.0.0.0 m.twitch.tv
47
+ 0.0.0.0 www.m.twitch.tv
48
+ 0.0.0.0 myanimelist.net
49
+ 0.0.0.0 www.myanimelist.net
50
+ 0.0.0.0 newgrounds.com
51
+ 0.0.0.0 www.newgrounds.com
52
+ 0.0.0.0 nintendo.com
53
+ 0.0.0.0 www.nintendo.com
54
+ 0.0.0.0 op.gg
55
+ 0.0.0.0 www.op.gg
56
+ 0.0.0.0 osu.ppy.sh
57
+ 0.0.0.0 www.osu.ppy.sh
58
+ 0.0.0.0 pokemonshowdown.com
59
+ 0.0.0.0 www.pokemonshowdown.com
60
+ 0.0.0.0 psnprofiles.com
61
+ 0.0.0.0 www.psnprofiles.com
62
+ 0.0.0.0 pubg.op.gg
63
+ 0.0.0.0 www.pubg.op.gg
64
+ 0.0.0.0 roblox.com
65
+ 0.0.0.0 www.roblox.com
66
+ 0.0.0.0 rockstargames.com
67
+ 0.0.0.0 www.rockstargames.com
68
+ 0.0.0.0 speedrun.com
69
+ 0.0.0.0 www.speedrun.com
70
+ 0.0.0.0 splits.io
71
+ 0.0.0.0 www.splits.io
72
+ 0.0.0.0 steamcommunity.com
73
+ 0.0.0.0 www.steamcommunity.com
74
+ 0.0.0.0 steampowered.com
75
+ 0.0.0.0 www.steampowered.com
76
+ 0.0.0.0 store.steampowered.com
77
+ 0.0.0.0 www.store.steampowered.com
78
+ 0.0.0.0 twitch.tv
79
+ 0.0.0.0 www.twitch.tv
80
+ 0.0.0.0 twitchtracker.com
81
+ 0.0.0.0 www.twitchtracker.com
82
+ 0.0.0.0 ubisoft.com
83
+ 0.0.0.0 www.ubisoft.com
84
+ 0.0.0.0 www.blizzard.com
85
+ 0.0.0.0 www.chess.com
86
+ 0.0.0.0 www.crunchyroll.com
87
+ 0.0.0.0 www.ea.com
88
+ 0.0.0.0 www.epicgames.com
89
+ 0.0.0.0 www.fandom.com
90
+ 0.0.0.0 www.gamespot.com
91
+ 0.0.0.0 www.gog.com
92
+ 0.0.0.0 www.humblebundle.com
93
+ 0.0.0.0 www.ign.com
94
+ 0.0.0.0 www.nintendo.com
95
+ 0.0.0.0 www.roblox.com
96
+ 0.0.0.0 www.rockstargames.com
97
+ 0.0.0.0 www.speedrun.com
98
+ 0.0.0.0 www.twitch.tv
99
+ 0.0.0.0 www.ubisoft.com
data/lists/messaging_domains.hosts ADDED
@@ -0,0 +1,96 @@
1
+ # Messaging Platform Domains - PiHole Compatible Hosts File
2
+ # Generated on: 2026-05-22 11:06:15 UTC
3
+ # Sources:
4
+ # - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
5
+ # - Manually curated domain lists
6
+ # Purpose: Identify and categorize messaging and chat platforms
7
+ # Format: 0.0.0.0 domain.com
8
+ #
9
+
10
+ 0.0.0.0 community.signalusers.org
11
+ 0.0.0.0 www.community.signalusers.org
12
+ 0.0.0.0 discord.com
13
+ 0.0.0.0 www.discord.com
14
+ 0.0.0.0 discord.gg
15
+ 0.0.0.0 www.discord.gg
16
+ 0.0.0.0 disqus.com
17
+ 0.0.0.0 www.disqus.com
18
+ 0.0.0.0 element.io
19
+ 0.0.0.0 www.element.io
20
+ 0.0.0.0 facebook.com
21
+ 0.0.0.0 www.facebook.com
22
+ 0.0.0.0 instagram.com
23
+ 0.0.0.0 www.instagram.com
24
+ 0.0.0.0 kakaotalk.com
25
+ 0.0.0.0 www.kakaotalk.com
26
+ 0.0.0.0 kik.me
27
+ 0.0.0.0 www.kik.me
28
+ 0.0.0.0 line.me
29
+ 0.0.0.0 www.line.me
30
+ 0.0.0.0 m.twitch.tv
31
+ 0.0.0.0 www.m.twitch.tv
32
+ 0.0.0.0 matrix.org
33
+ 0.0.0.0 www.matrix.org
34
+ 0.0.0.0 mattermost.com
35
+ 0.0.0.0 www.mattermost.com
36
+ 0.0.0.0 meet.google.com
37
+ 0.0.0.0 www.meet.google.com
38
+ 0.0.0.0 messenger.com
39
+ 0.0.0.0 www.messenger.com
40
+ 0.0.0.0 mobile.twitter.com
41
+ 0.0.0.0 www.mobile.twitter.com
42
+ 0.0.0.0 myspace.com
43
+ 0.0.0.0 www.myspace.com
44
+ 0.0.0.0 nitter.net
45
+ 0.0.0.0 www.nitter.net
46
+ 0.0.0.0 picuki.com
47
+ 0.0.0.0 www.picuki.com
48
+ 0.0.0.0 rocket.chat
49
+ 0.0.0.0 www.rocket.chat
50
+ 0.0.0.0 signal.org
51
+ 0.0.0.0 www.signal.org
52
+ 0.0.0.0 skype.com
53
+ 0.0.0.0 www.skype.com
54
+ 0.0.0.0 slack.com
55
+ 0.0.0.0 www.slack.com
56
+ 0.0.0.0 t.me
57
+ 0.0.0.0 www.t.me
58
+ 0.0.0.0 teams.microsoft.com
59
+ 0.0.0.0 www.teams.microsoft.com
60
+ 0.0.0.0 telegram.org
61
+ 0.0.0.0 www.telegram.org
62
+ 0.0.0.0 tellonym.me
63
+ 0.0.0.0 www.tellonym.me
64
+ 0.0.0.0 threads.net
65
+ 0.0.0.0 www.threads.net
66
+ 0.0.0.0 twitch.tv
67
+ 0.0.0.0 www.twitch.tv
68
+ 0.0.0.0 twitchtracker.com
69
+ 0.0.0.0 www.twitchtracker.com
70
+ 0.0.0.0 twitter.com
71
+ 0.0.0.0 www.twitter.com
72
+ 0.0.0.0 viber.com
73
+ 0.0.0.0 www.viber.com
74
+ 0.0.0.0 vk.com
75
+ 0.0.0.0 www.vk.com
76
+ 0.0.0.0 web.telegram.org
77
+ 0.0.0.0 www.web.telegram.org
78
+ 0.0.0.0 web.whatsapp.com
79
+ 0.0.0.0 www.web.whatsapp.com
80
+ 0.0.0.0 webex.com
81
+ 0.0.0.0 www.webex.com
82
+ 0.0.0.0 whatsapp.com
83
+ 0.0.0.0 www.whatsapp.com
84
+ 0.0.0.0 www.discord.com
85
+ 0.0.0.0 www.line.me
86
+ 0.0.0.0 www.messenger.com
87
+ 0.0.0.0 www.signal.org
88
+ 0.0.0.0 www.skype.com
89
+ 0.0.0.0 www.slack.com
90
+ 0.0.0.0 www.telegram.org
91
+ 0.0.0.0 www.viber.com
92
+ 0.0.0.0 www.whatsapp.com
93
+ 0.0.0.0 zoom.us
94
+ 0.0.0.0 www.zoom.us
95
+ 0.0.0.0 zulip.com
96
+ 0.0.0.0 www.zulip.com
data/lists/music_domains.hosts ADDED
@@ -0,0 +1,62 @@
1
+ # Music Platform Domains - PiHole Compatible Hosts File
2
+ # Generated on: 2026-05-22 11:06:15 UTC
3
+ # Sources:
4
+ # - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
5
+ # - Manually curated domain lists
6
+ # Purpose: Identify and categorize music streaming and sharing platforms
7
+ # Format: 0.0.0.0 domain.com
8
+ #
9
+
10
+ 0.0.0.0 audiojungle.net
11
+ 0.0.0.0 www.audiojungle.net
12
+ 0.0.0.0 bandcamp.com
13
+ 0.0.0.0 www.bandcamp.com
14
+ 0.0.0.0 deezer.com
15
+ 0.0.0.0 www.deezer.com
16
+ 0.0.0.0 discogs.com
17
+ 0.0.0.0 www.discogs.com
18
+ 0.0.0.0 freesound.org
19
+ 0.0.0.0 www.freesound.org
20
+ 0.0.0.0 genius.com
21
+ 0.0.0.0 www.genius.com
22
+ 0.0.0.0 house-mixes.com
23
+ 0.0.0.0 www.house-mixes.com
24
+ 0.0.0.0 last.fm
25
+ 0.0.0.0 www.last.fm
26
+ 0.0.0.0 mixcloud.com
27
+ 0.0.0.0 www.mixcloud.com
28
+ 0.0.0.0 music.amazon.com
29
+ 0.0.0.0 www.music.amazon.com
30
+ 0.0.0.0 music.apple.com
31
+ 0.0.0.0 www.music.apple.com
32
+ 0.0.0.0 musicbrainz.org
33
+ 0.0.0.0 www.musicbrainz.org
34
+ 0.0.0.0 musixmatch.com
35
+ 0.0.0.0 www.musixmatch.com
36
+ 0.0.0.0 open.spotify.com
37
+ 0.0.0.0 www.open.spotify.com
38
+ 0.0.0.0 ourdjtalk.com
39
+ 0.0.0.0 www.ourdjtalk.com
40
+ 0.0.0.0 pandora.com
41
+ 0.0.0.0 www.pandora.com
42
+ 0.0.0.0 rateyourmusic.com
43
+ 0.0.0.0 www.rateyourmusic.com
44
+ 0.0.0.0 smule.com
45
+ 0.0.0.0 www.smule.com
46
+ 0.0.0.0 songkick.com
47
+ 0.0.0.0 www.songkick.com
48
+ 0.0.0.0 soundcloud.com
49
+ 0.0.0.0 www.soundcloud.com
50
+ 0.0.0.0 spotify.com
51
+ 0.0.0.0 www.spotify.com
52
+ 0.0.0.0 tidal.com
53
+ 0.0.0.0 www.tidal.com
54
+ 0.0.0.0 www.bandcamp.com
55
+ 0.0.0.0 www.deezer.com
56
+ 0.0.0.0 www.discogs.com
57
+ 0.0.0.0 www.house-mixes.com
58
+ 0.0.0.0 www.last.fm
59
+ 0.0.0.0 www.pandora.com
60
+ 0.0.0.0 www.smule.com
61
+ 0.0.0.0 www.soundcloud.com
62
+ 0.0.0.0 www.tidal.com
data/lists/security_domains.hosts ADDED
@@ -0,0 +1,48 @@
1
+ # Security Community Domains - PiHole Compatible Hosts File
2
+ # Generated on: 2026-05-22 11:06:15 UTC
3
+ # Sources:
4
+ # - Mr.Holmes OSINT tool (https://github.com/Lucksi/Mr.Holmes)
5
+ # - Manually curated domain lists
6
+ # Purpose: Identify and categorize security research, bug bounty, and hacking community platforms
7
+ # Format: 0.0.0.0 domain.com
8
+ #
9
+
10
+ 0.0.0.0 app.hackthebox.com
11
+ 0.0.0.0 www.app.hackthebox.com
12
+ 0.0.0.0 bugcrowd.com
13
+ 0.0.0.0 www.bugcrowd.com
14
+ 0.0.0.0 censys.io
15
+ 0.0.0.0 www.censys.io
16
+ 0.0.0.0 cobalt.io
17
+ 0.0.0.0 www.cobalt.io
18
+ 0.0.0.0 cvedetails.com
19
+ 0.0.0.0 www.cvedetails.com
20
+ 0.0.0.0 exploit-db.com
21
+ 0.0.0.0 www.exploit-db.com
22
+ 0.0.0.0 forum.hackthebox.eu
23
+ 0.0.0.0 www.forum.hackthebox.eu
24
+ 0.0.0.0 hackerone.com
25
+ 0.0.0.0 www.hackerone.com
26
+ 0.0.0.0 hackthebox.eu
27
+ 0.0.0.0 www.hackthebox.eu
28
+ 0.0.0.0 intigriti.com
29
+ 0.0.0.0 www.intigriti.com
30
+ 0.0.0.0 otx.alienvault.com
31
+ 0.0.0.0 www.otx.alienvault.com
32
+ 0.0.0.0 portswigger.net
33
+ 0.0.0.0 www.portswigger.net
34
+ 0.0.0.0 shodan.io
35
+ 0.0.0.0 www.shodan.io
36
+ 0.0.0.0 synack.com
37
+ 0.0.0.0 www.synack.com
38
+ 0.0.0.0 tryhackme.com
39
+ 0.0.0.0 www.tryhackme.com
40
+ 0.0.0.0 urlscan.io
41
+ 0.0.0.0 www.urlscan.io
42
+ 0.0.0.0 virustotal.com
43
+ 0.0.0.0 www.virustotal.com
44
+ 0.0.0.0 vulnhub.com
45
+ 0.0.0.0 www.vulnhub.com
46
+ 0.0.0.0 www.hackthebox.eu
47
+ 0.0.0.0 yeswehack.com
48
+ 0.0.0.0 www.yeswehack.com