UrlCategorise 0.1.7 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/settings.local.json +9 -2
- data/.gitignore +1 -0
- data/.rubocop.yml +56 -0
- data/.sublime-project +14 -0
- data/Gemfile +2 -2
- data/Gemfile.lock +89 -41
- data/README.md +167 -9
- data/Rakefile +1 -1
- data/bin/check_lists +124 -22
- data/bin/export_csv +45 -36
- data/bin/export_hosts +34 -24
- data/bin/generate_categorised_lists +533 -0
- data/bin/generate_social_media_lists +398 -0
- data/bin/generate_video_lists +303 -124
- data/bin/generate_video_lists_focused +381 -0
- data/correct_usage_example.rb +18 -18
- data/lib/url_categorise/active_record_client.rb +11 -11
- data/lib/url_categorise/client.rb +856 -226
- data/lib/url_categorise/constants.rb +148 -70
- data/lib/url_categorise/dataset_processor.rb +68 -64
- data/lib/url_categorise/iab_compliance.rb +111 -99
- data/lib/url_categorise/models.rb +15 -15
- data/lib/url_categorise/version.rb +1 -1
- data/lib/url_categorise.rb +14 -14
- data/lists/blogging_domains.hosts +62 -0
- data/lists/crypto_domains.hosts +46 -0
- data/lists/developer_domains.hosts +108 -0
- data/lists/forum_domains.hosts +65 -0
- data/lists/gaming_domains.hosts +99 -0
- data/lists/messaging_domains.hosts +96 -0
- data/lists/music_domains.hosts +62 -0
- data/lists/security_domains.hosts +48 -0
- data/lists/social_media_domains.hosts +443 -0
- data/lists/streaming_domains.hosts +41 -0
- data/lists/video_hosting_domains.hosts +203 -175
- data/lists/video_url_patterns.txt +15 -15
- data/url_categorise.gemspec +11 -9
- metadata +69 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6ed2176041a871f6fae72dc1b7c1f266910f6233879e1359d02eead164f8f6ff
|
|
4
|
+
data.tar.gz: 5ff46c4dd9faaa22218a86d7b40f54bedaa6f91e9914b8e9671b14ba55ec6495
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 71aa116bb3de0c269ccaf18a3f159f78c722ab010589a2118c411f826f1ff4de48b9e5ed8c9936a67a6eac3cb17b3f52d71e41bac552e73a91c67edee804f9c4
|
|
7
|
+
data.tar.gz: 4c2d8f50a9cad1b1f32c56ddc51b44e8ca28f49af5718970bb0bea02a12e934cb2c8075589fa9ca8db56437e3a377a5e0cc1afe7a30e44aab5b393cb94a70f3a
|
data/.claude/settings.local.json
CHANGED
|
@@ -16,8 +16,15 @@
|
|
|
16
16
|
"Bash(timeout:*)",
|
|
17
17
|
"Bash(DEBUG=1 timeout 300 ruby correct_usage_example.rb)",
|
|
18
18
|
"Bash(chmod:*)",
|
|
19
|
-
"Bash(bundle exec bin/export_csv:*)"
|
|
19
|
+
"Bash(bundle exec bin/export_csv:*)",
|
|
20
|
+
"Bash(bundle exec rg:*)",
|
|
21
|
+
"WebFetch(domain:github.com)",
|
|
22
|
+
"WebFetch(domain:api.github.com)",
|
|
23
|
+
"WebFetch(domain:raw.githubusercontent.com)",
|
|
24
|
+
"WebSearch",
|
|
25
|
+
"WebFetch(domain:firebog.net)",
|
|
26
|
+
"Bash(curl:*)"
|
|
20
27
|
],
|
|
21
28
|
"deny": []
|
|
22
29
|
}
|
|
23
|
-
}
|
|
30
|
+
}
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
inherit_gem:
|
|
2
|
+
rubocop-rails-omakase: rubocop.yml
|
|
3
|
+
|
|
4
|
+
AllCops:
|
|
5
|
+
TargetRubyVersion: 3.0
|
|
6
|
+
NewCops: enable
|
|
7
|
+
Exclude:
|
|
8
|
+
- 'bin/*'
|
|
9
|
+
- 'vendor/**/*'
|
|
10
|
+
- 'tmp/**/*'
|
|
11
|
+
- 'db/migrate/*'
|
|
12
|
+
|
|
13
|
+
# Allow longer lines for URL constants
|
|
14
|
+
Layout/LineLength:
|
|
15
|
+
Max: 200
|
|
16
|
+
Exclude:
|
|
17
|
+
- 'lib/url_categorise/constants.rb'
|
|
18
|
+
- 'test/**/*'
|
|
19
|
+
|
|
20
|
+
# Allow complex methods in client due to categorization logic
|
|
21
|
+
Metrics/MethodLength:
|
|
22
|
+
Max: 30
|
|
23
|
+
Exclude:
|
|
24
|
+
- 'lib/url_categorise/client.rb'
|
|
25
|
+
- 'test/**/*'
|
|
26
|
+
|
|
27
|
+
# Allow complex classes for main client
|
|
28
|
+
Metrics/ClassLength:
|
|
29
|
+
Max: 500
|
|
30
|
+
Exclude:
|
|
31
|
+
- 'lib/url_categorise/client.rb'
|
|
32
|
+
- 'test/**/*'
|
|
33
|
+
|
|
34
|
+
# Allow higher complexity for categorization methods
|
|
35
|
+
Metrics/CyclomaticComplexity:
|
|
36
|
+
Max: 15
|
|
37
|
+
Exclude:
|
|
38
|
+
- 'lib/url_categorise/client.rb'
|
|
39
|
+
|
|
40
|
+
# Allow higher ABC size for complex categorization logic
|
|
41
|
+
Metrics/AbcSize:
|
|
42
|
+
Max: 25
|
|
43
|
+
Exclude:
|
|
44
|
+
- 'lib/url_categorise/client.rb'
|
|
45
|
+
- 'test/**/*'
|
|
46
|
+
|
|
47
|
+
# Allow higher parameter count for client initialization
|
|
48
|
+
Metrics/ParameterLists:
|
|
49
|
+
Max: 8
|
|
50
|
+
|
|
51
|
+
# Allow block length for tests and constants
|
|
52
|
+
Metrics/BlockLength:
|
|
53
|
+
Exclude:
|
|
54
|
+
- 'test/**/*'
|
|
55
|
+
- 'lib/url_categorise/constants.rb'
|
|
56
|
+
- 'url_categorise.gemspec'
|
data/.sublime-project
ADDED
data/Gemfile
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
source
|
|
1
|
+
source "https://rubygems.org"
|
|
2
2
|
|
|
3
|
-
git_source(:github) { |_repo_name|
|
|
3
|
+
git_source(:github) { |_repo_name| "https://github.com/TRex22/url_categorise" }
|
|
4
4
|
|
|
5
5
|
# Specify your gem's dependencies in url_categorise.gemspec
|
|
6
6
|
gemspec
|
data/Gemfile.lock
CHANGED
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
UrlCategorise (0.1.
|
|
4
|
+
UrlCategorise (0.1.10)
|
|
5
5
|
active_attr (>= 0.17.1, < 1.0)
|
|
6
6
|
api_pattern (>= 0.0.6, < 1.0)
|
|
7
7
|
csv (>= 3.3.0, < 4.0)
|
|
8
8
|
digest (>= 3.1.0, < 4.0)
|
|
9
9
|
fileutils (>= 1.7.0, < 2.0)
|
|
10
|
-
httparty (>= 0.
|
|
10
|
+
httparty (>= 0.24.0, < 1.0)
|
|
11
11
|
json (>= 2.7.0, < 3.0)
|
|
12
12
|
kaggle (>= 0.0.3, < 1.0)
|
|
13
|
-
nokogiri (>= 1.
|
|
14
|
-
reline (>= 0.6.2)
|
|
13
|
+
nokogiri (>= 1.19.1, < 2.0)
|
|
14
|
+
reline (>= 0.6.2, < 2.0)
|
|
15
15
|
resolv (>= 0.4.0, < 1.0)
|
|
16
16
|
rubyzip (>= 2.3.0, < 3.0)
|
|
17
17
|
|
|
18
18
|
GEM
|
|
19
19
|
remote: https://rubygems.org/
|
|
20
20
|
specs:
|
|
21
|
-
actionpack (8.
|
|
22
|
-
actionview (= 8.
|
|
23
|
-
activesupport (= 8.
|
|
21
|
+
actionpack (8.1.3)
|
|
22
|
+
actionview (= 8.1.3)
|
|
23
|
+
activesupport (= 8.1.3)
|
|
24
24
|
nokogiri (>= 1.8.5)
|
|
25
25
|
rack (>= 2.2.4)
|
|
26
26
|
rack-session (>= 1.0.1)
|
|
@@ -28,50 +28,53 @@ GEM
|
|
|
28
28
|
rails-dom-testing (~> 2.2)
|
|
29
29
|
rails-html-sanitizer (~> 1.6)
|
|
30
30
|
useragent (~> 0.16)
|
|
31
|
-
actionview (8.
|
|
32
|
-
activesupport (= 8.
|
|
31
|
+
actionview (8.1.3)
|
|
32
|
+
activesupport (= 8.1.3)
|
|
33
33
|
builder (~> 3.1)
|
|
34
34
|
erubi (~> 1.11)
|
|
35
35
|
rails-dom-testing (~> 2.2)
|
|
36
36
|
rails-html-sanitizer (~> 1.6)
|
|
37
|
-
active_attr (0.17.
|
|
38
|
-
actionpack (>= 3.0.2, < 8.
|
|
39
|
-
activemodel (>= 3.0.2, < 8.
|
|
40
|
-
activesupport (>= 3.0.2, < 8.
|
|
41
|
-
activemodel (8.
|
|
42
|
-
activesupport (= 8.
|
|
43
|
-
activerecord (8.
|
|
44
|
-
activemodel (= 8.
|
|
45
|
-
activesupport (= 8.
|
|
37
|
+
active_attr (0.17.2)
|
|
38
|
+
actionpack (>= 3.0.2, < 8.2)
|
|
39
|
+
activemodel (>= 3.0.2, < 8.2)
|
|
40
|
+
activesupport (>= 3.0.2, < 8.2)
|
|
41
|
+
activemodel (8.1.3)
|
|
42
|
+
activesupport (= 8.1.3)
|
|
43
|
+
activerecord (8.1.3)
|
|
44
|
+
activemodel (= 8.1.3)
|
|
45
|
+
activesupport (= 8.1.3)
|
|
46
46
|
timeout (>= 0.4.0)
|
|
47
|
-
activesupport (8.
|
|
47
|
+
activesupport (8.1.3)
|
|
48
48
|
base64
|
|
49
|
-
benchmark (>= 0.3)
|
|
50
49
|
bigdecimal
|
|
51
50
|
concurrent-ruby (~> 1.0, >= 1.3.1)
|
|
52
51
|
connection_pool (>= 2.2.5)
|
|
53
52
|
drb
|
|
54
53
|
i18n (>= 1.6, < 2)
|
|
54
|
+
json
|
|
55
55
|
logger (>= 1.4.2)
|
|
56
56
|
minitest (>= 5.1)
|
|
57
57
|
securerandom (>= 0.3)
|
|
58
58
|
tzinfo (~> 2.0, >= 2.0.5)
|
|
59
59
|
uri (>= 0.13.1)
|
|
60
|
-
addressable (2.
|
|
61
|
-
public_suffix (>= 2.0.2, <
|
|
60
|
+
addressable (2.9.0)
|
|
61
|
+
public_suffix (>= 2.0.2, < 8.0)
|
|
62
62
|
ansi (1.5.0)
|
|
63
63
|
api_pattern (0.0.6)
|
|
64
64
|
active_attr (>= 0.15.4)
|
|
65
65
|
csv (>= 3.3.0)
|
|
66
66
|
httparty (>= 0.22.0)
|
|
67
67
|
nokogiri (>= 1.16.0)
|
|
68
|
+
ast (2.4.3)
|
|
68
69
|
base64 (0.3.0)
|
|
69
|
-
|
|
70
|
-
bigdecimal (3.2.2)
|
|
70
|
+
bigdecimal (4.1.2)
|
|
71
71
|
builder (3.3.0)
|
|
72
|
+
bundler-audit (0.9.3)
|
|
73
|
+
bundler (>= 1.2.0)
|
|
74
|
+
thor (~> 1.0)
|
|
72
75
|
coderay (1.1.3)
|
|
73
|
-
concurrent-ruby (1.3.
|
|
74
|
-
connection_pool (
|
|
76
|
+
concurrent-ruby (1.3.6)
|
|
77
|
+
connection_pool (3.0.2)
|
|
75
78
|
crack (1.0.0)
|
|
76
79
|
bigdecimal
|
|
77
80
|
rexml
|
|
@@ -83,22 +86,24 @@ GEM
|
|
|
83
86
|
erubi (1.13.1)
|
|
84
87
|
fileutils (1.7.3)
|
|
85
88
|
hashdiff (1.2.0)
|
|
86
|
-
httparty (0.
|
|
89
|
+
httparty (0.24.2)
|
|
87
90
|
csv
|
|
88
91
|
mini_mime (>= 1.0.0)
|
|
89
92
|
multi_xml (>= 0.5.2)
|
|
90
|
-
i18n (1.14.
|
|
93
|
+
i18n (1.14.8)
|
|
91
94
|
concurrent-ruby (~> 1.0)
|
|
92
95
|
io-console (0.8.1)
|
|
93
|
-
json (2.
|
|
96
|
+
json (2.19.5)
|
|
94
97
|
kaggle (0.0.3)
|
|
95
98
|
csv (>= 3.3)
|
|
96
99
|
fileutils (>= 1.7)
|
|
97
100
|
httparty (>= 0.23)
|
|
98
101
|
oj (= 3.16.11)
|
|
99
102
|
rubyzip (>= 2.0)
|
|
103
|
+
language_server-protocol (3.17.0.5)
|
|
104
|
+
lint_roller (1.1.0)
|
|
100
105
|
logger (1.7.0)
|
|
101
|
-
loofah (2.
|
|
106
|
+
loofah (2.25.1)
|
|
102
107
|
crass (~> 1.0.2)
|
|
103
108
|
nokogiri (>= 1.12.0)
|
|
104
109
|
method_source (1.1.0)
|
|
@@ -113,36 +118,72 @@ GEM
|
|
|
113
118
|
ruby-progressbar
|
|
114
119
|
mocha (2.4.5)
|
|
115
120
|
ruby2_keywords (>= 0.0.5)
|
|
116
|
-
multi_xml (0.
|
|
117
|
-
bigdecimal (
|
|
118
|
-
nokogiri (1.
|
|
121
|
+
multi_xml (0.8.1)
|
|
122
|
+
bigdecimal (>= 3.1, < 5)
|
|
123
|
+
nokogiri (1.19.3-arm64-darwin)
|
|
119
124
|
racc (~> 1.4)
|
|
120
125
|
oj (3.16.11)
|
|
121
126
|
bigdecimal (>= 3.0)
|
|
122
127
|
ostruct (>= 0.2)
|
|
123
128
|
ostruct (0.6.3)
|
|
129
|
+
parallel (1.27.0)
|
|
130
|
+
parser (3.3.9.0)
|
|
131
|
+
ast (~> 2.4.1)
|
|
132
|
+
racc
|
|
133
|
+
prism (1.4.0)
|
|
124
134
|
pry (0.15.2)
|
|
125
135
|
coderay (~> 1.1)
|
|
126
136
|
method_source (~> 1.0)
|
|
127
|
-
public_suffix (
|
|
137
|
+
public_suffix (7.0.5)
|
|
128
138
|
racc (1.8.1)
|
|
129
|
-
rack (
|
|
130
|
-
rack-session (1.
|
|
131
|
-
|
|
139
|
+
rack (3.2.6)
|
|
140
|
+
rack-session (2.1.2)
|
|
141
|
+
base64 (>= 0.1.0)
|
|
142
|
+
rack (>= 3.0.0)
|
|
132
143
|
rack-test (2.2.0)
|
|
133
144
|
rack (>= 1.3)
|
|
134
145
|
rails-dom-testing (2.3.0)
|
|
135
146
|
activesupport (>= 5.0.0)
|
|
136
147
|
minitest
|
|
137
148
|
nokogiri (>= 1.6)
|
|
138
|
-
rails-html-sanitizer (1.
|
|
139
|
-
loofah (~> 2.
|
|
149
|
+
rails-html-sanitizer (1.7.0)
|
|
150
|
+
loofah (~> 2.25)
|
|
140
151
|
nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
|
|
152
|
+
rainbow (3.1.1)
|
|
141
153
|
rake (13.3.0)
|
|
154
|
+
regexp_parser (2.11.2)
|
|
142
155
|
reline (0.6.2)
|
|
143
156
|
io-console (~> 0.5)
|
|
144
157
|
resolv (0.6.2)
|
|
145
|
-
rexml (3.4.
|
|
158
|
+
rexml (3.4.4)
|
|
159
|
+
rubocop (1.80.1)
|
|
160
|
+
json (~> 2.3)
|
|
161
|
+
language_server-protocol (~> 3.17.0.2)
|
|
162
|
+
lint_roller (~> 1.1.0)
|
|
163
|
+
parallel (~> 1.10)
|
|
164
|
+
parser (>= 3.3.0.2)
|
|
165
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
166
|
+
regexp_parser (>= 2.9.3, < 3.0)
|
|
167
|
+
rubocop-ast (>= 1.46.0, < 2.0)
|
|
168
|
+
ruby-progressbar (~> 1.7)
|
|
169
|
+
unicode-display_width (>= 2.4.0, < 4.0)
|
|
170
|
+
rubocop-ast (1.46.0)
|
|
171
|
+
parser (>= 3.3.7.2)
|
|
172
|
+
prism (~> 1.4)
|
|
173
|
+
rubocop-performance (1.25.0)
|
|
174
|
+
lint_roller (~> 1.1)
|
|
175
|
+
rubocop (>= 1.75.0, < 2.0)
|
|
176
|
+
rubocop-ast (>= 1.38.0, < 2.0)
|
|
177
|
+
rubocop-rails (2.33.3)
|
|
178
|
+
activesupport (>= 4.2.0)
|
|
179
|
+
lint_roller (~> 1.1)
|
|
180
|
+
rack (>= 1.1)
|
|
181
|
+
rubocop (>= 1.75.0, < 2.0)
|
|
182
|
+
rubocop-ast (>= 1.44.0, < 2.0)
|
|
183
|
+
rubocop-rails-omakase (1.1.0)
|
|
184
|
+
rubocop (>= 1.72)
|
|
185
|
+
rubocop-performance (>= 1.24)
|
|
186
|
+
rubocop-rails (>= 2.30)
|
|
146
187
|
ruby-progressbar (1.13.0)
|
|
147
188
|
ruby2_keywords (0.0.5)
|
|
148
189
|
rubyzip (2.4.1)
|
|
@@ -154,11 +195,15 @@ GEM
|
|
|
154
195
|
simplecov-html (0.13.2)
|
|
155
196
|
simplecov_json_formatter (0.1.4)
|
|
156
197
|
sqlite3 (2.7.3-arm64-darwin)
|
|
198
|
+
thor (1.5.0)
|
|
157
199
|
timecop (0.9.10)
|
|
158
200
|
timeout (0.4.3)
|
|
159
201
|
tzinfo (2.0.6)
|
|
160
202
|
concurrent-ruby (~> 1.0)
|
|
161
|
-
|
|
203
|
+
unicode-display_width (3.1.5)
|
|
204
|
+
unicode-emoji (~> 4.0, >= 4.0.4)
|
|
205
|
+
unicode-emoji (4.0.4)
|
|
206
|
+
uri (1.1.1)
|
|
162
207
|
useragent (0.16.11)
|
|
163
208
|
webmock (3.24.0)
|
|
164
209
|
addressable (>= 2.8.0)
|
|
@@ -167,10 +212,12 @@ GEM
|
|
|
167
212
|
|
|
168
213
|
PLATFORMS
|
|
169
214
|
arm64-darwin-24
|
|
215
|
+
arm64-darwin-25
|
|
170
216
|
|
|
171
217
|
DEPENDENCIES
|
|
172
218
|
UrlCategorise!
|
|
173
219
|
activerecord (>= 8.0)
|
|
220
|
+
bundler-audit (~> 0.9)
|
|
174
221
|
logger
|
|
175
222
|
minitest (~> 5.25.5)
|
|
176
223
|
minitest-focus (~> 1.4.0)
|
|
@@ -178,6 +225,7 @@ DEPENDENCIES
|
|
|
178
225
|
mocha (~> 2.4.5)
|
|
179
226
|
pry (~> 0.15.2)
|
|
180
227
|
rake (~> 13.3.0)
|
|
228
|
+
rubocop-rails-omakase (~> 1.0)
|
|
181
229
|
simplecov (~> 0.22.0)
|
|
182
230
|
sqlite3 (>= 2.7)
|
|
183
231
|
timecop (~> 0.9.10)
|
data/README.md
CHANGED
|
@@ -6,6 +6,8 @@ A comprehensive Ruby gem for categorizing URLs and domains based on various secu
|
|
|
6
6
|
|
|
7
7
|
- **Comprehensive Coverage**: 60+ high-quality categories including security, content, and specialized lists
|
|
8
8
|
- **Video Content Detection**: Advanced regex-based categorization with `video_url?` method to distinguish video content from other website resources
|
|
9
|
+
- **Blog Content Detection**: Simple string-based `blog_url?` method to identify blog-related URLs and content
|
|
10
|
+
- **Debug Mode**: Comprehensive debug logging with timing information for initialization and operations
|
|
9
11
|
- **Custom Video Lists**: Generate and maintain comprehensive video hosting domain lists using yt-dlp extractors
|
|
10
12
|
- **Kaggle Dataset Integration**: Automatic loading and processing of machine learning datasets from Kaggle
|
|
11
13
|
- **Multiple Data Sources**: Supports blocklists, CSV datasets, and Kaggle ML datasets
|
|
@@ -162,11 +164,11 @@ $ bundle exec export_hosts --output /tmp/hosts --verbose
|
|
|
162
164
|
# Export CSV data with all features enabled
|
|
163
165
|
$ bundle exec export_csv --output /tmp/csv --iab-compliance --smart-categorization --auto-load-datasets --verbose
|
|
164
166
|
|
|
165
|
-
# Generate updated video hosting lists
|
|
166
|
-
$ ruby bin/generate_video_lists
|
|
167
|
+
# Generate updated video hosting lists (with custom worker configuration)
|
|
168
|
+
$ ruby bin/generate_video_lists --workers 8 --batch-size 50
|
|
167
169
|
|
|
168
|
-
# Check health of all blocklist URLs
|
|
169
|
-
$ bundle exec check_lists
|
|
170
|
+
# Check health of all blocklist URLs (with parallel processing)
|
|
171
|
+
$ bundle exec check_lists --parallel --threads 16
|
|
170
172
|
|
|
171
173
|
# Export with custom Kaggle credentials
|
|
172
174
|
$ bundle exec export_csv --auto-load-datasets --kaggle-credentials ~/my-kaggle.json --verbose
|
|
@@ -183,6 +185,8 @@ $ bundle exec check_lists
|
|
|
183
185
|
- `--kaggle-credentials FILE`: Specify custom Kaggle credentials file
|
|
184
186
|
- `--iab-compliance`: Enable IAB Content Taxonomy mapping
|
|
185
187
|
- `--smart-categorization`: Enable intelligent category filtering
|
|
188
|
+
- `--parallel` / `--threads NUM`: Enable parallel URL checking with configurable thread count
|
|
189
|
+
- `--workers NUM`: Configure number of Ractor workers for video list generation
|
|
186
190
|
|
|
187
191
|
## Advanced Configuration
|
|
188
192
|
|
|
@@ -244,10 +248,32 @@ client = UrlCategorise::Client.new(
|
|
|
244
248
|
iab_compliance: true, # Enable IAB compliance
|
|
245
249
|
iab_version: :v3, # Use IAB Content Taxonomy v3.0
|
|
246
250
|
auto_load_datasets: false, # Disable automatic dataset loading (default)
|
|
247
|
-
smart_categorization: false
|
|
251
|
+
smart_categorization: false, # Disable smart post-processing (default)
|
|
252
|
+
max_threads: 8, # Max threads for parallel processing (default: 8)
|
|
253
|
+
max_ractor_workers: 4, # Max Ractor workers for parallel processing (default: 4 or CPU count)
|
|
254
|
+
parallel_loading: true # Enable parallel loading when available (default: true)
|
|
248
255
|
)
|
|
249
256
|
```
|
|
250
257
|
|
|
258
|
+
### Parallel Processing Configuration
|
|
259
|
+
|
|
260
|
+
The gem uses Ruby Ractors (when available) or Threads for parallel processing to significantly improve performance:
|
|
261
|
+
|
|
262
|
+
```ruby
|
|
263
|
+
# Configure parallel processing parameters
|
|
264
|
+
client = UrlCategorise::Client.new(
|
|
265
|
+
parallel_loading: true, # Enable/disable parallel loading (default: true if Ractors available)
|
|
266
|
+
max_threads: 16, # Maximum threads for thread-based processing (default: 8)
|
|
267
|
+
max_ractor_workers: 8 # Maximum Ractor workers for Ractor-based processing (default: 4 or CPU count)
|
|
268
|
+
)
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
**Performance Notes:**
|
|
272
|
+
- **Ractors**: Used by default on Ruby 3.0+ when available, providing better isolation and performance
|
|
273
|
+
- **Threads**: Fallback method when Ractors are unavailable, still provides significant speedup
|
|
274
|
+
- **Test Environment**: Automatically switches to sequential processing during tests to avoid issues
|
|
275
|
+
- **Worker Pools**: Both Ractors and Threads use worker pool patterns to prevent resource exhaustion
|
|
276
|
+
|
|
251
277
|
### Custom Lists
|
|
252
278
|
|
|
253
279
|
Use your own curated lists or subset of categories:
|
|
@@ -273,7 +299,7 @@ The gem maintains a comprehensive list of video hosting domains extracted from y
|
|
|
273
299
|
|
|
274
300
|
```ruby
|
|
275
301
|
# Generate/update video hosting lists
|
|
276
|
-
system("ruby bin/generate_video_lists")
|
|
302
|
+
system("ruby bin/generate_video_lists --workers 4 --batch-size 25")
|
|
277
303
|
|
|
278
304
|
# Use video hosting categorization
|
|
279
305
|
client = UrlCategorise::Client.new
|
|
@@ -331,13 +357,56 @@ client.video_url?("https://google.com/search?q=cats") # => false
|
|
|
331
357
|
3. Returns `true` only if both conditions are met
|
|
332
358
|
4. Handles invalid URLs gracefully (returns `false`)
|
|
333
359
|
|
|
360
|
+
#### Additional Video URL Helper Methods
|
|
361
|
+
|
|
362
|
+
The gem provides specialized helper methods for different types of video content:
|
|
363
|
+
|
|
364
|
+
```ruby
|
|
365
|
+
client = UrlCategorise::Client.new(regex_categorization: true)
|
|
366
|
+
|
|
367
|
+
# Detect short-form video content
|
|
368
|
+
client.shorts_url?("https://youtube.com/shorts/abc123defgh") # => true
|
|
369
|
+
client.shorts_url?("https://tiktok.com/@user/video/123456789") # => true
|
|
370
|
+
client.shorts_url?("https://youtube.com/watch?v=test123") # => false
|
|
371
|
+
|
|
372
|
+
# Detect playlist URLs
|
|
373
|
+
client.playlist_url?("https://youtube.com/playlist?list=PLtest123") # => true
|
|
374
|
+
client.playlist_url?("https://youtube.com/watch?v=abc123&list=PLtest123") # => true
|
|
375
|
+
client.playlist_url?("https://vimeo.com/album/123456") # => true
|
|
376
|
+
client.playlist_url?("https://youtube.com/watch?v=test123") # => false
|
|
377
|
+
|
|
378
|
+
# Detect music content (works with video platforms hosting music)
|
|
379
|
+
client.music_url?("https://music.youtube.com/watch?v=abc123") # => true
|
|
380
|
+
client.music_url?("https://youtube.com/watch?v=abc123defgh&list=PLmusic") # => true
|
|
381
|
+
client.music_url?("https://youtube.com/c/musicchannel") # => true
|
|
382
|
+
client.music_url?("https://youtube.com/watch?v=regularvideo") # => false
|
|
383
|
+
|
|
384
|
+
# Detect channel/profile URLs
|
|
385
|
+
client.channel_url?("https://youtube.com/@channelname") # => true
|
|
386
|
+
client.channel_url?("https://tiktok.com/@username") # => true
|
|
387
|
+
client.channel_url?("https://twitch.tv/streamername") # => true
|
|
388
|
+
client.channel_url?("https://youtube.com/watch?v=test123") # => false
|
|
389
|
+
|
|
390
|
+
# Detect live stream URLs
|
|
391
|
+
client.live_stream_url?("https://youtube.com/live/streamid") # => true
|
|
392
|
+
client.live_stream_url?("https://twitch.tv/streamername") # => true
|
|
393
|
+
client.live_stream_url?("https://youtube.com/watch?v=test123") # => false
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
**All helper methods:**
|
|
397
|
+
- Require `regex_categorization: true` to be enabled
|
|
398
|
+
- First verify the URL is from a video hosting domain
|
|
399
|
+
- Use specific regex patterns for accurate detection
|
|
400
|
+
- Handle invalid URLs gracefully (return `false`)
|
|
401
|
+
- Work across multiple video platforms (YouTube, TikTok, Vimeo, Twitch, etc.)
|
|
402
|
+
|
|
334
403
|
#### Maintaining Video Lists
|
|
335
404
|
|
|
336
405
|
The gem includes a script to generate and maintain comprehensive video hosting lists:
|
|
337
406
|
|
|
338
407
|
```bash
|
|
339
|
-
# Generate updated video hosting lists
|
|
340
|
-
ruby bin/generate_video_lists
|
|
408
|
+
# Generate updated video hosting lists (configurable parallel processing)
|
|
409
|
+
ruby bin/generate_video_lists --workers 8 --batch-size 50 --threshold 20
|
|
341
410
|
|
|
342
411
|
# This creates:
|
|
343
412
|
# - lists/video_hosting_domains.hosts (PiHole compatible)
|
|
@@ -346,6 +415,92 @@ ruby bin/generate_video_lists
|
|
|
346
415
|
|
|
347
416
|
The script fetches data from yt-dlp extractors and combines it with manually curated major platforms to ensure comprehensive coverage.
|
|
348
417
|
|
|
418
|
+
### Blog Content Detection
|
|
419
|
+
|
|
420
|
+
The gem provides a `blog_url?` method to identify blog-related URLs using simple string matching patterns:
|
|
421
|
+
|
|
422
|
+
```ruby
|
|
423
|
+
client = UrlCategorise::Client.new
|
|
424
|
+
|
|
425
|
+
# Basic blog path detection
|
|
426
|
+
client.blog_url?("https://example.com/blog/") # => true
|
|
427
|
+
client.blog_url?("https://example.com/blogs/tech") # => true
|
|
428
|
+
client.blog_url?("https://example.com/blog?page=1") # => true
|
|
429
|
+
|
|
430
|
+
# Blog subdomains
|
|
431
|
+
client.blog_url?("https://blog.example.com/") # => true
|
|
432
|
+
client.blog_url?("https://blog.company.org/article") # => true
|
|
433
|
+
|
|
434
|
+
# Blog platforms
|
|
435
|
+
client.blog_url?("https://example.wordpress.com/") # => true
|
|
436
|
+
client.blog_url?("https://example.blogspot.com/post") # => true
|
|
437
|
+
client.blog_url?("https://medium.com/@user/article") # => true
|
|
438
|
+
client.blog_url?("https://user.substack.com/p/post") # => true
|
|
439
|
+
|
|
440
|
+
# Blog-like content paths
|
|
441
|
+
client.blog_url?("https://example.com/post/123") # => true
|
|
442
|
+
client.blog_url?("https://example.com/articles/tech") # => true
|
|
443
|
+
client.blog_url?("https://example.com/diary/entry") # => true
|
|
444
|
+
|
|
445
|
+
# Blog keywords in URLs
|
|
446
|
+
client.blog_url?("https://example.com/corporate-blog") # => true
|
|
447
|
+
client.blog_url?("https://example-blog.com/") # => true
|
|
448
|
+
|
|
449
|
+
# Non-blog URLs return false
|
|
450
|
+
client.blog_url?("https://example.com/") # => false
|
|
451
|
+
client.blog_url?("https://example.com/products") # => false
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
**Detection patterns include:**
|
|
455
|
+
- `/blog/` or `/blogs/` in URL paths
|
|
456
|
+
- `blog.` subdomains
|
|
457
|
+
- `blog-` or `-blog` in domain names
|
|
458
|
+
- Common blog platforms (WordPress, Blogspot, Medium, Substack)
|
|
459
|
+
- Blog-like content paths (`/post/`, `/articles/`, `/diary/`, `/journal/`)
|
|
460
|
+
- The word "blog" anywhere in the URL
|
|
461
|
+
- Case-insensitive matching
|
|
462
|
+
- Graceful handling of invalid URLs
|
|
463
|
+
|
|
464
|
+
### Debug Mode
|
|
465
|
+
|
|
466
|
+
The gem includes comprehensive debug functionality to help you understand what's happening during initialization and operation:
|
|
467
|
+
|
|
468
|
+
```ruby
|
|
469
|
+
# Enable debug mode during initialization
|
|
470
|
+
client = UrlCategorise::Client.new(debug: true)
|
|
471
|
+
|
|
472
|
+
# Or enable it dynamically using ActiveAttr
|
|
473
|
+
client = UrlCategorise::Client.new
|
|
474
|
+
client.debug_enabled = true
|
|
475
|
+
|
|
476
|
+
# Debug output shows:
|
|
477
|
+
# [UrlCategorise DEBUG] Initializing UrlCategorise Client with debug enabled
|
|
478
|
+
# [UrlCategorise DEBUG] Loading host lists from 15 categories
|
|
479
|
+
# [UrlCategorise DEBUG] Processing host list: https://example.com/malware.txt
|
|
480
|
+
# [UrlCategorise DEBUG] Cache miss for https://example.com/malware.txt
|
|
481
|
+
# [UrlCategorise DEBUG] Downloading and parsing https://example.com/malware.txt completed in 234.56ms
|
|
482
|
+
# [UrlCategorise DEBUG] Downloaded 1500 hosts from https://example.com/malware.txt
|
|
483
|
+
# [UrlCategorise DEBUG] Total unique hosts collected: 45000
|
|
484
|
+
# [UrlCategorise DEBUG] Host lists loading completed in 2543.21ms
|
|
485
|
+
# [UrlCategorise DEBUG] Client initialization completed
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
**Debug features include:**
|
|
489
|
+
- Initialization timing and progress
|
|
490
|
+
- Host list loading with individual URL timing
|
|
491
|
+
- Cache hit/miss information
|
|
492
|
+
- Download progress and host counts
|
|
493
|
+
- Dataset loading progress (when datasets are enabled)
|
|
494
|
+
- Regex pattern loading information
|
|
495
|
+
- Total timing for major operations
|
|
496
|
+
- Off by default, easily enabled via constructor or ActiveAttr
|
|
497
|
+
|
|
498
|
+
**Timing accuracy:**
|
|
499
|
+
- Millisecond precision timing
|
|
500
|
+
- Individual operation timing
|
|
501
|
+
- Cumulative timing for complex operations
|
|
502
|
+
- Helpful for performance optimization and debugging
|
|
503
|
+
|
|
349
504
|
### Smart Categorization (Post-Processing)
|
|
350
505
|
|
|
351
506
|
Smart categorization solves the problem of overly broad domain-level categorization. For example, `reddit.com` might appear in health & fitness blocklists, but not all Reddit content is health-related.
|
|
@@ -559,8 +714,11 @@ end
|
|
|
559
714
|
|
|
560
715
|
Use the included script to check all URLs:
|
|
561
716
|
```bash
|
|
562
|
-
# Check all URLs in constants
|
|
717
|
+
# Check all URLs in constants (sequential)
|
|
563
718
|
ruby bin/check_lists
|
|
719
|
+
|
|
720
|
+
# Check URLs in parallel with custom thread count
|
|
721
|
+
ruby bin/check_lists --parallel --threads 12 --verbose
|
|
564
722
|
```
|
|
565
723
|
|
|
566
724
|
[View all 60+ categories in constants.rb](lib/url_categorise/constants.rb)
|