udger 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/README.md +4 -0
- data/lib/udger/ip_parser.rb +11 -7
- data/lib/udger/parser.rb +4 -6
- data/lib/udger/ua_parser.rb +24 -25
- data/lib/udger/version.rb +1 -1
- data/udger.gemspec +2 -2
- metadata +10 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 47075e1239211f024f1cc5a0676aec3a3bec8cc52b3cb6cc8cf1cf37cf483c5e
|
4
|
+
data.tar.gz: c7ef9d033e8dea9e5fe212005ff31febf3214b6d4f6e8a303719b65c047f42ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab5956beaa3456264dc142666468d1042f9eeed9c7946bde8cb0357ef7ddbdad473f42de5de55d9b856238b04a8cff7654130d602cbb0e3917ba04f4529463c0
|
7
|
+
data.tar.gz: f9f8ed0f04af30e5c32b25e9f9ed81106656f1664b4d263d951f643dfc22d412eac3e297eadb1ea3cefa07253eed64299abd50608e7ee566ff7428d054f1bb78
|
data/.gitignore
CHANGED
data/README.md
CHANGED
data/lib/udger/ip_parser.rb
CHANGED
@@ -11,13 +11,15 @@ module Udger
|
|
11
11
|
|
12
12
|
def parse
|
13
13
|
return unless @ip
|
14
|
+
|
14
15
|
object.ip = @ip
|
15
16
|
parse_ip_object
|
16
17
|
end
|
17
18
|
|
18
19
|
def parse_ip_object
|
19
|
-
ip_object = fetch_ip
|
20
|
+
ip_object = fetch_ip
|
20
21
|
return if ip_object.nil?
|
22
|
+
|
21
23
|
object.ip_ver = ip_object.ipv4? ? 4 : 6
|
22
24
|
query = "SELECT udger_crawler_list.id as botid,ip_last_seen,ip_hostname,ip_country,ip_city,ip_country_code,ip_classification,ip_classification_code,
|
23
25
|
name,ver,ver_major,last_seen,respect_robotstxt,family,family_code,family_homepage,family_icon,vendor,vendor_code,vendor_homepage,crawler_classification,crawler_classification_code
|
@@ -27,7 +29,10 @@ module Udger
|
|
27
29
|
LEFT JOIN udger_crawler_class ON udger_crawler_class.id=udger_crawler_list.class_id
|
28
30
|
WHERE ip=? ORDER BY sequence"
|
29
31
|
data = db.execute(query, ip_object.to_s)
|
30
|
-
|
32
|
+
if data.empty?
|
33
|
+
object.ip_classification = 'Unrecognized'
|
34
|
+
object.ip_classification_code = 'Unrecognized'
|
35
|
+
else
|
31
36
|
result = data[0]
|
32
37
|
object.ip_classification = result['ip_classification']
|
33
38
|
object.ip_classification_code = result['ip_classification_code']
|
@@ -53,9 +58,6 @@ module Udger
|
|
53
58
|
object.crawler_category = result['crawler_classification']
|
54
59
|
object.crawler_category_code = result['crawler_classification_code']
|
55
60
|
object.crawler_respect_robotstxt = result['respect_robotstxt']
|
56
|
-
else
|
57
|
-
object.ip_classification = 'Unrecognized'
|
58
|
-
object.ip_classification_code = 'Unrecognized'
|
59
61
|
end
|
60
62
|
|
61
63
|
if object.ip_ver == 4
|
@@ -71,8 +73,9 @@ module Udger
|
|
71
73
|
JOIN udger_datacenter_list ON udger_datacenter_range.datacenter_id=udger_datacenter_list.id
|
72
74
|
WHERE iplong_from <= ? AND iplong_to >= ?'
|
73
75
|
ip_int = ip_object.to_i
|
74
|
-
data = db.execute(query, ip_int, ip_int)
|
76
|
+
data = db.execute(query, [ip_int, ip_int])
|
75
77
|
return if data.empty?
|
78
|
+
|
76
79
|
result = data[0]
|
77
80
|
object.datacenter_name = result['name']
|
78
81
|
object.datacenter_name_code = result['name_code']
|
@@ -92,6 +95,7 @@ module Udger
|
|
92
95
|
|
93
96
|
data = db.execute(query)
|
94
97
|
return if data.empty?
|
98
|
+
|
95
99
|
result = data[0]
|
96
100
|
object.datacenter_name = result['name']
|
97
101
|
object.datacenter_name_code = result['name_code']
|
@@ -100,7 +104,7 @@ module Udger
|
|
100
104
|
|
101
105
|
def fetch_ip
|
102
106
|
IPAddr.new @ip
|
103
|
-
rescue
|
107
|
+
rescue StandardError
|
104
108
|
nil
|
105
109
|
end
|
106
110
|
end
|
data/lib/udger/parser.rb
CHANGED
@@ -3,10 +3,8 @@ require 'lru_redux'
|
|
3
3
|
module Udger
|
4
4
|
class Parser
|
5
5
|
DB_FILENAME = 'udgerdb_v3.dat'.freeze
|
6
|
-
attr_reader :db
|
7
|
-
|
8
|
-
attr_accessor :cache
|
9
|
-
attr_accessor :lru_cache_size
|
6
|
+
attr_reader :db, :data_dir
|
7
|
+
attr_accessor :cache, :lru_cache_size
|
10
8
|
|
11
9
|
def initialize(data_dir = nil, lru_cache_size: 10_000, cache: true, ua_services: [])
|
12
10
|
@data_dir = data_dir || __dir__
|
@@ -17,12 +15,12 @@ module Udger
|
|
17
15
|
@lru_cache = lru_cache_size.zero? && !cache ? {} : LruRedux::Cache.new(lru_cache_size)
|
18
16
|
end
|
19
17
|
|
20
|
-
|
21
18
|
def parse_ua(ua_string)
|
22
19
|
if @cache
|
23
20
|
cache_string = ua_string.hash
|
24
21
|
cache_value = @lru_cache[cache_string]
|
25
22
|
return cache_value if cache_value
|
23
|
+
|
26
24
|
result = parse_ua_no_cache(ua_string)
|
27
25
|
@lru_cache[cache_string] = result
|
28
26
|
result
|
@@ -42,7 +40,7 @@ module Udger
|
|
42
40
|
def parse_ua_params(parse)
|
43
41
|
parsers = parse.empty?
|
44
42
|
@match_ua = {}
|
45
|
-
[
|
43
|
+
%i[crawler client os device device_market].each do |p|
|
46
44
|
@match_ua[p] = parsers || parse.include?(p)
|
47
45
|
end
|
48
46
|
end
|
data/lib/udger/ua_parser.rb
CHANGED
@@ -2,14 +2,14 @@ module Udger
|
|
2
2
|
class UaParser < BaseParser
|
3
3
|
attr_accessor :db, :ua_string, :object
|
4
4
|
|
5
|
-
def initialize(db, ua_string,
|
5
|
+
def initialize(db, ua_string, kargs = {})
|
6
6
|
super(db)
|
7
7
|
|
8
|
-
@match_crawler = crawler
|
9
|
-
@match_client = client
|
10
|
-
@match_os = os
|
11
|
-
@match_device = device
|
12
|
-
@match_device_market = device_market
|
8
|
+
@match_crawler = kargs.fetch(:crawler, true)
|
9
|
+
@match_client = kargs.fetch(:client, true)
|
10
|
+
@match_os = kargs.fetch(:os, true)
|
11
|
+
@match_device = kargs.fetch(:device, true)
|
12
|
+
@match_device_market = kargs.fetch(:device_market, true)
|
13
13
|
|
14
14
|
@ua_string = ua_string
|
15
15
|
@object = UserAgent.new
|
@@ -21,6 +21,7 @@ module Udger
|
|
21
21
|
|
22
22
|
def parse
|
23
23
|
return unless ua_string
|
24
|
+
|
24
25
|
object.ua_string = ua_string
|
25
26
|
crawler_data = @match_crawler ? parse_crawler : []
|
26
27
|
if !crawler_data.empty?
|
@@ -36,7 +37,6 @@ module Udger
|
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
39
|
-
|
40
40
|
private
|
41
41
|
|
42
42
|
def parse_crawler
|
@@ -135,6 +135,7 @@ module Udger
|
|
135
135
|
|
136
136
|
data = db.execute(query, @client_id)
|
137
137
|
return if data.empty?
|
138
|
+
|
138
139
|
result = data[0]
|
139
140
|
@os_id = result['os_id']
|
140
141
|
object.os = result['name']
|
@@ -151,7 +152,6 @@ module Udger
|
|
151
152
|
end
|
152
153
|
|
153
154
|
def parse_device
|
154
|
-
|
155
155
|
query = 'SELECT deviceclass_id,regstring,name,name_code,icon,icon_big
|
156
156
|
FROM udger_deviceclass_regex
|
157
157
|
JOIN udger_deviceclass_list ON udger_deviceclass_list.id=udger_deviceclass_regex.deviceclass_id
|
@@ -167,31 +167,30 @@ module Udger
|
|
167
167
|
end
|
168
168
|
|
169
169
|
# If there is no @client_class_id and @match_client is not enabled
|
170
|
-
if @client_class_id == -1 && !@match_client
|
171
|
-
parse_client
|
172
|
-
end
|
170
|
+
parse_client if @client_class_id == -1 && !@match_client
|
173
171
|
|
174
172
|
if @deviceclass_id.zero? && @client_class_id != -1
|
175
|
-
|
173
|
+
query = 'SELECT deviceclass_id,name,name_code,icon,icon_big
|
176
174
|
FROM udger_deviceclass_list
|
177
175
|
JOIN udger_client_class ON udger_client_class.deviceclass_id=udger_deviceclass_list.id
|
178
176
|
WHERE udger_client_class.id=?'
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
177
|
+
data = db.execute(query, @client_class_id)
|
178
|
+
unless data.empty?
|
179
|
+
result = data[0]
|
180
|
+
@deviceclass_id = result['deviceclass_id']
|
181
|
+
object.device_class = result['name']
|
182
|
+
object.device_class_code = result['name_code']
|
183
|
+
object.device_class_icon = result['icon']
|
184
|
+
object.device_class_icon_big = result['icon_big']
|
185
|
+
object.device_class_info_url = "https://udger.com/resources/ua-list/device-detail?device=#{result['name']}"
|
186
|
+
end
|
189
187
|
end
|
190
188
|
end
|
191
189
|
|
192
190
|
def devise_market_name
|
193
191
|
return unless object.os_family_code
|
194
|
-
|
192
|
+
|
193
|
+
# TODO: sanitize code
|
195
194
|
query = "SELECT id,regstring FROM udger_devicename_regex WHERE
|
196
195
|
((os_family_code='" + object.os_family_code + "' AND os_code='-all-')
|
197
196
|
OR
|
@@ -202,7 +201,8 @@ module Udger
|
|
202
201
|
FROM udger_devicename_list
|
203
202
|
JOIN udger_devicename_brand ON udger_devicename_brand.id=udger_devicename_list.brand_id
|
204
203
|
WHERE regex_id=? and code = ? COLLATE NOCASE"
|
205
|
-
qc = db.execute(sub_query, result['id'], match[0])
|
204
|
+
qc = db.execute(sub_query, [result['id'], match[0]])
|
205
|
+
|
206
206
|
unless qc.empty?
|
207
207
|
res = qc[0]
|
208
208
|
object.device_marketname = res['marketname']
|
@@ -216,6 +216,5 @@ module Udger
|
|
216
216
|
end
|
217
217
|
end
|
218
218
|
end
|
219
|
-
|
220
219
|
end
|
221
220
|
end
|
data/lib/udger/version.rb
CHANGED
data/udger.gemspec
CHANGED
@@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
31
|
spec.require_paths = ['lib']
|
32
32
|
|
33
|
-
spec.add_dependency 'sqlite3', '~>
|
33
|
+
spec.add_dependency 'sqlite3', '~> 2.0'
|
34
34
|
spec.add_dependency 'lru_redux', '~> 1.1'
|
35
35
|
|
36
|
-
spec.add_development_dependency 'bundler', '~> 1
|
36
|
+
spec.add_development_dependency 'bundler', '~> 2.1'
|
37
37
|
spec.add_development_dependency 'simplecov', '~> 0.14'
|
38
38
|
spec.add_development_dependency 'rake', '~> 10.0'
|
39
39
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: udger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bojan Milosavljevic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2025-06-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '2.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: lru_redux
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '1
|
47
|
+
version: '2.1'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '1
|
54
|
+
version: '2.1'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: simplecov
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,7 @@ homepage: https://github.com/TowerData/udger-ruby
|
|
122
122
|
licenses:
|
123
123
|
- MIT
|
124
124
|
metadata: {}
|
125
|
-
post_install_message:
|
125
|
+
post_install_message:
|
126
126
|
rdoc_options: []
|
127
127
|
require_paths:
|
128
128
|
- lib
|
@@ -137,9 +137,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
requirements: []
|
140
|
-
|
141
|
-
|
142
|
-
signing_key:
|
140
|
+
rubygems_version: 3.5.22
|
141
|
+
signing_key:
|
143
142
|
specification_version: 4
|
144
143
|
summary: Udger user agent library
|
145
144
|
test_files: []
|