udger 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/lib/udger/ip_parser.rb +10 -6
- data/lib/udger/parser.rb +4 -6
- data/lib/udger/ua_parser.rb +21 -23
- data/lib/udger/version.rb +1 -1
- data/udger.gemspec +2 -2
- metadata +10 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 902e1d53a718e5de8c5f96b4a9414703923c55e36d7041012ffabfad3bb33e31
|
4
|
+
data.tar.gz: c6556e4ad3985f031a5028d9a5874d987001de6bab2025eec7d23bcb369f2853
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b51f621303614ca70c1b9e5f7bc4bcc63a7e5bd77718ed90ca572703a252e07dddf5502bc5a49aeffd3a9766f351be434aaf27b9e7a28e2d61cf693cff2cb4a9
|
7
|
+
data.tar.gz: 97e244194d64c657381231cb7d75e2ec789a266acd4be3916de952dc21e8f424faf8f4f2179ae5724bdb50a86b56ca1ae03b44b88c58e63d58cd3f4bfda856a1
|
data/lib/udger/ip_parser.rb
CHANGED
@@ -11,13 +11,15 @@ module Udger
|
|
11
11
|
|
12
12
|
def parse
|
13
13
|
return unless @ip
|
14
|
+
|
14
15
|
object.ip = @ip
|
15
16
|
parse_ip_object
|
16
17
|
end
|
17
18
|
|
18
19
|
def parse_ip_object
|
19
|
-
ip_object = fetch_ip
|
20
|
+
ip_object = fetch_ip
|
20
21
|
return if ip_object.nil?
|
22
|
+
|
21
23
|
object.ip_ver = ip_object.ipv4? ? 4 : 6
|
22
24
|
query = "SELECT udger_crawler_list.id as botid,ip_last_seen,ip_hostname,ip_country,ip_city,ip_country_code,ip_classification,ip_classification_code,
|
23
25
|
name,ver,ver_major,last_seen,respect_robotstxt,family,family_code,family_homepage,family_icon,vendor,vendor_code,vendor_homepage,crawler_classification,crawler_classification_code
|
@@ -27,7 +29,10 @@ module Udger
|
|
27
29
|
LEFT JOIN udger_crawler_class ON udger_crawler_class.id=udger_crawler_list.class_id
|
28
30
|
WHERE ip=? ORDER BY sequence"
|
29
31
|
data = db.execute(query, ip_object.to_s)
|
30
|
-
|
32
|
+
if data.empty?
|
33
|
+
object.ip_classification = 'Unrecognized'
|
34
|
+
object.ip_classification_code = 'Unrecognized'
|
35
|
+
else
|
31
36
|
result = data[0]
|
32
37
|
object.ip_classification = result['ip_classification']
|
33
38
|
object.ip_classification_code = result['ip_classification_code']
|
@@ -53,9 +58,6 @@ module Udger
|
|
53
58
|
object.crawler_category = result['crawler_classification']
|
54
59
|
object.crawler_category_code = result['crawler_classification_code']
|
55
60
|
object.crawler_respect_robotstxt = result['respect_robotstxt']
|
56
|
-
else
|
57
|
-
object.ip_classification = 'Unrecognized'
|
58
|
-
object.ip_classification_code = 'Unrecognized'
|
59
61
|
end
|
60
62
|
|
61
63
|
if object.ip_ver == 4
|
@@ -73,6 +75,7 @@ module Udger
|
|
73
75
|
ip_int = ip_object.to_i
|
74
76
|
data = db.execute(query, ip_int, ip_int)
|
75
77
|
return if data.empty?
|
78
|
+
|
76
79
|
result = data[0]
|
77
80
|
object.datacenter_name = result['name']
|
78
81
|
object.datacenter_name_code = result['name_code']
|
@@ -92,6 +95,7 @@ module Udger
|
|
92
95
|
|
93
96
|
data = db.execute(query)
|
94
97
|
return if data.empty?
|
98
|
+
|
95
99
|
result = data[0]
|
96
100
|
object.datacenter_name = result['name']
|
97
101
|
object.datacenter_name_code = result['name_code']
|
@@ -100,7 +104,7 @@ module Udger
|
|
100
104
|
|
101
105
|
def fetch_ip
|
102
106
|
IPAddr.new @ip
|
103
|
-
rescue
|
107
|
+
rescue StandardError
|
104
108
|
nil
|
105
109
|
end
|
106
110
|
end
|
data/lib/udger/parser.rb
CHANGED
@@ -3,10 +3,8 @@ require 'lru_redux'
|
|
3
3
|
module Udger
|
4
4
|
class Parser
|
5
5
|
DB_FILENAME = 'udgerdb_v3.dat'.freeze
|
6
|
-
attr_reader :db
|
7
|
-
|
8
|
-
attr_accessor :cache
|
9
|
-
attr_accessor :lru_cache_size
|
6
|
+
attr_reader :db, :data_dir
|
7
|
+
attr_accessor :cache, :lru_cache_size
|
10
8
|
|
11
9
|
def initialize(data_dir = nil, lru_cache_size: 10_000, cache: true, ua_services: [])
|
12
10
|
@data_dir = data_dir || __dir__
|
@@ -17,12 +15,12 @@ module Udger
|
|
17
15
|
@lru_cache = lru_cache_size.zero? && !cache ? {} : LruRedux::Cache.new(lru_cache_size)
|
18
16
|
end
|
19
17
|
|
20
|
-
|
21
18
|
def parse_ua(ua_string)
|
22
19
|
if @cache
|
23
20
|
cache_string = ua_string.hash
|
24
21
|
cache_value = @lru_cache[cache_string]
|
25
22
|
return cache_value if cache_value
|
23
|
+
|
26
24
|
result = parse_ua_no_cache(ua_string)
|
27
25
|
@lru_cache[cache_string] = result
|
28
26
|
result
|
@@ -42,7 +40,7 @@ module Udger
|
|
42
40
|
def parse_ua_params(parse)
|
43
41
|
parsers = parse.empty?
|
44
42
|
@match_ua = {}
|
45
|
-
[
|
43
|
+
%i[crawler client os device device_market].each do |p|
|
46
44
|
@match_ua[p] = parsers || parse.include?(p)
|
47
45
|
end
|
48
46
|
end
|
data/lib/udger/ua_parser.rb
CHANGED
@@ -2,14 +2,14 @@ module Udger
|
|
2
2
|
class UaParser < BaseParser
|
3
3
|
attr_accessor :db, :ua_string, :object
|
4
4
|
|
5
|
-
def initialize(db, ua_string,
|
5
|
+
def initialize(db, ua_string, kargs = {})
|
6
6
|
super(db)
|
7
7
|
|
8
|
-
@match_crawler = crawler
|
9
|
-
@match_client = client
|
10
|
-
@match_os = os
|
11
|
-
@match_device = device
|
12
|
-
@match_device_market = device_market
|
8
|
+
@match_crawler = kargs.fetch(:crawler, true)
|
9
|
+
@match_client = kargs.fetch(:client, true)
|
10
|
+
@match_os = kargs.fetch(:os, true)
|
11
|
+
@match_device = kargs.fetch(:device, true)
|
12
|
+
@match_device_market = kargs.fetch(:device_market, true)
|
13
13
|
|
14
14
|
@ua_string = ua_string
|
15
15
|
@object = UserAgent.new
|
@@ -21,6 +21,7 @@ module Udger
|
|
21
21
|
|
22
22
|
def parse
|
23
23
|
return unless ua_string
|
24
|
+
|
24
25
|
object.ua_string = ua_string
|
25
26
|
crawler_data = @match_crawler ? parse_crawler : []
|
26
27
|
if !crawler_data.empty?
|
@@ -36,7 +37,6 @@ module Udger
|
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
39
|
-
|
40
40
|
private
|
41
41
|
|
42
42
|
def parse_crawler
|
@@ -135,6 +135,7 @@ module Udger
|
|
135
135
|
|
136
136
|
data = db.execute(query, @client_id)
|
137
137
|
return if data.empty?
|
138
|
+
|
138
139
|
result = data[0]
|
139
140
|
@os_id = result['os_id']
|
140
141
|
object.os = result['name']
|
@@ -151,7 +152,6 @@ module Udger
|
|
151
152
|
end
|
152
153
|
|
153
154
|
def parse_device
|
154
|
-
|
155
155
|
query = 'SELECT deviceclass_id,regstring,name,name_code,icon,icon_big
|
156
156
|
FROM udger_deviceclass_regex
|
157
157
|
JOIN udger_deviceclass_list ON udger_deviceclass_list.id=udger_deviceclass_regex.deviceclass_id
|
@@ -167,30 +167,29 @@ module Udger
|
|
167
167
|
end
|
168
168
|
|
169
169
|
# If there is no @client_class_id and @match_client is not enabled
|
170
|
-
if @client_class_id == -1 && !@match_client
|
171
|
-
parse_client
|
172
|
-
end
|
170
|
+
parse_client if @client_class_id == -1 && !@match_client
|
173
171
|
|
174
172
|
if @deviceclass_id.zero? && @client_class_id != -1
|
175
|
-
|
173
|
+
query = 'SELECT deviceclass_id,name,name_code,icon,icon_big
|
176
174
|
FROM udger_deviceclass_list
|
177
175
|
JOIN udger_client_class ON udger_client_class.deviceclass_id=udger_deviceclass_list.id
|
178
176
|
WHERE udger_client_class.id=?'
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
177
|
+
data = db.execute(query, @client_class_id)
|
178
|
+
unless data.empty?
|
179
|
+
result = data[0]
|
180
|
+
@deviceclass_id = result['deviceclass_id']
|
181
|
+
object.device_class = result['name']
|
182
|
+
object.device_class_code = result['name_code']
|
183
|
+
object.device_class_icon = result['icon']
|
184
|
+
object.device_class_icon_big = result['icon_big']
|
185
|
+
object.device_class_info_url = "https://udger.com/resources/ua-list/device-detail?device=#{result['name']}"
|
186
|
+
end
|
189
187
|
end
|
190
188
|
end
|
191
189
|
|
192
190
|
def devise_market_name
|
193
191
|
return unless object.os_family_code
|
192
|
+
|
194
193
|
# TODO: santize code
|
195
194
|
query = "SELECT id,regstring FROM udger_devicename_regex WHERE
|
196
195
|
((os_family_code='" + object.os_family_code + "' AND os_code='-all-')
|
@@ -216,6 +215,5 @@ module Udger
|
|
216
215
|
end
|
217
216
|
end
|
218
217
|
end
|
219
|
-
|
220
218
|
end
|
221
219
|
end
|
data/lib/udger/version.rb
CHANGED
data/udger.gemspec
CHANGED
@@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
31
|
spec.require_paths = ['lib']
|
32
32
|
|
33
|
-
spec.add_dependency 'sqlite3', '~> 1.
|
33
|
+
spec.add_dependency 'sqlite3', '~> 1.4'
|
34
34
|
spec.add_dependency 'lru_redux', '~> 1.1'
|
35
35
|
|
36
|
-
spec.add_development_dependency 'bundler', '~> 1
|
36
|
+
spec.add_development_dependency 'bundler', '~> 2.1'
|
37
37
|
spec.add_development_dependency 'simplecov', '~> 0.14'
|
38
38
|
spec.add_development_dependency 'rake', '~> 10.0'
|
39
39
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: udger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bojan Milosavljevic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.4'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.4'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: lru_redux
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '1
|
47
|
+
version: '2.1'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '1
|
54
|
+
version: '2.1'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: simplecov
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,7 @@ homepage: https://github.com/TowerData/udger-ruby
|
|
122
122
|
licenses:
|
123
123
|
- MIT
|
124
124
|
metadata: {}
|
125
|
-
post_install_message:
|
125
|
+
post_install_message:
|
126
126
|
rdoc_options: []
|
127
127
|
require_paths:
|
128
128
|
- lib
|
@@ -137,9 +137,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
requirements: []
|
140
|
-
|
141
|
-
|
142
|
-
signing_key:
|
140
|
+
rubygems_version: 3.2.33
|
141
|
+
signing_key:
|
143
142
|
specification_version: 4
|
144
143
|
summary: Udger user agent library
|
145
144
|
test_files: []
|