udger 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/udger/ip_parser.rb +10 -6
- data/lib/udger/parser.rb +4 -6
- data/lib/udger/ua_parser.rb +21 -23
- data/lib/udger/version.rb +1 -1
- data/udger.gemspec +2 -2
- metadata +10 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 902e1d53a718e5de8c5f96b4a9414703923c55e36d7041012ffabfad3bb33e31
|
4
|
+
data.tar.gz: c6556e4ad3985f031a5028d9a5874d987001de6bab2025eec7d23bcb369f2853
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b51f621303614ca70c1b9e5f7bc4bcc63a7e5bd77718ed90ca572703a252e07dddf5502bc5a49aeffd3a9766f351be434aaf27b9e7a28e2d61cf693cff2cb4a9
|
7
|
+
data.tar.gz: 97e244194d64c657381231cb7d75e2ec789a266acd4be3916de952dc21e8f424faf8f4f2179ae5724bdb50a86b56ca1ae03b44b88c58e63d58cd3f4bfda856a1
|
data/lib/udger/ip_parser.rb
CHANGED
@@ -11,13 +11,15 @@ module Udger
|
|
11
11
|
|
12
12
|
def parse
|
13
13
|
return unless @ip
|
14
|
+
|
14
15
|
object.ip = @ip
|
15
16
|
parse_ip_object
|
16
17
|
end
|
17
18
|
|
18
19
|
def parse_ip_object
|
19
|
-
ip_object = fetch_ip
|
20
|
+
ip_object = fetch_ip
|
20
21
|
return if ip_object.nil?
|
22
|
+
|
21
23
|
object.ip_ver = ip_object.ipv4? ? 4 : 6
|
22
24
|
query = "SELECT udger_crawler_list.id as botid,ip_last_seen,ip_hostname,ip_country,ip_city,ip_country_code,ip_classification,ip_classification_code,
|
23
25
|
name,ver,ver_major,last_seen,respect_robotstxt,family,family_code,family_homepage,family_icon,vendor,vendor_code,vendor_homepage,crawler_classification,crawler_classification_code
|
@@ -27,7 +29,10 @@ module Udger
|
|
27
29
|
LEFT JOIN udger_crawler_class ON udger_crawler_class.id=udger_crawler_list.class_id
|
28
30
|
WHERE ip=? ORDER BY sequence"
|
29
31
|
data = db.execute(query, ip_object.to_s)
|
30
|
-
|
32
|
+
if data.empty?
|
33
|
+
object.ip_classification = 'Unrecognized'
|
34
|
+
object.ip_classification_code = 'Unrecognized'
|
35
|
+
else
|
31
36
|
result = data[0]
|
32
37
|
object.ip_classification = result['ip_classification']
|
33
38
|
object.ip_classification_code = result['ip_classification_code']
|
@@ -53,9 +58,6 @@ module Udger
|
|
53
58
|
object.crawler_category = result['crawler_classification']
|
54
59
|
object.crawler_category_code = result['crawler_classification_code']
|
55
60
|
object.crawler_respect_robotstxt = result['respect_robotstxt']
|
56
|
-
else
|
57
|
-
object.ip_classification = 'Unrecognized'
|
58
|
-
object.ip_classification_code = 'Unrecognized'
|
59
61
|
end
|
60
62
|
|
61
63
|
if object.ip_ver == 4
|
@@ -73,6 +75,7 @@ module Udger
|
|
73
75
|
ip_int = ip_object.to_i
|
74
76
|
data = db.execute(query, ip_int, ip_int)
|
75
77
|
return if data.empty?
|
78
|
+
|
76
79
|
result = data[0]
|
77
80
|
object.datacenter_name = result['name']
|
78
81
|
object.datacenter_name_code = result['name_code']
|
@@ -92,6 +95,7 @@ module Udger
|
|
92
95
|
|
93
96
|
data = db.execute(query)
|
94
97
|
return if data.empty?
|
98
|
+
|
95
99
|
result = data[0]
|
96
100
|
object.datacenter_name = result['name']
|
97
101
|
object.datacenter_name_code = result['name_code']
|
@@ -100,7 +104,7 @@ module Udger
|
|
100
104
|
|
101
105
|
def fetch_ip
|
102
106
|
IPAddr.new @ip
|
103
|
-
rescue
|
107
|
+
rescue StandardError
|
104
108
|
nil
|
105
109
|
end
|
106
110
|
end
|
data/lib/udger/parser.rb
CHANGED
@@ -3,10 +3,8 @@ require 'lru_redux'
|
|
3
3
|
module Udger
|
4
4
|
class Parser
|
5
5
|
DB_FILENAME = 'udgerdb_v3.dat'.freeze
|
6
|
-
attr_reader :db
|
7
|
-
|
8
|
-
attr_accessor :cache
|
9
|
-
attr_accessor :lru_cache_size
|
6
|
+
attr_reader :db, :data_dir
|
7
|
+
attr_accessor :cache, :lru_cache_size
|
10
8
|
|
11
9
|
def initialize(data_dir = nil, lru_cache_size: 10_000, cache: true, ua_services: [])
|
12
10
|
@data_dir = data_dir || __dir__
|
@@ -17,12 +15,12 @@ module Udger
|
|
17
15
|
@lru_cache = lru_cache_size.zero? && !cache ? {} : LruRedux::Cache.new(lru_cache_size)
|
18
16
|
end
|
19
17
|
|
20
|
-
|
21
18
|
def parse_ua(ua_string)
|
22
19
|
if @cache
|
23
20
|
cache_string = ua_string.hash
|
24
21
|
cache_value = @lru_cache[cache_string]
|
25
22
|
return cache_value if cache_value
|
23
|
+
|
26
24
|
result = parse_ua_no_cache(ua_string)
|
27
25
|
@lru_cache[cache_string] = result
|
28
26
|
result
|
@@ -42,7 +40,7 @@ module Udger
|
|
42
40
|
def parse_ua_params(parse)
|
43
41
|
parsers = parse.empty?
|
44
42
|
@match_ua = {}
|
45
|
-
[
|
43
|
+
%i[crawler client os device device_market].each do |p|
|
46
44
|
@match_ua[p] = parsers || parse.include?(p)
|
47
45
|
end
|
48
46
|
end
|
data/lib/udger/ua_parser.rb
CHANGED
@@ -2,14 +2,14 @@ module Udger
|
|
2
2
|
class UaParser < BaseParser
|
3
3
|
attr_accessor :db, :ua_string, :object
|
4
4
|
|
5
|
-
def initialize(db, ua_string,
|
5
|
+
def initialize(db, ua_string, kargs = {})
|
6
6
|
super(db)
|
7
7
|
|
8
|
-
@match_crawler = crawler
|
9
|
-
@match_client = client
|
10
|
-
@match_os = os
|
11
|
-
@match_device = device
|
12
|
-
@match_device_market = device_market
|
8
|
+
@match_crawler = kargs.fetch(:crawler, true)
|
9
|
+
@match_client = kargs.fetch(:client, true)
|
10
|
+
@match_os = kargs.fetch(:os, true)
|
11
|
+
@match_device = kargs.fetch(:device, true)
|
12
|
+
@match_device_market = kargs.fetch(:device_market, true)
|
13
13
|
|
14
14
|
@ua_string = ua_string
|
15
15
|
@object = UserAgent.new
|
@@ -21,6 +21,7 @@ module Udger
|
|
21
21
|
|
22
22
|
def parse
|
23
23
|
return unless ua_string
|
24
|
+
|
24
25
|
object.ua_string = ua_string
|
25
26
|
crawler_data = @match_crawler ? parse_crawler : []
|
26
27
|
if !crawler_data.empty?
|
@@ -36,7 +37,6 @@ module Udger
|
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
39
|
-
|
40
40
|
private
|
41
41
|
|
42
42
|
def parse_crawler
|
@@ -135,6 +135,7 @@ module Udger
|
|
135
135
|
|
136
136
|
data = db.execute(query, @client_id)
|
137
137
|
return if data.empty?
|
138
|
+
|
138
139
|
result = data[0]
|
139
140
|
@os_id = result['os_id']
|
140
141
|
object.os = result['name']
|
@@ -151,7 +152,6 @@ module Udger
|
|
151
152
|
end
|
152
153
|
|
153
154
|
def parse_device
|
154
|
-
|
155
155
|
query = 'SELECT deviceclass_id,regstring,name,name_code,icon,icon_big
|
156
156
|
FROM udger_deviceclass_regex
|
157
157
|
JOIN udger_deviceclass_list ON udger_deviceclass_list.id=udger_deviceclass_regex.deviceclass_id
|
@@ -167,30 +167,29 @@ module Udger
|
|
167
167
|
end
|
168
168
|
|
169
169
|
# If there is no @client_class_id and @match_client is not enabled
|
170
|
-
if @client_class_id == -1 && !@match_client
|
171
|
-
parse_client
|
172
|
-
end
|
170
|
+
parse_client if @client_class_id == -1 && !@match_client
|
173
171
|
|
174
172
|
if @deviceclass_id.zero? && @client_class_id != -1
|
175
|
-
|
173
|
+
query = 'SELECT deviceclass_id,name,name_code,icon,icon_big
|
176
174
|
FROM udger_deviceclass_list
|
177
175
|
JOIN udger_client_class ON udger_client_class.deviceclass_id=udger_deviceclass_list.id
|
178
176
|
WHERE udger_client_class.id=?'
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
177
|
+
data = db.execute(query, @client_class_id)
|
178
|
+
unless data.empty?
|
179
|
+
result = data[0]
|
180
|
+
@deviceclass_id = result['deviceclass_id']
|
181
|
+
object.device_class = result['name']
|
182
|
+
object.device_class_code = result['name_code']
|
183
|
+
object.device_class_icon = result['icon']
|
184
|
+
object.device_class_icon_big = result['icon_big']
|
185
|
+
object.device_class_info_url = "https://udger.com/resources/ua-list/device-detail?device=#{result['name']}"
|
186
|
+
end
|
189
187
|
end
|
190
188
|
end
|
191
189
|
|
192
190
|
def devise_market_name
|
193
191
|
return unless object.os_family_code
|
192
|
+
|
194
193
|
# TODO: santize code
|
195
194
|
query = "SELECT id,regstring FROM udger_devicename_regex WHERE
|
196
195
|
((os_family_code='" + object.os_family_code + "' AND os_code='-all-')
|
@@ -216,6 +215,5 @@ module Udger
|
|
216
215
|
end
|
217
216
|
end
|
218
217
|
end
|
219
|
-
|
220
218
|
end
|
221
219
|
end
|
data/lib/udger/version.rb
CHANGED
data/udger.gemspec
CHANGED
@@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
31
|
spec.require_paths = ['lib']
|
32
32
|
|
33
|
-
spec.add_dependency 'sqlite3', '~> 1.
|
33
|
+
spec.add_dependency 'sqlite3', '~> 1.4'
|
34
34
|
spec.add_dependency 'lru_redux', '~> 1.1'
|
35
35
|
|
36
|
-
spec.add_development_dependency 'bundler', '~> 1
|
36
|
+
spec.add_development_dependency 'bundler', '~> 2.1'
|
37
37
|
spec.add_development_dependency 'simplecov', '~> 0.14'
|
38
38
|
spec.add_development_dependency 'rake', '~> 10.0'
|
39
39
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: udger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bojan Milosavljevic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.4'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.4'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: lru_redux
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '1
|
47
|
+
version: '2.1'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '1
|
54
|
+
version: '2.1'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: simplecov
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,7 @@ homepage: https://github.com/TowerData/udger-ruby
|
|
122
122
|
licenses:
|
123
123
|
- MIT
|
124
124
|
metadata: {}
|
125
|
-
post_install_message:
|
125
|
+
post_install_message:
|
126
126
|
rdoc_options: []
|
127
127
|
require_paths:
|
128
128
|
- lib
|
@@ -137,9 +137,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
requirements: []
|
140
|
-
|
141
|
-
|
142
|
-
signing_key:
|
140
|
+
rubygems_version: 3.2.33
|
141
|
+
signing_key:
|
143
142
|
specification_version: 4
|
144
143
|
summary: Udger user agent library
|
145
144
|
test_files: []
|