google_safe_browsing 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/google_safe_browsing.rb +32 -24
- data/lib/google_safe_browsing/api_v2.rb +19 -16
- data/lib/google_safe_browsing/binary_helper.rb +0 -6
- data/lib/google_safe_browsing/canonicalize.rb +39 -50
- data/lib/google_safe_browsing/google_safe_browsing_railtie.rb +4 -0
- data/lib/google_safe_browsing/http_helper.rb +2 -0
- data/lib/google_safe_browsing/rescheduler.rb +3 -3
- data/lib/google_safe_browsing/response_helper.rb +31 -18
- data/lib/google_safe_browsing/top_level_domain.rb +3 -4
- data/lib/google_safe_browsing/version.rb +1 -1
- metadata +9 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aaec9b161ce5eb3c820b5bd759c0a8b7c98844d1
|
4
|
+
data.tar.gz: 37dfc9d45dabf167e75cd7ec9dd8e5e4136bc42d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 619979715cecc6b6e782e15a080beed3b4a5919dbe6c0c64aa600427523e6111a1e3c5b851c3975032cbeb0a9911a9c74ce38bc4eeb00dd13d298df1b3f30d48
|
7
|
+
data.tar.gz: 064d1395118e4e768e290fbe9b9d15ccd99b9d96e2d1e7af2f5a601c721d17f22a5a2c809e9110dd4c2d602b759b567fc56fb1497c875e3e7d024a6a7bd65c79
|
data/README.md
CHANGED
@@ -98,7 +98,7 @@ not on Google's list.
|
|
98
98
|
## Contributing
|
99
99
|
|
100
100
|
We've already had some [great
|
101
|
-
|
101
|
+
contributors](https://github.com/mobiledefense/google_safe_browsing/graphs/contributors).
|
102
102
|
If you'd like to join us, we'd love to have you. When contributing please
|
103
103
|
|
104
104
|
1. [Fork](https://github.com/mobiledefense/google_safe_browsing/fork) the repo
|
data/lib/google_safe_browsing.rb
CHANGED
@@ -46,33 +46,41 @@ module GoogleSafeBrowsing
|
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
-
|
50
|
-
|
51
|
-
@@config ||= Config.new
|
52
|
-
end
|
49
|
+
class << self
|
50
|
+
attr_accessor :logger
|
53
51
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
end
|
52
|
+
def logger
|
53
|
+
@logger ||= Logger.new
|
54
|
+
end
|
58
55
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
56
|
+
# Returns of initializes the Module configuration
|
57
|
+
def config
|
58
|
+
@@config ||= Config.new
|
59
|
+
end
|
60
|
+
|
61
|
+
# Allows for setting config values via a block
|
62
|
+
def configure
|
63
|
+
yield config
|
64
|
+
end
|
65
|
+
|
66
|
+
# Adds the Rescheduler job to Resque
|
67
|
+
def kick_off
|
68
|
+
Resque.enqueue(Rescheduler)
|
69
|
+
end
|
63
70
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
71
|
+
# Converts the official Google list name into the name to return
|
72
|
+
#
|
73
|
+
# @param (String) list the 'official' list name
|
74
|
+
# @return (String) the friendly list name
|
75
|
+
def friendly_list_name(list)
|
76
|
+
case list
|
77
|
+
when 'goog-malware-shavar'
|
78
|
+
'malware'
|
79
|
+
when 'googpub-phish-shavar'
|
80
|
+
'phishing'
|
81
|
+
else
|
82
|
+
nil
|
83
|
+
end
|
76
84
|
end
|
77
85
|
end
|
78
86
|
end
|
@@ -13,7 +13,7 @@ module GoogleSafeBrowsing
|
|
13
13
|
|
14
14
|
to_do_array[:lists].each do |list|
|
15
15
|
to_do_array[:data_urls][list].each do |url|
|
16
|
-
|
16
|
+
GoogleSafeBrowsing.logger.info "#{list} - #{url}\n"
|
17
17
|
ResponseHelper.receive_data('http://' + url, list)
|
18
18
|
end
|
19
19
|
end
|
@@ -29,26 +29,26 @@ module GoogleSafeBrowsing
|
|
29
29
|
return nil if urls.empty?
|
30
30
|
|
31
31
|
hashes = HashHelper.urls_to_hashes(urls)
|
32
|
-
raw_hash_array = hashes.
|
32
|
+
raw_hash_array = hashes.map { |h| h.to_s }
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
end
|
34
|
+
full = FullHash.where(full_hash: raw_hash_array).first
|
35
|
+
return GoogleSafeBrowsing.friendly_list_name(full.list) if full
|
37
36
|
|
38
|
-
hits = AddShavar.where(:
|
39
|
-
safes = SubShavar.where(:
|
37
|
+
hits = AddShavar.where(prefix: hashes.map { |h| h.prefix }).map { |s| [s.list, s.prefix] }
|
38
|
+
safes = SubShavar.where(prefix: hashes.map { |h| h.prefix }).map { |s| [s.list, s.prefix] }
|
40
39
|
|
41
40
|
reals = hits - safes
|
42
41
|
|
43
42
|
if reals.any?
|
44
|
-
full_hashes = HttpHelper.request_full_hashes(reals.
|
43
|
+
full_hashes = HttpHelper.request_full_hashes(reals.map { |r| r[1] })
|
45
44
|
|
46
45
|
# save hashes first
|
47
46
|
# cannot return early because all FullHashes need to be saved
|
48
47
|
hit_list = nil
|
49
48
|
full_hashes.each do |hash|
|
50
|
-
FullHash.create!(:
|
51
|
-
|
49
|
+
FullHash.create!(list: hash[:list],
|
50
|
+
add_chunk_number: hash[:add_chunk_num],
|
51
|
+
full_hash: hash[:full_hash])
|
52
52
|
|
53
53
|
hit_list = hash[:list] if raw_hash_array.include?(hash[:full_hash])
|
54
54
|
end
|
@@ -59,16 +59,19 @@ module GoogleSafeBrowsing
|
|
59
59
|
|
60
60
|
# Can be used to force a delay into a script running updates
|
61
61
|
#
|
62
|
-
# @param (Integer) delay_seconds the number of seconds to delay, should be
|
62
|
+
# @param (Integer) delay_seconds the number of seconds to delay, should be
|
63
|
+
# the return value of {update}
|
63
64
|
def self.delay(delay_seconds)
|
64
|
-
|
65
|
-
|
65
|
+
GoogleSafeBrowsing.logger.info \
|
66
|
+
"Google told us to wait for #{delay_seconds} seconds"
|
67
|
+
GoogleSafeBrowsing.logger.info 'We will wait....'
|
66
68
|
start_time = Time.now
|
67
|
-
|
68
|
-
|
69
|
+
until start_time + delay_seconds <= Time.now
|
70
|
+
GoogleSafeBrowsing.logger.info \
|
71
|
+
"#{(delay_seconds - (Time.now - start_time)).to_i}..."
|
69
72
|
sleep(10)
|
70
73
|
end
|
71
|
-
|
74
|
+
GoogleSafeBrowsing.logger.info 'Thank you for being patient'
|
72
75
|
end
|
73
76
|
end
|
74
77
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
module GoogleSafeBrowsing
|
2
2
|
# Helper methods for working with binary encoded data from Forwarding URLs
|
3
3
|
class BinaryHelper
|
4
|
-
|
5
4
|
# Reads `counter` byes from byte iterator `iter` and returns the hex string represnetation
|
6
5
|
#
|
7
6
|
# @param [ByteIterator] iter byte iterator already at correct position
|
@@ -19,7 +18,6 @@ module GoogleSafeBrowsing
|
|
19
18
|
string.unpack('H8')[0]
|
20
19
|
end
|
21
20
|
|
22
|
-
|
23
21
|
# Read `count` bytes from `iter` without unpacking the result
|
24
22
|
#
|
25
23
|
# @param (see read_bytes_as_hex)
|
@@ -30,9 +28,6 @@ module GoogleSafeBrowsing
|
|
30
28
|
ret = ''
|
31
29
|
count.to_i.times { ret << iter.next }
|
32
30
|
ret
|
33
|
-
#rescue
|
34
|
-
# puts "Tried to read past chunk iterator++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
|
35
|
-
# return nil
|
36
31
|
end
|
37
32
|
|
38
33
|
# Returns the first four bytes of `string` as hexidecimal; for host key
|
@@ -66,6 +61,5 @@ module GoogleSafeBrowsing
|
|
66
61
|
def self.hex_to_bin(hex)
|
67
62
|
[hex].pack('H*')
|
68
63
|
end
|
69
|
-
|
70
64
|
end
|
71
65
|
end
|
@@ -5,7 +5,6 @@ require File.dirname(__FILE__) + '/top_level_domain.rb'
|
|
5
5
|
module GoogleSafeBrowsing
|
6
6
|
# Helpers to Canonicalize urls and generate url permutations for lookups
|
7
7
|
class Canonicalize
|
8
|
-
|
9
8
|
PROTOCOL_DELIMITER = '://'
|
10
9
|
DEFAULT_PROTOCOL = 'http'
|
11
10
|
|
@@ -15,14 +14,15 @@ module GoogleSafeBrowsing
|
|
15
14
|
# @return (String) canonicalized url string
|
16
15
|
def self.url(raw_url)
|
17
16
|
raw_url = raw_url.to_s
|
18
|
-
|
19
|
-
# Change encoding from UTF-8 to ASCII-8BIT to avoid InvalidByteSequenceError
|
20
|
-
raw_url = raw_url.force_encoding("ASCII-8BIT")
|
21
17
|
|
22
|
-
#
|
23
|
-
|
24
|
-
raw_url.
|
25
|
-
|
18
|
+
# Change encoding from UTF-8 to ASCII-8BIT to avoid
|
19
|
+
# InvalidByteSequenceError
|
20
|
+
raw_url = raw_url.force_encoding('ASCII-8BIT')
|
21
|
+
|
22
|
+
# remove tabs, carriage returns and line feeds
|
23
|
+
raw_url.gsub!("\t", '')
|
24
|
+
raw_url.gsub!("\r", '')
|
25
|
+
raw_url.gsub!("\n", '')
|
26
26
|
|
27
27
|
cann = raw_url.clone
|
28
28
|
cann.gsub!(/\A\s+|\s+\Z/, '')
|
@@ -35,10 +35,10 @@ module GoogleSafeBrowsing
|
|
35
35
|
# remove leading PROTOCOL
|
36
36
|
cann = remove_protocol(cann)
|
37
37
|
|
38
|
-
#split into host and path components
|
38
|
+
# split into host and path components
|
39
39
|
splits = split_host_path(cann)
|
40
40
|
|
41
|
-
cann = fix_host(
|
41
|
+
cann = fix_host(splits[:host]) + '/' + fix_path(splits[:path])
|
42
42
|
|
43
43
|
# add leading protocol
|
44
44
|
@protocol ||= DEFAULT_PROTOCOL
|
@@ -53,7 +53,7 @@ module GoogleSafeBrowsing
|
|
53
53
|
# @return (Array) array of cannonicalized url permutation strings
|
54
54
|
def self.urls_for_lookup(lookup_url)
|
55
55
|
lookup_url = url(lookup_url)
|
56
|
-
#return empty array if url returns nil; for invalid url
|
56
|
+
# return empty array if url returns nil; for invalid url
|
57
57
|
return [] if lookup_url.blank?
|
58
58
|
|
59
59
|
lookup_url = remove_protocol(lookup_url)
|
@@ -62,12 +62,12 @@ module GoogleSafeBrowsing
|
|
62
62
|
|
63
63
|
host_string = strip_username_password_and_port_from_host(splits[:host])
|
64
64
|
|
65
|
-
#return empty array unless host_string has at least one period
|
65
|
+
# return empty array unless host_string has at least one period
|
66
66
|
return [] unless host_string.include?('.')
|
67
67
|
|
68
68
|
host_strings = [host_string]
|
69
69
|
host = TopLevelDomain.split_from_host(host_string).last(5)
|
70
|
-
(
|
70
|
+
(host.length - 1).times do
|
71
71
|
host_strings << host.join('.')
|
72
72
|
host.shift
|
73
73
|
end
|
@@ -78,52 +78,43 @@ module GoogleSafeBrowsing
|
|
78
78
|
cart_prod(host_strings, path_strings)
|
79
79
|
end
|
80
80
|
|
81
|
-
# private
|
82
|
-
|
83
81
|
# Generates the path permutations from the raw path string
|
84
82
|
#
|
85
83
|
# @param (String) raw_path path split from the full url string
|
86
84
|
# @return (Array) array of path permutation strings
|
87
85
|
def self.generate_path_strings(raw_path)
|
88
|
-
return [
|
86
|
+
return ['/', ''] if raw_path == ''
|
89
87
|
|
90
88
|
path_split = raw_path.split('?')
|
91
89
|
path = path_split[0] || ''
|
92
90
|
params = path_split[1] || ''
|
93
91
|
|
94
|
-
|
95
92
|
path_components = path.split('/').first(3)
|
96
|
-
path_strings = [
|
93
|
+
path_strings = ['/']
|
97
94
|
path_components.length.times do
|
98
95
|
path_strings << '/' + path_components.join('/')
|
99
96
|
path_components.pop
|
100
97
|
end
|
101
98
|
|
102
99
|
path_strings.map! do |p|
|
103
|
-
|
104
|
-
p + '/'
|
105
|
-
else
|
100
|
+
if p.index('.')
|
106
101
|
p
|
102
|
+
else
|
103
|
+
p + '/'
|
107
104
|
end
|
108
105
|
end
|
109
|
-
path_strings.map!{ |p| p.to_s.gsub!(/\/+/, '/') }
|
106
|
+
path_strings.map! { |p| p.to_s.gsub!(/\/+/, '/') }
|
110
107
|
path_strings.compact!
|
111
108
|
path_strings.uniq!
|
112
109
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
p
|
117
|
-
else
|
118
|
-
"#{p}?#{params}"
|
119
|
-
end
|
120
|
-
end
|
121
|
-
else
|
122
|
-
return path_strings
|
110
|
+
return path_strings if params.blank?
|
111
|
+
path_strings | path_strings.map do |p|
|
112
|
+
p[-1] == '/' ? p : "#{p}?#{params}"
|
123
113
|
end
|
124
114
|
end
|
125
115
|
|
126
|
-
# Returns the cartesian product of two arrays by concatination of the
|
116
|
+
# Returns the cartesian product of two arrays by concatination of the
|
117
|
+
# string representation of the elements
|
127
118
|
#
|
128
119
|
# @param (Array) a_one array of strings
|
129
120
|
# @param (Array) a_two array of strings
|
@@ -141,24 +132,25 @@ module GoogleSafeBrowsing
|
|
141
132
|
# Takes the canonicalized url and splits the host and the path apart
|
142
133
|
#
|
143
134
|
# @param (String) cann canonicalized url string
|
144
|
-
# @return (Hash) !{ :
|
135
|
+
# @return (Hash) !{ host: host_part, path: path_part }
|
145
136
|
def self.split_host_path(cann)
|
146
|
-
ret= { :
|
137
|
+
ret = { host: cann, path: '' }
|
147
138
|
split_point = cann.index('/')
|
148
139
|
if split_point
|
149
|
-
ret[:host] = cann[0..split_point-1]
|
150
|
-
ret[:path] = cann[split_point+1..-1]
|
140
|
+
ret[:host] = cann[0..split_point - 1]
|
141
|
+
ret[:path] = cann[(split_point + 1)..-1]
|
151
142
|
end
|
152
143
|
|
153
144
|
ret
|
154
145
|
end
|
155
146
|
|
156
|
-
# Strips the fragment portion of the url string (the last '#' and
|
147
|
+
# Strips the fragment portion of the url string (the last '#' and
|
148
|
+
# everything after)
|
157
149
|
#
|
158
150
|
# @param (String) string url
|
159
151
|
# @return (String) parameter with the fragment removed
|
160
152
|
def self.remove_fragment(string)
|
161
|
-
string = string[0..string.index('#')-1] if string.index('#')
|
153
|
+
string = string[0..(string.index('#') - 1)] if string.index('#')
|
162
154
|
string
|
163
155
|
end
|
164
156
|
|
@@ -167,9 +159,9 @@ module GoogleSafeBrowsing
|
|
167
159
|
# @param (String) url url string
|
168
160
|
# @return (String) fully unescaped url string
|
169
161
|
def self.recursively_unescape(url)
|
170
|
-
compare_url = url.clone
|
162
|
+
compare_url = url.clone
|
171
163
|
url = URI.unescape(url)
|
172
|
-
|
164
|
+
until compare_url == url
|
173
165
|
compare_url = url.clone
|
174
166
|
url = URI.unescape(url)
|
175
167
|
end
|
@@ -181,7 +173,6 @@ module GoogleSafeBrowsing
|
|
181
173
|
# @param (String) host host string
|
182
174
|
# @return (String) standardized host string
|
183
175
|
def self.fix_host(host)
|
184
|
-
#puts "In Host: #{host}"
|
185
176
|
# remove leading and trailing dots, multiple dots to one
|
186
177
|
host.gsub!(/\A\.+|\.+\Z/, '')
|
187
178
|
host.gsub!(/\.+/, '.')
|
@@ -194,7 +185,7 @@ module GoogleSafeBrowsing
|
|
194
185
|
host_splits[:host] = IP::V4.new(host.to_i).to_addr
|
195
186
|
elsif host_splits[:host] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
|
196
187
|
begin
|
197
|
-
host_splits[:host] = IP.new(host).to_addr
|
188
|
+
host_splits[:host] = IP.new(host).to_addr
|
198
189
|
rescue ArgumentError
|
199
190
|
end
|
200
191
|
end
|
@@ -210,17 +201,15 @@ module GoogleSafeBrowsing
|
|
210
201
|
# @param (String) path path string
|
211
202
|
# @return (String) standardized path string
|
212
203
|
def self.fix_path(path)
|
213
|
-
#
|
214
|
-
|
215
|
-
#remove leading slash
|
204
|
+
# remove leading slash
|
216
205
|
path = path[1..-1] if path[0..0] == '/'
|
217
206
|
|
218
|
-
preserve_trailing_slash = (
|
207
|
+
preserve_trailing_slash = (path[-1..-1] == '/')
|
219
208
|
|
220
209
|
if path.index('?')
|
221
210
|
first_ques = path.index('?')
|
222
211
|
params = path[first_ques..-1]
|
223
|
-
path = path[0..first_ques-1]
|
212
|
+
path = path[0..(first_ques - 1)]
|
224
213
|
end
|
225
214
|
|
226
215
|
# remove multiple '/'
|
@@ -247,7 +236,7 @@ module GoogleSafeBrowsing
|
|
247
236
|
url = URI.escape url
|
248
237
|
|
249
238
|
# unescape carat, may need other optionally escapeable chars
|
250
|
-
url.gsub!('%5E','^')
|
239
|
+
url.gsub!('%5E', '^')
|
251
240
|
|
252
241
|
url
|
253
242
|
end
|
@@ -259,7 +248,7 @@ module GoogleSafeBrowsing
|
|
259
248
|
def self.remove_protocol(cann)
|
260
249
|
if cann.index(PROTOCOL_DELIMITER)
|
261
250
|
delimiting_index = cann.index(PROTOCOL_DELIMITER)
|
262
|
-
@protocol = cann[0..delimiting_index-1]
|
251
|
+
@protocol = cann[0..(delimiting_index - 1)]
|
263
252
|
protocol_end_index = delimiting_index + PROTOCOL_DELIMITER.length
|
264
253
|
cann = cann[protocol_end_index..-1]
|
265
254
|
end
|
@@ -3,11 +3,11 @@ module GoogleSafeBrowsing
|
|
3
3
|
@queue = :google_safe_browsing
|
4
4
|
|
5
5
|
def self.perform
|
6
|
-
|
6
|
+
GoogleSafeBrowsing.logger.info "Running Update"
|
7
7
|
delay = APIv2.update
|
8
|
-
|
8
|
+
GoogleSafeBrowsing.logger.info "Scheduling new update in #{delay} seconds"
|
9
9
|
Resque.enqueue_in(delay.seconds, Rescheduler)
|
10
|
-
|
10
|
+
GoogleSafeBrowsing.logger.info "Update scheduled"
|
11
11
|
end
|
12
12
|
|
13
13
|
end
|
@@ -49,6 +49,9 @@ module GoogleSafeBrowsing
|
|
49
49
|
# we no longer have to report that we received these chunks
|
50
50
|
SubShavar.delete_chunks_from_list(current_list,
|
51
51
|
ChunkList.new(vals[1]))
|
52
|
+
else
|
53
|
+
GoogleSafeBrowsing.logger.warn \
|
54
|
+
"Unknown data response directive: #{vals[0]}"
|
52
55
|
end
|
53
56
|
end
|
54
57
|
|
@@ -75,9 +78,11 @@ module GoogleSafeBrowsing
|
|
75
78
|
chunk = f.read(line_actions[:chunk_length])
|
76
79
|
# f iterator is now set for next chunk
|
77
80
|
|
78
|
-
add_attrs = { :
|
79
|
-
|
80
|
-
|
81
|
+
add_attrs = { chunk_number: line_actions[:chunk_number],
|
82
|
+
list: list,
|
83
|
+
prefix: nil,
|
84
|
+
host_key: nil
|
85
|
+
}
|
81
86
|
|
82
87
|
case line_actions[:action]
|
83
88
|
when 'a'
|
@@ -102,11 +107,14 @@ module GoogleSafeBrowsing
|
|
102
107
|
counter += 1
|
103
108
|
end
|
104
109
|
rescue StopIteration
|
105
|
-
|
110
|
+
GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
|
111
|
+
Added #{counter} host_keys for add chunk number
|
112
|
+
#{line_actions[:chunk_number]}
|
113
|
+
LOG
|
106
114
|
end
|
107
115
|
end
|
108
116
|
when 's'
|
109
|
-
sub_attrs = add_attrs.merge({ :
|
117
|
+
sub_attrs = add_attrs.merge({ add_chunk_number: nil })
|
110
118
|
if line_actions[:chunk_length] == 0
|
111
119
|
record_sub_shavar_to_insert(sub_attrs)
|
112
120
|
else
|
@@ -130,11 +138,16 @@ module GoogleSafeBrowsing
|
|
130
138
|
counter += 1
|
131
139
|
end
|
132
140
|
rescue StopIteration
|
133
|
-
|
141
|
+
GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
|
142
|
+
Added #{counter} host_keys for sub chunk number
|
143
|
+
#{line_actions[:chunk_number]}
|
144
|
+
LOG
|
134
145
|
end
|
135
146
|
end
|
136
147
|
else
|
137
|
-
|
148
|
+
GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
|
149
|
+
neither a nor s =================================================
|
150
|
+
LOG
|
138
151
|
end
|
139
152
|
|
140
153
|
end
|
@@ -165,16 +178,16 @@ module GoogleSafeBrowsing
|
|
165
178
|
|
166
179
|
def self.record_add_shavar_to_insert(h)
|
167
180
|
@add_shavar_values ||= []
|
168
|
-
values = [
|
181
|
+
values = [h[:prefix], h[:host_key], h[:chunk_number], h[:list]]
|
169
182
|
@add_shavar_values << "(#{escape_and_join values})"
|
170
183
|
end
|
171
184
|
def self.record_sub_shavar_to_insert(h)
|
172
185
|
@sub_shavar_values ||= []
|
173
|
-
values = [
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
186
|
+
values = [h[:prefix],
|
187
|
+
h[:host_key],
|
188
|
+
h[:add_chunk_number],
|
189
|
+
h[:chunk_number],
|
190
|
+
h[:list]]
|
178
191
|
@sub_shavar_values << "(#{escape_and_join values})"
|
179
192
|
end
|
180
193
|
|
@@ -182,10 +195,10 @@ module GoogleSafeBrowsing
|
|
182
195
|
split_line = line.split(':')
|
183
196
|
|
184
197
|
ret = {}
|
185
|
-
ret[
|
186
|
-
ret[
|
187
|
-
ret[
|
188
|
-
ret[
|
198
|
+
ret[:action] = split_line[0]
|
199
|
+
ret[:chunk_number] = split_line[1].to_i
|
200
|
+
ret[:hash_length] = split_line[2].to_i
|
201
|
+
ret[:chunk_length] = split_line[3].to_i
|
189
202
|
ret
|
190
203
|
end
|
191
204
|
|
@@ -196,7 +209,7 @@ module GoogleSafeBrowsing
|
|
196
209
|
end
|
197
210
|
|
198
211
|
def self.pop_and_join(records)
|
199
|
-
records.pop(
|
212
|
+
records.pop(10_000).join(', ')
|
200
213
|
end
|
201
214
|
end
|
202
215
|
end
|
@@ -38,13 +38,12 @@ module GoogleSafeBrowsing
|
|
38
38
|
components
|
39
39
|
end
|
40
40
|
|
41
|
-
|
42
|
-
private
|
41
|
+
private
|
43
42
|
|
44
43
|
def self.parse_tld_to_hash
|
45
44
|
hash = Hash.new(nil)
|
46
|
-
|
47
|
-
|
45
|
+
file_name = File.dirname(__FILE__) + '/effective_tld_names.dat.txt'
|
46
|
+
File.readlines(file_name, 'r').each do |line|
|
48
47
|
hash[line.chomp] = true unless line[0..1] == '//'
|
49
48
|
end
|
50
49
|
hash
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google_safe_browsing
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Marshall
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-07-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '3.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '3.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: ruby-ip
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -157,7 +157,8 @@ files:
|
|
157
157
|
- lib/google_safe_browsing/version.rb
|
158
158
|
- lib/tasks/google_safe_browsing_tasks.rake
|
159
159
|
homepage: https://github.com/mobiledefense/google_safe_browsing
|
160
|
-
licenses:
|
160
|
+
licenses:
|
161
|
+
- Apache 2.0
|
161
162
|
metadata: {}
|
162
163
|
post_install_message:
|
163
164
|
rdoc_options: []
|
@@ -175,7 +176,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
175
176
|
version: '0'
|
176
177
|
requirements: []
|
177
178
|
rubyforge_project:
|
178
|
-
rubygems_version: 2.
|
179
|
+
rubygems_version: 2.4.5
|
179
180
|
signing_key:
|
180
181
|
specification_version: 4
|
181
182
|
summary: Rails 3 plugin for Google's Safe Browsing API v2
|