miteru 0.14.7 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Miteru
6
+ class Record < ActiveRecord::Base
7
+ class << self
8
+ #
9
+ # Check uniqueness of a record by a hash
10
+ #
11
+ # @param [String] hash
12
+ #
13
+ # @return [Boolean] true if it is unique. Otherwise false.
14
+ #
15
+ def unique_hash?(hash)
16
+ record = find_by(hash: hash)
17
+ return true if record.nil?
18
+
19
+ false
20
+ end
21
+
22
+ #
23
+ # Create a new record based on a kit
24
+ #
25
+ # @param [Miteru::Kit] kit
26
+ # @param [String] hash
27
+ #
28
+ # @return [Miteru::Record]
29
+ #
30
+ def create_by_kit_and_hash(kit, hash)
31
+ record = new(
32
+ hash: hash,
33
+ source: kit.source,
34
+ hostname: kit.hostname,
35
+ url: kit.decoded_url,
36
+ headers: kit.headers,
37
+ filename: kit.filename,
38
+ filesize: kit.filesize,
39
+ mime_type: kit.mime_type,
40
+ downloaded_as: kit.filepath_to_download
41
+ )
42
+ record.save
43
+ record
44
+ rescue TypeError, ActiveRecord::RecordNotUnique => _e
45
+ nil
46
+ end
47
+ end
48
+ end
49
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.7"
4
+ VERSION = "1.1.0"
5
5
  end
@@ -6,10 +6,15 @@ module Miteru
6
6
  class Website
7
7
  VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
8
 
9
+ # @return [String]
9
10
  attr_reader :url
10
11
 
11
- def initialize(url)
12
+ # @return [String]
13
+ attr_reader :source
14
+
15
+ def initialize(url, source)
12
16
  @url = url
17
+ @source = source
13
18
  end
14
19
 
15
20
  def title
@@ -17,10 +22,10 @@ module Miteru
17
22
  end
18
23
 
19
24
  def kits
20
- @kits ||= links.map do |link|
21
- kit = Kit.new(link)
25
+ @kits ||= links.filter_map do |link|
26
+ kit = Kit.new(link, source)
22
27
  kit.valid? ? kit : nil
23
- end.compact
28
+ end
24
29
  end
25
30
 
26
31
  def ok?
@@ -42,11 +47,11 @@ module Miteru
42
47
  end
43
48
 
44
49
  def message
45
- return "It doesn't contain a phishing kit." unless kits?
50
+ return "it doesn't contain a phishing kit." unless kits?
46
51
 
47
52
  filename_with_sizes = kits.map(&:filename_with_size).join(", ")
48
53
  noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
49
- "It might contain #{noun}: #{filename_with_sizes}."
54
+ "it might contain #{noun}: #{filename_with_sizes}."
50
55
  end
51
56
 
52
57
  def links
@@ -75,7 +80,7 @@ module Miteru
75
80
 
76
81
  def href_links
77
82
  if doc && ok? && index?
78
- doc.css("a").map { |a| a.get("href") }.compact.map do |href|
83
+ doc.css("a").filter_map { |a| a.get("href") }.map do |href|
79
84
  href = href.start_with?("/") ? href : "/#{href}"
80
85
  url + href
81
86
  end
data/lib/miteru.rb CHANGED
@@ -3,6 +3,12 @@
3
3
  require "miteru/version"
4
4
 
5
5
  require "miteru/configuration"
6
+ require "miteru/database"
7
+
8
+ require "miteru/record"
9
+
10
+ require "miteru/mixin"
11
+
6
12
  require "miteru/error"
7
13
  require "miteru/http_client"
8
14
  require "miteru/kit"
@@ -14,6 +20,4 @@ require "miteru/notifier"
14
20
  require "miteru/crawler"
15
21
  require "miteru/cli"
16
22
 
17
- module Miteru
18
- # Your code goes here...
19
- end
23
+ module Miteru; end
data/miteru.gemspec CHANGED
@@ -1,43 +1,51 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- lib = File.expand_path('lib', __dir__)
3
+ lib = File.expand_path("lib", __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require "miteru/version"
6
6
 
7
7
  Gem::Specification.new do |spec|
8
- spec.name = "miteru"
9
- spec.version = Miteru::VERSION
10
- spec.authors = ["Manabu Niseki"]
11
- spec.email = ["manabu.niseki@gmail.com"]
8
+ spec.name = "miteru"
9
+ spec.version = Miteru::VERSION
10
+ spec.authors = ["Manabu Niseki"]
11
+ spec.email = ["manabu.niseki@gmail.com"]
12
12
 
13
- spec.summary = "An experimental phishing kit detector"
14
- spec.description = "An experimental phishing kit detector"
15
- spec.homepage = "https://github.com/ninoseki/miteru"
16
- spec.license = "MIT"
13
+ spec.summary = "An experimental phishing kit detector"
14
+ spec.description = "An experimental phishing kit detector"
15
+ spec.homepage = "https://github.com/ninoseki/miteru"
16
+ spec.license = "MIT"
17
17
 
18
18
  # Specify which files should be added to the gem when it is released.
19
19
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
20
20
  spec.files = Dir.chdir(File.expand_path(__dir__)) do
21
21
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
22
22
  end
23
- spec.bindir = "exe"
24
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
23
+ spec.bindir = "exe"
24
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 2.1"
28
- spec.add_development_dependency "coveralls", "~> 0.8"
27
+ spec.add_development_dependency "bundler", "~> 2.2"
28
+ spec.add_development_dependency "coveralls_reborn", "~> 0.22"
29
29
  spec.add_development_dependency "glint", "~> 0.1"
30
+ spec.add_development_dependency "mysql2", "~> 0.5"
31
+ spec.add_development_dependency "overcommit", "~> 0.58"
32
+ spec.add_development_dependency "pg", "~> 1.2"
30
33
  spec.add_development_dependency "rake", "~> 13.0"
31
- spec.add_development_dependency "rspec", "~> 3.9"
34
+ spec.add_development_dependency "rspec", "~> 3.10"
35
+ spec.add_development_dependency "standard", "~> 1.3"
32
36
  spec.add_development_dependency "vcr", "~> 6.0"
33
- spec.add_development_dependency "webmock", "~> 3.9"
37
+ spec.add_development_dependency "webmock", "~> 3.14"
38
+ spec.add_development_dependency "webrick", "~> 1.7.0"
34
39
 
40
+ spec.add_dependency "activerecord", "~> 6.1"
35
41
  spec.add_dependency "colorize", "~> 0.8"
36
42
  spec.add_dependency "down", "~> 5.2"
37
- spec.add_dependency "http", "~> 4.4"
43
+ spec.add_dependency "http", "~> 5.0"
38
44
  spec.add_dependency "oga", "~> 3.3"
39
- spec.add_dependency "parallel", "~> 1.19"
40
- spec.add_dependency "slack-notifier", "~> 2.3"
41
- spec.add_dependency "thor", "~> 1.0"
42
- spec.add_dependency "urlscan", "~> 0.6"
45
+ spec.add_dependency "parallel", "~> 1.20"
46
+ spec.add_dependency "slack-notifier", "~> 2.4"
47
+ spec.add_dependency "sqlite3", "~> 1.4"
48
+ spec.add_dependency "thor", "~> 1.1"
49
+ spec.add_dependency "urlscan", "~> 0.7"
50
+ spec.add_dependency "uuidtools", "~> 2.2"
43
51
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.7
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-10-18 00:00:00.000000000 Z
11
+ date: 2021-09-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,28 +16,28 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '2.1'
19
+ version: '2.2'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.1'
26
+ version: '2.2'
27
27
  - !ruby/object:Gem::Dependency
28
- name: coveralls
28
+ name: coveralls_reborn
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0.8'
33
+ version: '0.22'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0.8'
40
+ version: '0.22'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: glint
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,48 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0.1'
55
+ - !ruby/object:Gem::Dependency
56
+ name: mysql2
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.5'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.5'
69
+ - !ruby/object:Gem::Dependency
70
+ name: overcommit
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.58'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.58'
83
+ - !ruby/object:Gem::Dependency
84
+ name: pg
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.2'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.2'
55
97
  - !ruby/object:Gem::Dependency
56
98
  name: rake
57
99
  requirement: !ruby/object:Gem::Requirement
@@ -72,14 +114,28 @@ dependencies:
72
114
  requirements:
73
115
  - - "~>"
74
116
  - !ruby/object:Gem::Version
75
- version: '3.9'
117
+ version: '3.10'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '3.10'
125
+ - !ruby/object:Gem::Dependency
126
+ name: standard
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.3'
76
132
  type: :development
77
133
  prerelease: false
78
134
  version_requirements: !ruby/object:Gem::Requirement
79
135
  requirements:
80
136
  - - "~>"
81
137
  - !ruby/object:Gem::Version
82
- version: '3.9'
138
+ version: '1.3'
83
139
  - !ruby/object:Gem::Dependency
84
140
  name: vcr
85
141
  requirement: !ruby/object:Gem::Requirement
@@ -100,14 +156,42 @@ dependencies:
100
156
  requirements:
101
157
  - - "~>"
102
158
  - !ruby/object:Gem::Version
103
- version: '3.9'
159
+ version: '3.14'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '3.14'
167
+ - !ruby/object:Gem::Dependency
168
+ name: webrick
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: 1.7.0
104
174
  type: :development
105
175
  prerelease: false
106
176
  version_requirements: !ruby/object:Gem::Requirement
107
177
  requirements:
108
178
  - - "~>"
109
179
  - !ruby/object:Gem::Version
110
- version: '3.9'
180
+ version: 1.7.0
181
+ - !ruby/object:Gem::Dependency
182
+ name: activerecord
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '6.1'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '6.1'
111
195
  - !ruby/object:Gem::Dependency
112
196
  name: colorize
113
197
  requirement: !ruby/object:Gem::Requirement
@@ -142,14 +226,14 @@ dependencies:
142
226
  requirements:
143
227
  - - "~>"
144
228
  - !ruby/object:Gem::Version
145
- version: '4.4'
229
+ version: '5.0'
146
230
  type: :runtime
147
231
  prerelease: false
148
232
  version_requirements: !ruby/object:Gem::Requirement
149
233
  requirements:
150
234
  - - "~>"
151
235
  - !ruby/object:Gem::Version
152
- version: '4.4'
236
+ version: '5.0'
153
237
  - !ruby/object:Gem::Dependency
154
238
  name: oga
155
239
  requirement: !ruby/object:Gem::Requirement
@@ -170,56 +254,84 @@ dependencies:
170
254
  requirements:
171
255
  - - "~>"
172
256
  - !ruby/object:Gem::Version
173
- version: '1.19'
257
+ version: '1.20'
174
258
  type: :runtime
175
259
  prerelease: false
176
260
  version_requirements: !ruby/object:Gem::Requirement
177
261
  requirements:
178
262
  - - "~>"
179
263
  - !ruby/object:Gem::Version
180
- version: '1.19'
264
+ version: '1.20'
181
265
  - !ruby/object:Gem::Dependency
182
266
  name: slack-notifier
183
267
  requirement: !ruby/object:Gem::Requirement
184
268
  requirements:
185
269
  - - "~>"
186
270
  - !ruby/object:Gem::Version
187
- version: '2.3'
271
+ version: '2.4'
188
272
  type: :runtime
189
273
  prerelease: false
190
274
  version_requirements: !ruby/object:Gem::Requirement
191
275
  requirements:
192
276
  - - "~>"
193
277
  - !ruby/object:Gem::Version
194
- version: '2.3'
278
+ version: '2.4'
279
+ - !ruby/object:Gem::Dependency
280
+ name: sqlite3
281
+ requirement: !ruby/object:Gem::Requirement
282
+ requirements:
283
+ - - "~>"
284
+ - !ruby/object:Gem::Version
285
+ version: '1.4'
286
+ type: :runtime
287
+ prerelease: false
288
+ version_requirements: !ruby/object:Gem::Requirement
289
+ requirements:
290
+ - - "~>"
291
+ - !ruby/object:Gem::Version
292
+ version: '1.4'
195
293
  - !ruby/object:Gem::Dependency
196
294
  name: thor
197
295
  requirement: !ruby/object:Gem::Requirement
198
296
  requirements:
199
297
  - - "~>"
200
298
  - !ruby/object:Gem::Version
201
- version: '1.0'
299
+ version: '1.1'
202
300
  type: :runtime
203
301
  prerelease: false
204
302
  version_requirements: !ruby/object:Gem::Requirement
205
303
  requirements:
206
304
  - - "~>"
207
305
  - !ruby/object:Gem::Version
208
- version: '1.0'
306
+ version: '1.1'
209
307
  - !ruby/object:Gem::Dependency
210
308
  name: urlscan
211
309
  requirement: !ruby/object:Gem::Requirement
212
310
  requirements:
213
311
  - - "~>"
214
312
  - !ruby/object:Gem::Version
215
- version: '0.6'
313
+ version: '0.7'
314
+ type: :runtime
315
+ prerelease: false
316
+ version_requirements: !ruby/object:Gem::Requirement
317
+ requirements:
318
+ - - "~>"
319
+ - !ruby/object:Gem::Version
320
+ version: '0.7'
321
+ - !ruby/object:Gem::Dependency
322
+ name: uuidtools
323
+ requirement: !ruby/object:Gem::Requirement
324
+ requirements:
325
+ - - "~>"
326
+ - !ruby/object:Gem::Version
327
+ version: '2.2'
216
328
  type: :runtime
217
329
  prerelease: false
218
330
  version_requirements: !ruby/object:Gem::Requirement
219
331
  requirements:
220
332
  - - "~>"
221
333
  - !ruby/object:Gem::Version
222
- version: '0.6'
334
+ version: '2.2'
223
335
  description: An experimental phishing kit detector
224
336
  email:
225
337
  - manabu.niseki@gmail.com
@@ -228,9 +340,11 @@ executables:
228
340
  extensions: []
229
341
  extra_rdoc_files: []
230
342
  files:
343
+ - ".github/workflows/test.yml"
231
344
  - ".gitignore"
345
+ - ".overcommit.yml"
232
346
  - ".rspec"
233
- - ".travis.yml"
347
+ - ".standard.yml"
234
348
  - Gemfile
235
349
  - LICENSE
236
350
  - README.md
@@ -244,6 +358,7 @@ files:
244
358
  - lib/miteru/cli.rb
245
359
  - lib/miteru/configuration.rb
246
360
  - lib/miteru/crawler.rb
361
+ - lib/miteru/database.rb
247
362
  - lib/miteru/downloader.rb
248
363
  - lib/miteru/error.rb
249
364
  - lib/miteru/feeds.rb
@@ -255,7 +370,9 @@ files:
255
370
  - lib/miteru/feeds/urlscan_pro.rb
256
371
  - lib/miteru/http_client.rb
257
372
  - lib/miteru/kit.rb
373
+ - lib/miteru/mixin.rb
258
374
  - lib/miteru/notifier.rb
375
+ - lib/miteru/record.rb
259
376
  - lib/miteru/version.rb
260
377
  - lib/miteru/website.rb
261
378
  - miteru.gemspec
@@ -280,7 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
280
397
  - !ruby/object:Gem::Version
281
398
  version: '0'
282
399
  requirements: []
283
- rubygems_version: 3.1.2
400
+ rubygems_version: 3.2.22
284
401
  signing_key:
285
402
  specification_version: 4
286
403
  summary: An experimental phishing kit detector