miteru 0.14.7 → 1.1.0

data/lib/miteru/record.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Miteru
+  class Record < ActiveRecord::Base
+    class << self
+      #
+      # Check uniqueness of a record by a hash
+      #
+      # @param [String] hash
+      #
+      # @return [Boolean] true if it is unique. Otherwise false.
+      #
+      def unique_hash?(hash)
+        record = find_by(hash: hash)
+        return true if record.nil?
+
+        false
+      end
+
+      #
+      # Create a new record based on a kit
+      #
+      # @param [Miteru::Kit] kit
+      # @param [String] hash
+      #
+      # @return [Miteru::Record]
+      #
+      def create_by_kit_and_hash(kit, hash)
+        record = new(
+          hash: hash,
+          source: kit.source,
+          hostname: kit.hostname,
+          url: kit.decoded_url,
+          headers: kit.headers,
+          filename: kit.filename,
+          filesize: kit.filesize,
+          mime_type: kit.mime_type,
+          downloaded_as: kit.filepath_to_download
+        )
+        record.save
+        record
+      rescue TypeError, ActiveRecord::RecordNotUnique => _e
+        nil
+      end
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.14.7"
+  VERSION = "1.1.0"
 end

data/lib/miteru/website.rb

@@ -6,10 +6,15 @@ module Miteru
   class Website
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
 
+    # @return [String]
     attr_reader :url
 
-    def initialize(url)
+    # @return [String]
+    attr_reader :source
+
+    def initialize(url, source)
       @url = url
+      @source = source
     end
 
     def title
@@ -17,10 +22,10 @@ module Miteru
     end
 
     def kits
-      @kits ||= links.map do |link|
-        kit = Kit.new(link)
+      @kits ||= links.filter_map do |link|
+        kit = Kit.new(link, source)
         kit.valid? ? kit : nil
-      end.compact
+      end
     end
 
     def ok?
@@ -42,11 +47,11 @@ module Miteru
     end
 
     def message
-      return "It doesn't contain a phishing kit." unless kits?
+      return "it doesn't contain a phishing kit." unless kits?
 
       filename_with_sizes = kits.map(&:filename_with_size).join(", ")
       noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
-      "It might contain #{noun}: #{filename_with_sizes}."
+      "it might contain #{noun}: #{filename_with_sizes}."
     end
 
     def links
@@ -75,7 +80,7 @@ module Miteru
 
     def href_links
       if doc && ok? && index?
-        doc.css("a").map { |a| a.get("href") }.compact.map do |href|
+        doc.css("a").filter_map { |a| a.get("href") }.map do |href|
           href = href.start_with?("/") ? href : "/#{href}"
           url + href
         end

data/lib/miteru.rb

@@ -3,6 +3,12 @@
 require "miteru/version"
 
 require "miteru/configuration"
+require "miteru/database"
+
+require "miteru/record"
+
+require "miteru/mixin"
+
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/kit"
@@ -14,6 +20,4 @@ require "miteru/notifier"
 require "miteru/crawler"
 require "miteru/cli"
 
-module Miteru
-  # Your code goes here...
-end
+module Miteru; end

data/miteru.gemspec

@@ -1,43 +1,51 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('lib', __dir__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "miteru/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "miteru"
-  spec.version       = Miteru::VERSION
-  spec.authors       = ["Manabu Niseki"]
-  spec.email         = ["manabu.niseki@gmail.com"]
+  spec.name = "miteru"
+  spec.version = Miteru::VERSION
+  spec.authors = ["Manabu Niseki"]
+  spec.email = ["manabu.niseki@gmail.com"]
 
-  spec.summary       = "An experimental phishing kit detector"
-  spec.description   = "An experimental phishing kit detector"
-  spec.homepage      = "https://github.com/ninoseki/miteru"
-  spec.license       = "MIT"
+  spec.summary = "An experimental phishing kit detector"
+  spec.description = "An experimental phishing kit detector"
+  spec.homepage = "https://github.com/ninoseki/miteru"
+  spec.license = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "bundler", "~> 2.2"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.22"
   spec.add_development_dependency "glint", "~> 0.1"
+  spec.add_development_dependency "mysql2", "~> 0.5"
+  spec.add_development_dependency "overcommit", "~> 0.58"
+  spec.add_development_dependency "pg", "~> 1.2"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "standard", "~> 1.3"
   spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.9"
+  spec.add_development_dependency "webmock", "~> 3.14"
+  spec.add_development_dependency "webrick", "~> 1.7.0"
 
+  spec.add_dependency "activerecord", "~> 6.1"
   spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "down", "~> 5.2"
-  spec.add_dependency "http", "~> 4.4"
+  spec.add_dependency "http", "~> 5.0"
   spec.add_dependency "oga", "~> 3.3"
-  spec.add_dependency "parallel", "~> 1.19"
-  spec.add_dependency "slack-notifier", "~> 2.3"
-  spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "urlscan", "~> 0.6"
+  spec.add_dependency "parallel", "~> 1.20"
+  spec.add_dependency "slack-notifier", "~> 2.4"
+  spec.add_dependency "sqlite3", "~> 1.4"
+  spec.add_dependency "thor", "~> 1.1"
+  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "uuidtools", "~> 2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.14.7
+  version: 1.1.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-18 00:00:00.000000000 Z
+date: 2021-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.22'
 - !ruby/object:Gem::Dependency
   name: glint
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: overcommit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.58'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.58'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +156,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.14'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +226,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
@@ -170,56 +254,84 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: urlscan
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.2'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com
@@ -228,9 +340,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
+- ".overcommit.yml"
 - ".rspec"
-- ".travis.yml"
+- ".standard.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -244,6 +358,7 @@ files:
 - lib/miteru/cli.rb
 - lib/miteru/configuration.rb
 - lib/miteru/crawler.rb
+- lib/miteru/database.rb
 - lib/miteru/downloader.rb
 - lib/miteru/error.rb
 - lib/miteru/feeds.rb
@@ -255,7 +370,9 @@ files:
 - lib/miteru/feeds/urlscan_pro.rb
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
+- lib/miteru/mixin.rb
 - lib/miteru/notifier.rb
+- lib/miteru/record.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb
 - miteru.gemspec
@@ -280,7 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: An experimental phishing kit detector