RubyGems - miteru - Versions diffs - 1.2.2 → 2.0.1 - Mend

miteru 1.2.2 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (55) hide show

checksums.yaml +4 -4
data/.github/workflows/gem.yml +36 -0
data/.github/workflows/{test.yml → ruby.yml} +4 -13
data/.gitignore +7 -2
data/.rspec +1 -1
data/README.md +7 -17
data/docker-compose.yml +12 -0
data/exe/miteru +3 -3
data/lefthook.yml +9 -0
data/lib/miteru/cli/application.rb +27 -0
data/lib/miteru/cli/base.rb +16 -0
data/lib/miteru/cli/database.rb +11 -0
data/lib/miteru/commands/database.rb +23 -0
data/lib/miteru/commands/main.rb +37 -0
data/lib/miteru/commands/sidekiq.rb +35 -0
data/lib/miteru/commands/web.rb +37 -0
data/lib/miteru/concerns/database_connectable.rb +16 -0
data/lib/miteru/concerns/error_unwrappable.rb +30 -0
data/lib/miteru/config.rb +98 -0
data/lib/miteru/crawler.rb +28 -44
data/lib/miteru/database.rb +50 -38
data/lib/miteru/downloader.rb +52 -41
data/lib/miteru/errors.rb +37 -0
data/lib/miteru/feeds/ayashige.rb +9 -20
data/lib/miteru/feeds/base.rb +141 -0
data/lib/miteru/feeds/phishing_database.rb +11 -10
data/lib/miteru/feeds/urlscan.rb +47 -19
data/lib/miteru/feeds/urlscan_pro.rb +20 -18
data/lib/miteru/http.rb +51 -0
data/lib/miteru/kit.rb +28 -20
data/lib/miteru/mixin.rb +2 -29
data/lib/miteru/notifiers/base.rb +10 -3
data/lib/miteru/notifiers/slack.rb +85 -10
data/lib/miteru/notifiers/urlscan.rb +29 -14
data/lib/miteru/orchestrator.rb +58 -0
data/lib/miteru/record.rb +8 -15
data/lib/miteru/service.rb +28 -0
data/lib/miteru/sidekiq/application.rb +13 -0
data/lib/miteru/sidekiq/jobs.rb +21 -0
data/lib/miteru/version.rb +1 -1
data/lib/miteru/web/application.rb +42 -0
data/lib/miteru/website.rb +48 -48
data/lib/miteru.rb +130 -22
data/miteru.gemspec +49 -38
metadata +262 -97
data/.overcommit.yml +0 -12
data/.standard.yml +0 -4
data/lib/miteru/attachement.rb +0 -74
data/lib/miteru/cli.rb +0 -41
data/lib/miteru/configuration.rb +0 -122
data/lib/miteru/error.rb +0 -7
data/lib/miteru/feeds/feed.rb +0 -53
data/lib/miteru/feeds/phishstats.rb +0 -28
data/lib/miteru/feeds.rb +0 -45
data/lib/miteru/http_client.rb +0 -85

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 2.0.1
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire:
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-09 00:00:00.000000000 Z
+date: 2024-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,42 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.40'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.40'
 - !ruby/object:Gem::Dependency
   name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.25'
+        version: '0.28'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.25'
+        version: '0.28'
 - !ruby/object:Gem::Dependency
-  name: glint
+  name: fuubar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '2.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
@@ -67,145 +81,159 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: overcommit
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.59'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.59'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '13.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '13.1'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '3.12'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: simplecov-lcov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.33'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '1.33'
 - !ruby/object:Gem::Dependency
-  name: vcr
+  name: test-prof
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.17'
+        version: '6.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.17'
+        version: '6.2'
 - !ruby/object:Gem::Dependency
-  name: webrick
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: '3.19'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: '3.19'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: 7.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: 7.1.3
+- !ruby/object:Gem::Dependency
+  name: anyway_config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.2
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -224,157 +252,283 @@ dependencies:
   name: down
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: 5.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '5.3'
+        version: 5.4.1
+- !ruby/object:Gem::Dependency
+  name: dry-files
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: dry-monads
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: 5.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: 5.1.1
 - !ruby/object:Gem::Dependency
-  name: memist
+  name: memo_wise
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
         version: '3.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
         version: '3.4'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: 1.24.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: 1.24.0
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 6.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 6.4.2
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: rack-session
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.11.0
+        version: 4.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.15.0
+- !ruby/object:Gem::Dependency
+  name: sentry-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.16.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.16.1
+- !ruby/object:Gem::Dependency
+  name: sentry-sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.16.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.16.1
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.11.0
+        version: 7.2.1
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: 2.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 1.7.1
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 1.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
-  name: urlscan
+  name: thor-hollaback
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2.2'
-description: An experimental phishing kit detector
+        version: 2.2.0
+description: A phishing kit collector for scavengers
 email:
 - manabu.niseki@gmail.com
 executables:
@@ -382,42 +536,53 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/test.yml"
+- ".github/workflows/gem.yml"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
-- ".overcommit.yml"
 - ".rspec"
-- ".standard.yml"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yml
 - docker/Dockerfile
 - exe/miteru
+- lefthook.yml
 - lib/miteru.rb
-- lib/miteru/attachement.rb
-- lib/miteru/cli.rb
-- lib/miteru/configuration.rb
+- lib/miteru/cli/application.rb
+- lib/miteru/cli/base.rb
+- lib/miteru/cli/database.rb
+- lib/miteru/commands/database.rb
+- lib/miteru/commands/main.rb
+- lib/miteru/commands/sidekiq.rb
+- lib/miteru/commands/web.rb
+- lib/miteru/concerns/database_connectable.rb
+- lib/miteru/concerns/error_unwrappable.rb
+- lib/miteru/config.rb
 - lib/miteru/crawler.rb
 - lib/miteru/database.rb
 - lib/miteru/downloader.rb
-- lib/miteru/error.rb
-- lib/miteru/feeds.rb
+- lib/miteru/errors.rb
 - lib/miteru/feeds/ayashige.rb
-- lib/miteru/feeds/feed.rb
+- lib/miteru/feeds/base.rb
 - lib/miteru/feeds/phishing_database.rb
-- lib/miteru/feeds/phishstats.rb
 - lib/miteru/feeds/urlscan.rb
 - lib/miteru/feeds/urlscan_pro.rb
-- lib/miteru/http_client.rb
+- lib/miteru/http.rb
 - lib/miteru/kit.rb
 - lib/miteru/mixin.rb
 - lib/miteru/notifiers/base.rb
 - lib/miteru/notifiers/slack.rb
 - lib/miteru/notifiers/urlscan.rb
+- lib/miteru/orchestrator.rb
 - lib/miteru/record.rb
+- lib/miteru/service.rb
+- lib/miteru/sidekiq/application.rb
+- lib/miteru/sidekiq/jobs.rb
 - lib/miteru/version.rb
+- lib/miteru/web/application.rb
 - lib/miteru/website.rb
 - miteru.gemspec
 - renovate.json
@@ -427,7 +592,7 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message:
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -442,8 +607,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.14
-signing_key:
+rubygems_version: 3.5.3
+signing_key:
 specification_version: 4
-summary: An experimental phishing kit detector
+summary: A phishing kit collector for scavengers
 test_files: []

data/.overcommit.yml DELETED Viewed

@@ -1,12 +0,0 @@
-PreCommit:
-  BundleCheck:
-    enabled: true
-  RuboCop:
-    enabled: true
-    required_executable: bundle
-    command: ["bundle", "exec", "standardrb"]
-    on_warn: fail
-  YamlSyntax:
-    enabled: true

data/.standard.yml DELETED Viewed

@@ -1,4 +0,0 @@
-ignore:
-  - "**/*":
-      - Layout/SpaceInsideHashLiteralBraces
-      - Style/RescueStandardError