miteru 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5c223747e06cb6d010f4a5e3157b1a4999662761ccaef108521b1627489b800
-  data.tar.gz: 403ed81289db89582add2420290b6f8c343da88babf2c548940871f0791655dd
+  metadata.gz: 7428f741cd791e37e5f201c978f4ee6b5f6fe239d350b69d3377a6af4604305d
+  data.tar.gz: 0057c9ee7ebe6dd5ee039939b30ff897d3298605c3843c980f2b30081ad8bc2d
 SHA512:
-  metadata.gz: 255a86779cdb3473ba4b7683159b1290b0774eae7863db2cd9e770610b6a14f1397ff1a07823132009daafc5b0c75aea2489cd39c051067311242cf30535517d
-  data.tar.gz: 3662c8946aba747ad6dcda2fbedc1dc34b0cba8fd680e9d573b3a1c24114b5658f5029ecb3bf5e222c2d4cced01dd0f6f1542d4878566fab0880456bb500997e
+  metadata.gz: b4f69cf18c8db256f035466f669835810e53329f7af7df5fd3d58689002c2e7175abb9456b31e21c0b23369e31a1559040c82f92c662202661c50cc8cc702334
+  data.tar.gz: d7188fe6bc2372560031fb964254f5ea6df56672edd0f112a6ebe7b41d5ad7efa2a52e6c1deb666891573990f0cf75e8b4ea768f5240cd5d25127a544e06e26c

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Style/StringLiterals:
+  EnforcedStyle: double_quotes

data/README.md

@@ -1,7 +1,7 @@
 # Miteru
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
-[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/test.yml)
+[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/ruby.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/ruby.yml)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
@@ -14,12 +14,12 @@ I take no responsibility and/or liability for how you choose to use this tool.
 
 ## How It Works
 
-- It collects phishy URLs from the following feeds:
-  - urlscan.io's automatic submissions. (`task.method:automatic`)
+- Collect phishy URLs from the following feeds:
+  - urlscan.io's automatic submissions. (`task.method:automatic AND NOT task.source:urlscan-observe`)
   - urlscan.io phish feed. (available for Pro users)
   - [mitchellkrogza/Phishing.Database](https://github.com/mitchellkrogza/Phishing.Database)'s `phishing-links-ACTIVE-NOW.txt`.
   - [ninoseki/ayashige](https://github.com/ninoseki/ayashige) feed.
-- It checks each phishy URL whether it enables directory listing and contains phishing kits (compressed files) or not.
+- Check each phishy URL whether it enables directory listing and contains phishing kits (compressed files) or not.
   - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
 ## Docs

data/lib/miteru/config.rb

@@ -27,7 +27,8 @@ module Miteru
       threads: Parallel.processor_count,
       urlscan_api_key: nil,
       urlscan_submit_visibility: "public",
-      urlscan_date_condition: ">now-1h",
+      urlscan_date_condition: "date:>now-1h",
+      urlscan_base_condition: "task.method:automatic AND NOT task.source:urlscan-observe",
       verbose: false
     )
 
@@ -91,6 +92,9 @@ module Miteru
     # @!attribute [r] urlscan_date_condition
     #   @return [String]
 
+    # @!attribute [r] urlscan_base_condition
+    #   @return [String]
+
     def database_url=(val)
       super(URI(val.to_s))
     end

data/lib/miteru/feeds/ayashige.rb

@@ -3,9 +3,6 @@
 module Miteru
   class Feeds
     class Ayashige < Base
-      HOST = "ayashige.herokuapp.com"
-      URL = "https://#{HOST}".freeze
-
       def initialize(base_url = "https://ayashige.herokuapp.com")
         super(base_url)
       end

data/lib/miteru/feeds/urlscan.rb

@@ -31,7 +31,7 @@ module Miteru
       end
 
       def q
-        "task.method:automatic AND date:#{Miteru.config.urlscan_date_condition}"
+        "#{base_condition} AND #{date_condition}"
       end
 
       #
@@ -59,6 +59,14 @@ module Miteru
           end
         end
       end
+
+      def base_condition
+        Miteru.config.urlscan_base_condition
+      end
+
+      def date_condition
+        Miteru.config.urlscan_date_condition
+      end
     end
   end
 end

data/lib/miteru/feeds/urlscan_pro.rb

@@ -23,7 +23,7 @@ module Miteru
       end
 
       def q
-        "date:#{Miteru.config.urlscan_date_condition}"
+        Miteru.config.urlscan_date_condition
       end
 
       def format

data/lib/miteru/notifiers/slack.rb

@@ -87,7 +87,7 @@ module Miteru
       end
 
       def callable?
-        !Miteru.config.slack_webhook_url.nil?
+        !slack_webhook_url.nil?
       end
 
       private

data/lib/miteru/orchestrator.rb

@@ -40,7 +40,14 @@ module Miteru
             Miteru.logger.warn("Feed:#{feed.source} failed - #{result.failure}")
           end
         end
-      end.flatten
+      end.flatten.uniq(&:url)
+    end
+
+    #
+    # @return [Array<Miteru::Feeds::Base>]
+    #
+    def feeds
+      Miteru.feeds.map(&:new)
     end
 
     private
@@ -52,12 +59,5 @@ module Miteru
     def verbose?
       Miteru.config.verbose
     end
-
-    #
-    # @return [Array<Miteru::Feeds::Base>]
-    #
-    def feeds
-      Miteru.feeds.map(&:new)
-    end
   end
 end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.0.2"
+  VERSION = "2.1.0"
 end

data/miteru.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'mysql2', '~> 0.5'
   spec.add_development_dependency 'pg', '~> 1.5'
   spec.add_development_dependency 'rake', '~> 13.1'
-  spec.add_development_dependency 'rspec', '~> 3.12'
+  spec.add_development_dependency 'rspec', '~> 3.13'
   spec.add_development_dependency 'simplecov-lcov', '~> 0.8'
   spec.add_development_dependency 'standard', '~> 1.33'
   spec.add_development_dependency 'test-prof', '~> 1.3'
@@ -47,12 +47,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'down', '5.4.1'
   spec.add_dependency 'dry-files', '1.1.0'
   spec.add_dependency 'dry-monads', '1.6.0'
-  spec.add_dependency 'http', '5.1.1'
+  spec.add_dependency 'http', '5.2.0'
   spec.add_dependency 'memo_wise', '1.8.0'
   spec.add_dependency 'oga', '3.4'
   spec.add_dependency 'parallel', '1.24.0'
   spec.add_dependency 'puma', '6.4.2'
-  spec.add_dependency 'rack', '3.0.8'
+  spec.add_dependency 'rack', '3.0.9'
   spec.add_dependency 'rack-session', '2.0.0'
   spec.add_dependency 'rackup', '2.1.0'
   spec.add_dependency 'semantic_logger', '4.15.0'
@@ -60,7 +60,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sentry-sidekiq', '5.16.1'
   spec.add_dependency 'sidekiq', '7.2.1'
   spec.add_dependency 'slack-notifier', '2.4.0'
-  spec.add_dependency 'sqlite3', '1.7.1'
+  spec.add_dependency 'sqlite3', '1.7.2'
   spec.add_dependency 'thor', '1.3.0'
   spec.add_dependency 'thor-hollaback', '0.2.1'
   spec.add_dependency 'uuidtools', '2.2.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-29 00:00:00.000000000 Z
+date: 2024-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '3.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '3.13'
 - !ruby/object:Gem::Dependency
   name: simplecov-lcov
   requirement: !ruby/object:Gem::Requirement
@@ -310,14 +310,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.1.1
+        version: 5.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.1.1
+        version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: memo_wise
   requirement: !ruby/object:Gem::Requirement
@@ -380,14 +380,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.8
+        version: 3.0.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.8
+        version: 3.0.9
 - !ruby/object:Gem::Dependency
   name: rack-session
   requirement: !ruby/object:Gem::Requirement
@@ -492,14 +492,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: 1.7.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: 1.7.2
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -554,6 +554,7 @@ files:
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -600,7 +601,6 @@ files:
 - lib/miteru/website.rb
 - miteru.gemspec
 - renovate.json
-- screenshots/slack.png
 homepage: https://github.com/ninoseki/miteru
 licenses:
 - MIT