miteru 2.0.3 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4133dc72d0d07c5719999c526b8534a2dd7b5db5fe74172042b747c5cc557d4a
-  data.tar.gz: d2f59be0a18bdaee1b74633a58b8ed59a396862defd4b80dff4ae8ae9d8199f4
+  metadata.gz: 509957b241020f76fcd9c85a6acbb9e80c0f8b3224fe5c1b4af41dcd2faa6ca5
+  data.tar.gz: 4e507a9974264a9b98d1ded16c09496a06ed4ca83409f75fc49719c37f934f77
 SHA512:
-  metadata.gz: 4a311f7ffa407ed11951f8f3b7ce696510ba31f8888dff459d3a04e4a19ec892f6f7e5651b7616e70a4482cf07d92f657dc4765ef17b6f42e66a3b526c7e1e26
-  data.tar.gz: 70b43504f29dc3646a343a87029e5c7a785d777099352717a4d2c2f26801879d2541d90ec5568f04d56c6635a35067397c4042e64a3177b6375cb74e51b5eae3
+  metadata.gz: 3dc8e2b4569555f5551a1b7fd2407162949cc7a459a5a7f83c5a787206b5cf39b34f16235466b6bcd74f34bac1878e1e4df6797daebe407eda2c03a9e7379e3a
+  data.tar.gz: 69d83a4c964ebbcd87b416675ed03bc18e270c0d604a19ef98b8988a0f344388eb49d0fb290edaa2f8637fa5f100608a3e9a76a1571503bda529611aea332bc0

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Style/StringLiterals:
+  EnforcedStyle: double_quotes

data/README.md

@@ -1,7 +1,7 @@
 # Miteru
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
-[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/test.yml)
+[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/ruby.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/ruby.yml)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
@@ -14,12 +14,12 @@ I take no responsibility and/or liability for how you choose to use this tool.
 
 ## How It Works
 
-- It collects phishy URLs from the following feeds:
-  - urlscan.io's automatic submissions. (`task.method:automatic`)
+- Collect phishy URLs from the following feeds:
+  - urlscan.io's automatic submissions. (`task.method:automatic AND NOT task.source:urlscan-observe`)
   - urlscan.io phish feed. (available for Pro users)
   - [mitchellkrogza/Phishing.Database](https://github.com/mitchellkrogza/Phishing.Database)'s `phishing-links-ACTIVE-NOW.txt`.
   - [ninoseki/ayashige](https://github.com/ninoseki/ayashige) feed.
-- It checks each phishy URL whether it enables directory listing and contains phishing kits (compressed files) or not.
+- Check each phishy URL whether it enables directory listing and contains phishing kits (compressed files) or not.
   - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
 ## Docs

data/lib/miteru/config.rb

@@ -27,7 +27,8 @@ module Miteru
       threads: Parallel.processor_count,
       urlscan_api_key: nil,
       urlscan_submit_visibility: "public",
-      urlscan_date_condition: ">now-1h",
+      urlscan_date_condition: "date:>now-1h",
+      urlscan_base_condition: "task.method:automatic AND NOT task.source:urlscan-observe",
       verbose: false
     )
 
@@ -91,6 +92,9 @@ module Miteru
     # @!attribute [r] urlscan_date_condition
     #   @return [String]
 
+    # @!attribute [r] urlscan_base_condition
+    #   @return [String]
+
     def database_url=(val)
       super(URI(val.to_s))
     end

data/lib/miteru/crawler.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "colorize"
+
 module Miteru
   class Crawler < Service
     #
@@ -7,7 +9,10 @@ module Miteru
     #
     def call(website)
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
-        Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
+        info = "Website:#{website.info}."
+        info = info.colorize(:red) if website.kits?
+
+        Miteru.logger.info(info)
         return unless website.kits?
 
         notify website

data/lib/miteru/feeds/urlscan.rb

@@ -31,7 +31,7 @@ module Miteru
       end
 
       def q
-        "task.method:automatic AND date:#{Miteru.config.urlscan_date_condition}"
+        "#{base_condition} AND #{date_condition}"
       end
 
       #
@@ -59,6 +59,14 @@ module Miteru
           end
         end
       end
+
+      def base_condition
+        Miteru.config.urlscan_base_condition
+      end
+
+      def date_condition
+        Miteru.config.urlscan_date_condition
+      end
     end
   end
 end

data/lib/miteru/feeds/urlscan_pro.rb

@@ -23,7 +23,7 @@ module Miteru
       end
 
       def q
-        "date:#{Miteru.config.urlscan_date_condition}"
+        Miteru.config.urlscan_date_condition
       end
 
       def format

data/lib/miteru/notifiers/slack.rb

@@ -5,6 +5,7 @@ require "slack-notifier"
 module Miteru
   module Notifiers
     class SlackAttachment
+      # @return [String]
       attr_reader :url
 
       def initialize(url)
@@ -14,7 +15,7 @@ module Miteru
       def to_a
         [
           {
-            text: defanged_url,
+            text:,
             fallback: "VT & urlscan.io links",
             actions:
           }
@@ -47,10 +48,6 @@ module Miteru
         }
       end
 
-      def defanged_url
-        @defanged_url ||= url.to_s.gsub(".", "[.]")
-      end
-
       def domain
         @domain ||= [].tap do |out|
           out << URI(url).hostname
@@ -59,6 +56,10 @@ module Miteru
         end.first
       end
 
+      def text
+        domain.to_s.gsub(".", "[.]")
+      end
+
       def _urlscan_link
         return nil unless domain
 
@@ -82,12 +83,11 @@ module Miteru
         return unless callable?
 
         attachment = SlackAttachment.new(website.url)
-        kits = website.kits.select(&:downloaded?)
-        notifier.post(text: website.message.capitalize, attachments: attachment.to_a) if kits.any?
+        notifier.post(text: website.info, attachments: attachment.to_a) if website.kits?
       end
 
       def callable?
-        !slack_webhook_url.nil?
+        !webhook_url.nil?
       end
 
       private

data/lib/miteru/notifiers/urlscan.rb

@@ -9,7 +9,7 @@ module Miteru
       def call(website)
         return unless callable?
 
-        website.kits.each { |kit| submit(kit.url) }
+        website.kits.each { |kit| submit(kit.url, source: website.source) }
       end
 
       def callable?
@@ -41,8 +41,12 @@ module Miteru
         Miteru.config.urlscan_submit_visibility
       end
 
-      def submit(url)
-        http.post("https://urlscan.io/api/v1/scan/", json: {tags:, visibility:, url:})
+      #
+      # @param [String] url
+      # @param [String] source
+      #
+      def submit(url, source:)
+        http.post("https://urlscan.io/api/v1/scan/", json: {tags: tags + ["source:#{source}"], visibility:, url:})
       end
     end
   end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.0.3"
+  VERSION = "2.1.1"
 end

data/lib/miteru/website.rb

@@ -48,6 +48,14 @@ module Miteru
       url.truncate(64)
     end
 
+    def defanged_truncated_url
+      truncated_url.to_s.gsub(".", "[.]")
+    end
+
+    def info
+      "#{defanged_truncated_url} has #{kits.length} kit(s) (Source: #{source})"
+    end
+
     private
 
     def timeout

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.1.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -554,6 +554,7 @@ files:
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - Gemfile
 - LICENSE
 - README.md