miteru 0.12.13 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: feb9c62c511e26e85a86755680ed77ddb34aca15c6d17d8c57dd2c60aa84becf
-  data.tar.gz: d9e86a727e12e171b65accb829b07602847986ba81ea6457400ad8527b862324
+  metadata.gz: 16f41a7076d910face109e154bbc1a90b3d5a5e1e964be7236b8c798939dee7b
+  data.tar.gz: d16c59e0e78897c96d78a2e51f84903449c53355eb2436b4e700ba7eb4a07887
 SHA512:
-  metadata.gz: 8402163c1decbcee1c352a2ca9004f52b8c0262c5adc18d10866abe190151c8bf56d1e7ed03018f33539d1f108a1ccfc265d289dcaeb167606a10217ec112d6a
-  data.tar.gz: 6ecde10d8a4d1842995a2281e1b30663cb25c3fdd9edefa5c8f883c94f2da3b973372a84adfa70bd8537c126ab12b870b2bf6046f3021e329129428974fc96c0
+  metadata.gz: e83f7f493b2f1015ccfed05b8b1acde61fc093a3e75f29dcb6a1be8f20203f10ccbddbfd4f770277821821bf0ec31eab4e1bcf5587d77c435510a3b583302fc9
+  data.tar.gz: c9b68154de357f2563fc80ca0059e476e1c919cffa6b67ff4d1a2658d317b6ce7277c168717a26be2b020716da94c0b751dc1e9a6ebac085dc3e2729847c7b0e

data/.travis.yml

@@ -3,6 +3,4 @@ language: ruby
 cache: bundler
 rvm:
   - 2.6
-before_install:
-  - gem update --system
-  - gem install bundler
+before_install: gem install bundler -v 2.1

data/README.md

@@ -2,7 +2,7 @@
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
 [![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
-![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)
+[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
@@ -15,6 +15,7 @@ Miteru is an experimental phishing kit detection tool.
   - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
   - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
   - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
+  - urlscan.io phish feed (available for Pro users)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
   - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
@@ -85,6 +86,10 @@ For using `--post-to-slack` feature, you should set the following environment va
 - `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
 - `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
 
+If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
+
+It enables you to subscribe the urlscan.io phish feed.
+
 ## Examples
 
 ### Aasciinema cast

data/lib/miteru/feeds.rb

@@ -3,6 +3,7 @@
 require_relative "./feeds/feed"
 require_relative "./feeds/ayashige"
 require_relative "./feeds/urlscan"
+require_relative "./feeds/urlscan_pro"
 
 module Miteru
   class Feeds
@@ -11,6 +12,7 @@ module Miteru
     def initialize
       @feeds = [
         UrlScan.new(Miteru.configuration.size),
+        UrlScanPro.new,
         Miteru.configuration.ayashige? ? Ayashige.new : nil
       ].compact
     end

data/lib/miteru/feeds/urlscan.rb

@@ -1,39 +1,35 @@
 # frozen_string_literal: true
 
-require "json"
-require "uri"
+require "urlscan"
 
 module Miteru
   class Feeds
     class UrlScan < Feed
-      HOST = "urlscan.io"
-      VERSION = 1
-      URL = "https://#{HOST}/api/v#{VERSION}"
-
       attr_reader :size
+
       def initialize(size = 100)
         @size = size
         raise ArgumentError, "size must be less than 10,000" if size > 10_000
       end
 
+      def api
+        @api ||= ::UrlScan::API.new
+      end
+
       def urls
-        url = url_for("/search/")
-        url.query = URI.encode_www_form(
-          q: "task.method:automatic",
-          size: size
-        )
-
-        res = JSON.parse(get(url))
-        res["results"].map { |result| result.dig("task", "url") }
-      rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
+        urls_from_community_feed
+      rescue ::UrlScan::ResponseError => e
         puts "Failed to load urlscan.io feed (#{e})"
         []
       end
 
       private
 
-      def url_for(path)
-        URI(URL + path)
+      def urls_from_community_feed
+        res = api.search("task.method:automatic", size: size)
+
+        results = res["results"] || []
+        results.map { |result| result.dig("task", "url") }
       end
     end
   end

data/lib/miteru/feeds/urlscan_pro.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "urlscan"
+
+module Miteru
+  class Feeds
+    class UrlScanPro < Feed
+      def api
+        @api ||= ::UrlScan::API.new
+      end
+
+      def urls
+        urls_from_pro_feed
+      rescue ::UrlScan::ResponseError => e
+        puts "Failed to load urlscan.io pro feed (#{e})"
+        []
+      end
+
+      private
+
+      def api_key?
+        ENV.key? "URLSCAN_API_KEY"
+      end
+
+      def urls_from_pro_feed
+        return [] unless api_key?
+
+        res = api.pro.phishfeed
+        results = res["results"] || []
+        results.map { |result| result.dig("page_url") }
+      rescue ArgumentError => _e
+        []
+      end
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.13"
+  VERSION = "0.13.0"
 end

data/miteru.gemspec

@@ -35,8 +35,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "down", "~> 5.0"
   spec.add_dependency "http", "~> 4.2"
-  spec.add_dependency "oga", "~> 2.15"
+  spec.add_dependency "oga", "~> 3.0"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "slack-notifier", "~> 2.3"
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "urlscan", "~> 0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.13
+  version: 0.13.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-27 00:00:00.000000000 Z
+date: 2019-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -156,14 +156,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +198,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: urlscan
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com
@@ -236,6 +250,7 @@ files:
 - lib/miteru/feeds/ayashige.rb
 - lib/miteru/feeds/feed.rb
 - lib/miteru/feeds/urlscan.rb
+- lib/miteru/feeds/urlscan_pro.rb
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
 - lib/miteru/notifier.rb