mihari 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3cf190c27b1be3c0eeb690354361c3bb3295beb8609661a861fcc31e227cb7f0
-  data.tar.gz: e989209e3668e9342803ed1b91c9e668bcf0519e659f9fb88687cc38460818cb
+  metadata.gz: dff04f8065ceff1c6b9fa68bd606db9cc4789ed034a330019143a69c4ec2c037
+  data.tar.gz: d307e3e48eedf5d17245da14a20bd578468e03a70b2ba302fcc4ea68aab5651c
 SHA512:
-  metadata.gz: 0a40de96264eaf211240b0020aa3494e6a64bddf8e2aa74461e91122fff29d72a5ab66032e226fa80c19c984fcb7a3e37d1e78aa0da317f6fd1445945b93d200
-  data.tar.gz: b0b55e75e84037cf7761fd28128504d45fc049de01d0f4db65184b2e8ff0e9bc3948e06b8165e90c19244fb1c8b8af33efa77d260bb323d4084d85e21f72245f
+  metadata.gz: 97f8237d491778739e307e15d0b5b96fba510a548a0a42ae1760e41469cef7fad551b6e876025d7daed58e449c77bee6d3573d659c85e4b2886712229e2ee7ef
+  data.tar.gz: 2d5f9125f2039ce7d8727661a8f71f1a7c0b9021a0eaf613242598371fb6066234fa5a8959528190bc7275b35da026932700d13929cfd9f94aa04eded6881951

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: Bug report
+about: Create a bug report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+<!--
+Thank you for taking the time to report a bug.
+Please make sure there is no existing issue about this kind of bug.
+-->
+
+### **Describe the bug**
+
+A clear and concise description of what the bug is.
+
+### **Steps to reproduce**
+
+- ...
+
+### **Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+### **Actual behavior**
+
+A clear and concise description of what actually happened.
+
+### **Screenshots**
+
+Add screenshots to help explain your problem.
+
+### **System Information:**
+
+- OS: [e.g. Windows10]
+- Ruby version: [e.g. 3.0]
+- Mihari version: [e.g. 2.0.0]
+
+### **Additional context**
+
+Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,15 @@
+---
+name: Feature request
+about: Suggest a new Feature for Mihari
+title: "[Feature Request]"
+labels: enhancement
+assignees: ''
+
+---
+<!--
+
+1. Make sure your requested feature makes sense for Mihari.
+
+2. If you want to suggest a new integration of a service, please provide detailed information of it. (e.g. API docs)
+
+-->

data/.standard.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "**/*":
+      - Layout/SpaceInsideHashLiteralBraces
+      - Style/RescueStandardError

data/README.md

@@ -50,9 +50,11 @@ See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
 
 - [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
 - [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
+- [Built-in Web App](https://github.com/ninoseki/mihari/wiki/Built-in-Web-App)
 - [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
-- [Custom script](https://github.com/ninoseki/mihari/wiki/Custom-script)
+- [Custom Script](https://github.com/ninoseki/mihari/wiki/Custom-Script)
 - [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
+- [GitHub Actions](https://github.com/ninoseki/mihari/wiki/GitHub-Actions)
 
 ## License
 

data/docker/Dockerfile

@@ -1,6 +1,7 @@
 FROM ruby:3.0.0-alpine3.13
 
-RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
+RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
+  && gem install pg mysql2 \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
   && cd mihari \

data/lib/mihari/analyzers/base.rb

@@ -42,7 +42,7 @@ module Mihari
 
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
-      rescue => e
+      rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
 

data/lib/mihari/analyzers/securitytrails.rb

@@ -58,13 +58,13 @@ module Mihari
       end
 
       def ip_lookup
-        result = api.domains.search(filter: {ipv4: query})
+        result = api.domains.search(filter: { ipv4: query })
         records = result["records"] || []
         records.map { |record| record["hostname"] }.compact.uniq
       end
 
       def mail_lookup
-        result = api.domains.search(filter: {whois_email: query})
+        result = api.domains.search(filter: { whois_email: query })
         records = result["records"] || []
         records.map { |record| record["hostname"] }.compact.uniq
       end

data/lib/mihari/cli.rb

@@ -1,293 +1,73 @@
 # frozen_string_literal: true
 
-require "json"
-require "rack/builder"
-require "rack/handler/webrick"
 require "thor"
 
+require "mihari/commands/binaryedge"
+require "mihari/commands/censys"
+require "mihari/commands/circl"
+require "mihari/commands/crtsh"
+require "mihari/commands/dnpedia"
+require "mihari/commands/dnstwister"
+require "mihari/commands/onyphe"
+require "mihari/commands/otx"
+require "mihari/commands/passivetotal"
+require "mihari/commands/pulsedive"
+require "mihari/commands/securitytrails_domain_feed"
+require "mihari/commands/securitytrails"
+require "mihari/commands/shodan"
+require "mihari/commands/spyse"
+require "mihari/commands/urlscan"
+require "mihari/commands/virustotal"
+require "mihari/commands/zoomeye"
+
+require "mihari/commands/free_text"
+require "mihari/commands/http_hash"
+require "mihari/commands/passive_dns"
+require "mihari/commands/passive_ssl"
+require "mihari/commands/reverse_whois"
+require "mihari/commands/ssh_fingerprint"
+
+require "mihari/commands/config"
+require "mihari/commands/json"
+require "mihari/commands/web"
+
 module Mihari
   class CLI < Thor
-    class_option :config, type: :string, desc: "path to config file"
-
-    def self.exit_on_failure?
-      true
-    end
-
-    desc "censys [QUERY]", "Censys IPv4 search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search (ipv4 / websites / certificates)", default: "ipv4"
-    def censys(query)
-      with_error_handling do
-        run_analyzer Analyzers::Censys, query: query, options: options
+    class_option :config, type: :string, desc: "path to the config file"
+
+    include Mihari::Commands::BinaryEdge
+    include Mihari::Commands::Censys
+    include Mihari::Commands::CIRCL
+    include Mihari::Commands::Config
+    include Mihari::Commands::Crtsh
+    include Mihari::Commands::DNPedia
+    include Mihari::Commands::DNSTwister
+    include Mihari::Commands::FreeText
+    include Mihari::Commands::HTTPHash
+    include Mihari::Commands::JSON
+    include Mihari::Commands::Onyphe
+    include Mihari::Commands::OTX
+    include Mihari::Commands::PassiveDNS
+    include Mihari::Commands::PassiveSSL
+    include Mihari::Commands::PassiveTotal
+    include Mihari::Commands::Pulsedive
+    include Mihari::Commands::ReverseWhois
+    include Mihari::Commands::SecurityTrails
+    include Mihari::Commands::SecurityTrailsDomainFeed
+    include Mihari::Commands::Shodan
+    include Mihari::Commands::Spyse
+    include Mihari::Commands::SSHFingerprint
+    include Mihari::Commands::Urlscan
+    include Mihari::Commands::VirusTotal
+    include Mihari::Commands::Web
+    include Mihari::Commands::ZoomEye
+
+    class << self
+      def exit_on_failure?
+        true
       end
     end
 
-    desc "shodan [QUERY]", "Shodan host search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def shodan(query)
-      with_error_handling do
-        run_analyzer Analyzers::Shodan, query: query, options: options
-      end
-    end
-
-    desc "onyphe [QUERY]", "Onyphe datascan search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def onyphe(query)
-      with_error_handling do
-        run_analyzer Analyzers::Onyphe, query: query, options: options
-      end
-    end
-
-    desc "urlscan [QUERY]", "urlscan search by a given query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :filter, type: :string, desc: "filter for urlscan pro search"
-    method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
-    method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
-    method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
-    def urlscan(query)
-      with_error_handling do
-        run_analyzer Analyzers::Urlscan, query: query, options: options
-      end
-    end
-
-    desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def virustotal(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
-      end
-    end
-
-    desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails lookup by an ip, domain or email"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def securitytrails(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
-      end
-    end
-    map "st" => :securitytrails
-
-    desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
-    def securitytrails_domain_feed(regexp)
-      with_error_handling do
-        run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
-      end
-    end
-    map "st_domain_feed" => :securitytrails_domain_feed
-
-    desc "crtsh [QUERY]", "crt.sh search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :exclude_expired, type: :boolean, desc: "exclude expired certificates"
-    def crtsh(query)
-      with_error_handling do
-        run_analyzer Analyzers::Crtsh, query: query, options: options
-      end
-    end
-
-    desc "dnpedia [QUERY]", "DNPedia domain search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def dnpedia(query)
-      with_error_handling do
-        run_analyzer Analyzers::DNPedia, query: query, options: options
-      end
-    end
-
-    desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def circl(query)
-      with_error_handling do
-        run_analyzer Analyzers::CIRCL, query: refang(query), options: options
-      end
-    end
-
-    desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passivetotal(indicator)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
-      end
-    end
-
-    desc "zoomeye [QUERY]", "ZoomEye search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
-    def zoomeye(query)
-      with_error_handling do
-        run_analyzer Analyzers::ZoomEye, query: query, options: options
-      end
-    end
-
-    desc "binaryedge [QUERY]", "BinaryEdge host search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def binaryedge(query)
-      with_error_handling do
-        run_analyzer Analyzers::BinaryEdge, query: query, options: options
-      end
-    end
-
-    desc "pulsedive [IP|DOMAIN]", "Pulsedive lookup by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def pulsedive(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::Pulsedive, query: refang(indiactor), options: options
-      end
-    end
-
-    desc "dnstwister [DOMAIN]", "dnstwister lookup by a domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def dnstwister(domain)
-      with_error_handling do
-        run_analyzer Analyzers::DNSTwister, query: refang(domain), options: options
-      end
-    end
-
-    desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def otx(domain)
-      with_error_handling do
-        run_analyzer Analyzers::OTX, query: refang(domain), options: options
-      end
-    end
-
-    desc "spyse [QUERY]", "Spyse search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
-    def spyse(query)
-      with_error_handling do
-        run_analyzer Analyzers::Spyse, query: query, options: options
-      end
-    end
-
-    desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passive_dns(query)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveDNS, query: refang(query), options: options
-      end
-    end
-
-    desc "passive_ssl [SHA1]", "Cross search with passive SSL services by an SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passive_ssl(query)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveSSL, query: query, options: options
-      end
-    end
-
-    desc "reverse_whois [EMAIL]", "Cross search with reverse whois services by an email"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def reverse_whois(query)
-      with_error_handling do
-        run_analyzer Analyzers::ReveseWhois, query: refang(query), options: options
-      end
-    end
-
-    desc "http_hash", "Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :md5, type: :string, desc: "MD5 hash"
-    method_option :sha256, type: :string, desc: "SHA256 hash"
-    method_option :mmh3, type: :numeric, desc: "MurmurHash3 hash"
-    method_option :html, type: :string, desc: "path to an HTML file"
-    def http_hash
-      with_error_handling do
-        run_analyzer Analyzers::HTTPHash, query: nil, options: options
-      end
-    end
-
-    desc "free_text [TEXT]", "Cross search with search engines by a free text"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def free_text(text)
-      with_error_handling do
-        run_analyzer Analyzers::FreeText, query: text, options: options
-      end
-    end
-
-    desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def ssh_fingerprint(fingerprint)
-      with_error_handling do
-        run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
-      end
-    end
-
-    desc "import_from_json", "Give a JSON input via STDIN"
-    def import_from_json(input = nil)
-      with_error_handling do
-        json = input || $stdin.gets.chomp
-        raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
-
-        json = parse_as_json(json)
-        raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
-
-        title = json["title"]
-        description = json["description"]
-        artifacts = json["artifacts"]
-        tags = json["tags"] || []
-
-        basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
-        basic.run
-      end
-    end
-
-    desc "web", "Launch the web app"
-    method_option :port, type: :numeric, default: 9292
-    method_option :host, type: :string, default: "localhost"
-    def web
-      port = options["port"].to_i || 9292
-      host = options["host"] || "localhost"
-
-      load_configuration
-      Mihari::App.run!(port: port, host: host)
-    end
-
     no_commands do
       def with_error_handling
         yield
@@ -296,12 +76,6 @@ module Mihari
         notifier.notify e
       end
 
-      def parse_as_json(input)
-        JSON.parse input
-      rescue JSON::ParserError => _e
-        nil
-      end
-
       # @return [true, false]
       def valid_json?(json)
         %w[title description artifacts].all? { |key| json.key? key }