mihari 0.17.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 004a0d8f2ddeda5059f6748657b786b9470a290eb4d593ff0bb0632ebb495d3d
-  data.tar.gz: c4f51acdb2dc76e52a445e382ce3c15b215b29f9abd860ca1c05fe814b24bf96
+  metadata.gz: '099c3dece3c31a745979b619c361d3bcff09e67abb3214c32747a935d57be963'
+  data.tar.gz: 1f777f2ca61721716e32e85846817a5183d11063573813adec757919bb27f457
 SHA512:
-  metadata.gz: f3d1b8959e726240a7257f999347ff2fc81b9bf359948dfc9f09a9edad66d76225f8a4e16b105d1bda2ee5675100e13705fdb448dc6986493234280f849c4637
-  data.tar.gz: 1b6dd5276812e5ba6f6c7997cb921acd261dbcced5af3599253b3995ced1450a8a998e8bdd88155ebe44ce3e724fb71da6e59fb72c2c6ed12222f1e1efd07dcc
+  metadata.gz: b8e0babf78935a90d1d4d39493ed7db77d498697a1ea8d2bbdf3277745544a0f7a59de6dc444e93065f66c339b9c5208df408748194cb490ffe3901681a9aa0b
+  data.tar.gz: 0b430d270f34f1fae64f384b672a4270500407e725f0f8f99bba278b134d74196fda894b4ed425ed21d363583da93e40648dce77b9ff39a9b0da07c7f5a7a5d2

data/.gitignore

@@ -54,3 +54,6 @@ Gemfile.lock
 
 # solargraph
 .solargraph.yml
+
+# SQLite
+*.db

data/.rubocop.yml

@@ -0,0 +1,155 @@
+# Relaxed.Ruby.Style
+## Version 2.5
+
+require:
+  - rubocop-performance
+
+Style/Alias:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylealias
+
+Style/AsciiComments:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleasciicomments
+
+Style/BeginBlock:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylebeginblock
+
+Style/BlockDelimiters:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleblockdelimiters
+
+Style/CommentAnnotation:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylecommentannotation
+
+Style/Documentation:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styledocumentation
+
+Layout/DotPosition:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#layoutdotposition
+
+Style/DoubleNegation:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styledoublenegation
+
+Style/EndBlock:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleendblock
+
+Style/FormatString:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleformatstring
+
+Style/IfUnlessModifier:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleifunlessmodifier
+
+Style/Lambda:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylelambda
+
+Style/ModuleFunction:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylemodulefunction
+
+Style/MultilineBlockChain:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylemultilineblockchain
+
+Style/NegatedIf:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylenegatedif
+
+Style/NegatedWhile:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylenegatedwhile
+
+Style/NumericPredicate:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylenumericpredicate
+
+Style/ParallelAssignment:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleparallelassignment
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylepercentliteraldelimiters
+
+Style/PerlBackrefs:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styleperlbackrefs
+
+Style/Semicolon:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylesemicolon
+
+Style/SignalException:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylesignalexception
+
+Style/SingleLineBlockParams:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylesinglelineblockparams
+
+Style/SingleLineMethods:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylesinglelinemethods
+
+Layout/SpaceBeforeBlockBraces:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#layoutspacebeforeblockbraces
+
+Layout/SpaceInsideParens:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#layoutspaceinsideparens
+
+Style/SpecialGlobalVars:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylespecialglobalvars
+
+Style/StringLiterals:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylestringliterals
+
+Style/TrailingCommaInArguments:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styletrailingcommainarguments
+
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styletrailingcommainarrayliteral
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#styletrailingcommainhashliteral
+
+Style/SymbolArray:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesymbolarray
+
+Style/WhileUntilModifier:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylewhileuntilmodifier
+
+Style/WordArray:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#stylewordarray
+
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#lintambiguousregexpliteral
+
+Lint/AssignmentInCondition:
+  Enabled: false
+  StyleGuide: https://relaxed.ruby.style/#lintassignmentincondition
+
+Layout/LineLength:
+  Enabled: false
+
+Metrics:
+  Enabled: false

data/.travis.yml

@@ -4,4 +4,5 @@ language: ruby
 cache: bundler
 rvm:
   - 2.6
+  - 2.7
 before_install: gem install bundler -v 2.1

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 # Specify your gem's dependencies in mihari.gemspec

data/README.md

@@ -10,19 +10,15 @@ Mihari is a helper to run queries & manage results continuously. Mihari can be u
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts from the results.
-- Mihari checks whether [TheHive](https://thehive-project.org/) contains the artifacts or not.
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes) from the results.
+- Mihari checks whether a DB (SQLite3) contains the artifacts or not.
   - If it doesn't contain the artifacts:
-    - Mihari creates an alert on TheHive.
+    - Mihari creates an alert on TheHive. (Optional)
     - Mihari sends a notification to Slack. (Optional)
     - Mihari creates an event on MISP. (Optional)
 
 ![img](https://github.com/ninoseki/mihari/raw/master/screenshots/eyecatch.png)
 
-Check this blog post for more details: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
-
-You can use mihari without TheHive but note that mihari depends on TheHive to manage artifacts. It means mihari might make duplications when without TheHive.
-
 ### Screenshots
 
 - TheHive alert example
@@ -156,49 +152,13 @@ mihari http_hash --html /tmp/index.html
 
 ```bash
 # Censys lookup for PANDA C2
-$ mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
-{
-  "title": "PANDA C2",
-  "description": "query = (\"PANDA\" AND \"SMAdmin\" AND \"layui\")",
-  "artifacts": [
-    "154.223.165.223",
-    "154.194.2.31",
-    "45.114.127.119",
-    "..."
-  ],
-  "tags": []
-}
+mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
 
 # VirusTotal passive DNS lookup of a FAKESPY host
-$ mihari virustotal "jppost-hi.top" --title "FAKESPY host passive DNS results"
-{
-  "title": "FAKESPY host passive DNS results",
-  "description": "indicator = jppost-hi.top",
-  "artifacts": [
-    "185.22.152.28",
-    "192.236.200.44",
-    "193.148.69.12",
-    "..."
-  ],
-  "tags": []
-}
+mihari virustotal "jppost-hi.top" --title "FAKESPY passive DNS"
 
 # You can pass a "defanged" indicator as an input
-$ mihari virustotal "jppost-hi[.]top" --title "FAKESPY host passive DNS results"
-
-# SecurityTrails domain feed lookup for finding (possibly) Apple phishing websites
-$ mihari securitytrails_domain_feed "apple-" --type new
-{
-  "title": "SecurityTrails domain feed lookup",
-  "description": "Regexp = /apple-/",
-  "artifacts": [
-    "apple-sign.online",
-    "apple-log-in.com",
-    "apple-locator-id.info",
-    "..."
-  ],
-  "tags": []
-}
+mihari virustotal "jppost-hi[.]top" --title "FAKESPY passive DNS"
 ```
 
 ### Import from JSON
@@ -229,28 +189,29 @@ The input is a JSON data should have `title`, `description` and `artifacts` key.
 
 Configuration can be done via environment variables or a YAML file.
 
-| Key                    | Desc.                          | Recommended or optional        |
-|------------------------|--------------------------------|--------------------------------|
-| THEHIVE_API_ENDPOINT   | TheHive URL                    | Recommended                    |
-| THEHIVE_API_KEY        | TheHive API key                | Recommended                    |
-| MISP_API_ENDPOINT      | MISP URL                       | Optional                       |
-| MISP_API_KEY           | MISP API key                   | Optional                       |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL              | Optional                       |
-| SLACK_CHANNEL          | Slack channel name             | Optional (default: `#general`) |
-| BINARYEDGE_API_KEY     | BinaryEdge API key             | Optional                       |
-| CENSYS_ID              | Censys API ID                  | Optional                       |
-| CENSYS_SECRET          | Censys secret                  | Optional                       |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password | Optional                       |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username | Optional                       |
-| ONYPHE_API_KEY         | Onyphe API key                 | Optional                       |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key           | Optional                       |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username          | Optional                       |
-| PULSEDIVE_API_KEY      | Pulsedive API key              | Optional                       |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key         | Optional                       |
-| SHODAN_API_KEY         | Shodan API key                 | Optional                       |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key             | Optional                       |
-| ZOOMEYE_USERNAMME      | ZoomEye username               | Optional                       |
-| ZOOMEYE_PASSWORD       | ZoomEye password               | Optional                       |
+| Key                    | Description                    | Default     |
+|------------------------|--------------------------------|-------------|
+| DATABASE               | A path to the SQLite database  | `mihari.db` |
+| BINARYEDGE_API_KEY     | BinaryEdge API key             |             |
+| CENSYS_ID              | Censys API ID                  |             |
+| CENSYS_SECRET          | Censys secret                  |             |
+| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password |             |
+| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username |             |
+| MISP_API_ENDPOINT      | MISP URL                       |             |
+| MISP_API_KEY           | MISP API key                   |             |
+| ONYPHE_API_KEY         | Onyphe API key                 |             |
+| PASSIVETOTAL_API_KEY   | PassiveTotal API key           |             |
+| PASSIVETOTAL_USERNAME  | PassiveTotal username          |             |
+| PULSEDIVE_API_KEY      | Pulsedive API key              |             |
+| SECURITYTRAILS_API_KEY | SecurityTrails API key         |             |
+| SHODAN_API_KEY         | Shodan API key                 |             |
+| SLACK_CHANNEL          | Slack channel name             | `#general`  |
+| SLACK_WEBHOOK_URL      | Slack Webhook URL              |             |
+| THEHIVE_API_ENDPOINT   | TheHive URL                    |             |
+| THEHIVE_API_KEY        | TheHive API key                |             |
+| VIRUSTOTAL_API_KEY     | VirusTotal API key             |             |
+| ZOOMEYE_PASSWORD       | ZoomEye password               |             |
+| ZOOMEYE_USERNAMME      | ZoomEye username               |             |
 
 Instead of using environment variables, you can use a YAML file for configuration.
 
@@ -261,6 +222,7 @@ mihari virustotal 1.1.1.1 --config /path/to/yaml.yml
 The YAML file should be a YAML hash like below:
 
 ```yaml
+database: /tmp/mihari.db
 thehive_api_endpoint: https://localhost
 thehive_api_key: foo
 virustotal_api_key: foo
@@ -314,10 +276,6 @@ example.run
 
 See `/examples` for more.
 
-## Caching
-
-Mihari caches execution results in `/tmp/mihari` and the default cache duration is 7 days. If you want to clear the cache, please clear `/tmp/mihari`.
-
 ## Using it with Docker
 
 ```bash

data/config/pre_commit.yml

@@ -0,0 +1,3 @@
+---
+:checks_add:
+  - :rubocop

data/lib/mihari.rb

@@ -19,24 +19,27 @@ module Mihari
 end
 
 require "mihari/version"
-
 require "mihari/errors"
 
-require "mihari/artifact"
-require "mihari/cache"
 require "mihari/config"
+
+require "mihari/database"
 require "mihari/type_checker"
 
+require "mihari/models/alert"
+require "mihari/models/artifact"
+require "mihari/models/tag"
+require "mihari/models/tagging"
+
+require "mihari/serializers/alert"
+require "mihari/serializers/artifact"
+require "mihari/serializers/tag"
+
 require "mihari/html"
 
 require "mihari/configurable"
 require "mihari/retriable"
 
-require "mihari/the_hive/base"
-require "mihari/the_hive/alert"
-require "mihari/the_hive/artifact"
-require "mihari/the_hive"
-
 require "mihari/analyzers/base"
 require "mihari/analyzers/basic"
 
@@ -70,6 +73,7 @@ require "mihari/notifiers/exception_notifier"
 require "mihari/emitters/base"
 require "mihari/emitters/misp"
 require "mihari/emitters/slack"
+require "mihari/emitters/sqlite"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 

data/lib/mihari/alert_viewer.rb

@@ -3,39 +3,17 @@
 module Mihari
   class AlertViewer
     attr_reader :limit
-    attr_reader :the_hive
-
-    ALERT_KEYS = %w(title description artifacts tags createdAt status).freeze
 
     def initialize(limit: 5)
-      @limit = limit
-      validate_limit
-
-      @the_hive = TheHive.new
-      raise Error, "Cannot connect to the TheHive instance" unless the_hive.valid?
+      @limit = limit.to_i
+      raise ArgumentError, "limit should be bigger than zero" unless @limit.positive?
     end
 
     def list
-      range = limit == "all" ? "all" : "0-#{limit}"
-      alerts = the_hive.alert.list(range: range)
-      alerts.map { |alert| convert alert }
-    end
-
-    private
-
-    def validate_limit
-      return true if limit == "all"
-
-      raise ArgumentError, "limit should be bigger than zero" unless limit.to_i.positive?
-    end
-
-    def convert(alert)
-      attributes = alert.select { |k, _v| ALERT_KEYS.include? k }
-      attributes["createdAt"] = Time.at(attributes["createdAt"] / 1000).to_s
-      attributes["artifacts"] = (attributes.dig("artifacts") || []).map do |artifact|
-        artifact.dig("data")
-      end.sort
-      attributes
+      alerts = Alert.order(id: :desc).limit(limit).includes(:tags, :artifacts)
+      alerts.map do |alert|
+        AlertSerializer.new(alert).as_json
+      end
     end
   end
 end

data/lib/mihari/analyzers/base.rb

@@ -23,6 +23,10 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      def source
+        self.class.to_s.split("::").last
+      end
+
       # @return [Array<String>]
       def tags
         []
@@ -37,7 +41,7 @@ module Mihari
       end
 
       def run_emitter(emitter)
-        emitter.run(title: title, description: description, artifacts: unique_artifacts, tags: tags)
+        emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
       rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
@@ -48,32 +52,16 @@ module Mihari
 
       private
 
-      def the_hive
-        @the_hive ||= TheHive.new
-      end
-
-      def cache
-        @cache ||= Cache.new
-      end
-
       # @return [Array<Mihari::Artifact>]
       def normalized_artifacts
         @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
-          artifact.is_a?(Artifact) ? artifact : Artifact.new(artifact)
+          artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
         end.select(&:valid?)
       end
 
-      def uncached_artifacts
-        @uncached_artifacts ||= normalized_artifacts.reject do |artifact|
-          cache.cached? artifact.data
-        end
-      end
-
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        return uncached_artifacts unless the_hive.valid?
-
-        @unique_artifacts ||= the_hive.artifact.find_non_existing_artifacts(uncached_artifacts)
+        @unique_artifacts ||= normalized_artifacts.select(&:unique?)
       end
 
       def set_unique_artifacts