mihari 3.12.0 → 4.0.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 3.12.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-01-03 00:00:00.000000000 Z
+date: 2022-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -240,14 +240,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 7.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 7.0.1
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -332,20 +332,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-- !ruby/object:Gem::Dependency
-  name: cymbal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: dnpedia
   requirement: !ruby/object:Gem::Requirement
@@ -394,14 +380,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.13.0
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.13.0
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
   name: dry-container
   requirement: !ruby/object:Gem::Requirement
@@ -436,14 +422,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.4
+        version: 3.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.4
+        version: 3.1.1
 - !ruby/object:Gem::Dependency
   name: dry-struct
   requirement: !ruby/object:Gem::Requirement
@@ -612,20 +598,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
-- !ruby/object:Gem::Dependency
-  name: mem
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: memist
   requirement: !ruby/object:Gem::Requirement
@@ -1106,36 +1078,15 @@ files:
 - lib/mihari/analyzers/virustotal.rb
 - lib/mihari/analyzers/virustotal_intelligence.rb
 - lib/mihari/analyzers/zoomeye.rb
-- lib/mihari/cli/analyzer.rb
 - lib/mihari/cli/base.rb
 - lib/mihari/cli/init.rb
 - lib/mihari/cli/main.rb
 - lib/mihari/cli/mixins/utils.rb
 - lib/mihari/cli/validator.rb
-- lib/mihari/commands/binaryedge.rb
-- lib/mihari/commands/censys.rb
-- lib/mihari/commands/circl.rb
-- lib/mihari/commands/crtsh.rb
-- lib/mihari/commands/dnpedia.rb
-- lib/mihari/commands/dnstwister.rb
-- lib/mihari/commands/feed.rb
-- lib/mihari/commands/greynoise.rb
 - lib/mihari/commands/init.rb
-- lib/mihari/commands/json.rb
-- lib/mihari/commands/onyphe.rb
-- lib/mihari/commands/otx.rb
-- lib/mihari/commands/passivetotal.rb
-- lib/mihari/commands/pulsedive.rb
 - lib/mihari/commands/search.rb
-- lib/mihari/commands/securitytrails.rb
-- lib/mihari/commands/shodan.rb
-- lib/mihari/commands/spyse.rb
-- lib/mihari/commands/urlscan.rb
 - lib/mihari/commands/validator.rb
-- lib/mihari/commands/virustotal.rb
-- lib/mihari/commands/virustotal_intelligence.rb
 - lib/mihari/commands/web.rb
-- lib/mihari/commands/zoomeye.rb
 - lib/mihari/constants.rb
 - lib/mihari/database.rb
 - lib/mihari/emitters/base.rb
@@ -1152,9 +1103,8 @@ files:
 - lib/mihari/feed/reader.rb
 - lib/mihari/mixins/autonomous_system.rb
 - lib/mihari/mixins/configurable.rb
-- lib/mihari/mixins/configuration.rb
+- lib/mihari/mixins/database.rb
 - lib/mihari/mixins/disallowed_data_value.rb
-- lib/mihari/mixins/hash.rb
 - lib/mihari/mixins/refang.rb
 - lib/mihari/mixins/retriable.rb
 - lib/mihari/mixins/rule.rb
@@ -1164,6 +1114,7 @@ files:
 - lib/mihari/models/dns.rb
 - lib/mihari/models/geolocation.rb
 - lib/mihari/models/reverse_dns.rb
+- lib/mihari/models/rule.rb
 - lib/mihari/models/tag.rb
 - lib/mihari/models/tagging.rb
 - lib/mihari/models/whois.rb
@@ -1171,7 +1122,6 @@ files:
 - lib/mihari/notifiers/exception_notifier.rb
 - lib/mihari/notifiers/slack.rb
 - lib/mihari/schemas/analyzer.rb
-- lib/mihari/schemas/configuration.rb
 - lib/mihari/schemas/macros.rb
 - lib/mihari/schemas/rule.rb
 - lib/mihari/status.rb
@@ -1180,6 +1130,7 @@ files:
 - lib/mihari/structs/greynoise.rb
 - lib/mihari/structs/ipinfo.rb
 - lib/mihari/structs/onyphe.rb
+- lib/mihari/structs/rule.rb
 - lib/mihari/structs/shodan.rb
 - lib/mihari/structs/urlscan.rb
 - lib/mihari/structs/virustotal_intelligence.rb
@@ -1194,6 +1145,7 @@ files:
 - lib/mihari/web/endpoints/command.rb
 - lib/mihari/web/endpoints/configs.rb
 - lib/mihari/web/endpoints/ip_addresses.rb
+- lib/mihari/web/endpoints/rules.rb
 - lib/mihari/web/endpoints/sources.rb
 - lib/mihari/web/endpoints/tags.rb
 - lib/mihari/web/entities/alert.rb
@@ -1206,10 +1158,11 @@ files:
 - lib/mihari/web/entities/ip_address.rb
 - lib/mihari/web/entities/message.rb
 - lib/mihari/web/entities/reverse_dns.rb
+- lib/mihari/web/entities/rule.rb
 - lib/mihari/web/entities/source.rb
 - lib/mihari/web/entities/tag.rb
 - lib/mihari/web/entities/whois.rb
-- lib/mihari/web/public/grape.rb
+- lib/mihari/web/middleware/connection_adapter.rb
 - lib/mihari/web/public/index.html
 - lib/mihari/web/public/redoc-static.html
 - lib/mihari/web/public/static/favicon.ico
@@ -1253,6 +1206,8 @@ files:
 - lib/mihari/web/public/static/js/app.365f1907.js.map
 - lib/mihari/web/public/static/js/app.378da3dc.js
 - lib/mihari/web/public/static/js/app.378da3dc.js.map
+- lib/mihari/web/public/static/js/app.49ab738a.js
+- lib/mihari/web/public/static/js/app.49ab738a.js.map
 - lib/mihari/web/public/static/js/app.5dc97aae.js
 - lib/mihari/web/public/static/js/app.5dc97aae.js.map
 - lib/mihari/web/public/static/js/app.6b636b62.js
@@ -1295,34 +1250,16 @@ files:
 - sig/lib/mihari/analyzers/virustotal.rbs
 - sig/lib/mihari/analyzers/virustotal_intelligence.rbs
 - sig/lib/mihari/analyzers/zoomeye.rbs
-- sig/lib/mihari/cli/analyzer.rbs
 - sig/lib/mihari/cli/base.rbs
 - sig/lib/mihari/cli/init.rbs
 - sig/lib/mihari/cli/main.rbs
 - sig/lib/mihari/cli/mixins/utils.rbs
 - sig/lib/mihari/cli/validator.rbs
-- sig/lib/mihari/commands/binaryedge.rbs
-- sig/lib/mihari/commands/censys.rbs
-- sig/lib/mihari/commands/circl.rbs
-- sig/lib/mihari/commands/crtsh.rbs
-- sig/lib/mihari/commands/dnpedia.rbs
-- sig/lib/mihari/commands/dnstwister.rbs
-- sig/lib/mihari/commands/feed.rbs
 - sig/lib/mihari/commands/init.rbs
 - sig/lib/mihari/commands/json.rbs
-- sig/lib/mihari/commands/onyphe.rbs
-- sig/lib/mihari/commands/otx.rbs
-- sig/lib/mihari/commands/passivetotal.rbs
-- sig/lib/mihari/commands/pulsedive.rbs
 - sig/lib/mihari/commands/search.rbs
-- sig/lib/mihari/commands/securitytrails.rbs
-- sig/lib/mihari/commands/shodan.rbs
-- sig/lib/mihari/commands/spyse.rbs
-- sig/lib/mihari/commands/urlscan.rbs
 - sig/lib/mihari/commands/validator.rbs
-- sig/lib/mihari/commands/virustotal.rbs
 - sig/lib/mihari/commands/web.rbs
-- sig/lib/mihari/commands/zoomeye.rbs
 - sig/lib/mihari/constants.rbs
 - sig/lib/mihari/database.rbs
 - sig/lib/mihari/emitters/base.rbs
@@ -1351,6 +1288,7 @@ files:
 - sig/lib/mihari/models/dns.rbs
 - sig/lib/mihari/models/geolocation.rbs
 - sig/lib/mihari/models/reverse_dns.rbs
+- sig/lib/mihari/models/rule.rbs
 - sig/lib/mihari/models/tag.rbs
 - sig/lib/mihari/models/tagging.rbs
 - sig/lib/mihari/models/whois.rbs
@@ -1363,6 +1301,7 @@ files:
 - sig/lib/mihari/structs/greynoise.rbs
 - sig/lib/mihari/structs/ipinfo.rbs
 - sig/lib/mihari/structs/onyphe.rbs
+- sig/lib/mihari/structs/rule.rbs
 - sig/lib/mihari/structs/shodan.rbs
 - sig/lib/mihari/structs/urlscan.rbs
 - sig/lib/mihari/structs/virustotal_intelligence.rbs

data/lib/mihari/cli/analyzer.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-require "mihari/commands/binaryedge"
-require "mihari/commands/censys"
-require "mihari/commands/circl"
-require "mihari/commands/crtsh"
-require "mihari/commands/dnpedia"
-require "mihari/commands/dnstwister"
-require "mihari/commands/feed"
-require "mihari/commands/greynoise"
-require "mihari/commands/onyphe"
-require "mihari/commands/otx"
-require "mihari/commands/passivetotal"
-require "mihari/commands/pulsedive"
-require "mihari/commands/securitytrails"
-require "mihari/commands/shodan"
-require "mihari/commands/spyse"
-require "mihari/commands/urlscan"
-require "mihari/commands/virustotal_intelligence"
-require "mihari/commands/virustotal"
-require "mihari/commands/zoomeye"
-
-require "mihari/commands/json"
-
-module Mihari
-  module CLI
-    class Analyzer < Base
-      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
-      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
-      class_option :interval, type: :numeric, default: 0, desc: "Seconds of the interval while calling API in a row."
-      class_option :config, type: :string, desc: "Path to the config file"
-
-      include Mihari::Commands::BinaryEdge
-      include Mihari::Commands::Censys
-      include Mihari::Commands::CIRCL
-      include Mihari::Commands::Crtsh
-      include Mihari::Commands::DNPedia
-      include Mihari::Commands::DNSTwister
-      include Mihari::Commands::Feed
-      include Mihari::Commands::GreyNoise
-      include Mihari::Commands::JSON
-      include Mihari::Commands::Onyphe
-      include Mihari::Commands::OTX
-      include Mihari::Commands::PassiveTotal
-      include Mihari::Commands::Pulsedive
-      include Mihari::Commands::SecurityTrails
-      include Mihari::Commands::Shodan
-      include Mihari::Commands::Spyse
-      include Mihari::Commands::Urlscan
-      include Mihari::Commands::VirusTotal
-      include Mihari::Commands::VirusTotalIntelligence
-      include Mihari::Commands::ZoomEye
-    end
-  end
-end

data/lib/mihari/commands/binaryedge.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module BinaryEdge
-      def self.included(thor)
-        thor.class_eval do
-          desc "binaryedge [QUERY]", "BinaryEdge host search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def binaryedge(query)
-            with_error_handling do
-              run_analyzer Analyzers::BinaryEdge, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/censys.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Censys
-      def self.included(thor)
-        thor.class_eval do
-          desc "censys [QUERY]", "Censys IPv4 search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          method_option :type, type: :string, desc: "type to search (ipv4 / websites / certificates)", default: "ipv4"
-          def censys(query)
-            with_error_handling do
-              run_analyzer Analyzers::Censys, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/circl.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module CIRCL
-      def self.included(thor)
-        thor.class_eval do
-          desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL search by a domain or SHA1 certificate fingerprint"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def circl(query)
-            with_error_handling do
-              run_analyzer Analyzers::CIRCL, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/crtsh.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Crtsh
-      def self.included(thor)
-        thor.class_eval do
-          desc "crtsh [QUERY]", "crt.sh search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          method_option :exclude_expired, type: :boolean, desc: "exclude expired certificates"
-          def crtsh(query)
-            with_error_handling do
-              run_analyzer Analyzers::Crtsh, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/dnpedia.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module DNPedia
-      def self.included(thor)
-        thor.class_eval do
-          desc "dnpedia [QUERY]", "DNPedia domain search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def dnpedia(query)
-            with_error_handling do
-              run_analyzer Analyzers::DNPedia, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/dnstwister.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module DNSTwister
-      def self.included(thor)
-        thor.class_eval do
-          desc "dnstwister [DOMAIN]", "dnstwister search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def dnstwister(domain)
-            with_error_handling do
-              run_analyzer Analyzers::DNSTwister, query: domain, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/feed.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Feed
-      def self.included(thor)
-        thor.class_eval do
-          desc "feed [URL]", "ingest feed"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          method_option :http_request_method, type: :string, desc: "HTTP request method"
-          method_option :http_request_headers, type: :hash, desc: "HTTP request headers"
-          method_option :http_request_payload_type, type: :string, desc: "HTTP request payload type"
-          method_option :http_request_payload, type: :hash, desc: "HTTP request payload"
-          method_option :selector, type: :string, desc: "jr selector", required: true
-          def feed(query)
-            with_error_handling do
-              run_analyzer Analyzers::Feed, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/greynoise.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module GreyNoise
-      def self.included(thor)
-        thor.class_eval do
-          desc "greynoise [QUERY]", "GreyNoise search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def greynoise(query)
-            with_error_handling do
-              run_analyzer Analyzers::GreyNoise, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/json.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module JSON
-      def self.included(thor)
-        thor.class_eval do
-          desc "import_from_json", "Give a JSON input via STDIN"
-          def import_from_json(input = nil)
-            with_error_handling do
-              json = input || $stdin.gets.chomp
-              raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
-
-              json = parse_as_json(json)
-              raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless required_alert_keys?(json)
-
-              title = json["title"]
-              description = json["description"]
-              artifacts = json["artifacts"]
-              tags = json["tags"] || []
-
-              basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
-
-              basic.ignore_old_artifacts = options["ignore_old_artifacts"] || false
-              basic.ignore_threshold = options["ignore_threshold"] || 0
-
-              basic.run
-            end
-          end
-
-          no_commands do
-            def parse_as_json(input)
-              ::JSON.parse input
-            rescue ::JSON::ParserError => _e
-              nil
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/onyphe.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Onyphe
-      def self.included(thor)
-        thor.class_eval do
-          desc "onyphe [QUERY]", "Onyphe datascan search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def onyphe(query)
-            with_error_handling do
-              run_analyzer Analyzers::Onyphe, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/otx.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module OTX
-      def self.included(thor)
-        thor.class_eval do
-          desc "otx [IP|DOMAIN]", "OTX search by an IP or domain"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def otx(domain)
-            with_error_handling do
-              run_analyzer Analyzers::OTX, query: domain, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/passivetotal.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module PassiveTotal
-      def self.included(thor)
-        thor.class_eval do
-          desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal search by an ip, domain, email or SHA1 certificate fingerprint"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def passivetotal(indicator)
-            with_error_handling do
-              run_analyzer Analyzers::PassiveTotal, query: indicator, options: options
-            end
-          end
-          map "pt" => :passivetotal
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/pulsedive.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Pulsedive
-      def self.included(thor)
-        thor.class_eval do
-          desc "pulsedive [IP|DOMAIN]", "Pulsedive search by an ip or domain"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def pulsedive(indiactor)
-            with_error_handling do
-              run_analyzer Analyzers::Pulsedive, query: indiactor, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/securitytrails.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module SecurityTrails
-      def self.included(thor)
-        thor.class_eval do
-          desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails search by an ip, domain or email"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def securitytrails(indiactor)
-            with_error_handling do
-              run_analyzer Analyzers::SecurityTrails, query: indiactor, options: options
-            end
-          end
-          map "st" => :securitytrails
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/shodan.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Shodan
-      def self.included(thor)
-        thor.class_eval do
-          desc "shodan [QUERY]", "Shodan host search"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def shodan(query)
-            with_error_handling do
-              run_analyzer Analyzers::Shodan, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end