mihari 6.2.0 → 7.0.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 6.2.0
+  version: 7.0.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-03 00:00:00.000000000 Z
+date: 2023-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: better_errors
@@ -44,14 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.39'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '3.39'
 - !ruby/object:Gem::Dependency
   name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
@@ -240,14 +268,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
 - !ruby/object:Gem::Dependency
   name: simplecov-lcov
   requirement: !ruby/object:Gem::Requirement
@@ -268,14 +324,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.32'
+        version: '1.33'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.32'
+        version: '1.33'
 - !ruby/object:Gem::Dependency
   name: test-prof
   requirement: !ruby/object:Gem::Requirement
@@ -350,16 +406,16 @@ dependencies:
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.5
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.5
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: anyway_config
   requirement: !ruby/object:Gem::Requirement
@@ -632,14 +688,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.23.0
+        version: 1.24.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.23.0
+        version: 1.24.0
 - !ruby/object:Gem::Dependency
   name: plissken
   requirement: !ruby/object:Gem::Requirement
@@ -710,6 +766,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: rack-session
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: rackup
   requirement: !ruby/object:Gem::Requirement
@@ -724,6 +794,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: search_cop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -744,14 +828,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: '5.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: '5.15'
+- !ruby/object:Gem::Dependency
+  name: sentry-sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.15'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.2.0
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -772,14 +884,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -862,6 +974,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".standard.yml"
+- Dockerfile
 - Gemfile
 - LICENSE
 - README.md
@@ -870,6 +983,7 @@ files:
 - bin/setup
 - build_frontend.sh
 - config.ru
+- docker-compose.yml
 - exe/mihari
 - lefthook.yml
 - lib/mihari.rb
@@ -895,10 +1009,13 @@ files:
 - lib/mihari/analyzers/virustotal_intelligence.rb
 - lib/mihari/analyzers/zoomeye.rb
 - lib/mihari/cli/alert.rb
+- lib/mihari/cli/application.rb
+- lib/mihari/cli/artifact.rb
 - lib/mihari/cli/base.rb
+- lib/mihari/cli/config.rb
 - lib/mihari/cli/database.rb
-- lib/mihari/cli/main.rb
 - lib/mihari/cli/rule.rb
+- lib/mihari/cli/tag.rb
 - lib/mihari/clients/base.rb
 - lib/mihari/clients/binaryedge.rb
 - lib/mihari/clients/censys.rb
@@ -910,23 +1027,35 @@ files:
 - lib/mihari/clients/greynoise.rb
 - lib/mihari/clients/hunterhow.rb
 - lib/mihari/clients/misp.rb
+- lib/mihari/clients/mmdb.rb
 - lib/mihari/clients/onyphe.rb
 - lib/mihari/clients/otx.rb
 - lib/mihari/clients/passivetotal.rb
 - lib/mihari/clients/publsedive.rb
 - lib/mihari/clients/securitytrails.rb
 - lib/mihari/clients/shodan.rb
+- lib/mihari/clients/shodan_internet_db.rb
 - lib/mihari/clients/the_hive.rb
 - lib/mihari/clients/urlscan.rb
 - lib/mihari/clients/virustotal.rb
 - lib/mihari/clients/zoomeye.rb
 - lib/mihari/commands/alert.rb
+- lib/mihari/commands/artifact.rb
+- lib/mihari/commands/config.rb
 - lib/mihari/commands/database.rb
-- lib/mihari/commands/mixins.rb
 - lib/mihari/commands/rule.rb
 - lib/mihari/commands/search.rb
+- lib/mihari/commands/sidekiq.rb
+- lib/mihari/commands/tag.rb
 - lib/mihari/commands/version.rb
 - lib/mihari/commands/web.rb
+- lib/mihari/concerns/autonomous_system_normalizable.rb
+- lib/mihari/concerns/configurable.rb
+- lib/mihari/concerns/database_connectable.rb
+- lib/mihari/concerns/error_unwrappable.rb
+- lib/mihari/concerns/falsepositive_validatable.rb
+- lib/mihari/concerns/refangable.rb
+- lib/mihari/concerns/retriable.rb
 - lib/mihari/config.rb
 - lib/mihari/constants.rb
 - lib/mihari/data_type.rb
@@ -939,7 +1068,7 @@ files:
 - lib/mihari/emitters/webhook.rb
 - lib/mihari/enrichers/base.rb
 - lib/mihari/enrichers/google_public_dns.rb
-- lib/mihari/enrichers/ipinfo.rb
+- lib/mihari/enrichers/mmdb.rb
 - lib/mihari/enrichers/shodan.rb
 - lib/mihari/enrichers/whois.rb
 - lib/mihari/entities/alert.rb
@@ -950,25 +1079,19 @@ files:
 - lib/mihari/entities/dns.rb
 - lib/mihari/entities/geolocation.rb
 - lib/mihari/entities/ip_address.rb
-- lib/mihari/entities/message.rb
+- lib/mihari/entities/messages.rb
+- lib/mihari/entities/pagination.rb
 - lib/mihari/entities/port.rb
 - lib/mihari/entities/reverse_dns.rb
 - lib/mihari/entities/rule.rb
 - lib/mihari/entities/tag.rb
 - lib/mihari/entities/whois.rb
 - lib/mihari/errors.rb
-- lib/mihari/feed/parser.rb
-- lib/mihari/feed/reader.rb
 - lib/mihari/http.rb
-- lib/mihari/mixins/autonomous_system.rb
-- lib/mihari/mixins/configurable.rb
-- lib/mihari/mixins/falsepositive.rb
-- lib/mihari/mixins/refang.rb
-- lib/mihari/mixins/retriable.rb
-- lib/mihari/mixins/unwrap_error.rb
 - lib/mihari/models/alert.rb
 - lib/mihari/models/artifact.rb
 - lib/mihari/models/autonomous_system.rb
+- lib/mihari/models/concerns/searchable.rb
 - lib/mihari/models/cpe.rb
 - lib/mihari/models/dns.rb
 - lib/mihari/models/geolocation.rb
@@ -981,16 +1104,24 @@ files:
 - lib/mihari/rule.rb
 - lib/mihari/schemas/alert.rb
 - lib/mihari/schemas/analyzer.rb
+- lib/mihari/schemas/concerns/orrable.rb
 - lib/mihari/schemas/emitter.rb
 - lib/mihari/schemas/enricher.rb
 - lib/mihari/schemas/macros.rb
-- lib/mihari/schemas/mixins.rb
 - lib/mihari/schemas/options.rb
 - lib/mihari/schemas/rule.rb
 - lib/mihari/service.rb
-- lib/mihari/services/alert_builder.rb
-- lib/mihari/services/alert_runner.rb
-- lib/mihari/services/rule_builder.rb
+- lib/mihari/services/builders.rb
+- lib/mihari/services/creators.rb
+- lib/mihari/services/destroyers.rb
+- lib/mihari/services/enrichers.rb
+- lib/mihari/services/feed.rb
+- lib/mihari/services/getters.rb
+- lib/mihari/services/initializers.rb
+- lib/mihari/services/proxies.rb
+- lib/mihari/services/searchers.rb
+- lib/mihari/sidekiq/application.rb
+- lib/mihari/sidekiq/jobs.rb
 - lib/mihari/structs/binaryedge.rb
 - lib/mihari/structs/censys.rb
 - lib/mihari/structs/config.rb
@@ -999,7 +1130,7 @@ files:
 - lib/mihari/structs/google_public_dns.rb
 - lib/mihari/structs/greynoise.rb
 - lib/mihari/structs/hunterhow.rb
-- lib/mihari/structs/ipinfo.rb
+- lib/mihari/structs/mmdb.rb
 - lib/mihari/structs/onyphe.rb
 - lib/mihari/structs/shodan.rb
 - lib/mihari/structs/urlscan.rb
@@ -1007,18 +1138,18 @@ files:
 - lib/mihari/types.rb
 - lib/mihari/version.rb
 - lib/mihari/web/api.rb
-- lib/mihari/web/app.rb
+- lib/mihari/web/application.rb
 - lib/mihari/web/endpoints/alerts.rb
 - lib/mihari/web/endpoints/artifacts.rb
 - lib/mihari/web/endpoints/configs.rb
 - lib/mihari/web/endpoints/ip_addresses.rb
 - lib/mihari/web/endpoints/rules.rb
 - lib/mihari/web/endpoints/tags.rb
-- lib/mihari/web/middleware/connection_adapter.rb
-- lib/mihari/web/middleware/error_notification_adapter.rb
-- lib/mihari/web/public/assets/index-1d77cd61.js
-- lib/mihari/web/public/assets/index-4c8509ee.css
-- lib/mihari/web/public/assets/mode-yaml-24faa242.js
+- lib/mihari/web/middleware/capture_exceptions.rb
+- lib/mihari/web/middleware/connection.rb
+- lib/mihari/web/public/assets/index-cQUcyII5.js
+- lib/mihari/web/public/assets/index-dVaNxqTC.css
+- lib/mihari/web/public/assets/mode-yaml-BC4MIiYj.js
 - lib/mihari/web/public/favicon.ico
 - lib/mihari/web/public/index.html
 - lib/mihari/web/public/redoc-static.html
@@ -1046,7 +1177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: A query aggregator for OSINT based threat hunting

data/lib/mihari/commands/mixins.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Mixins
-      def with_db_connection(&block)
-        Mihari::Database.with_db_connection(&block)
-      end
-    end
-  end
-end

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Enrichers
-    #
-    # IPInfo enricher
-    #
-    class IPInfo < Base
-      # @return [String, nil]
-      attr_reader :api_key
-
-      #
-      # @param [Hash, nil] options
-      # @param [String, nil] api_key
-      #
-      def initialize(options: nil, api_key: nil)
-        @api_key = api_key || Mihari.config.ipinfo_api_key
-
-        super(options: options)
-      end
-
-      def configuration_keys
-        %w[ipinfo_api_key]
-      end
-
-      #
-      # Query IPInfo
-      #
-      # @param [String] ip
-      #
-      # @return [Mihari::Structs::IPInfo::Response]
-      #
-      def call(ip)
-        url = "https://ipinfo.io/#{ip}/json"
-        res = http.get(url)
-        Structs::IPInfo::Response.from_dynamic! JSON.parse(res.body.to_s)
-      end
-      memo_wise :call
-
-      private
-
-      def headers
-        authorization = api_key.nil? ? nil : "Bearer #{api_key}"
-        { authorization: authorization }.compact
-      end
-
-      def http
-        HTTP::Factory.build headers: headers, timeout: timeout
-      end
-    end
-  end
-end

data/lib/mihari/entities/message.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Entities
-    class Message < Grape::Entity
-      expose :message, documentation: { type: String, required: true }
-    end
-  end
-end

data/lib/mihari/feed/parser.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require "jr/cli/core_ext"
-
-module Mihari
-  module Feed
-    #
-    # Feed parser
-    #
-    class Parser
-      # @return [Array<Hash>, Array<Array<String>>]
-      attr_reader :data
-
-      #
-      # @param [Array<Hash>, Array<Array<String>>] data
-      #
-      def initialize(data)
-        @data = data
-      end
-
-      #
-      # Parse data by selector
-      #
-      # @param [String] selector
-      #
-      # @return [Array<String>]
-      #
-      def parse(selector)
-        parsed = data.instance_eval(selector)
-
-        raise TypeError unless parsed.is_a?(Array) || parsed.is_a?(Enumerator)
-        raise TypeError unless parsed.all?(String)
-
-        parsed.to_a
-      end
-    end
-  end
-end

data/lib/mihari/feed/reader.rb

@@ -1,111 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module Mihari
-  module Feed
-    #
-    # Feed reader
-    #
-    class Reader
-      # @return [String]
-      attr_reader :url
-
-      # @return [Hash]
-      attr_reader :headers
-
-      # @return [Hash, nil]
-      attr_reader :params
-
-      # @return [Hash, nil]
-      attr_reader :json
-
-      # @return [Hash, nil]
-      attr_reader :data
-
-      # @return [String]
-      attr_reader :method
-
-      # @return [Integer, nil]
-      attr_reader :timeout
-
-      def initialize(url, headers: {}, method: "GET", params: nil, json: nil, data: nil, timeout: nil)
-        @url = Addressable::URI.parse(url)
-        @headers = headers
-        @method = method
-        @timeout = timeout
-
-        @params = params
-        @json = json
-        @data = data
-      end
-
-      #
-      # @return [Array<Hash>]
-      #
-      def read
-        return read_file(url.path) if url.scheme == "file"
-
-        res = nil
-        res = http.get(url, params: params) if method == "GET"
-        res = http.post(url, params: params, json: json, form: data) if method == "POST"
-
-        return [] if res.nil?
-
-        body = res.body.to_s
-        content_type = res["Content-Type"].to_s
-        return convert_as_json(body) if content_type.include?("application/json")
-
-        convert_as_csv(body)
-      end
-
-      #
-      # Convert text as JSON
-      #
-      # @param [String] text
-      #
-      # @return [Array<Hash>]
-      #
-      def convert_as_json(text)
-        parsed = JSON.parse(text, symbolize_names: true)
-        return parsed if parsed.is_a?(Array)
-
-        [parsed]
-      end
-
-      #
-      # Convert text as CSV
-      #
-      # @param [String] text
-      #
-      # @return [Array<Hash>]
-      #
-      def convert_as_csv(text)
-        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
-
-        CSV.new(text_without_comments).to_a.reject(&:empty?)
-      end
-
-      #
-      # Read & convert a file
-      #
-      # @param [String] path
-      #
-      # @return [Array<Hash>]
-      #
-      def read_file(path)
-        text = File.read(path)
-
-        return convert_as_json(text) if path.end_with?(".json")
-
-        convert_as_csv text
-      end
-
-      private
-
-      def http
-        HTTP::Factory.build headers: headers, timeout: timeout
-      end
-    end
-  end
-end