mihari 6.3.0 → 7.0.1

data/lib/mihari.rb

@@ -1,21 +1,22 @@
 # frozen_string_literal: true
 
 # standard libs
+require "date"
+require "erb"
 require "ipaddr"
 require "json"
+require "pathname"
 require "resolv"
+require "securerandom"
 require "yaml"
 
 # Active Support & Active Record
 require "active_support"
-
-require "active_support/core_ext/hash"
-require "active_support/core_ext/integer/time"
-require "active_support/core_ext/numeric/time"
-require "active_support/core_ext/object/deep_dup"
-
 require "active_record"
 
+# Search Cop
+require "search_cop"
+
 # dry-rb
 require "dry/files"
 require "dry/monads"
@@ -49,13 +50,14 @@ require "mihari/errors"
 
 require "mihari/config"
 
-# Mixins
-require "mihari/mixins/autonomous_system"
-require "mihari/mixins/configurable"
-require "mihari/mixins/falsepositive"
-require "mihari/mixins/refang"
-require "mihari/mixins/retriable"
-require "mihari/mixins/unwrap_error"
+# Concerns
+require "mihari/concerns/autonomous_system_normalizable"
+require "mihari/concerns/configurable"
+require "mihari/concerns/database_connectable"
+require "mihari/concerns/error_unwrappable"
+require "mihari/concerns/falsepositive_validatable"
+require "mihari/concerns/refangable"
+require "mihari/concerns/retriable"
 
 #
 # Mihari module
@@ -129,6 +131,27 @@ module Mihari
     end
     memo_wise :logger
 
+    #
+    # @return [String]
+    #
+    def env
+      ENV["APP_ENV"] || ENV["RACK_ENV"]
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def development?
+      env == "development"
+    end
+
+    #
+    # @return [Boolean]
+    #
+    def sidekiq?
+      !Mihari.config.sidekiq_redis_url.nil?
+    end
+
     def initialize_sentry
       return if Mihari.config.sentry_dsn.nil?
       return if Sentry.initialized?
@@ -136,6 +159,7 @@ module Mihari
       Sentry.init do |config|
         config.dsn = Mihari.config.sentry_dsn
         config.traces_sample_rate = Mihari.config.sentry_trace_sample_rate
+        config.breadcrumbs_logger = %i[sentry_logger http_logger]
       end
     end
   end
@@ -154,11 +178,13 @@ require "mihari/rule"
 # Enrichers
 require "mihari/enrichers/base"
 require "mihari/enrichers/google_public_dns"
-require "mihari/enrichers/ipinfo"
+require "mihari/enrichers/mmdb"
 require "mihari/enrichers/shodan"
 require "mihari/enrichers/whois"
 
 # Models
+require "mihari/models/concerns/searchable"
+
 require "mihari/models/alert"
 require "mihari/models/artifact"
 require "mihari/models/autonomous_system"
@@ -194,11 +220,13 @@ require "mihari/clients/google_public_dns"
 require "mihari/clients/greynoise"
 require "mihari/clients/hunterhow"
 require "mihari/clients/misp"
+require "mihari/clients/mmdb"
 require "mihari/clients/onyphe"
 require "mihari/clients/otx"
 require "mihari/clients/passivetotal"
 require "mihari/clients/publsedive"
 require "mihari/clients/securitytrails"
+require "mihari/clients/shodan_internet_db"
 require "mihari/clients/shodan"
 require "mihari/clients/the_hive"
 require "mihari/clients/urlscan"
@@ -242,8 +270,8 @@ require "mihari/structs/filters"
 require "mihari/structs/fofa"
 require "mihari/structs/google_public_dns"
 require "mihari/structs/greynoise"
-require "mihari/structs/ipinfo"
 require "mihari/structs/hunterhow"
+require "mihari/structs/mmdb"
 require "mihari/structs/onyphe"
 require "mihari/structs/shodan"
 require "mihari/structs/urlscan"
@@ -251,7 +279,7 @@ require "mihari/structs/virustotal_intelligence"
 
 # Schemas
 require "mihari/schemas/macros"
-require "mihari/schemas/mixins"
+require "mihari/schemas/concerns/orrable"
 
 require "mihari/schemas/options"
 
@@ -260,13 +288,18 @@ require "mihari/schemas/analyzer"
 require "mihari/schemas/rule"
 
 # Services
-require "mihari/services/rule_builder"
-
-require "mihari/services/alert_builder"
-require "mihari/services/alert_runner"
+require "mihari/services/builders"
+require "mihari/services/creators"
+require "mihari/services/destroyers"
+require "mihari/services/enrichers"
+require "mihari/services/feed"
+require "mihari/services/getters"
+require "mihari/services/initializers"
+require "mihari/services/proxies"
+require "mihari/services/searchers"
 
 # Entities
-require "mihari/entities/message"
+require "mihari/entities/pagination"
 
 require "mihari/entities/autonomous_system"
 require "mihari/entities/config"
@@ -274,6 +307,7 @@ require "mihari/entities/cpe"
 require "mihari/entities/dns"
 require "mihari/entities/geolocation"
 require "mihari/entities/ip_address"
+require "mihari/entities/messages"
 require "mihari/entities/port"
 require "mihari/entities/reverse_dns"
 require "mihari/entities/tag"
@@ -285,11 +319,5 @@ require "mihari/entities/alert"
 
 require "mihari/entities/rule"
 
-# Web app
-require "mihari/web/app"
-
-# CLIs
-require "mihari/cli/main"
-
-# initialize Sentry (if it's possible)
+# Initialize Sentry (if it's possible)
 Mihari.initialize_sentry

data/mihari.gemspec

@@ -39,7 +39,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "better_errors", "~> 2.10"
   spec.add_development_dependency "binding_of_caller", "~> 1.0"
   spec.add_development_dependency "bundler", "~> 2.5"
+  spec.add_development_dependency "capybara", "~> 3.39"
   spec.add_development_dependency "coveralls_reborn", "~> 0.28"
+  spec.add_development_dependency "factory_bot", "~> 6.4"
   spec.add_development_dependency "fakefs", "~> 2.5"
   spec.add_development_dependency "faker", "~> 3.2"
   spec.add_development_dependency "fuubar", "~> 2.5"
@@ -52,9 +54,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.12"
   spec.add_development_dependency "rspec-parameterized", "~> 1.0"
   spec.add_development_dependency "rubocop-rspec", "~> 2.25"
-  spec.add_development_dependency "rubocop-yard", "~> 0.8"
+  spec.add_development_dependency "rubocop-yard", "~> 0.9"
+  spec.add_development_dependency "rubocop-rake", "~> 0.6"
+  spec.add_development_dependency "rubocop-factory_bot", "~> 2.24"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
-  spec.add_development_dependency "standard", "~> 1.32"
+  spec.add_development_dependency "standard", "~> 1.33"
   spec.add_development_dependency "test-prof", "~> 1.3"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.2"
@@ -93,11 +97,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency "puma", "6.4.0"
   spec.add_dependency "rack", "3.0.8"
   spec.add_dependency "rack-cors", "2.0.1"
+  spec.add_dependency "rack-session", "2.0.0"
   spec.add_dependency "rackup", "2.1.0"
+  spec.add_dependency "search_cop", "1.2.3"
   spec.add_dependency "semantic_logger", "4.15.0"
-  spec.add_dependency "sentry-ruby", "~> 5.14"
+  spec.add_dependency "sentry-ruby", "~> 5.15"
+  spec.add_dependency "sentry-sidekiq", "~> 5.15"
+  spec.add_dependency "sidekiq", "7.2.0"
   spec.add_dependency "slack-notifier", "2.4.0"
-  spec.add_dependency "sqlite3", "~> 1.6"
+  spec.add_dependency "sqlite3", "~> 1.7"
   spec.add_dependency "thor", "1.3.0"
   spec.add_dependency "thor-hollaback", "0.2.1"
   spec.add_dependency "uuidtools", "2.2.0"

data/mkdocs.yml

@@ -11,11 +11,13 @@ plugins:
 
 nav:
   - Requirements: requirements.md
+  - Overview: overview.md
   - Installation: installation.md
-  - How to Write a Rule: rule.md
-  - Usage: usage.md
+  - Rule: rule.md
   - Configuration: configuration.md
+  - Usage: usage.md
   - Tips:
+      - Sidekiq: tips/sidekiq.md
       - Docker: tips/docker.md
       - GitHub Actions: tips/github_actions.md
       - Superset: tips/superset.md
@@ -27,6 +29,7 @@ nav:
       - Emitters: emitters/index.md
       - Database: database.md
       - Tags: tags.md
+  - Development: dev.md
 
 markdown_extensions:
   - toc:

data/requirements.txt

@@ -1,2 +1,2 @@
 mkdocs==1.5.3
-mkdocs-material==9.5.1
+mkdocs-material==9.5.3

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 6.3.0
+  version: 7.0.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-16 00:00:00.000000000 Z
+date: 2024-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: better_errors
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.39'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.39'
 - !ruby/object:Gem::Dependency
   name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
@@ -240,14 +268,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
 - !ruby/object:Gem::Dependency
   name: simplecov-lcov
   requirement: !ruby/object:Gem::Requirement
@@ -268,14 +324,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.32'
+        version: '1.33'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.32'
+        version: '1.33'
 - !ruby/object:Gem::Dependency
   name: test-prof
   requirement: !ruby/object:Gem::Requirement
@@ -710,6 +766,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: rack-session
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: rackup
   requirement: !ruby/object:Gem::Requirement
@@ -724,6 +794,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: search_cop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -744,14 +828,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: '5.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.15'
+- !ruby/object:Gem::Dependency
+  name: sentry-sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.14'
+        version: '5.15'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 7.2.0
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -772,14 +884,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -862,6 +974,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".standard.yml"
+- Dockerfile
 - Gemfile
 - LICENSE
 - README.md
@@ -870,6 +983,7 @@ files:
 - bin/setup
 - build_frontend.sh
 - config.ru
+- docker-compose.yml
 - exe/mihari
 - lefthook.yml
 - lib/mihari.rb
@@ -895,10 +1009,13 @@ files:
 - lib/mihari/analyzers/virustotal_intelligence.rb
 - lib/mihari/analyzers/zoomeye.rb
 - lib/mihari/cli/alert.rb
+- lib/mihari/cli/application.rb
+- lib/mihari/cli/artifact.rb
 - lib/mihari/cli/base.rb
+- lib/mihari/cli/config.rb
 - lib/mihari/cli/database.rb
-- lib/mihari/cli/main.rb
 - lib/mihari/cli/rule.rb
+- lib/mihari/cli/tag.rb
 - lib/mihari/clients/base.rb
 - lib/mihari/clients/binaryedge.rb
 - lib/mihari/clients/censys.rb
@@ -910,23 +1027,35 @@ files:
 - lib/mihari/clients/greynoise.rb
 - lib/mihari/clients/hunterhow.rb
 - lib/mihari/clients/misp.rb
+- lib/mihari/clients/mmdb.rb
 - lib/mihari/clients/onyphe.rb
 - lib/mihari/clients/otx.rb
 - lib/mihari/clients/passivetotal.rb
 - lib/mihari/clients/publsedive.rb
 - lib/mihari/clients/securitytrails.rb
 - lib/mihari/clients/shodan.rb
+- lib/mihari/clients/shodan_internet_db.rb
 - lib/mihari/clients/the_hive.rb
 - lib/mihari/clients/urlscan.rb
 - lib/mihari/clients/virustotal.rb
 - lib/mihari/clients/zoomeye.rb
 - lib/mihari/commands/alert.rb
+- lib/mihari/commands/artifact.rb
+- lib/mihari/commands/config.rb
 - lib/mihari/commands/database.rb
-- lib/mihari/commands/mixins.rb
 - lib/mihari/commands/rule.rb
 - lib/mihari/commands/search.rb
+- lib/mihari/commands/sidekiq.rb
+- lib/mihari/commands/tag.rb
 - lib/mihari/commands/version.rb
 - lib/mihari/commands/web.rb
+- lib/mihari/concerns/autonomous_system_normalizable.rb
+- lib/mihari/concerns/configurable.rb
+- lib/mihari/concerns/database_connectable.rb
+- lib/mihari/concerns/error_unwrappable.rb
+- lib/mihari/concerns/falsepositive_validatable.rb
+- lib/mihari/concerns/refangable.rb
+- lib/mihari/concerns/retriable.rb
 - lib/mihari/config.rb
 - lib/mihari/constants.rb
 - lib/mihari/data_type.rb
@@ -939,7 +1068,7 @@ files:
 - lib/mihari/emitters/webhook.rb
 - lib/mihari/enrichers/base.rb
 - lib/mihari/enrichers/google_public_dns.rb
-- lib/mihari/enrichers/ipinfo.rb
+- lib/mihari/enrichers/mmdb.rb
 - lib/mihari/enrichers/shodan.rb
 - lib/mihari/enrichers/whois.rb
 - lib/mihari/entities/alert.rb
@@ -950,25 +1079,19 @@ files:
 - lib/mihari/entities/dns.rb
 - lib/mihari/entities/geolocation.rb
 - lib/mihari/entities/ip_address.rb
-- lib/mihari/entities/message.rb
+- lib/mihari/entities/messages.rb
+- lib/mihari/entities/pagination.rb
 - lib/mihari/entities/port.rb
 - lib/mihari/entities/reverse_dns.rb
 - lib/mihari/entities/rule.rb
 - lib/mihari/entities/tag.rb
 - lib/mihari/entities/whois.rb
 - lib/mihari/errors.rb
-- lib/mihari/feed/parser.rb
-- lib/mihari/feed/reader.rb
 - lib/mihari/http.rb
-- lib/mihari/mixins/autonomous_system.rb
-- lib/mihari/mixins/configurable.rb
-- lib/mihari/mixins/falsepositive.rb
-- lib/mihari/mixins/refang.rb
-- lib/mihari/mixins/retriable.rb
-- lib/mihari/mixins/unwrap_error.rb
 - lib/mihari/models/alert.rb
 - lib/mihari/models/artifact.rb
 - lib/mihari/models/autonomous_system.rb
+- lib/mihari/models/concerns/searchable.rb
 - lib/mihari/models/cpe.rb
 - lib/mihari/models/dns.rb
 - lib/mihari/models/geolocation.rb
@@ -981,16 +1104,24 @@ files:
 - lib/mihari/rule.rb
 - lib/mihari/schemas/alert.rb
 - lib/mihari/schemas/analyzer.rb
+- lib/mihari/schemas/concerns/orrable.rb
 - lib/mihari/schemas/emitter.rb
 - lib/mihari/schemas/enricher.rb
 - lib/mihari/schemas/macros.rb
-- lib/mihari/schemas/mixins.rb
 - lib/mihari/schemas/options.rb
 - lib/mihari/schemas/rule.rb
 - lib/mihari/service.rb
-- lib/mihari/services/alert_builder.rb
-- lib/mihari/services/alert_runner.rb
-- lib/mihari/services/rule_builder.rb
+- lib/mihari/services/builders.rb
+- lib/mihari/services/creators.rb
+- lib/mihari/services/destroyers.rb
+- lib/mihari/services/enrichers.rb
+- lib/mihari/services/feed.rb
+- lib/mihari/services/getters.rb
+- lib/mihari/services/initializers.rb
+- lib/mihari/services/proxies.rb
+- lib/mihari/services/searchers.rb
+- lib/mihari/sidekiq/application.rb
+- lib/mihari/sidekiq/jobs.rb
 - lib/mihari/structs/binaryedge.rb
 - lib/mihari/structs/censys.rb
 - lib/mihari/structs/config.rb
@@ -999,7 +1130,7 @@ files:
 - lib/mihari/structs/google_public_dns.rb
 - lib/mihari/structs/greynoise.rb
 - lib/mihari/structs/hunterhow.rb
-- lib/mihari/structs/ipinfo.rb
+- lib/mihari/structs/mmdb.rb
 - lib/mihari/structs/onyphe.rb
 - lib/mihari/structs/shodan.rb
 - lib/mihari/structs/urlscan.rb
@@ -1007,18 +1138,17 @@ files:
 - lib/mihari/types.rb
 - lib/mihari/version.rb
 - lib/mihari/web/api.rb
-- lib/mihari/web/app.rb
+- lib/mihari/web/application.rb
 - lib/mihari/web/endpoints/alerts.rb
 - lib/mihari/web/endpoints/artifacts.rb
 - lib/mihari/web/endpoints/configs.rb
-- lib/mihari/web/endpoints/exports.rb
 - lib/mihari/web/endpoints/ip_addresses.rb
 - lib/mihari/web/endpoints/rules.rb
 - lib/mihari/web/endpoints/tags.rb
-- lib/mihari/web/middleware/connection_adapter.rb
-- lib/mihari/web/middleware/error_notification_adapter.rb
-- lib/mihari/web/public/assets/index-81613_nX.js
-- lib/mihari/web/public/assets/index-Wv6xUrTI.css
+- lib/mihari/web/middleware/capture_exceptions.rb
+- lib/mihari/web/middleware/connection.rb
+- lib/mihari/web/public/assets/index-cQUcyII5.js
+- lib/mihari/web/public/assets/index-dVaNxqTC.css
 - lib/mihari/web/public/assets/mode-yaml-BC4MIiYj.js
 - lib/mihari/web/public/favicon.ico
 - lib/mihari/web/public/index.html
@@ -1047,7 +1177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: A query aggregator for OSINT based threat hunting

data/lib/mihari/commands/mixins.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module Mixins
-      def with_db_connection(&block)
-        Mihari::Database.with_db_connection(&block)
-      end
-    end
-  end
-end