mihari 7.0.0 → 7.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3f4b3b5ca761f5ca5e6f3697593f8253b1e6aa03f21924726ad5dc78ac3f011
-  data.tar.gz: f673dd7775f31c2fa134102c8cf9707e0d6fc74467f496c86202de43db7b3c5a
+  metadata.gz: 184d0084b9ab68c359ddaa10436a2a6172fa09e71f0cc4492af57a1afcf924d8
+  data.tar.gz: 035620673b2351bd6c70a837c7e73f36819e320a868298b6e9a07271759bf6ed
 SHA512:
-  metadata.gz: 0f139c2a0795bd6e741ce93681f8be47cb75fcb5b707d187d1fd2c758bfe2877388d0d5a25fe5920ec2e7565532c142b585b0029816899664426becea2f0776c
-  data.tar.gz: 6dad7972edcba2090bfa12170539efcd51a20dc1ba94757383546ae630002d374e33dcdbc99675cc719801a80b92a11d2eff308a553d627050f8596d6a877c9a
+  metadata.gz: 8f16c21d9475313b8d6ef81cd12fe79c18964552f27633aa2c23fd530c92f9eece651af02fc51b3059f4e57acd6a7c5080607f2822488d26d56a0945c97c424b
+  data.tar.gz: 25640e747ea9113d294dcb910574d7958d0883187e03f0fa891f809b1bc997722bc9af48299685d4f67d51562a6a9d6cdbec0eb499d6e1afb38410e38a7c1936

data/Dockerfile

@@ -2,10 +2,11 @@ FROM ruby:3.2.2-alpine3.19
 
 ARG MIHARI_VERSION=0.0.0
 
-RUN apk --no-cache add git build-base ruby-dev postgresql-dev && \
+RUN apk --no-cache add build-base ruby-dev libpq-dev && \
+  echo 'gem: --no-document' >> /usr/local/etc/gemrc && \
   gem install pg && \
   gem install mihari -v ${MIHARI_VERSION} && \
-  apk del --purge git build-base ruby-dev && \
+  apk del --purge build-base ruby-dev && \
   rm -rf /usr/local/bundle/cache/*
 
 ENTRYPOINT ["mihari"]

data/docker-compose.yml

@@ -47,8 +47,7 @@ services:
     image: ghcr.io/ninoseki/mihari:latest
     environment:
       - DATABASE_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@database:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-mihari}}
-      - REDIS_URL=${REDIS_URL:-redis://redis:${REDIS_PORT:-6379}}
-      - USE_SIDEKIQ=true
+      - SIDEKIQ_REDIS_URL=${REDIS_URL:-redis://redis:${REDIS_PORT:-6379}}
     env_file:
       - .env
     entrypoint: ["mihari", "sidekiq"]

data/lib/mihari/analyzers/crtsh.rb

@@ -9,20 +9,25 @@ module Mihari
       # @return [Boolean]
       attr_reader :exclude_expired
 
+      # @return [String, nil]
+      attr_reader :match
+
       #
       # @param [String] query
       # @param [Hash, nil] options
       # @param [Bool] exclude_expired
+      # @param [String, nil] match
       #
-      def initialize(query, options: nil, exclude_expired: true)
+      def initialize(query, options: nil, exclude_expired: true, match: nil)
         super(query, options: options)
 
         @exclude_expired = exclude_expired
+        @match = match
       end
 
       def artifacts
         exclude = exclude_expired ? "expired" : nil
-        client.search(query, exclude: exclude).map do |result|
+        client.search(query, exclude: exclude, match: match).map do |result|
           values = result["name_value"].to_s.lines.map(&:chomp).reject { |value| value.starts_with?("*.") }
           values.map { |value| Models::Artifact.new(data: value, metadata: result) }
         end.flatten

data/lib/mihari/clients/crtsh.rb

@@ -19,7 +19,7 @@ module Mihari
       # Search crt.sh by a given identity
       #
       # @param [String] identity
-      # @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any" or nil
+      # @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any", "FTS" or nil
       # @param [String, nil] exclude "expired" or nil
       #
       # @return [Array<Hash>]

data/lib/mihari/models/artifact.rb

@@ -87,7 +87,7 @@ module Mihari
       def enrich_whois(enricher = Enrichers::Whois.new)
         return unless can_enrich_whois?
 
-        self.whois_record = WhoisRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
+        self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
       end
 
       #
@@ -98,7 +98,7 @@ module Mihari
       def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
         return unless can_enrich_dns?
 
-        self.dns_records = DnsRecord.build_by_domain(normalize_as_domain(data), enricher: enricher)
+        self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
       end
 
       #
@@ -109,7 +109,7 @@ module Mihari
       def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
         return unless can_enrich_reverse_dns?
 
-        self.reverse_dns_names = ReverseDnsName.build_by_ip(data, enricher: enricher)
+        self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -120,7 +120,7 @@ module Mihari
       def enrich_geolocation(enricher = Enrichers::MMDB.new)
         return unless can_enrich_geolocation?
 
-        self.geolocation = Geolocation.build_by_ip(data, enricher: enricher)
+        self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -131,7 +131,7 @@ module Mihari
       def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
         return unless can_enrich_autonomous_system?
 
-        self.autonomous_system = AutonomousSystem.build_by_ip(data, enricher: enricher)
+        self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -142,7 +142,7 @@ module Mihari
       def enrich_ports(enricher = Enrichers::Shodan.new)
         return unless can_enrich_ports?
 
-        self.ports = Port.build_by_ip(data, enricher: enricher)
+        self.ports = Services::PortBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -153,7 +153,7 @@ module Mihari
       def enrich_cpes(enricher = Enrichers::Shodan.new)
         return unless can_enrich_cpes?
 
-        self.cpes = CPE.build_by_ip(data, enricher: enricher)
+        self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
       end
 
       #
@@ -225,10 +225,16 @@ module Mihari
         @shodan ||= Enrichers::Shodan.new
       end
 
-      def normalize_as_domain(url_or_domain)
-        return url_or_domain if data_type == "domain"
-
-        Addressable::URI.parse(url_or_domain).host
+      #
+      # @return [String, nil]
+      #
+      def domain
+        case data_type
+        when "domain"
+          data
+        when "url"
+          Addressable::URI.parse(data).host
+        end
       end
 
       def can_enrich_whois?

data/lib/mihari/models/autonomous_system.rb

@@ -7,22 +7,6 @@ module Mihari
     #
     class AutonomousSystem < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        #
-        # Build AS
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::MMDB] enricher
-        #
-        # @return [Mihari::AutonomousSystem, nil]
-        #
-        def build_by_ip(ip, enricher: Enrichers::MMDB.new)
-          enricher.result(ip).fmap do |res|
-            new(asn: res.asn) if res.asn
-          end.value_or nil
-        end
-      end
     end
   end
 end

data/lib/mihari/models/cpe.rb

@@ -7,22 +7,6 @@ module Mihari
     #
     class CPE < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        #
-        # Build CPEs
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::Shodan] enricher
-        #
-        # @return [Array<Mihari::CPE>]
-        #
-        def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          enricher.result(ip).fmap do |res|
-            (res&.cpes || []).map { |cpe| new(cpe: cpe) }
-          end.value_or []
-        end
-      end
     end
   end
 end

data/lib/mihari/models/dns.rb

@@ -7,22 +7,6 @@ module Mihari
     #
     class DnsRecord < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        #
-        # Build DNS records
-        #
-        # @param [String] domain
-        # @param [Mihari::Enrichers::Shodan] enricher
-        #
-        # @return [Array<Mihari::Models::DnsRecord>]
-        #
-        def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
-          enricher.result(domain).fmap do |res|
-            res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
-          end.value_or([])
-        end
-      end
     end
   end
 end

data/lib/mihari/models/geolocation.rb

@@ -9,25 +9,6 @@ module Mihari
     #
     class Geolocation < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        #
-        # Build Geolocation
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::MMDB] enricher
-        #
-        # @return [Mihari::Geolocation, nil]
-        #
-        def build_by_ip(ip, enricher: Enrichers::MMDB.new)
-          enricher.result(ip).fmap do |res|
-            if res.country_code
-              new(country: NormalizeCountry(res.country_code, to: :short),
-                country_code: res.country_code)
-            end
-          end.value_or nil
-        end
-      end
     end
   end
 end

data/lib/mihari/models/reverse_dns.rb

@@ -7,24 +7,6 @@ module Mihari
     #
     class ReverseDnsName < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        include Dry::Monads[:result]
-
-        #
-        # Build reverse DNS names
-        #
-        # @param [String] ip
-        # @param [Mihari::Enrichers::Shodan] enricher
-        #
-        # @return [Array<Mihari::Models::ReverseDnsName>]
-        #
-        def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          enricher.result(ip).fmap do |res|
-            (res&.hostnames || []).map { |name| new(name: name) }
-          end.value_or []
-        end
-      end
     end
   end
 end

data/lib/mihari/models/whois.rb

@@ -7,20 +7,6 @@ module Mihari
     #
     class WhoisRecord < ActiveRecord::Base
       belongs_to :artifact
-
-      class << self
-        #
-        # Build whois record
-        #
-        # @param [String] domain
-        # @param [Mihari::Enrichers::Whois] enricher
-        #
-        # @return [WhoisRecord, nil]
-        #
-        def build_by_domain(domain, enricher: Enrichers::Whois.new)
-          enricher.result(domain).value_or nil
-        end
-      end
     end
   end
 end

data/lib/mihari/schemas/analyzer.rb

@@ -91,6 +91,7 @@ module Mihari
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.class_keys))
         required(:query).value(:string)
         optional(:exclude_expired).value(:bool).default(true)
+        optional(:match).value(Types::String.enum("=", "ILIKE", "LIKE", "single", "any", "FTS")).default(nil)
         optional(:options).hash(AnalyzerOptions)
       end
 

data/lib/mihari/services/builders.rb

@@ -6,35 +6,152 @@ module Mihari
     # Rule builder
     #
     class RuleBuilder < Service
-      # @return [String]
-      attr_reader :path_or_id
-
       #
       # @param [String] path_or_id
       #
-      # @return [Hash]
+      # @return [Mihari::Rule]
       #
-      def data
-        result = Try { Mihari::Models::Rule.find path_or_id }.to_result
-        return result.value! if result.success?
+      def call(path_or_id)
+        res = Try { Rule.from_model Mihari::Models::Rule.find(path_or_id) }
+        return res.value! if res.value?
 
         raise ArgumentError, "#{path_or_id} not found" unless Pathname(path_or_id).exist?
 
-        YAML.safe_load(
-          ERB.new(File.read(path_or_id)).result,
-          permitted_classes: [Date, Symbol]
-        )
+        Rule.from_yaml ERB.new(File.read(path_or_id)).result
       end
+    end
 
+    #
+    # Autonomous system builder
+    #
+    class AutonomousSystemBuilder < Service
       #
-      # @param [String] path_or_id
+      # @param [String] ip
+      # @param [Mihari::Enrichers::MMDB] enricher
       #
-      # @return [Mihari::Rule]
+      # @return [Mihari::Models::AutonomousSystem, nil]
       #
-      def call(path_or_id)
-        @path_or_id = path_or_id
+      def call(ip, enricher: Enrichers::MMDB.new)
+        enricher.result(ip).fmap do |res|
+          Models::AutonomousSystem.new(asn: res.asn) if res.asn
+        end.value_or nil
+      end
+    end
+
+    #
+    # CPE builder
+    #
+    class CPEBuilder < Service
+      #
+      # Build CPEs
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::CPE>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.cpes || []).map { |cpe| Models::CPE.new(cpe: cpe) }
+        end.value_or []
+      end
+    end
+
+    #
+    # DNS record builder
+    #
+    class DnsRecordBuilder < Service
+      #
+      # Build DNS records
+      #
+      # @param [String] domain
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::DnsRecord>]
+      #
+      def call(domain, enricher: Enrichers::GooglePublicDNS.new)
+        enricher.result(domain).fmap do |res|
+          res.answers.map { |answer| Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) }
+        end.value_or []
+      end
+    end
+
+    #
+    # Geolocation builder
+    #
+    class GeolocationBuilder < Service
+      #
+      # Build Geolocation
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::MMDB] enricher
+      #
+      # @return [Mihari::Models::Geolocation, nil]
+      #
+      def call(ip, enricher: Enrichers::MMDB.new)
+        enricher.result(ip).fmap do |res|
+          if res.country_code
+            Models::Geolocation.new(
+              country: NormalizeCountry(res.country_code, to: :short),
+              country_code: res.country_code
+            )
+          end
+        end.value_or nil
+      end
+    end
+
+    #
+    # Port builder
+    #
+    class PortBuilder < Service
+      #
+      # Build ports
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::Port>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.ports || []).map { |port| Models::Port.new(port: port) }
+        end.value_or []
+      end
+    end
 
-        Rule.new(**data)
+    #
+    # Reverse DNS name builder
+    #
+    class ReverseDnsNameBuilder < Service
+      #
+      # Build reverse DNS names
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::ReverseDnsName>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.hostnames || []).map { |name| Models::ReverseDnsName.new(name: name) }
+        end.value_or []
+      end
+    end
+
+    #
+    # Whois record builder
+    #
+    class WhoisRecordBuilder < Service
+      #
+      # Build whois record
+      #
+      # @param [String] domain
+      # @param [Mihari::Enrichers::Whois] enricher
+      #
+      # @return [Mihari::Models::WhoisRecord, nil]
+      #
+      def call(domain, enricher: Enrichers::Whois.new)
+        enricher.result(domain).value_or nil
       end
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "7.0.0"
+  VERSION = "7.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.0.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-31 00:00:00.000000000 Z
+date: 2024-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: better_errors