mihari 3.9.0 → 3.9.1

data/lib/mihari.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
+require "awrence"
 require "colorize"
 require "dry/configurable"
 require "dry/files"
 require "mem"
+require "plissken"
 require "yaml"
 
 # Load .env
@@ -135,18 +137,6 @@ require "mihari/models/tag"
 require "mihari/models/tagging"
 require "mihari/models/whois"
 
-# Serializers
-require "mihari/serializers/autonomous_system"
-require "mihari/serializers/dns"
-require "mihari/serializers/geolocation"
-require "mihari/serializers/reverse_dns"
-require "mihari/serializers/tag"
-require "mihari/serializers/whois"
-
-require "mihari/serializers/artifact"
-
-require "mihari/serializers/alert"
-
 # Analyzers
 require "mihari/analyzers/base"
 require "mihari/analyzers/basic"

data/mihari.gemspec

@@ -42,7 +42,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_dependency "active_model_serializers", "~> 0.10"
   spec.add_dependency "activerecord", "~> 6.1"
   spec.add_dependency "activerecord-filter", "~> 6.1"
   spec.add_dependency "addressable", "~> 2.8"
@@ -61,6 +60,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-struct", "~> 1.4"
   spec.add_dependency "dry-validation", "~> 1.7"
   spec.add_dependency "email_address", "~> 0.2"
+  spec.add_dependency "grape", "~> 1.5"
+  spec.add_dependency "grape-entity", "~> 0.10"
+  spec.add_dependency "grape-swagger", "~> 1.4"
+  spec.add_dependency "grape-swagger-entity", "~> 0.5"
   spec.add_dependency "hachi", "~> 1.0"
   spec.add_dependency "http", "~> 5.0"
   spec.add_dependency "launchy", "~> 2.5"
@@ -71,20 +74,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency "normalize_country", "0.3"
   spec.add_dependency "onyphe", "~> 2.0"
   spec.add_dependency "otx_ruby", "~> 0.9"
-  spec.add_dependency "parallel", "~> 1.20"
+  spec.add_dependency "parallel", "~> 1.21"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
+  spec.add_dependency "plissken", "~> 1.4"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
   spec.add_dependency "puma", "~> 5.5"
   spec.add_dependency "rack", "~> 2.2"
   spec.add_dependency "rack-contrib", "~> 2.3"
+  spec.add_dependency "rack-cors", "~> 1.1"
   spec.add_dependency "safe_shell", "~> 1.1"
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
-  spec.add_dependency "sinatra", "~> 2.1"
-  spec.add_dependency "sinatra-contrib", "~> 2.1"
-  spec.add_dependency "sinatra-param", "~> 1.6"
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"
   spec.add_dependency "sqlite3", "~> 1.4"

data/sig/lib/mihari/web/app.rbs

@@ -1,5 +1,5 @@
 module Mihari
-  class App # < Sinatra::Base
+  class App
     def self.run!: (?port: ::Integer port, ?host: ::String host, ?threads: ::String threads, ?verbose: bool verbose) -> void
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 3.9.0
+  version: 3.9.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-23 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -234,20 +234,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.14'
-- !ruby/object:Gem::Dependency
-  name: active_model_serializers
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.10'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -500,6 +486,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: grape
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: grape-entity
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: grape-swagger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: grape-swagger-entity
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: hachi
   requirement: !ruby/object:Gem::Requirement
@@ -646,14 +688,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.20'
+        version: '1.21'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.20'
+        version: '1.21'
 - !ruby/object:Gem::Dependency
   name: passive_circl
   requirement: !ruby/object:Gem::Requirement
@@ -682,6 +724,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: plissken
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
@@ -753,7 +809,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.3'
 - !ruby/object:Gem::Dependency
-  name: safe_shell
+  name: rack-cors
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -767,75 +823,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: securitytrails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: shodanx
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
-- !ruby/object:Gem::Dependency
-  name: sinatra
+  name: safe_shell
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: sinatra-contrib
+  name: securitytrails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: sinatra-param
+  name: shodanx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -1112,14 +1140,6 @@ files:
 - lib/mihari/schemas/configuration.rb
 - lib/mihari/schemas/macros.rb
 - lib/mihari/schemas/rule.rb
-- lib/mihari/serializers/alert.rb
-- lib/mihari/serializers/artifact.rb
-- lib/mihari/serializers/autonomous_system.rb
-- lib/mihari/serializers/dns.rb
-- lib/mihari/serializers/geolocation.rb
-- lib/mihari/serializers/reverse_dns.rb
-- lib/mihari/serializers/tag.rb
-- lib/mihari/serializers/whois.rb
 - lib/mihari/status.rb
 - lib/mihari/structs/alert.rb
 - lib/mihari/structs/censys.rb
@@ -1131,17 +1151,29 @@ files:
 - lib/mihari/type_checker.rb
 - lib/mihari/types.rb
 - lib/mihari/version.rb
+- lib/mihari/web/api.rb
 - lib/mihari/web/app.rb
-- lib/mihari/web/controllers/alerts_controller.rb
-- lib/mihari/web/controllers/analyzers_controller.rb
-- lib/mihari/web/controllers/artifacts_controller.rb
-- lib/mihari/web/controllers/base_controller.rb
-- lib/mihari/web/controllers/command_controller.rb
-- lib/mihari/web/controllers/config_controller.rb
-- lib/mihari/web/controllers/ip_address_controller.rb
-- lib/mihari/web/controllers/sources_controller.rb
-- lib/mihari/web/controllers/tags_controller.rb
-- lib/mihari/web/helpers/json.rb
+- lib/mihari/web/endpoints/alerts.rb
+- lib/mihari/web/endpoints/artifacts.rb
+- lib/mihari/web/endpoints/command.rb
+- lib/mihari/web/endpoints/configs.rb
+- lib/mihari/web/endpoints/ip_addresses.rb
+- lib/mihari/web/endpoints/sources.rb
+- lib/mihari/web/endpoints/tags.rb
+- lib/mihari/web/entities/alert.rb
+- lib/mihari/web/entities/artifact.rb
+- lib/mihari/web/entities/autonomous_system.rb
+- lib/mihari/web/entities/command.rb
+- lib/mihari/web/entities/config.rb
+- lib/mihari/web/entities/dns.rb
+- lib/mihari/web/entities/geolocation.rb
+- lib/mihari/web/entities/ip_address.rb
+- lib/mihari/web/entities/message.rb
+- lib/mihari/web/entities/reverse_dns.rb
+- lib/mihari/web/entities/source.rb
+- lib/mihari/web/entities/tag.rb
+- lib/mihari/web/entities/whois.rb
+- lib/mihari/web/public/grape.rb
 - lib/mihari/web/public/index.html
 - lib/mihari/web/public/redoc-static.html
 - lib/mihari/web/public/static/favicon.ico
@@ -1181,6 +1213,8 @@ files:
 - lib/mihari/web/public/static/js/app.365f1907.js.map
 - lib/mihari/web/public/static/js/app.378da3dc.js
 - lib/mihari/web/public/static/js/app.378da3dc.js.map
+- lib/mihari/web/public/static/js/app.6b636b62.js
+- lib/mihari/web/public/static/js/app.6b636b62.js.map
 - lib/mihari/web/public/static/js/app.8e3e5150.js
 - lib/mihari/web/public/static/js/app.8e3e5150.js.map
 - lib/mihari/web/public/static/js/app.a862ebca.js

data/lib/mihari/serializers/alert.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class AlertSerializer < ActiveModel::Serializer
-      attributes :id, :title, :description, :source, :created_at
-
-      has_many :artifacts, serializer: ArtifactSerializer
-      has_many :tags, through: :taggings, serializer: TagSerializer
-    end
-  end
-end

data/lib/mihari/serializers/artifact.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class ArtifactSerializer < ActiveModel::Serializer
-      attributes :id, :data, :data_type, :source
-
-      has_one :autonomous_system, serializer: AutonomousSystemSerializer
-      has_one :geolocation, serializer: GeolocationSerializer
-      has_one :whois_record, serializer: WhoisRecordSerializer
-
-      has_many :dns_records, serializer: DnsRecordSerializer
-      has_many :reverse_dns_names, serializer: ReverseDnsNameSerializer
-    end
-  end
-end

data/lib/mihari/serializers/autonomous_system.rb

@@ -1,9 +0,0 @@
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class AutonomousSystemSerializer < ActiveModel::Serializer
-      attributes :asn
-    end
-  end
-end

data/lib/mihari/serializers/dns.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class DnsRecordSerializer < ActiveModel::Serializer
-      attributes :resource, :value
-    end
-  end
-end

data/lib/mihari/serializers/geolocation.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class GeolocationSerializer < ActiveModel::Serializer
-      attributes :country, :country_code
-    end
-  end
-end

data/lib/mihari/serializers/reverse_dns.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class ReverseDnsNameSerializer < ActiveModel::Serializer
-      attributes :name
-    end
-  end
-end

data/lib/mihari/serializers/tag.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class TagSerializer < ActiveModel::Serializer
-      attributes :id, :name
-    end
-  end
-end

data/lib/mihari/serializers/whois.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "active_model_serializers"
-
-module Mihari
-  module Serializers
-    class WhoisRecordSerializer < ActiveModel::Serializer
-      attributes :domain, :created_on, :updated_on, :expires_on, :registrar, :contacts
-    end
-  end
-end

data/lib/mihari/web/controllers/alerts_controller.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Controllers
-    class AlertsController < BaseController
-      get "/api/alerts" do
-        param :page, Integer
-        param :artifact, String
-        param :description, String
-        param :source, String
-        param :tag, String
-
-        param :from_at, DateTime
-        param :fromAt, DateTime
-        param :to_at, DateTime
-        param :toAt, DateTime
-
-        param :asn, Integer
-        param :dns_record, String
-        param :dnsRecord, String
-        param :reverse_dns_name, String
-        param :reverseDnsName, String
-
-        # set page & limit
-        page = params["page"] || 1
-        params["page"] = page.to_i
-
-        limit = 10
-        params["limit"] = 10
-
-        # normalize keys
-        params["artifact_data"] = params["artifact"]
-        params["from_at"] = params["from_at"] || params["fromAt"]
-        params["to_at"] = params["to_at"] || params["toAt"]
-        params["dns_record"] = params["dns_record"] || params["dnsRecord"]
-        params["reverse_dns_name"] = params["reverse_dns_name"] || params["reverseDnsName"]
-
-        # symbolize hash keys
-        filter = params.to_h.transform_keys(&:to_sym)
-
-        search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
-        alerts = Mihari::Alert.search(search_filter_with_pagenation)
-        total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)
-
-        json({ alerts: alerts, total: total, current_page: page, page_size: limit })
-      end
-
-      delete "/api/alerts/:id" do
-        param :id, Integer, required: true
-
-        id = params["id"].to_i
-
-        begin
-          alert = Mihari::Alert.find(id)
-          alert.destroy
-
-          status 204
-          body ""
-        rescue ActiveRecord::RecordNotFound
-          status 404
-
-          json({ message: "ID:#{id} is not found" })
-        end
-      end
-    end
-  end
-end

data/lib/mihari/web/controllers/analyzers_controller.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Controllers
-    class AnalyzersController < BaseController
-      post "/api/analyzer" do
-        contract = Mihari::Schemas::AnalyzerRunContract.new
-        result = contract.call(params)
-
-        unless result.errors.empty?
-          status 400
-
-          return json(result.errors.to_h)
-        end
-
-        args = result.to_h
-
-        ignore_old_artifacts = args[:ignoreOldArtifacts]
-        ignore_threshold = args[:ignoreThreshold]
-
-        analyzer = Mihari::Analyzers::Basic.new(
-          title: args[:title],
-          description: args[:description],
-          source: args[:source],
-          artifacts: args[:artifacts],
-          tags: args[:tags]
-        )
-        analyzer.ignore_old_artifacts = ignore_old_artifacts
-        analyzer.ignore_threshold = ignore_threshold
-
-        analyzer.run
-
-        status 201
-        body ""
-      end
-    end
-  end
-end

data/lib/mihari/web/controllers/artifacts_controller.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Controllers
-    class ArtifactsController < BaseController
-      get "/api/artifacts/:id" do
-        param :id, Integer, required: true
-
-        id = params["id"].to_i
-
-        begin
-          artifact = Mihari::Artifact.includes(
-            :autonomous_system,
-            :geolocation,
-            :whois_record,
-            :dns_records,
-            :reverse_dns_names
-          ).find(id)
-        rescue ActiveRecord::RecordNotFound
-          status 404
-
-          return json({ message: "ID:#{id} is not found" })
-        end
-
-        # TODO: improve queries
-        alert_ids = Mihari::Artifact.where(data: artifact.data).pluck(:alert_id)
-        tag_ids = Mihari::Tagging.where(alert_id: alert_ids).pluck(:tag_id)
-        tag_names = Mihari::Tag.where(id: tag_ids).distinct.pluck(:name)
-
-        artifact_json = Serializers::ArtifactSerializer.new(artifact).as_json
-
-        # convert reverse DNS names into an array of string
-        # also change it as nil if it is empty
-        reverse_dns_names = (artifact_json[:reverse_dns_names] || []).filter_map { |v| v[:name] }
-        reverse_dns_names = nil if reverse_dns_names.empty?
-        artifact_json[:reverse_dns_names] = reverse_dns_names
-
-        # change DNS records as nil if it is empty
-        dns_records = artifact_json[:dns_records] || []
-        dns_records = nil if dns_records.empty?
-        artifact_json[:dns_records] = dns_records
-
-        # set tags
-        artifact_json[:tags] = tag_names
-
-        json artifact_json
-      end
-
-      get "/api/artifacts/:id/enrich" do
-        param :id, Integer, required: true
-
-        id = params["id"].to_i
-
-        begin
-          artifact = Mihari::Artifact.includes(
-            :autonomous_system,
-            :geolocation,
-            :whois_record,
-            :dns_records,
-            :reverse_dns_names
-          ).find(id)
-        rescue ActiveRecord::RecordNotFound
-          status 404
-
-          return json({ message: "ID:#{id} is not found" })
-        end
-
-        artifact.enrich_all
-        artifact.save
-
-        status 201
-        body ""
-      end
-
-      delete "/api/artifacts/:id" do
-        param :id, Integer, required: true
-
-        id = params["id"].to_i
-
-        begin
-          alert = Mihari::Artifact.find(id)
-          alert.destroy
-
-          status 204
-          body ""
-        rescue ActiveRecord::RecordNotFound
-          status 404
-
-          json({ message: "ID:#{id} is not found" })
-        end
-      end
-    end
-  end
-end