mihari 5.7.0 → 5.7.2

data/lib/mihari.rb

@@ -53,7 +53,6 @@ require "mihari/config"
 require "mihari/mixins/autonomous_system"
 require "mihari/mixins/configurable"
 require "mihari/mixins/falsepositive"
-require "mihari/mixins/error_notification"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
 
@@ -264,7 +263,6 @@ require "mihari/services/rule_builder"
 require "mihari/services/rule_runner"
 
 require "mihari/services/alert_builder"
-require "mihari/services/alert_proxy"
 require "mihari/services/alert_runner"
 
 # Entities

data/mihari.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|images|docker|.github)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(.github|.vscode|docs|docker|frontend|images|spec|)/}) }
   end
   # Include frontend assets in lib/mihari/web/public
   spec.files += Dir.glob("lib/mihari/web/public/**/*")
@@ -67,7 +67,7 @@ Gem::Specification.new do |spec|
     spec.add_development_dependency "solargraph", "~> 0.49"
   end
 
-  spec.add_dependency "activerecord", "7.1.1"
+  spec.add_dependency "activerecord", "7.1.2"
   spec.add_dependency "addressable", "2.8.5"
   spec.add_dependency "anyway_config", "2.5.4"
   spec.add_dependency "awrence", "2.0.1"
@@ -95,13 +95,13 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack", "3.0.8"
   spec.add_dependency "rack-cors", "2.0.1"
   spec.add_dependency "rackup", "2.1.0"
-  spec.add_dependency "semantic_logger", "4.14.0"
-  spec.add_dependency "sentry-ruby", "5.12.0"
+  spec.add_dependency "semantic_logger", "4.15.0"
+  spec.add_dependency "sentry-ruby", "5.13.0"
   spec.add_dependency "slack-notifier", "2.4.0"
   spec.add_dependency "sqlite3", "1.6.8"
   spec.add_dependency "thor", "1.3.0"
   spec.add_dependency "thor-hollaback", "0.2.1"
   spec.add_dependency "uuidtools", "2.2.0"
-  spec.add_dependency "whois", "5.1.0"
+  spec.add_dependency "whois", "5.1.1"
   spec.add_dependency "whois-parser", "2.0.0"
 end

data/mkdocs.yml

@@ -16,13 +16,16 @@ nav:
   - Usage: usage.md
   - Configuration: configuration.md
   - Tips:
-      - GitHub Actions: github_actions.md
-      - Alternatives: alternatives.md
+      - Docker: tips/docker.md
+      - GitHub Actions: tips/github_actions.md
+      - Superset: tips/superset.md
+      - Alternatives: tips/alternatives.md
   - References:
-      - Analyzers: "analyzers/index.md"
-      - Enrichers: "enrichers/index.md"
-      - Emitters: "emitters/index.md"
-      - Tags: "./tags.md"
+      - Analyzers: analyzers/index.md
+      - Enrichers: enrichers/index.md
+      - Emitters: emitters/index.md
+      - Database: database.md
+      - Tags: tags.md
 
 markdown_extensions:
   - toc:
@@ -32,4 +35,8 @@ markdown_extensions:
   - pymdownx.extra
   - pymdownx.highlight
   - pymdownx.magiclink
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 5.7.0
+  version: 5.7.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-03 00:00:00.000000000 Z
+date: 2023-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -324,14 +324,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.1.1
+        version: 7.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.1.1
+        version: 7.1.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -716,28 +716,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.14.0
+        version: 4.15.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.14.0
+        version: 4.15.0
 - !ruby/object:Gem::Dependency
   name: sentry-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.12.0
+        version: 5.13.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.12.0
+        version: 5.13.0
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -814,14 +814,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: 5.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: 5.1.1
 - !ruby/object:Gem::Dependency
   name: whois-parser
   requirement: !ruby/object:Gem::Requirement
@@ -856,133 +856,7 @@ files:
 - bin/setup
 - build_frontend.sh
 - config.ru
-- docs/alternatives.md
-- docs/analyzers/binaryedge.md
-- docs/analyzers/censys.md
-- docs/analyzers/circl.md
-- docs/analyzers/crtsh.md
-- docs/analyzers/dnstwister.md
-- docs/analyzers/feed.md
-- docs/analyzers/fofa.md
-- docs/analyzers/greynoise.md
-- docs/analyzers/hunterhow.md
-- docs/analyzers/index.md
-- docs/analyzers/onyphe.md
-- docs/analyzers/otx.md
-- docs/analyzers/passivetotal.md
-- docs/analyzers/pulsedive.md
-- docs/analyzers/securitytrails.md
-- docs/analyzers/shodan.md
-- docs/analyzers/urlscan.md
-- docs/analyzers/virustotal.md
-- docs/analyzers/virustotal_intelligence.md
-- docs/analyzers/zoomeye.md
-- docs/configuration.md
-- docs/emitters/database.md
-- docs/emitters/hive.md
-- docs/emitters/index.md
-- docs/emitters/misp.md
-- docs/emitters/slack.md
-- docs/emitters/webhook.md
-- docs/enrichers/google_public_dns.md
-- docs/enrichers/index.md
-- docs/enrichers/ipinfo.md
-- docs/enrichers/shodan.md
-- docs/enrichers/whois.md
-- docs/github_actions.md
-- docs/index.md
-- docs/installation.md
-- docs/requirements.md
-- docs/rule.md
-- docs/tags.md
-- docs/usage.md
 - exe/mihari
-- frontend/.eslintrc.cjs
-- frontend/.gitignore
-- frontend/.prettierrc.json
-- frontend/README.md
-- frontend/env.d.ts
-- frontend/index.html
-- frontend/package-lock.json
-- frontend/package.json
-- frontend/public/favicon.ico
-- frontend/scripts/swagger_doc_to_yaml.rb
-- frontend/src/App.vue
-- frontend/src/ace-config.ts
-- frontend/src/api-helper.ts
-- frontend/src/api.ts
-- frontend/src/components/ErrorMessage.vue
-- frontend/src/components/Loading.vue
-- frontend/src/components/Navbar.vue
-- frontend/src/components/Pagination.vue
-- frontend/src/components/alert/Alert.vue
-- frontend/src/components/alert/Alerts.vue
-- frontend/src/components/alert/AlertsWithPagination.vue
-- frontend/src/components/alert/AlertsWrapper.vue
-- frontend/src/components/alert/Form.vue
-- frontend/src/components/artifact/AS.vue
-- frontend/src/components/artifact/Artifact.vue
-- frontend/src/components/artifact/ArtifactTag.vue
-- frontend/src/components/artifact/ArtifactTags.vue
-- frontend/src/components/artifact/ArtifactWrapper.vue
-- frontend/src/components/artifact/CPEs.vue
-- frontend/src/components/artifact/DnsRecords.vue
-- frontend/src/components/artifact/Ports.vue
-- frontend/src/components/artifact/ReverseDnsNames.vue
-- frontend/src/components/artifact/Tags.vue
-- frontend/src/components/artifact/WhoisRecord.vue
-- frontend/src/components/config/Configs.vue
-- frontend/src/components/config/ConfigsWrapper.vue
-- frontend/src/components/link/Link.vue
-- frontend/src/components/link/Links.vue
-- frontend/src/components/rule/EditRule.vue
-- frontend/src/components/rule/EditRuleWrapper.vue
-- frontend/src/components/rule/Form.vue
-- frontend/src/components/rule/InputForm.vue
-- frontend/src/components/rule/NewRule.vue
-- frontend/src/components/rule/Rule.vue
-- frontend/src/components/rule/RuleWrapper.vue
-- frontend/src/components/rule/Rules.vue
-- frontend/src/components/rule/RulesWrapper.vue
-- frontend/src/components/rule/YAML.vue
-- frontend/src/components/tag/Tag.vue
-- frontend/src/components/tag/Tags.vue
-- frontend/src/countries.ts
-- frontend/src/index.ts
-- frontend/src/links/anyrun.ts
-- frontend/src/links/base.ts
-- frontend/src/links/censys.ts
-- frontend/src/links/crtsh.ts
-- frontend/src/links/dnslytics.ts
-- frontend/src/links/greynoise.ts
-- frontend/src/links/index.ts
-- frontend/src/links/intezer.ts
-- frontend/src/links/otx.ts
-- frontend/src/links/securitytrails.ts
-- frontend/src/links/shodan.ts
-- frontend/src/links/urlscan.ts
-- frontend/src/links/virustotal.ts
-- frontend/src/main.ts
-- frontend/src/router/index.ts
-- frontend/src/rule.ts
-- frontend/src/shims-vue.d.ts
-- frontend/src/swagger.yaml
-- frontend/src/types.ts
-- frontend/src/utils.ts
-- frontend/src/views/Alerts.vue
-- frontend/src/views/Artifact.vue
-- frontend/src/views/Configs.vue
-- frontend/src/views/EditRule.vue
-- frontend/src/views/NewRule.vue
-- frontend/src/views/Rule.vue
-- frontend/src/views/Rules.vue
-- frontend/tests/utils.spec.ts
-- frontend/tsconfig.app.json
-- frontend/tsconfig.json
-- frontend/tsconfig.node.json
-- frontend/tsconfig.vitest.json
-- frontend/vite.config.ts
-- frontend/vitest.config.ts
 - lefthook.yml
 - lib/mihari.rb
 - lib/mihari/actor.rb
@@ -1033,6 +907,7 @@ files:
 - lib/mihari/clients/zoomeye.rb
 - lib/mihari/commands/alert.rb
 - lib/mihari/commands/database.rb
+- lib/mihari/commands/mixins.rb
 - lib/mihari/commands/rule.rb
 - lib/mihari/commands/search.rb
 - lib/mihari/commands/version.rb
@@ -1071,7 +946,6 @@ files:
 - lib/mihari/http.rb
 - lib/mihari/mixins/autonomous_system.rb
 - lib/mihari/mixins/configurable.rb
-- lib/mihari/mixins/error_notification.rb
 - lib/mihari/mixins/falsepositive.rb
 - lib/mihari/mixins/refang.rb
 - lib/mihari/mixins/retriable.rb
@@ -1098,7 +972,6 @@ files:
 - lib/mihari/schemas/rule.rb
 - lib/mihari/service.rb
 - lib/mihari/services/alert_builder.rb
-- lib/mihari/services/alert_proxy.rb
 - lib/mihari/services/alert_runner.rb
 - lib/mihari/services/rule_builder.rb
 - lib/mihari/services/rule_runner.rb
@@ -1129,7 +1002,7 @@ files:
 - lib/mihari/web/middleware/connection_adapter.rb
 - lib/mihari/web/middleware/error_notification_adapter.rb
 - lib/mihari/web/public/assets/index-56fc2187.css
-- lib/mihari/web/public/assets/index-821134e2.js
+- lib/mihari/web/public/assets/index-ec641cb0.js
 - lib/mihari/web/public/assets/mode-yaml-24faa242.js
 - lib/mihari/web/public/favicon.ico
 - lib/mihari/web/public/index.html
@@ -1158,7 +1031,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.20
+rubygems_version: 3.4.21
 signing_key:
 specification_version: 4
 summary: A query aggregator for OSINT based threat hunting

data/docs/alternatives.md

@@ -1,5 +0,0 @@
-# Alternatives
-
-- [InQuest/ThreatIngestor](https://github.com/InQuest/ThreatIngestor) - Extract and aggregate threat intelligence.
-- [thalesgroup-cert/Watcher](https://github.com/thalesgroup-cert/Watcher) - Watcher - Open Source Cybersecurity Threat Hunting Platform. Developed with Django & React JS.
-- [projectdiscovery/uncover](https://github.com/projectdiscovery/uncover) - Quickly discover exposed hosts on the internet using multiple search engines.

data/docs/analyzers/binaryedge.md

@@ -1,26 +0,0 @@
----
-tags:
-  - Artifact:IP
----
-
-# BinaryEdge
-
-- [https://www.binaryedge.io/](https://www.binaryedge.io/)
-
-This analyzer uses [BinaryEdge API V2](https://docs.binaryedge.io/api-v2/) (`/v2/query/search`) to search. Pagination is supported.
-
-```yaml
-analyzer: binaryedge
-query: ...
-api_key: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a search query.
-
-### API Key
-
-`api_key` (`string`) is an API key. Optional. Defaults to `ENV[”BINARYEDGE_API_KEY"]`.

data/docs/analyzers/censys.md

@@ -1,31 +0,0 @@
----
-tags:
-  - Artifact:IP
----
-
-# Censys
-
-- [https://censys.io/](https://censys.io/)
-
-This analyzer uses [Censys Search 2.0 REST API](https://search.censys.io/api) to search. Pagination is supported.
-
-```yaml
-analyzer: censys
-query: ...
-id: ...
-secret: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a search query.
-
-### ID
-
-`id` (`string`) is a Cencys ID. Optional. Defaults to `ENV[”CENSYS_ID”]`.
-
-### Secret
-
-`secret` (`string`) is a Cencys secret. Optional. Defaults to `ENV[”CENSYS_SECRET”]`.

data/docs/analyzers/circl.md

@@ -1,37 +0,0 @@
----
-tags:
-  - Artifact:IP
-  - Passive DNS
-  - Passive SSL
----
-
-# CIRCL Passive DNS/SSL
-
-- [https://www.circl.lu/services/passive-dns/](https://www.circl.lu/services/passive-dns/)
-- [https://www.circl.lu/services/passive-ssl/](https://www.circl.lu/services/passive-ssl/)
-
-This analyzer uses CIRCL passive DNS API or passive SSL API:
-
-- Use passive DNS API if a query(input) is a domain
-- Use passive SSL API if a query(input) is a SHA1 certificate fingerprint
-
-```yaml
-analyzer: circl
-query: ...
-password: ...
-username: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a domain or SHA1 certificate fingerprint.
-
-### Username
-
-`username` (`string`) is a username. Optional. Defaults to `ENV[”CIRCL_PASSIVE_USERNAME”]`.
-
-### Password
-
-`password` (`string`) is a password. Optional. Defaults to `ENV[”CIRCL_PASSIVE_PASSWORD”]`.

data/docs/analyzers/crtsh.md

@@ -1,26 +0,0 @@
----
-tags:
-  - Artifact:Domain
----
-
-# crt.sh
-
-- [https://crt.sh/](https://crt.sh/)
-
-This analyzer uses [crt.sh](http://crt.sh)'s (unofficial?) REST API.
-
-```yaml
-analyzer: crtsh
-query: ...
-exclude_expired: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a search query.
-
-### Exclude Expired
-
-`exclude_expired` (`boolean`) determines whether to exclude expired domains or not. Optional. Defaults to `true`.

data/docs/analyzers/dnstwister.md

@@ -1,25 +0,0 @@
----
-tags:
-  - Artifact:Domain
----
-
-# dnstwister
-
-- [https://dnstwister.report/](https://dnstwister.report/)
-
-This analyzer uses [dnstwister API](https://dnstwister.report/api/) to search.
-
-```yaml
-analyzer: dnstwister
-query: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a search query.
-
-!!! tip
-
-    There is no need to input a domain in hexadecimal format. This analyzer automatically converts a domain (in string format) into a hexadecimal value.

data/docs/analyzers/feed.md

@@ -1,73 +0,0 @@
-# Feed
-
-This analyzer can ingest a feed (JSON or CSV) by specifying conditions.
-
-Note that you should write a selector to get proper IoCs from a feed. A selector is based on [jr](https://github.com/yuya-takeyama/jr).
-
-```yaml
-analyzer: feed
-query: ...
-selector: ...
-method: ...
-headers: ...
-params: ...
-data: ...
-json: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a URL of a feed.
-
-!!! note
-
-    I know this is a strange naming. It's just for keeping the convention with other analyzers.
-
-### Method
-
-`method` (`string`) is an HTTP method. Defaults to `GET`.
-
-### Selector
-
-`selector` (`string`) is a `jr` selector.
-
-### Headers
-
-`headers` (`hash`) is an HTTP headers. Optional.
-
-### Params
-
-`params` (`hash`) is an HTTP query params. Optional.
-
-### Data
-
-`data` (`hash`) is an HTTP form data. Optional.
-
-### JSON
-
-`json` (`hash`) is an JSON body. Optional.
-
-## Examples
-
-### ThreatFox
-
-```yaml
-analyzer: feed
-query: "https://threatfox-api.abuse.ch/api/v1/"
-method: POST
-json:
-  query: get_iocs
-  days: 1
-headers:
-selector: "map(&:data).unwrap.map(&:ioc).map { |v| v.start_with?('http://', 'https://') ? v :  v.split(':').first }"
-```
-
-### URLhaus
-
-```yaml
-analyzer: feed
-query: "https://urlhaus.abuse.ch/feeds/country/JP/"
-selector: "map { |v| v[1] }"
-```

data/docs/analyzers/fofa.md

@@ -1,31 +0,0 @@
----
-tags:
-  - Artifact:IP
----
-
-# Fofa
-
-- https://en.fofa.info/
-
-This analyzer uses Fofa API (`/api/v1/search/all`) to search. Pagination is supported.
-
-```yaml
-analyzer: fofa
-query: ...
-api_key: ...
-email: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a search query.
-
-### API Key
-
-`api_key` (`string`) is an API key. Optional. Defaults to `ENV[”FOFA_API_KEY"]`.
-
-### Email
-
-`email` (`string`) is an email. Optional. Defaults to `ENV[”FOFA_EMAIL"]`.

data/docs/analyzers/greynoise.md

@@ -1,26 +0,0 @@
----
-tags:
-  - Artifact:IP
----
-
-# GreyNoise
-
-- [https://www.greynoise.io/](https://www.greynoise.io/)
-
-This analyzer uses GreyNoise API (`/v2/experimental/gnql`) to search. Pagination is supported.
-
-```yaml
-analyzer: greynoise
-query: ...
-api_key: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a GNQL search query.
-
-### API Key
-
-`api_key` (`string`) is an API key. Optional. Defaults to `ENV[”GREYNOISE_API_KEY"]`.

data/docs/analyzers/hunterhow.md

@@ -1,33 +0,0 @@
----
-tags:
-  - Artifact:IP
----
-
-# Hunter How
-
-- [https://hunter.how/](https://hunter.how/)
-
-This analyzer uses Hunter How API (`https://api.hunter.how/search`) to search. Pagination is supported.
-
-```yaml
-analyzer: hunterhow
-query: ...
-api_key: ...
-start_time: ...
-end_time: ...
-```
-
-## Components
-
-### Query
-
-`query` (`string`) is a search query.
-
-### Start/End Time
-
-- `start_time` (`date`): Only show results after the given date.
-- `end_time` (`date`): Only show results after the given date.
-
-### API key
-
-`api_key` (`string`) is an API key. Optional. Defaults to `ENV[”HUNTERHOW_API_KEY"]`.