mihari 2.2.0 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +7 -0
  3. data/.overcommit.yml +12 -0
  4. data/README.md +7 -9
  5. data/exe/mihari +1 -1
  6. data/images/tines.png +0 -0
  7. data/lib/mihari.rb +89 -15
  8. data/lib/mihari/analyzers/base.rb +49 -8
  9. data/lib/mihari/analyzers/basic.rb +1 -2
  10. data/lib/mihari/analyzers/binaryedge.rb +7 -13
  11. data/lib/mihari/analyzers/censys.rb +26 -63
  12. data/lib/mihari/analyzers/circl.rb +20 -17
  13. data/lib/mihari/analyzers/crtsh.rb +6 -13
  14. data/lib/mihari/analyzers/dnpedia.rb +6 -12
  15. data/lib/mihari/analyzers/dnstwister.rb +13 -10
  16. data/lib/mihari/analyzers/onyphe.rb +6 -12
  17. data/lib/mihari/analyzers/otx.rb +22 -19
  18. data/lib/mihari/analyzers/passivetotal.rb +22 -21
  19. data/lib/mihari/analyzers/pulsedive.rb +16 -13
  20. data/lib/mihari/analyzers/rule.rb +99 -0
  21. data/lib/mihari/analyzers/securitytrails.rb +22 -19
  22. data/lib/mihari/analyzers/shodan.rb +7 -13
  23. data/lib/mihari/analyzers/spyse.rb +12 -19
  24. data/lib/mihari/analyzers/urlscan.rb +20 -30
  25. data/lib/mihari/analyzers/virustotal.rb +25 -22
  26. data/lib/mihari/analyzers/zoomeye.rb +16 -22
  27. data/lib/mihari/cli/analyzer.rb +44 -0
  28. data/lib/mihari/cli/base.rb +27 -0
  29. data/lib/mihari/cli/init.rb +13 -0
  30. data/lib/mihari/cli/main.rb +30 -0
  31. data/lib/mihari/cli/mixins/utils.rb +88 -0
  32. data/lib/mihari/cli/validator.rb +11 -0
  33. data/lib/mihari/commands/binaryedge.rb +1 -1
  34. data/lib/mihari/commands/censys.rb +1 -1
  35. data/lib/mihari/commands/circl.rb +2 -2
  36. data/lib/mihari/commands/crtsh.rb +1 -1
  37. data/lib/mihari/commands/dnpedia.rb +1 -1
  38. data/lib/mihari/commands/dnstwister.rb +2 -2
  39. data/lib/mihari/commands/init.rb +46 -0
  40. data/lib/mihari/commands/json.rb +1 -1
  41. data/lib/mihari/commands/onyphe.rb +1 -1
  42. data/lib/mihari/commands/otx.rb +2 -2
  43. data/lib/mihari/commands/passivetotal.rb +2 -2
  44. data/lib/mihari/commands/pulsedive.rb +2 -2
  45. data/lib/mihari/commands/search.rb +77 -0
  46. data/lib/mihari/commands/securitytrails.rb +2 -2
  47. data/lib/mihari/commands/shodan.rb +1 -1
  48. data/lib/mihari/commands/spyse.rb +1 -1
  49. data/lib/mihari/commands/urlscan.rb +2 -4
  50. data/lib/mihari/commands/validator.rb +38 -0
  51. data/lib/mihari/commands/virustotal.rb +2 -2
  52. data/lib/mihari/commands/zoomeye.rb +1 -1
  53. data/lib/mihari/constraints.rb +5 -0
  54. data/lib/mihari/database.rb +13 -2
  55. data/lib/mihari/emitters/base.rb +2 -2
  56. data/lib/mihari/emitters/database.rb +1 -1
  57. data/lib/mihari/emitters/misp.rb +1 -1
  58. data/lib/mihari/emitters/slack.rb +5 -9
  59. data/lib/mihari/emitters/the_hive.rb +1 -1
  60. data/lib/mihari/emitters/webhook.rb +53 -0
  61. data/lib/mihari/mixins/configurable.rb +38 -0
  62. data/lib/mihari/mixins/configuration.rb +85 -0
  63. data/lib/mihari/mixins/hash.rb +20 -0
  64. data/lib/mihari/mixins/refang.rb +21 -0
  65. data/lib/mihari/mixins/retriable.rb +27 -0
  66. data/lib/mihari/mixins/rule.rb +79 -0
  67. data/lib/mihari/models/alert.rb +28 -1
  68. data/lib/mihari/models/artifact.rb +10 -0
  69. data/lib/mihari/notifiers/base.rb +9 -1
  70. data/lib/mihari/notifiers/exception_notifier.rb +50 -0
  71. data/lib/mihari/notifiers/slack.rb +29 -1
  72. data/lib/mihari/schemas/configuration.rb +42 -0
  73. data/lib/mihari/schemas/macros.rb +17 -0
  74. data/lib/mihari/schemas/rule.rb +72 -0
  75. data/lib/mihari/serializers/artifact.rb +1 -1
  76. data/lib/mihari/status.rb +14 -0
  77. data/lib/mihari/templates/rule.yml.erb +19 -0
  78. data/lib/mihari/type_checker.rb +8 -3
  79. data/lib/mihari/version.rb +1 -1
  80. data/lib/mihari/web/app.rb +2 -0
  81. data/lib/mihari/web/controllers/alerts_controller.rb +12 -3
  82. data/lib/mihari/web/controllers/artifacts_controller.rb +1 -3
  83. data/lib/mihari/web/controllers/base_controller.rb +22 -0
  84. data/lib/mihari/web/controllers/command_controller.rb +3 -4
  85. data/lib/mihari/web/controllers/config_controller.rb +1 -3
  86. data/lib/mihari/web/controllers/sources_controller.rb +1 -3
  87. data/lib/mihari/web/controllers/tags_controller.rb +1 -3
  88. data/lib/mihari/web/helpers/json.rb +2 -0
  89. data/lib/mihari/web/public/index.html +1 -21
  90. data/lib/mihari/web/public/redoc-static.html +2 -2
  91. data/lib/mihari/web/public/static/js/app.ab213f7c.js +12 -0
  92. data/lib/mihari/web/public/static/js/app.ab213f7c.js.map +1 -0
  93. data/lib/mihari/web/public/static/js/{app.bcc595df.js → app.cccddb2b.js} +2 -2
  94. data/lib/mihari/web/public/static/js/app.cccddb2b.js.map +1 -0
  95. data/mihari.gemspec +20 -10
  96. metadata +180 -63
  97. data/.rubocop.yml +0 -161
  98. data/lib/mihari/analyzers/free_text.rb +0 -48
  99. data/lib/mihari/analyzers/http_hash.rb +0 -100
  100. data/lib/mihari/analyzers/passive_dns.rb +0 -59
  101. data/lib/mihari/analyzers/passive_ssl.rb +0 -55
  102. data/lib/mihari/analyzers/reverse_whois.rb +0 -55
  103. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +0 -59
  104. data/lib/mihari/analyzers/ssh_fingerprint.rb +0 -58
  105. data/lib/mihari/cli.rb +0 -124
  106. data/lib/mihari/commands/config.rb +0 -27
  107. data/lib/mihari/commands/free_text.rb +0 -21
  108. data/lib/mihari/commands/http_hash.rb +0 -25
  109. data/lib/mihari/commands/passive_dns.rb +0 -21
  110. data/lib/mihari/commands/passive_ssl.rb +0 -21
  111. data/lib/mihari/commands/reverse_whois.rb +0 -21
  112. data/lib/mihari/commands/securitytrails_domain_feed.rb +0 -23
  113. data/lib/mihari/commands/ssh_fingerprint.rb +0 -21
  114. data/lib/mihari/config.rb +0 -84
  115. data/lib/mihari/configurable.rb +0 -21
  116. data/lib/mihari/html.rb +0 -43
  117. data/lib/mihari/retriable.rb +0 -17
  118. data/lib/mihari/slack_monkeypatch.rb +0 -16
  119. data/lib/mihari/web/public/static/js/app.bcc595df.js.map +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c5859ab6b5d161359ea07c4af14de087013befa3b51d91844a60127137882e28
4
- data.tar.gz: 481f34619d14e04f3187a9d2c3a05b3a1d109c7809622accebc47b100a506bc4
3
+ metadata.gz: 513324945758cadacf94345cb8a2a410cab99a08abc0ad02148c7737660d7348
4
+ data.tar.gz: 5c031467f3a6da288c5fceb137c17102654de072b6b34b2374dd15c523d09a03
5
5
  SHA512:
6
- metadata.gz: 13693510f6f7d3560d207bb97dec50bb4851514e3b35918a82924e4e73d18e4bdcf963c3cd8928cd17559187b69419c8aa2a8e11e9d857965473dabfdf12ac59
7
- data.tar.gz: 8786ad5973687848a89c8737eb92e52dd636e912b6af0ddf339dccccbff28c42f55e4111d1967b1d9a7b8aca07618788e82d98c24a56ca54102b36c84231d21e
6
+ metadata.gz: bf28354ab901254085d6e055e9c7a4989ce3d75ede0a6e978c009e69eb8f8a60a198e3fe505b82b9c5eddfa29b1866cc51da4bb8b41cc1727352d5bacd04e4d1
7
+ data.tar.gz: fa111a82851f3e695ce1b5c7a582b030459e3dac3598cff1b4d943f03fea8bdd000d21f17434e080f80109d2f90a19d154e4291efced316bf9e3c2d59880f747
data/.gitignore CHANGED
@@ -57,3 +57,10 @@ Gemfile.lock
57
57
 
58
58
  # SQLite
59
59
  *.db
60
+
61
+ # Config
62
+ mihari.yml
63
+
64
+ # Rule
65
+ rule.yml
66
+ !lib/mihari/templates/rule.yml
data/.overcommit.yml ADDED
@@ -0,0 +1,12 @@
1
+ PreCommit:
2
+ BundleCheck:
3
+ enabled: true
4
+
5
+ RuboCop:
6
+ enabled: true
7
+ required_executable: bundle
8
+ command: ["bundle", "exec", "standardrb"]
9
+ on_warn: fail
10
+
11
+ YamlSyntax:
12
+ enabled: true
data/README.md CHANGED
@@ -8,6 +8,8 @@
8
8
 
9
9
  ![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
10
10
 
11
+ [![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
12
+
11
13
  Mihari is a framework for continuous OSINT based threat hunting.
12
14
 
13
15
  ## How it works
@@ -46,18 +48,14 @@ Mihari supports the following services by default.
46
48
  - [VirusTotal](http://virustotal.com)
47
49
  - [ZoomEye](https://zoomeye.org)
48
50
 
49
- See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
50
-
51
51
  ## Docs
52
52
 
53
- - [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
54
- - [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
55
- - [Built-in Web App](https://github.com/ninoseki/mihari/wiki/Built-in-Web-App)
56
- - [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
57
- - [Custom Script](https://github.com/ninoseki/mihari/wiki/Custom-Script)
58
- - [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
59
- - [GitHub Actions](https://github.com/ninoseki/mihari/wiki/GitHub-Actions)
53
+ - [Mihari Knowledge Base](https://www.notion.so/Mihari-Knowledge-Base-266994ff61204428ba6cfcebe40b0bd1)
60
54
 
61
55
  ## License
62
56
 
63
57
  The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
58
+
59
+ ## Acknowledgement
60
+
61
+ Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.
data/exe/mihari CHANGED
@@ -5,4 +5,4 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
5
5
 
6
6
  require "mihari"
7
7
 
8
- Mihari::CLI.start
8
+ Mihari::CLI::Main.start
data/images/tines.png ADDED
Binary file
data/lib/mihari.rb CHANGED
@@ -1,8 +1,56 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "colorize"
4
+ require "dry/configurable"
5
+ require "dry/files"
3
6
  require "mem"
7
+ require "yaml"
8
+
9
+ # Mixins
10
+ require "mihari/mixins/configurable"
11
+ require "mihari/mixins/configuration"
12
+ require "mihari/mixins/hash"
13
+ require "mihari/mixins/refang"
14
+ require "mihari/mixins/retriable"
15
+ require "mihari/mixins/rule"
16
+
17
+ def truthy?(value)
18
+ return true if value == "true"
19
+ return true if value == true
20
+
21
+ false
22
+ end
4
23
 
5
24
  module Mihari
25
+ extend Dry::Configurable
26
+ extend Mixins::Configuration
27
+
28
+ setting :binaryedge_api_key, ENV["BINARYEDGE_API_KEY"]
29
+ setting :censys_id, ENV["CENSYS_ID"]
30
+ setting :censys_secret, ENV["CENSYS_SECRET"]
31
+ setting :circl_passive_password, ENV["CIRCL_PASSIVE_PASSWORD"]
32
+ setting :circl_passive_username, ENV["CIRCL_PASSIVE_USERNAME"]
33
+ setting :misp_api_endpoint, ENV["MISP_API_ENDPOINT"]
34
+ setting :misp_api_key, ENV["MISP_API_KEY"]
35
+ setting :onyphe_api_key, ENV["ONYPHE_API_KEY"]
36
+ setting :otx_api_key, ENV["OTX_API_KEY"]
37
+ setting :passivetotal_api_key, ENV["PASSIVETOTAL_API_KEY"]
38
+ setting :passivetotal_username, ENV["PASSIVETOTAL_USERNAME"]
39
+ setting :pulsedive_api_key, ENV["PULSEDIVE_API_KEY"]
40
+ setting :securitytrails_api_key, ENV["SECURITYTRAILS_API_KEY"]
41
+ setting :shodan_api_key, ENV["SHODAN_API_KEY"]
42
+ setting :slack_channel, ENV["SLACK_CHANNEL"]
43
+ setting :slack_webhook_url, ENV["SLACK_WEBHOOK_URL"]
44
+ setting :spyse_api_key, ENV["SPYSE_API_KEY"]
45
+ setting :thehive_api_endpoint, ENV["THEHIVE_API_ENDPOINT"]
46
+ setting :thehive_api_key, ENV["THEHIVE_API_KEY"]
47
+ setting :urlscan_api_key, ENV["URLSCAN_API_KEY"]
48
+ setting :virustotal_api_key, ENV["VIRUSTOTAL_API_KEY"]
49
+ setting :zoomeye_api_key, ENV["ZOOMEYE_API_KEY"]
50
+ setting :webhook_url, ENV["WEBHOOK_URL"]
51
+ setting(:webhook_use_json_body, ENV["WEBHOOK_USE_JSON_BODY"]) { |value| truthy?(value) }
52
+ setting :database, ENV["DATABASE"] || "mihari.db"
53
+
6
54
  class << self
7
55
  include Mem
8
56
 
@@ -15,31 +63,52 @@ module Mihari
15
63
  []
16
64
  end
17
65
  memoize :analyzers
66
+
67
+ #
68
+ # Load configuration from YAML file
69
+ #
70
+ # @param [String] path Path to YAML file
71
+ #
72
+ # @return [nil]
73
+ #
74
+ def load_config_from_yaml(path)
75
+ config = load_config(path)
76
+
77
+ # validate loaded yaml data
78
+ validate_config config
79
+
80
+ config.each do |key, value|
81
+ Mihari.config.send("#{key.downcase}=".to_sym, value)
82
+ end
83
+ end
18
84
  end
19
85
  end
20
86
 
21
87
  require "mihari/version"
22
88
  require "mihari/errors"
23
89
 
24
- require "mihari/config"
25
-
26
90
  require "mihari/database"
27
91
  require "mihari/type_checker"
28
92
 
93
+ # Constraints
94
+ require "mihari/constraints"
95
+
96
+ # Schemas
97
+ require "mihari/schemas/configuration"
98
+ require "mihari/schemas/rule"
99
+
100
+ # Models
29
101
  require "mihari/models/alert"
30
102
  require "mihari/models/artifact"
31
103
  require "mihari/models/tag"
32
104
  require "mihari/models/tagging"
33
105
 
106
+ # Serializers
34
107
  require "mihari/serializers/alert"
35
108
  require "mihari/serializers/artifact"
36
109
  require "mihari/serializers/tag"
37
110
 
38
- require "mihari/html"
39
-
40
- require "mihari/configurable"
41
- require "mihari/retriable"
42
-
111
+ # Analyzers
43
112
  require "mihari/analyzers/base"
44
113
  require "mihari/analyzers/basic"
45
114
 
@@ -53,7 +122,6 @@ require "mihari/analyzers/onyphe"
53
122
  require "mihari/analyzers/otx"
54
123
  require "mihari/analyzers/passivetotal"
55
124
  require "mihari/analyzers/pulsedive"
56
- require "mihari/analyzers/securitytrails_domain_feed"
57
125
  require "mihari/analyzers/securitytrails"
58
126
  require "mihari/analyzers/shodan"
59
127
  require "mihari/analyzers/spyse"
@@ -61,26 +129,32 @@ require "mihari/analyzers/urlscan"
61
129
  require "mihari/analyzers/virustotal"
62
130
  require "mihari/analyzers/zoomeye"
63
131
 
64
- require "mihari/analyzers/free_text"
65
- require "mihari/analyzers/http_hash"
66
- require "mihari/analyzers/passive_dns"
67
- require "mihari/analyzers/passive_ssl"
68
- require "mihari/analyzers/reverse_whois"
69
- require "mihari/analyzers/ssh_fingerprint"
132
+ require "mihari/analyzers/rule"
70
133
 
134
+ # Notifiers
71
135
  require "mihari/notifiers/base"
72
136
  require "mihari/notifiers/slack"
73
137
  require "mihari/notifiers/exception_notifier"
74
138
 
139
+ # Emitters
75
140
  require "mihari/emitters/base"
76
141
  require "mihari/emitters/database"
77
142
  require "mihari/emitters/misp"
78
143
  require "mihari/emitters/slack"
79
144
  require "mihari/emitters/stdout"
80
145
  require "mihari/emitters/the_hive"
146
+ require "mihari/emitters/webhook"
81
147
 
148
+ # Status checker
82
149
  require "mihari/status"
83
150
 
151
+ # Web app
84
152
  require "mihari/web/app"
85
153
 
86
- require "mihari/cli"
154
+ # CLIs
155
+ require "mihari/cli/base"
156
+
157
+ require "mihari/cli/analyzer"
158
+ require "mihari/cli/init"
159
+
160
+ require "mihari/cli/main"
@@ -1,16 +1,21 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "dry-initializer"
3
4
  require "parallel"
4
5
 
5
6
  module Mihari
6
7
  module Analyzers
7
8
  class Base
8
- include Configurable
9
- include Retriable
9
+ extend Dry::Initializer
10
+
11
+ include Mixins::Configurable
12
+ include Mixins::Retriable
10
13
 
11
14
  attr_accessor :ignore_old_artifacts, :ignore_threshold
12
15
 
13
- def initialize
16
+ def initialize(*args, **kwargs)
17
+ super
18
+
14
19
  @ignore_old_artifacts = false
15
20
  @ignore_threshold = 0
16
21
  end
@@ -30,6 +35,7 @@ module Mihari
30
35
  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
31
36
  end
32
37
 
38
+ # @return [String]
33
39
  def source
34
40
  self.class.to_s.split("::").last
35
41
  end
@@ -39,6 +45,11 @@ module Mihari
39
45
  []
40
46
  end
41
47
 
48
+ #
49
+ # Set artifacts & run emitters in parallel
50
+ #
51
+ # @return [nil]
52
+ #
42
53
  def run
43
54
  set_unique_artifacts
44
55
 
@@ -47,6 +58,13 @@ module Mihari
47
58
  end
48
59
  end
49
60
 
61
+ #
62
+ # Run emitter
63
+ #
64
+ # @param [Mihari::Emitters::Base] emitter
65
+ #
66
+ # @return [nil]
67
+ #
50
68
  def run_emitter(emitter)
51
69
  emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
52
70
  rescue StandardError => e
@@ -57,22 +75,40 @@ module Mihari
57
75
  Mihari.analyzers << child
58
76
  end
59
77
 
60
- private
61
-
78
+ #
79
+ # Normalize artifacts
80
+ # - Uniquefy artifacts by native #uniq
81
+ # - Convert data (string) into an artifact
82
+ # - Reject an invalid artifact
83
+ #
62
84
  # @return [Array<Mihari::Artifact>]
85
+ #
63
86
  def normalized_artifacts
64
87
  @normalized_artifacts ||= artifacts.compact.uniq.sort.map do |artifact|
65
- artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact)
88
+ # No need to set data_type manually
89
+ # It is set automatically in #initialize
90
+ artifact.is_a?(Artifact) ? artifact : Artifact.new(data: artifact, source: source)
66
91
  end.select(&:valid?)
67
92
  end
68
93
 
94
+ private
95
+
96
+ #
97
+ # Uniquefy artifacts
98
+ #
69
99
  # @return [Array<Mihari::Artifact>]
100
+ #
70
101
  def unique_artifacts
71
102
  @unique_artifacts ||= normalized_artifacts.select do |artifact|
72
103
  artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
73
104
  end
74
105
  end
75
106
 
107
+ #
108
+ # Set unique artifacts
109
+ #
110
+ # @return [nil]
111
+ #
76
112
  def set_unique_artifacts
77
113
  retry_on_error { unique_artifacts }
78
114
  rescue ArgumentError => _e
@@ -80,11 +116,16 @@ module Mihari
80
116
  raise Error, "Please configure #{klass} API settings properly"
81
117
  end
82
118
 
119
+ #
120
+ # Select valid emitters
121
+ #
122
+ # @return [Array<Mihari::Emitters::Base>]
123
+ #
83
124
  def valid_emitters
84
- @valid_emitters ||= Mihari.emitters.map do |klass|
125
+ @valid_emitters ||= Mihari.emitters.filter_map do |klass|
85
126
  emitter = klass.new
86
127
  emitter.valid? ? emitter : nil
87
- end.compact
128
+ end
88
129
  end
89
130
  end
90
131
  end
@@ -3,8 +3,7 @@
3
3
  module Mihari
4
4
  module Analyzers
5
5
  class Basic < Base
6
- attr_accessor :title
7
- attr_reader :description, :artifacts, :source, :tags
6
+ attr_reader :title, :description, :artifacts, :source, :tags
8
7
 
9
8
  def initialize(title:, description:, artifacts:, source:, tags: [])
10
9
  super()
@@ -5,16 +5,10 @@ require "binaryedge"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class BinaryEdge < Base
8
- attr_reader :title, :description, :query, :tags
9
-
10
- def initialize(query, title: nil, description: nil, tags: [])
11
- super()
12
-
13
- @query = query
14
- @title = title || "BinaryEdge lookup"
15
- @description = description || "query = #{query}"
16
- @tags = tags
17
- end
8
+ param :query
9
+ option :title, default: proc { "BinaryEdge search" }
10
+ option :description, default: proc { "query = #{query}" }
11
+ option :tags, default: proc { [] }
18
12
 
19
13
  def artifacts
20
14
  results = search
@@ -22,9 +16,9 @@ module Mihari
22
16
 
23
17
  results.map do |result|
24
18
  events = result["events"] || []
25
- events.map do |event|
19
+ events.filter_map do |event|
26
20
  event.dig "target", "ip"
27
- end.compact
21
+ end
28
22
  end.flatten.compact.uniq
29
23
  end
30
24
 
@@ -52,7 +46,7 @@ module Mihari
52
46
  responses
53
47
  end
54
48
 
55
- def config_keys
49
+ def configuration_keys
56
50
  %w[binaryedge_api_key]
57
51
  end
58
52
 
@@ -1,87 +1,50 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "censu"
3
+ require "censysx"
4
4
 
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class Censys < Base
8
- attr_reader :title, :description, :query, :tags, :type
9
-
10
- def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
11
- super()
12
-
13
- @query = query
14
- @title = title || "Censys lookup"
15
- @description = description || "query = #{query}"
16
- @tags = tags
17
- @type = type
18
- end
8
+ param :query
9
+ option :title, default: proc { "Censys search" }
10
+ option :description, default: proc { "query = #{query}" }
11
+ option :tags, default: proc { [] }
19
12
 
20
13
  def artifacts
21
- case type
22
- when "ipv4"
23
- ipv4_lookup
24
- when "websites"
25
- websites_lookup
26
- when "certificates"
27
- certificates_lookup
28
- else
29
- raise InvalidInputError, "#{type} type is not supported." unless valid_type?
30
- end
14
+ search
31
15
  end
32
16
 
33
17
  private
34
18
 
35
- def valid_type?
36
- %w[ipv4 websites certificates].include? type
37
- end
38
-
39
- def normalize(domain)
40
- return domain unless domain.start_with?("*.")
41
-
42
- domain.sub("*.", "")
43
- end
44
-
45
- def ipv4_lookup
19
+ def search
46
20
  ipv4s = []
47
21
 
48
- res = api.ipv4.search(query: query)
49
- res.each_page do |page|
50
- ipv4s << page.map(&:ip)
51
- end
22
+ cursor = nil
23
+ loop do
24
+ response = api.search(query, cursor: cursor)
25
+ ipv4s << response_to_ipv4s(response)
52
26
 
53
- ipv4s.flatten
54
- end
55
-
56
- def websites_lookup
57
- domains = []
58
-
59
- res = api.websites.search(query: query)
60
- res.each_page do |page|
61
- domains << page.map(&:domain)
27
+ links = response.dig("result", "links")
28
+ cursor = links["next"]
29
+ break if cursor == ""
62
30
  end
63
31
 
64
- domains.flatten
32
+ ipv4s.flatten
65
33
  end
66
34
 
67
- def certificates_lookup
68
- domains = []
69
-
70
- res = api.certificates.search(query: query)
71
- res.each_page do |page|
72
- page.each do |result|
73
- subject_dn = result.subject_dn
74
- names = subject_dn.scan(/CN=(.+)/).flatten.first
75
- next unless names
76
-
77
- domains << names.split(",").map { |domain| normalize(domain) }
78
- end
79
- end
80
-
81
- domains.flatten
35
+ #
36
+ # Extract IPv4s from Censys search API response
37
+ #
38
+ # @param [Hash] response
39
+ #
40
+ # @return [Array<String>]
41
+ #
42
+ def response_to_ipv4s(response)
43
+ hits = response.dig("result", "hits") || []
44
+ hits.map { |hit| hit["ip"] }
82
45
  end
83
46
 
84
- def config_keys
47
+ def configuration_keys
85
48
  %w[censys_id censys_secret]
86
49
  end
87
50