fluent-plugin-shodan 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -3
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_shodan_search.rb +41 -7
- data/test/test_in_shodan_search.rb +107 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6433f3162ef921f1b577f5489f802a1a6d88d561026a4274440732d326e55e88
|
4
|
+
data.tar.gz: 6d97984422e0d0bcaada2ac521a47d8fcd4049e437327fd03e3c39aac11858e5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 775543570f5613db0515f258ca8b88b034f87e6265d954b82f59b21e09befee57e0f162f79d8ab6fd8b6e932427bd8d4e242c2dbec8b96a57477c0535d97fd06
|
7
|
+
data.tar.gz: 82e80191ff18df2b6f393e6b2330f61a273d1a786863f58041c0d0aeef14c79e16f41bbd36febe625d9f5b7005b7ffd1dbd3bf3a792d464b89126d4715eaef6b
|
data/README.md
CHANGED
@@ -83,9 +83,11 @@ Default value: `3600`.
|
|
83
83
|
|
84
84
|
The tag to apply to each shodan entries.
|
85
85
|
|
86
|
-
#### query (string) (
|
86
|
+
#### query (string) (optional)
|
87
87
|
|
88
|
-
The Shodan query to execute.
|
88
|
+
The Shodan query to execute. The query can be empty if at least one filter is set.
|
89
|
+
|
90
|
+
Default: `nil`
|
89
91
|
|
90
92
|
#### max_pages (integer) (optional)
|
91
93
|
|
@@ -93,6 +95,16 @@ The maximum amount of pages to crawl. A 0 or negative value means to crawl all p
|
|
93
95
|
|
94
96
|
Default value: `1`.
|
95
97
|
|
98
|
+
#### filter (optional) (multi)
|
99
|
+
|
100
|
+
##### name (string) (required)
|
101
|
+
|
102
|
+
The name of the filter to be added to the query. Full filters list is available on the [Shodan filter reference page](https://www.shodan.io/search/filters). The filter can be negated by prepending `-` to the filter name (ex: `name -port`).
|
103
|
+
|
104
|
+
##### value (string) (required)
|
105
|
+
|
106
|
+
The value to be passed to the filter.
|
107
|
+
|
96
108
|
## Shodan Stream
|
97
109
|
|
98
110
|
WIP
|
@@ -103,7 +115,7 @@ WIP
|
|
103
115
|
|
104
116
|
```
|
105
117
|
<source>
|
106
|
-
@type
|
118
|
+
@type shodan_alert
|
107
119
|
interval 15m
|
108
120
|
alert_id GA3FRJ1HJNDPORHV
|
109
121
|
api_key 1234567890AZERTYUIOP
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
@@ -7,6 +7,25 @@ module Fluent::Plugin
|
|
7
7
|
|
8
8
|
helpers :timer
|
9
9
|
|
10
|
+
SUPPORTED_FILTERS = [
|
11
|
+
'asn','city','country','cpe','device','geo','has_ipv6','has_screenshot',
|
12
|
+
'has_ssl','has_vuln','hash','hostname','ip','isp','link','net','org','os',
|
13
|
+
'port','postal','product','region','scan','shodan.module','state',
|
14
|
+
'version','screenshot.label','cloud.provider','cloud.region',
|
15
|
+
'cloud.service','http.component','http.component_category',
|
16
|
+
'http.favicon.hash','http.html','http.html_hash','http.robots_hash',
|
17
|
+
'http.securitytxt','http.status','http.title','http.waf','bitcoin.ip',
|
18
|
+
'bitcoin.ip_count','bitcoin.port','bitcoin.version','snmp.contact',
|
19
|
+
'snmp.location','snmp.name','ssl','ssl.alpn','ssl.cert.alg',
|
20
|
+
'ssl.cert.expired','ssl.cert.extension','ssl.cert.fingerprint',
|
21
|
+
'ssl.cert.issuer.cn','ssl.cert.pubkey.bits','ssl.cert.pubkey.type',
|
22
|
+
'ssl.cert.serial','ssl.cert.subject.cn','ssl.chain_count',
|
23
|
+
'ssl.cipher.bits','ssl.cipher.name','ssl.cipher.version','ssl.ja3s',
|
24
|
+
'ssl.jarm','ssl.version','ntp.ip','ntp.ip_count','ntp.more','ntp.port',
|
25
|
+
'telnet.do','telnet.dont','telnet.option','telnet.will','telnet.wont',
|
26
|
+
'ssh.hassh','ssh.type', 'tag', 'vuln'
|
27
|
+
]
|
28
|
+
|
10
29
|
desc "The API key to connect to the Shodan API."
|
11
30
|
config_param :api_key, :string, secret: true
|
12
31
|
desc "The interval time between running queries."
|
@@ -14,9 +33,16 @@ module Fluent::Plugin
|
|
14
33
|
desc "The tag to apply to each shodan entries."
|
15
34
|
config_param :tag, :string, default: nil
|
16
35
|
desc "The Shodan query to execute."
|
17
|
-
config_param :query, :string
|
36
|
+
config_param :query, :string, default: ''
|
18
37
|
desc "The maximum amount of pages to crawl. A 0 or negative value means to crawl all pages."
|
19
38
|
config_param :max_pages, :integer, default: 1
|
39
|
+
desc "Search filters configuration."
|
40
|
+
config_section :filter, param_name: 'filters', required: false, multi: true do
|
41
|
+
desc "Name of the filter. See https://www.shodan.io/search/filters for a list of supported filters"
|
42
|
+
config_param :name, :enum, list: (SUPPORTED_FILTERS + SUPPORTED_FILTERS.map {|filter| "-#{filter}"}).map { |filter| filter.to_sym }
|
43
|
+
desc "Value to be given to the filter"
|
44
|
+
config_param :value, :string
|
45
|
+
end
|
20
46
|
|
21
47
|
def configure(conf)
|
22
48
|
super
|
@@ -27,6 +53,13 @@ module Fluent::Plugin
|
|
27
53
|
rescue RuntimeError => exception
|
28
54
|
raise Fluent::ConfigError.new "Invalid Shodan API key"
|
29
55
|
end
|
56
|
+
|
57
|
+
raise Fluent::ConfigError.new("At least a query or one filter should be configured") if @query.empty? and @filters.empty?
|
58
|
+
|
59
|
+
@search_filters = {}
|
60
|
+
@filters.each do |filter|
|
61
|
+
@search_filters[filter.name] = filter.value
|
62
|
+
end
|
30
63
|
end
|
31
64
|
|
32
65
|
def multi_workers_ready?
|
@@ -44,20 +77,21 @@ module Fluent::Plugin
|
|
44
77
|
def run
|
45
78
|
log.debug "Starting Shodan search", query: @query, max_pages: @max_pages
|
46
79
|
es_time = Fluent::EventTime.now
|
47
|
-
|
80
|
+
opts = @search_filters.merge({page: 0})
|
48
81
|
read_entries = 0
|
49
82
|
loop do
|
50
|
-
|
51
|
-
|
83
|
+
opts[:page] += 1
|
84
|
+
log.trace query: @query, opts: opts
|
85
|
+
result = @client.host_search(@query.dup, **opts)
|
52
86
|
result['matches'].each do |rec|
|
53
87
|
router.emit(@tag, es_time, rec)
|
54
88
|
end
|
55
89
|
read_entries += result['matches'].length
|
56
|
-
break if (@max_pages >= 0 &&
|
90
|
+
break if (@max_pages >= 0 && opts[:page] >= @max_pages) || read_entries >= result['total']
|
57
91
|
end
|
58
|
-
log.debug "Shodan search ending", query: @query, total_read: read_entries
|
92
|
+
log.debug "Shodan search ending", query: @query, filters: @search_filters, total_read: read_entries
|
59
93
|
rescue RuntimeError => re
|
60
|
-
log.error "Unable to execute Shodan query", query: @query, page: current_page, error: re
|
94
|
+
log.error "Unable to execute Shodan query", query: @query, filters: @search_filters, page: current_page, error: re
|
61
95
|
rescue => exception
|
62
96
|
log.error "Error executing Shodan query", error: exception
|
63
97
|
end
|
@@ -9,6 +9,13 @@ class ShodanSearchInputTest < Test::Unit::TestCase
|
|
9
9
|
include Fluent::Test::Helpers
|
10
10
|
|
11
11
|
API_KEY = ENV['SHODAN_TEST_API_KEY']
|
12
|
+
CONFIG = %[
|
13
|
+
api_key #{API_KEY}
|
14
|
+
query 8.8.8.8
|
15
|
+
]
|
16
|
+
|
17
|
+
private_constant :API_KEY
|
18
|
+
private_constant :CONFIG
|
12
19
|
|
13
20
|
def setup
|
14
21
|
Fluent::Test.setup
|
@@ -30,10 +37,21 @@ class ShodanSearchInputTest < Test::Unit::TestCase
|
|
30
37
|
])
|
31
38
|
end
|
32
39
|
end
|
33
|
-
test 'is invalid without a query' do
|
40
|
+
test 'is invalid without a query or a filter' do
|
41
|
+
assert_raise Fluent::ConfigError do
|
42
|
+
create_driver(%[
|
43
|
+
api_key #{API_KEY}
|
44
|
+
])
|
45
|
+
end
|
46
|
+
end
|
47
|
+
test 'is invalid with an unsupported filter' do
|
34
48
|
assert_raise Fluent::ConfigError do
|
35
49
|
create_driver(%[
|
36
50
|
api_key #{API_KEY}
|
51
|
+
<filter>
|
52
|
+
name some.dumb.filter
|
53
|
+
value 42
|
54
|
+
</filter>
|
37
55
|
])
|
38
56
|
end
|
39
57
|
end
|
@@ -41,18 +59,100 @@ class ShodanSearchInputTest < Test::Unit::TestCase
|
|
41
59
|
d = create_driver
|
42
60
|
assert_equal '8.8.8.8', d.instance.query
|
43
61
|
end
|
62
|
+
test 'is valid with one filter and no query' do
|
63
|
+
assert_nothing_raised do
|
64
|
+
d = create_driver(%[
|
65
|
+
api_key #{API_KEY}
|
66
|
+
<filter>
|
67
|
+
name product
|
68
|
+
value ssh
|
69
|
+
</filter>
|
70
|
+
])
|
71
|
+
end
|
72
|
+
end
|
44
73
|
end
|
45
|
-
|
74
|
+
|
46
75
|
sub_test_case 'Plugin emission' do
|
76
|
+
test 'with simple query' do
|
77
|
+
expected_tag = 'test_shodan'
|
78
|
+
d = create_driver(CONFIG + "\ninterval 5\ntag #{expected_tag}")
|
79
|
+
d.run(expect_emits: 10, timeout: 30)
|
80
|
+
events = d.events
|
81
|
+
assert_not_empty(events)
|
82
|
+
assert_all(events, "Events are not properly tagged") {|evt| expected_tag == evt[0]}
|
83
|
+
assert_all(events, "Events do not match query") {|evt| evt[2]['data'].downcase.include?('8.8.8.8')}
|
84
|
+
assert_all(events, "Events do not have a '_shodan' key") {|evt| evt[2].has_key?('_shodan')}
|
85
|
+
end
|
86
|
+
|
87
|
+
test 'with a single filter' do
|
88
|
+
config = %[
|
89
|
+
api_key #{API_KEY}
|
90
|
+
query ssh
|
91
|
+
interval 5
|
92
|
+
tag shodan_test
|
93
|
+
<filter>
|
94
|
+
name product
|
95
|
+
value ssh
|
96
|
+
</filter>
|
97
|
+
]
|
98
|
+
d = create_driver(config)
|
99
|
+
d.run(expect_emits: 10, timeout: 30)
|
100
|
+
events = d.events
|
101
|
+
assert_not_empty(events)
|
102
|
+
assert_all(events) {|evt| evt[2]['product'].downcase.include?('ssh')}
|
103
|
+
end
|
104
|
+
|
105
|
+
test 'with multiple filters' do
|
106
|
+
config = %[
|
107
|
+
api_key #{API_KEY}
|
108
|
+
query ssh
|
109
|
+
interval 5
|
110
|
+
tag shodan_test
|
111
|
+
<filter>
|
112
|
+
name product
|
113
|
+
value ssh
|
114
|
+
</filter>
|
115
|
+
<filter>
|
116
|
+
name -port
|
117
|
+
value 22
|
118
|
+
</filter>
|
119
|
+
]
|
120
|
+
d = create_driver(config)
|
121
|
+
d.run(expect_emits: 10, timeout: 30)
|
122
|
+
events = d.events
|
123
|
+
assert_not_empty(events)
|
124
|
+
assert_all(events) {|evt| evt[2]['product'].downcase.include?('ssh')}
|
125
|
+
assert_all(events) {|evt| evt[2]['port'] != 22}
|
126
|
+
end
|
127
|
+
|
128
|
+
test 'combine query and filters' do
|
129
|
+
config = %[
|
130
|
+
api_key #{API_KEY}
|
131
|
+
query ssh
|
132
|
+
interval 5
|
133
|
+
tag shodan_test
|
134
|
+
query plex
|
135
|
+
<filter>
|
136
|
+
name country
|
137
|
+
value FR
|
138
|
+
</filter>
|
139
|
+
<filter>
|
140
|
+
name -port
|
141
|
+
value 32400
|
142
|
+
</filter>
|
143
|
+
]
|
144
|
+
d = create_driver(config)
|
145
|
+
d.run(expect_emits: 10, timeout: 30)
|
146
|
+
events = d.events
|
147
|
+
assert_not_empty(events)
|
148
|
+
assert_all(events) {|evt| evt[2]['data'].downcase.include?('plex')}
|
149
|
+
assert_all(events) {|evt| evt[2]['location']['country_code'] == 'FR'}
|
150
|
+
assert_all(events) {|evt| evt[2]['port'] != 32400}
|
151
|
+
end
|
47
152
|
end
|
48
153
|
|
49
154
|
private
|
50
155
|
|
51
|
-
CONFIG = %[
|
52
|
-
api_key #{API_KEY}
|
53
|
-
query 8.8.8.8
|
54
|
-
]
|
55
|
-
|
56
156
|
def create_driver(conf = CONFIG)
|
57
157
|
Fluent::Test::Driver::Input.new(Fluent::Plugin::ShodanSearch).configure(conf)
|
58
158
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-shodan
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- srilumpa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-02-
|
11
|
+
date: 2022-02-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|