shodanz 1.0.6 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +85 -56
- data/examples/async_honeypot_detector.rb +19 -0
- data/examples/async_host_search_example.rb +29 -0
- data/examples/async_stream_example.rb +33 -0
- data/examples/streaming_banner_product_stats.rb +3 -0
- data/lib/shodanz.rb +9 -2
- data/lib/shodanz/apis/exploits.rb +12 -31
- data/lib/shodanz/apis/rest.rb +50 -80
- data/lib/shodanz/apis/streaming.rb +44 -82
- data/lib/shodanz/apis/utils.rb +175 -0
- data/lib/shodanz/client.rb +88 -1
- data/lib/shodanz/errors.rb +29 -0
- data/lib/shodanz/version.rb +3 -1
- data/shodanz.gemspec +21 -18
- metadata +60 -27
data/lib/shodanz/apis/rest.rb
CHANGED
@@ -1,27 +1,38 @@
|
|
1
|
-
|
1
|
+
require_relative 'utils.rb'
|
2
|
+
|
3
|
+
# frozen_string_literal: true
|
2
4
|
|
5
|
+
module Shodanz
|
3
6
|
module API
|
4
|
-
# The REST API provides methods to search Shodan, look up
|
5
|
-
# hosts, get summary information on queries and a variety
|
7
|
+
# The REST API provides methods to search Shodan, look up
|
8
|
+
# hosts, get summary information on queries and a variety
|
6
9
|
# of other utilities. This requires you to have an API key
|
7
10
|
# which you can get from Shodan.
|
8
11
|
#
|
9
12
|
# @author Kent 'picat' Gruber
|
10
13
|
class REST
|
14
|
+
include Shodanz::API::Utils
|
15
|
+
|
16
|
+
# @return [String]
|
11
17
|
attr_accessor :key
|
12
18
|
|
13
19
|
# The path to the REST API endpoint.
|
14
|
-
URL =
|
20
|
+
URL = 'https://api.shodan.io/'
|
15
21
|
|
16
22
|
# @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
|
17
23
|
def initialize(key: ENV['SHODAN_API_KEY'])
|
18
|
-
|
19
|
-
|
24
|
+
@url = URL
|
25
|
+
@internet = Async::HTTP::Internet.new
|
26
|
+
self.key = key
|
27
|
+
|
28
|
+
warn 'No key has been found or provided!' unless key?
|
20
29
|
end
|
21
30
|
|
22
31
|
# Check if there's an API key.
|
23
32
|
def key?
|
24
|
-
return true if @key
|
33
|
+
return true if @key
|
34
|
+
|
35
|
+
false
|
25
36
|
end
|
26
37
|
|
27
38
|
# Returns all services that have been found on the given host IP.
|
@@ -31,92 +42,91 @@ module Shodanz
|
|
31
42
|
# == Examples
|
32
43
|
# # Typical usage.
|
33
44
|
# rest_api.host("8.8.8.8")
|
34
|
-
#
|
45
|
+
#
|
35
46
|
# # All historical banners should be returned.
|
36
|
-
# rest_api.host("8.8.8.8", history: true)
|
47
|
+
# rest_api.host("8.8.8.8", history: true)
|
37
48
|
#
|
38
49
|
# # Only return the list of ports and the general host information, no banners.
|
39
|
-
# rest_api.host("8.8.8.8", minify: true)
|
50
|
+
# rest_api.host("8.8.8.8", minify: true)
|
40
51
|
def host(ip, **params)
|
41
52
|
get("shodan/host/#{ip}", params)
|
42
53
|
end
|
43
54
|
|
44
|
-
# This method behaves identical to "/shodan/host/search" with the only
|
45
|
-
# difference that this method does not return any host results, it only
|
46
|
-
# returns the total number of results that matched the query and any
|
47
|
-
# facet information that was requested. As a result this method does
|
55
|
+
# This method behaves identical to "/shodan/host/search" with the only
|
56
|
+
# difference that this method does not return any host results, it only
|
57
|
+
# returns the total number of results that matched the query and any
|
58
|
+
# facet information that was requested. As a result this method does
|
48
59
|
# not consume query credits.
|
49
60
|
# == Examples
|
50
61
|
# rest_api.host_count("apache")
|
51
62
|
# rest_api.host_count("apache", country: "US")
|
52
63
|
# rest_api.host_count("apache", country: "US", state: "MI")
|
53
64
|
# rest_api.host_count("apache", country: "US", state: "MI", city: "Detroit")
|
54
|
-
def host_count(query =
|
65
|
+
def host_count(query = '', facets: {}, **params)
|
55
66
|
params[:query] = query
|
56
67
|
params = turn_into_query(params)
|
57
68
|
facets = turn_into_facets(facets)
|
58
|
-
get(
|
69
|
+
get('shodan/host/count', params.merge(facets))
|
59
70
|
end
|
60
71
|
|
61
|
-
# Search Shodan using the same query syntax as the website and use facets
|
72
|
+
# Search Shodan using the same query syntax as the website and use facets
|
62
73
|
# to get summary information for different properties.
|
63
74
|
# == Example
|
64
75
|
# rest_api.host_search("apache", country: "US", facets: { city: "Detroit" }, page: 1, minify: false)
|
65
|
-
def host_search(query =
|
76
|
+
def host_search(query = '', facets: {}, page: 1, minify: true, **params)
|
66
77
|
params[:query] = query
|
67
78
|
params = turn_into_query(params)
|
68
79
|
facets = turn_into_facets(facets)
|
69
80
|
params[:page] = page
|
70
81
|
params[:minify] = minify
|
71
|
-
get(
|
82
|
+
get('shodan/host/search', params.merge(facets))
|
72
83
|
end
|
73
84
|
|
74
|
-
# This method lets you determine which filters are being used by
|
85
|
+
# This method lets you determine which filters are being used by
|
75
86
|
# the query string and what parameters were provided to the filters.
|
76
|
-
def host_search_tokens(query =
|
87
|
+
def host_search_tokens(query = '', **params)
|
77
88
|
params[:query] = query
|
78
89
|
params = turn_into_query(params)
|
79
|
-
get(
|
90
|
+
get('shodan/host/search/tokens', params)
|
80
91
|
end
|
81
92
|
|
82
93
|
# This method returns a list of port numbers that the crawlers are looking for.
|
83
94
|
def ports
|
84
|
-
get(
|
95
|
+
get('shodan/ports')
|
85
96
|
end
|
86
97
|
|
87
98
|
# List all protocols that can be used when performing on-demand Internet scans via Shodan.
|
88
|
-
def protocols
|
89
|
-
get(
|
99
|
+
def protocols
|
100
|
+
get('shodan/protocols')
|
90
101
|
end
|
91
102
|
|
92
103
|
# Use this method to request Shodan to crawl a network.
|
93
104
|
#
|
94
|
-
# This method uses API scan credits: 1 IP consumes 1 scan credit. You
|
95
|
-
# must have a paid API plan (either one-time payment or subscription)
|
105
|
+
# This method uses API scan credits: 1 IP consumes 1 scan credit. You
|
106
|
+
# must have a paid API plan (either one-time payment or subscription)
|
96
107
|
# in order to use this method.
|
97
108
|
#
|
98
109
|
# IP, IPs or netblocks (in CIDR notation) that should get crawled.
|
99
110
|
def scan(*ips)
|
100
|
-
|
101
|
-
post("shodan/scan", ips: ips.join(","))
|
111
|
+
post('shodan/scan', ips: ips.join(','))
|
102
112
|
end
|
103
113
|
|
104
114
|
# Use this method to request Shodan to crawl the Internet for a specific port.
|
105
115
|
#
|
106
|
-
# This method is restricted to security researchers and companies with
|
107
|
-
# a Shodan Data license. To apply for access to this method as a researcher,
|
108
|
-
# please email jmath@shodan.io with information about your project.
|
116
|
+
# This method is restricted to security researchers and companies with
|
117
|
+
# a Shodan Data license. To apply for access to this method as a researcher,
|
118
|
+
# please email jmath@shodan.io with information about your project.
|
109
119
|
# Access is restricted to prevent abuse.
|
110
120
|
#
|
111
121
|
# == Example
|
112
122
|
# rest_api.crawl_for(port: 80, protocol: "http")
|
113
123
|
def crawl_for(**params)
|
114
|
-
params[:query] =
|
124
|
+
params[:query] = ''
|
115
125
|
params = turn_into_query(params)
|
116
126
|
post('shodan/scan/internet', params)
|
117
127
|
end
|
118
128
|
|
119
|
-
# Check the progress of a previously submitted scan request.
|
129
|
+
# Check the progress of a previously submitted scan request.
|
120
130
|
def scan_status(id)
|
121
131
|
get("shodan/scan/#{id}")
|
122
132
|
end
|
@@ -130,7 +140,7 @@ module Shodanz
|
|
130
140
|
def search_for_community_query(query, **params)
|
131
141
|
params[:query] = query
|
132
142
|
params = turn_into_query(params)
|
133
|
-
get('shodan/query/search', params)
|
143
|
+
get('shodan/query/search', params)
|
134
144
|
end
|
135
145
|
|
136
146
|
# Use this method to obtain a list of popular tags for the saved search queries in Shodan.
|
@@ -147,16 +157,16 @@ module Shodanz
|
|
147
157
|
|
148
158
|
# Look up the IP address for the provided list of hostnames.
|
149
159
|
def resolve(*hostnames)
|
150
|
-
get('dns/resolve', hostnames: hostnames.join(
|
160
|
+
get('dns/resolve', hostnames: hostnames.join(','))
|
151
161
|
end
|
152
162
|
|
153
|
-
# Look up the hostnames that have been defined for the
|
163
|
+
# Look up the hostnames that have been defined for the
|
154
164
|
# given list of IP addresses.
|
155
165
|
def reverse_lookup(*ips)
|
156
|
-
get('dns/reverse', ips: ips.join(
|
166
|
+
get('dns/reverse', ips: ips.join(','))
|
157
167
|
end
|
158
168
|
|
159
|
-
# Shows the HTTP headers that your client sends when
|
169
|
+
# Shows the HTTP headers that your client sends when
|
160
170
|
# connecting to a webserver.
|
161
171
|
def http_headers
|
162
172
|
get('tools/httpheaders')
|
@@ -167,7 +177,7 @@ module Shodanz
|
|
167
177
|
get('tools/myip')
|
168
178
|
end
|
169
179
|
|
170
|
-
# Calculates a honeypot probability score ranging
|
180
|
+
# Calculates a honeypot probability score ranging
|
171
181
|
# from 0 (not a honeypot) to 1.0 (is a honeypot).
|
172
182
|
def honeypot_score(ip)
|
173
183
|
get("labs/honeyscore/#{ip}")
|
@@ -177,46 +187,6 @@ module Shodanz
|
|
177
187
|
def info
|
178
188
|
get('api-info')
|
179
189
|
end
|
180
|
-
|
181
|
-
# Perform a direct GET HTTP request to the REST API.
|
182
|
-
def get(path, **params)
|
183
|
-
resp = Unirest.get "#{URL}#{path}?key=#{@key}", parameters: params
|
184
|
-
if resp.code != 200
|
185
|
-
raise resp.body['error'] if resp.body.key?('error')
|
186
|
-
raise resp
|
187
|
-
end
|
188
|
-
resp.body
|
189
|
-
end
|
190
|
-
|
191
|
-
# Perform a direct POST HTTP request to the REST API.
|
192
|
-
def post(path, **params)
|
193
|
-
resp = Unirest.post "#{URL}#{path}?key=#{@key}", parameters: params
|
194
|
-
if resp.code != 200
|
195
|
-
raise resp.body['error'] if resp.body.key?('error')
|
196
|
-
raise resp
|
197
|
-
end
|
198
|
-
resp.body
|
199
|
-
end
|
200
|
-
|
201
|
-
private
|
202
|
-
|
203
|
-
def turn_into_query(params)
|
204
|
-
filters = params.reject { |key, _| key == :query }
|
205
|
-
filters.each do |key, value|
|
206
|
-
params[:query] << " #{key}:#{value}"
|
207
|
-
end
|
208
|
-
params.select { |key, _| key == :query }
|
209
|
-
end
|
210
|
-
|
211
|
-
def turn_into_facets(facets)
|
212
|
-
filters = facets.reject { |key, _| key == :facets }
|
213
|
-
facets[:facets] = []
|
214
|
-
filters.each do |key, value|
|
215
|
-
facets[:facets] << "#{key}:#{value}"
|
216
|
-
end
|
217
|
-
facets[:facets] = facets[:facets].join(',')
|
218
|
-
facets.select { |key, _| key == :facets }
|
219
|
-
end
|
220
190
|
end
|
221
191
|
end
|
222
192
|
end
|
@@ -1,38 +1,48 @@
|
|
1
|
-
|
1
|
+
require_relative 'utils.rb'
|
2
|
+
|
3
|
+
# frozen_string_literal: true
|
2
4
|
|
5
|
+
module Shodanz
|
3
6
|
module API
|
4
|
-
# The REST API provides methods to search Shodan, look up
|
5
|
-
# hosts, get summary information on queries and a variety
|
7
|
+
# The REST API provides methods to search Shodan, look up
|
8
|
+
# hosts, get summary information on queries and a variety
|
6
9
|
# of other utilities. This requires you to have an API key
|
7
10
|
# which you can get from Shodan.
|
8
11
|
#
|
9
|
-
# Note: Only 1-5% of the data is currently provided to
|
10
|
-
# subscription-based API plans. If your company is interested
|
11
|
-
# in large-scale, real-time access to all of the Shodan data
|
12
|
+
# Note: Only 1-5% of the data is currently provided to
|
13
|
+
# subscription-based API plans. If your company is interested
|
14
|
+
# in large-scale, real-time access to all of the Shodan data
|
12
15
|
# please contact us for pricing information (sales@shodan.io).
|
13
|
-
#
|
16
|
+
#
|
14
17
|
# @author Kent 'picat' Gruber
|
15
18
|
class Streaming
|
19
|
+
include Shodanz::API::Utils
|
20
|
+
|
21
|
+
# @return [String]
|
16
22
|
attr_accessor :key
|
17
23
|
|
18
|
-
# The Streaming API is an HTTP-based service that returns
|
24
|
+
# The Streaming API is an HTTP-based service that returns
|
19
25
|
# a real-time stream of data collected by Shodan.
|
20
|
-
URL =
|
26
|
+
URL = 'https://stream.shodan.io/'
|
21
27
|
|
22
28
|
# @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
|
23
29
|
def initialize(key: ENV['SHODAN_API_KEY'])
|
24
|
-
|
25
|
-
|
30
|
+
@url = URL
|
31
|
+
@internet = Async::HTTP::Internet.new
|
32
|
+
self.key = key
|
33
|
+
|
34
|
+
warn 'No key has been found or provided!' unless key?
|
26
35
|
end
|
27
36
|
|
28
37
|
# Check if there's an API key.
|
29
38
|
def key?
|
30
39
|
return true if @key; false
|
40
|
+
|
31
41
|
end
|
32
42
|
|
33
|
-
# This stream provides ALL of the data that Shodan collects.
|
34
|
-
# Use this stream if you need access to everything and/ or want to
|
35
|
-
# store your own Shodan database locally. If you only care about specific
|
43
|
+
# This stream provides ALL of the data that Shodan collects.
|
44
|
+
# Use this stream if you need access to everything and/ or want to
|
45
|
+
# store your own Shodan database locally. If you only care about specific
|
36
46
|
# ports, please use the Ports stream.
|
37
47
|
#
|
38
48
|
# Sometimes data may be piped down stream that is weird to parse. You can choose
|
@@ -44,28 +54,28 @@ module Shodanz
|
|
44
54
|
# puts data
|
45
55
|
# end
|
46
56
|
def banners(**params)
|
47
|
-
slurp_stream(
|
57
|
+
slurp_stream('shodan/banners', params) do |data|
|
48
58
|
yield data
|
49
59
|
end
|
50
60
|
end
|
51
61
|
|
52
|
-
# This stream provides a filtered, bandwidth-saving view of the Banners
|
62
|
+
# This stream provides a filtered, bandwidth-saving view of the Banners
|
53
63
|
# stream in case you are only interested in devices located in certain ASNs.
|
54
64
|
# == Example
|
55
|
-
# api.banners_within_asns(3303, 32475) do |data|
|
65
|
+
# api.banners_within_asns(3303, 32475) do |data|
|
56
66
|
# # do something with the banner hash
|
57
67
|
# puts data
|
58
68
|
# end
|
59
69
|
def banners_within_asns(*asns, **params)
|
60
|
-
slurp_stream("shodan/asn/#{asns.join(
|
70
|
+
slurp_stream("shodan/asn/#{asns.join(',')}", params) do |data|
|
61
71
|
yield data
|
62
72
|
end
|
63
73
|
end
|
64
74
|
|
65
|
-
# This stream provides a filtered, bandwidth-saving view of the Banners
|
75
|
+
# This stream provides a filtered, bandwidth-saving view of the Banners
|
66
76
|
# stream in case you are only interested in devices located in a certain ASN.
|
67
77
|
# == Example
|
68
|
-
# api.banners_within_asn(3303) do |data|
|
78
|
+
# api.banners_within_asn(3303) do |data|
|
69
79
|
# # do something with the banner hash
|
70
80
|
# puts data
|
71
81
|
# end
|
@@ -75,38 +85,38 @@ module Shodanz
|
|
75
85
|
end
|
76
86
|
end
|
77
87
|
|
78
|
-
# Only returns banner data for the list of specified ports. This
|
79
|
-
# stream provides a filtered, bandwidth-saving view of the Banners
|
88
|
+
# Only returns banner data for the list of specified ports. This
|
89
|
+
# stream provides a filtered, bandwidth-saving view of the Banners
|
80
90
|
# stream in case you are only interested in a specific list of ports.
|
81
91
|
# == Example
|
82
|
-
# api.banners_within_countries("US","DE","JP") do |data|
|
92
|
+
# api.banners_within_countries("US","DE","JP") do |data|
|
83
93
|
# # do something with the banner hash
|
84
94
|
# puts data
|
85
95
|
# end
|
86
96
|
def banners_within_countries(*params)
|
87
|
-
slurp_stream("shodan/countries/#{params.join(
|
97
|
+
slurp_stream("shodan/countries/#{params.join(',')}") do |data|
|
88
98
|
yield data
|
89
99
|
end
|
90
100
|
end
|
91
101
|
|
92
|
-
# Only returns banner data for the list of specified ports. This
|
93
|
-
# stream provides a filtered, bandwidth-saving view of the
|
94
|
-
# Banners stream in case you are only interested in a
|
102
|
+
# Only returns banner data for the list of specified ports. This
|
103
|
+
# stream provides a filtered, bandwidth-saving view of the
|
104
|
+
# Banners stream in case you are only interested in a
|
95
105
|
# specific list of ports.
|
96
106
|
# == Example
|
97
|
-
# api.banners_on_port(80, 443) do |data|
|
107
|
+
# api.banners_on_port(80, 443) do |data|
|
98
108
|
# # do something with the banner hash
|
99
109
|
# puts data
|
100
110
|
# end
|
101
111
|
def banners_on_ports(*params)
|
102
|
-
slurp_stream("shodan/ports/#{params.join(
|
112
|
+
slurp_stream("shodan/ports/#{params.join(',')}") do |data|
|
103
113
|
yield data
|
104
114
|
end
|
105
115
|
end
|
106
116
|
|
107
|
-
# Only returns banner data for a specific port. This
|
108
|
-
# stream provides a filtered, bandwidth-saving view of the
|
109
|
-
# Banners stream in case you are only interested in a
|
117
|
+
# Only returns banner data for a specific port. This
|
118
|
+
# stream provides a filtered, bandwidth-saving view of the
|
119
|
+
# Banners stream in case you are only interested in a
|
110
120
|
# specific list of ports.
|
111
121
|
# == Example
|
112
122
|
# api.banners_on_port(80) do |banner|
|
@@ -120,10 +130,10 @@ module Shodanz
|
|
120
130
|
end
|
121
131
|
|
122
132
|
# Subscribe to banners discovered on all IP ranges described in the network alerts.
|
123
|
-
# Use the REST API methods to create/ delete/ manage your network alerts and
|
133
|
+
# Use the REST API methods to create/ delete/ manage your network alerts and
|
124
134
|
# use the Streaming API to subscribe to them.
|
125
135
|
def alerts
|
126
|
-
slurp_stream(
|
136
|
+
slurp_stream('alert') do |data|
|
127
137
|
yield data
|
128
138
|
end
|
129
139
|
end
|
@@ -134,54 +144,6 @@ module Shodanz
|
|
134
144
|
yield data
|
135
145
|
end
|
136
146
|
end
|
137
|
-
|
138
|
-
private
|
139
|
-
|
140
|
-
# Perform the main function of consuming the streaming API.
|
141
|
-
def slurp_stream(path, **params)
|
142
|
-
uri = URI("#{URL}#{path}?key=#{@key}")
|
143
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
|
144
|
-
request = Net::HTTP::Get.new uri
|
145
|
-
begin
|
146
|
-
http.request request do |resp|
|
147
|
-
raise "Unable to connect to Streaming API" if resp.code != "200"
|
148
|
-
# Buffer for Shodan's bullshit.
|
149
|
-
raw_body = ""
|
150
|
-
resp.read_body do |chunk|
|
151
|
-
if /^\{"product":.*\}\}\n/.match(chunk)
|
152
|
-
begin
|
153
|
-
yield Oj.load(chunk)
|
154
|
-
rescue
|
155
|
-
# yolo
|
156
|
-
end
|
157
|
-
elsif /.*\}\}\n$/.match(chunk)
|
158
|
-
next if raw_body.empty?
|
159
|
-
raw_body << chunk
|
160
|
-
raw_body
|
161
|
-
elsif /^\{.*\b/.match(chunk)
|
162
|
-
raw_body << chunk
|
163
|
-
end
|
164
|
-
if m = /^\{"product":.*\}\}\n/.match(raw_body)
|
165
|
-
index = 0
|
166
|
-
while matched = m[index]
|
167
|
-
index += 1
|
168
|
-
raw_body = raw_body.gsub(/^\{"product":.*\}\}\n/, "")
|
169
|
-
begin
|
170
|
-
yield Oj.load(matched)
|
171
|
-
rescue
|
172
|
-
# yolo
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
176
|
-
end
|
177
|
-
end
|
178
|
-
ensure
|
179
|
-
http.finish
|
180
|
-
end
|
181
|
-
end
|
182
|
-
end
|
183
|
-
|
184
147
|
end
|
185
|
-
|
186
148
|
end
|
187
149
|
end
|