shodanz 1.0.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +85 -56
- data/examples/async_honeypot_detector.rb +19 -0
- data/examples/async_host_search_example.rb +29 -0
- data/examples/async_stream_example.rb +33 -0
- data/examples/streaming_banner_product_stats.rb +3 -0
- data/lib/shodanz.rb +9 -2
- data/lib/shodanz/apis/exploits.rb +12 -31
- data/lib/shodanz/apis/rest.rb +50 -80
- data/lib/shodanz/apis/streaming.rb +44 -82
- data/lib/shodanz/apis/utils.rb +175 -0
- data/lib/shodanz/client.rb +88 -1
- data/lib/shodanz/errors.rb +29 -0
- data/lib/shodanz/version.rb +3 -1
- data/shodanz.gemspec +21 -18
- metadata +60 -27
data/lib/shodanz/apis/rest.rb
CHANGED
@@ -1,27 +1,38 @@
|
|
1
|
-
|
1
|
+
require_relative 'utils.rb'
|
2
|
+
|
3
|
+
# frozen_string_literal: true
|
2
4
|
|
5
|
+
module Shodanz
|
3
6
|
module API
|
4
|
-
# The REST API provides methods to search Shodan, look up
|
5
|
-
# hosts, get summary information on queries and a variety
|
7
|
+
# The REST API provides methods to search Shodan, look up
|
8
|
+
# hosts, get summary information on queries and a variety
|
6
9
|
# of other utilities. This requires you to have an API key
|
7
10
|
# which you can get from Shodan.
|
8
11
|
#
|
9
12
|
# @author Kent 'picat' Gruber
|
10
13
|
class REST
|
14
|
+
include Shodanz::API::Utils
|
15
|
+
|
16
|
+
# @return [String]
|
11
17
|
attr_accessor :key
|
12
18
|
|
13
19
|
# The path to the REST API endpoint.
|
14
|
-
URL =
|
20
|
+
URL = 'https://api.shodan.io/'
|
15
21
|
|
16
22
|
# @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
|
17
23
|
def initialize(key: ENV['SHODAN_API_KEY'])
|
18
|
-
|
19
|
-
|
24
|
+
@url = URL
|
25
|
+
@internet = Async::HTTP::Internet.new
|
26
|
+
self.key = key
|
27
|
+
|
28
|
+
warn 'No key has been found or provided!' unless key?
|
20
29
|
end
|
21
30
|
|
22
31
|
# Check if there's an API key.
|
23
32
|
def key?
|
24
|
-
return true if @key
|
33
|
+
return true if @key
|
34
|
+
|
35
|
+
false
|
25
36
|
end
|
26
37
|
|
27
38
|
# Returns all services that have been found on the given host IP.
|
@@ -31,92 +42,91 @@ module Shodanz
|
|
31
42
|
# == Examples
|
32
43
|
# # Typical usage.
|
33
44
|
# rest_api.host("8.8.8.8")
|
34
|
-
#
|
45
|
+
#
|
35
46
|
# # All historical banners should be returned.
|
36
|
-
# rest_api.host("8.8.8.8", history: true)
|
47
|
+
# rest_api.host("8.8.8.8", history: true)
|
37
48
|
#
|
38
49
|
# # Only return the list of ports and the general host information, no banners.
|
39
|
-
# rest_api.host("8.8.8.8", minify: true)
|
50
|
+
# rest_api.host("8.8.8.8", minify: true)
|
40
51
|
def host(ip, **params)
|
41
52
|
get("shodan/host/#{ip}", params)
|
42
53
|
end
|
43
54
|
|
44
|
-
# This method behaves identical to "/shodan/host/search" with the only
|
45
|
-
# difference that this method does not return any host results, it only
|
46
|
-
# returns the total number of results that matched the query and any
|
47
|
-
# facet information that was requested. As a result this method does
|
55
|
+
# This method behaves identical to "/shodan/host/search" with the only
|
56
|
+
# difference that this method does not return any host results, it only
|
57
|
+
# returns the total number of results that matched the query and any
|
58
|
+
# facet information that was requested. As a result this method does
|
48
59
|
# not consume query credits.
|
49
60
|
# == Examples
|
50
61
|
# rest_api.host_count("apache")
|
51
62
|
# rest_api.host_count("apache", country: "US")
|
52
63
|
# rest_api.host_count("apache", country: "US", state: "MI")
|
53
64
|
# rest_api.host_count("apache", country: "US", state: "MI", city: "Detroit")
|
54
|
-
def host_count(query =
|
65
|
+
def host_count(query = '', facets: {}, **params)
|
55
66
|
params[:query] = query
|
56
67
|
params = turn_into_query(params)
|
57
68
|
facets = turn_into_facets(facets)
|
58
|
-
get(
|
69
|
+
get('shodan/host/count', params.merge(facets))
|
59
70
|
end
|
60
71
|
|
61
|
-
# Search Shodan using the same query syntax as the website and use facets
|
72
|
+
# Search Shodan using the same query syntax as the website and use facets
|
62
73
|
# to get summary information for different properties.
|
63
74
|
# == Example
|
64
75
|
# rest_api.host_search("apache", country: "US", facets: { city: "Detroit" }, page: 1, minify: false)
|
65
|
-
def host_search(query =
|
76
|
+
def host_search(query = '', facets: {}, page: 1, minify: true, **params)
|
66
77
|
params[:query] = query
|
67
78
|
params = turn_into_query(params)
|
68
79
|
facets = turn_into_facets(facets)
|
69
80
|
params[:page] = page
|
70
81
|
params[:minify] = minify
|
71
|
-
get(
|
82
|
+
get('shodan/host/search', params.merge(facets))
|
72
83
|
end
|
73
84
|
|
74
|
-
# This method lets you determine which filters are being used by
|
85
|
+
# This method lets you determine which filters are being used by
|
75
86
|
# the query string and what parameters were provided to the filters.
|
76
|
-
def host_search_tokens(query =
|
87
|
+
def host_search_tokens(query = '', **params)
|
77
88
|
params[:query] = query
|
78
89
|
params = turn_into_query(params)
|
79
|
-
get(
|
90
|
+
get('shodan/host/search/tokens', params)
|
80
91
|
end
|
81
92
|
|
82
93
|
# This method returns a list of port numbers that the crawlers are looking for.
|
83
94
|
def ports
|
84
|
-
get(
|
95
|
+
get('shodan/ports')
|
85
96
|
end
|
86
97
|
|
87
98
|
# List all protocols that can be used when performing on-demand Internet scans via Shodan.
|
88
|
-
def protocols
|
89
|
-
get(
|
99
|
+
def protocols
|
100
|
+
get('shodan/protocols')
|
90
101
|
end
|
91
102
|
|
92
103
|
# Use this method to request Shodan to crawl a network.
|
93
104
|
#
|
94
|
-
# This method uses API scan credits: 1 IP consumes 1 scan credit. You
|
95
|
-
# must have a paid API plan (either one-time payment or subscription)
|
105
|
+
# This method uses API scan credits: 1 IP consumes 1 scan credit. You
|
106
|
+
# must have a paid API plan (either one-time payment or subscription)
|
96
107
|
# in order to use this method.
|
97
108
|
#
|
98
109
|
# IP, IPs or netblocks (in CIDR notation) that should get crawled.
|
99
110
|
def scan(*ips)
|
100
|
-
|
101
|
-
post("shodan/scan", ips: ips.join(","))
|
111
|
+
post('shodan/scan', ips: ips.join(','))
|
102
112
|
end
|
103
113
|
|
104
114
|
# Use this method to request Shodan to crawl the Internet for a specific port.
|
105
115
|
#
|
106
|
-
# This method is restricted to security researchers and companies with
|
107
|
-
# a Shodan Data license. To apply for access to this method as a researcher,
|
108
|
-
# please email jmath@shodan.io with information about your project.
|
116
|
+
# This method is restricted to security researchers and companies with
|
117
|
+
# a Shodan Data license. To apply for access to this method as a researcher,
|
118
|
+
# please email jmath@shodan.io with information about your project.
|
109
119
|
# Access is restricted to prevent abuse.
|
110
120
|
#
|
111
121
|
# == Example
|
112
122
|
# rest_api.crawl_for(port: 80, protocol: "http")
|
113
123
|
def crawl_for(**params)
|
114
|
-
params[:query] =
|
124
|
+
params[:query] = ''
|
115
125
|
params = turn_into_query(params)
|
116
126
|
post('shodan/scan/internet', params)
|
117
127
|
end
|
118
128
|
|
119
|
-
# Check the progress of a previously submitted scan request.
|
129
|
+
# Check the progress of a previously submitted scan request.
|
120
130
|
def scan_status(id)
|
121
131
|
get("shodan/scan/#{id}")
|
122
132
|
end
|
@@ -130,7 +140,7 @@ module Shodanz
|
|
130
140
|
def search_for_community_query(query, **params)
|
131
141
|
params[:query] = query
|
132
142
|
params = turn_into_query(params)
|
133
|
-
get('shodan/query/search', params)
|
143
|
+
get('shodan/query/search', params)
|
134
144
|
end
|
135
145
|
|
136
146
|
# Use this method to obtain a list of popular tags for the saved search queries in Shodan.
|
@@ -147,16 +157,16 @@ module Shodanz
|
|
147
157
|
|
148
158
|
# Look up the IP address for the provided list of hostnames.
|
149
159
|
def resolve(*hostnames)
|
150
|
-
get('dns/resolve', hostnames: hostnames.join(
|
160
|
+
get('dns/resolve', hostnames: hostnames.join(','))
|
151
161
|
end
|
152
162
|
|
153
|
-
# Look up the hostnames that have been defined for the
|
163
|
+
# Look up the hostnames that have been defined for the
|
154
164
|
# given list of IP addresses.
|
155
165
|
def reverse_lookup(*ips)
|
156
|
-
get('dns/reverse', ips: ips.join(
|
166
|
+
get('dns/reverse', ips: ips.join(','))
|
157
167
|
end
|
158
168
|
|
159
|
-
# Shows the HTTP headers that your client sends when
|
169
|
+
# Shows the HTTP headers that your client sends when
|
160
170
|
# connecting to a webserver.
|
161
171
|
def http_headers
|
162
172
|
get('tools/httpheaders')
|
@@ -167,7 +177,7 @@ module Shodanz
|
|
167
177
|
get('tools/myip')
|
168
178
|
end
|
169
179
|
|
170
|
-
# Calculates a honeypot probability score ranging
|
180
|
+
# Calculates a honeypot probability score ranging
|
171
181
|
# from 0 (not a honeypot) to 1.0 (is a honeypot).
|
172
182
|
def honeypot_score(ip)
|
173
183
|
get("labs/honeyscore/#{ip}")
|
@@ -177,46 +187,6 @@ module Shodanz
|
|
177
187
|
def info
|
178
188
|
get('api-info')
|
179
189
|
end
|
180
|
-
|
181
|
-
# Perform a direct GET HTTP request to the REST API.
|
182
|
-
def get(path, **params)
|
183
|
-
resp = Unirest.get "#{URL}#{path}?key=#{@key}", parameters: params
|
184
|
-
if resp.code != 200
|
185
|
-
raise resp.body['error'] if resp.body.key?('error')
|
186
|
-
raise resp
|
187
|
-
end
|
188
|
-
resp.body
|
189
|
-
end
|
190
|
-
|
191
|
-
# Perform a direct POST HTTP request to the REST API.
|
192
|
-
def post(path, **params)
|
193
|
-
resp = Unirest.post "#{URL}#{path}?key=#{@key}", parameters: params
|
194
|
-
if resp.code != 200
|
195
|
-
raise resp.body['error'] if resp.body.key?('error')
|
196
|
-
raise resp
|
197
|
-
end
|
198
|
-
resp.body
|
199
|
-
end
|
200
|
-
|
201
|
-
private
|
202
|
-
|
203
|
-
def turn_into_query(params)
|
204
|
-
filters = params.reject { |key, _| key == :query }
|
205
|
-
filters.each do |key, value|
|
206
|
-
params[:query] << " #{key}:#{value}"
|
207
|
-
end
|
208
|
-
params.select { |key, _| key == :query }
|
209
|
-
end
|
210
|
-
|
211
|
-
def turn_into_facets(facets)
|
212
|
-
filters = facets.reject { |key, _| key == :facets }
|
213
|
-
facets[:facets] = []
|
214
|
-
filters.each do |key, value|
|
215
|
-
facets[:facets] << "#{key}:#{value}"
|
216
|
-
end
|
217
|
-
facets[:facets] = facets[:facets].join(',')
|
218
|
-
facets.select { |key, _| key == :facets }
|
219
|
-
end
|
220
190
|
end
|
221
191
|
end
|
222
192
|
end
|
@@ -1,38 +1,48 @@
|
|
1
|
-
|
1
|
+
require_relative 'utils.rb'
|
2
|
+
|
3
|
+
# frozen_string_literal: true
|
2
4
|
|
5
|
+
module Shodanz
|
3
6
|
module API
|
4
|
-
# The REST API provides methods to search Shodan, look up
|
5
|
-
# hosts, get summary information on queries and a variety
|
7
|
+
# The REST API provides methods to search Shodan, look up
|
8
|
+
# hosts, get summary information on queries and a variety
|
6
9
|
# of other utilities. This requires you to have an API key
|
7
10
|
# which you can get from Shodan.
|
8
11
|
#
|
9
|
-
# Note: Only 1-5% of the data is currently provided to
|
10
|
-
# subscription-based API plans. If your company is interested
|
11
|
-
# in large-scale, real-time access to all of the Shodan data
|
12
|
+
# Note: Only 1-5% of the data is currently provided to
|
13
|
+
# subscription-based API plans. If your company is interested
|
14
|
+
# in large-scale, real-time access to all of the Shodan data
|
12
15
|
# please contact us for pricing information (sales@shodan.io).
|
13
|
-
#
|
16
|
+
#
|
14
17
|
# @author Kent 'picat' Gruber
|
15
18
|
class Streaming
|
19
|
+
include Shodanz::API::Utils
|
20
|
+
|
21
|
+
# @return [String]
|
16
22
|
attr_accessor :key
|
17
23
|
|
18
|
-
# The Streaming API is an HTTP-based service that returns
|
24
|
+
# The Streaming API is an HTTP-based service that returns
|
19
25
|
# a real-time stream of data collected by Shodan.
|
20
|
-
URL =
|
26
|
+
URL = 'https://stream.shodan.io/'
|
21
27
|
|
22
28
|
# @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
|
23
29
|
def initialize(key: ENV['SHODAN_API_KEY'])
|
24
|
-
|
25
|
-
|
30
|
+
@url = URL
|
31
|
+
@internet = Async::HTTP::Internet.new
|
32
|
+
self.key = key
|
33
|
+
|
34
|
+
warn 'No key has been found or provided!' unless key?
|
26
35
|
end
|
27
36
|
|
28
37
|
# Check if there's an API key.
|
29
38
|
def key?
|
30
39
|
return true if @key; false
|
40
|
+
|
31
41
|
end
|
32
42
|
|
33
|
-
# This stream provides ALL of the data that Shodan collects.
|
34
|
-
# Use this stream if you need access to everything and/ or want to
|
35
|
-
# store your own Shodan database locally. If you only care about specific
|
43
|
+
# This stream provides ALL of the data that Shodan collects.
|
44
|
+
# Use this stream if you need access to everything and/ or want to
|
45
|
+
# store your own Shodan database locally. If you only care about specific
|
36
46
|
# ports, please use the Ports stream.
|
37
47
|
#
|
38
48
|
# Sometimes data may be piped down stream that is weird to parse. You can choose
|
@@ -44,28 +54,28 @@ module Shodanz
|
|
44
54
|
# puts data
|
45
55
|
# end
|
46
56
|
def banners(**params)
|
47
|
-
slurp_stream(
|
57
|
+
slurp_stream('shodan/banners', params) do |data|
|
48
58
|
yield data
|
49
59
|
end
|
50
60
|
end
|
51
61
|
|
52
|
-
# This stream provides a filtered, bandwidth-saving view of the Banners
|
62
|
+
# This stream provides a filtered, bandwidth-saving view of the Banners
|
53
63
|
# stream in case you are only interested in devices located in certain ASNs.
|
54
64
|
# == Example
|
55
|
-
# api.banners_within_asns(3303, 32475) do |data|
|
65
|
+
# api.banners_within_asns(3303, 32475) do |data|
|
56
66
|
# # do something with the banner hash
|
57
67
|
# puts data
|
58
68
|
# end
|
59
69
|
def banners_within_asns(*asns, **params)
|
60
|
-
slurp_stream("shodan/asn/#{asns.join(
|
70
|
+
slurp_stream("shodan/asn/#{asns.join(',')}", params) do |data|
|
61
71
|
yield data
|
62
72
|
end
|
63
73
|
end
|
64
74
|
|
65
|
-
# This stream provides a filtered, bandwidth-saving view of the Banners
|
75
|
+
# This stream provides a filtered, bandwidth-saving view of the Banners
|
66
76
|
# stream in case you are only interested in devices located in a certain ASN.
|
67
77
|
# == Example
|
68
|
-
# api.banners_within_asn(3303) do |data|
|
78
|
+
# api.banners_within_asn(3303) do |data|
|
69
79
|
# # do something with the banner hash
|
70
80
|
# puts data
|
71
81
|
# end
|
@@ -75,38 +85,38 @@ module Shodanz
|
|
75
85
|
end
|
76
86
|
end
|
77
87
|
|
78
|
-
# Only returns banner data for the list of specified ports. This
|
79
|
-
# stream provides a filtered, bandwidth-saving view of the Banners
|
88
|
+
# Only returns banner data for the list of specified ports. This
|
89
|
+
# stream provides a filtered, bandwidth-saving view of the Banners
|
80
90
|
# stream in case you are only interested in a specific list of ports.
|
81
91
|
# == Example
|
82
|
-
# api.banners_within_countries("US","DE","JP") do |data|
|
92
|
+
# api.banners_within_countries("US","DE","JP") do |data|
|
83
93
|
# # do something with the banner hash
|
84
94
|
# puts data
|
85
95
|
# end
|
86
96
|
def banners_within_countries(*params)
|
87
|
-
slurp_stream("shodan/countries/#{params.join(
|
97
|
+
slurp_stream("shodan/countries/#{params.join(',')}") do |data|
|
88
98
|
yield data
|
89
99
|
end
|
90
100
|
end
|
91
101
|
|
92
|
-
# Only returns banner data for the list of specified ports. This
|
93
|
-
# stream provides a filtered, bandwidth-saving view of the
|
94
|
-
# Banners stream in case you are only interested in a
|
102
|
+
# Only returns banner data for the list of specified ports. This
|
103
|
+
# stream provides a filtered, bandwidth-saving view of the
|
104
|
+
# Banners stream in case you are only interested in a
|
95
105
|
# specific list of ports.
|
96
106
|
# == Example
|
97
|
-
# api.banners_on_port(80, 443) do |data|
|
107
|
+
# api.banners_on_port(80, 443) do |data|
|
98
108
|
# # do something with the banner hash
|
99
109
|
# puts data
|
100
110
|
# end
|
101
111
|
def banners_on_ports(*params)
|
102
|
-
slurp_stream("shodan/ports/#{params.join(
|
112
|
+
slurp_stream("shodan/ports/#{params.join(',')}") do |data|
|
103
113
|
yield data
|
104
114
|
end
|
105
115
|
end
|
106
116
|
|
107
|
-
# Only returns banner data for a specific port. This
|
108
|
-
# stream provides a filtered, bandwidth-saving view of the
|
109
|
-
# Banners stream in case you are only interested in a
|
117
|
+
# Only returns banner data for a specific port. This
|
118
|
+
# stream provides a filtered, bandwidth-saving view of the
|
119
|
+
# Banners stream in case you are only interested in a
|
110
120
|
# specific list of ports.
|
111
121
|
# == Example
|
112
122
|
# api.banners_on_port(80) do |banner|
|
@@ -120,10 +130,10 @@ module Shodanz
|
|
120
130
|
end
|
121
131
|
|
122
132
|
# Subscribe to banners discovered on all IP ranges described in the network alerts.
|
123
|
-
# Use the REST API methods to create/ delete/ manage your network alerts and
|
133
|
+
# Use the REST API methods to create/ delete/ manage your network alerts and
|
124
134
|
# use the Streaming API to subscribe to them.
|
125
135
|
def alerts
|
126
|
-
slurp_stream(
|
136
|
+
slurp_stream('alert') do |data|
|
127
137
|
yield data
|
128
138
|
end
|
129
139
|
end
|
@@ -134,54 +144,6 @@ module Shodanz
|
|
134
144
|
yield data
|
135
145
|
end
|
136
146
|
end
|
137
|
-
|
138
|
-
private
|
139
|
-
|
140
|
-
# Perform the main function of consuming the streaming API.
|
141
|
-
def slurp_stream(path, **params)
|
142
|
-
uri = URI("#{URL}#{path}?key=#{@key}")
|
143
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
|
144
|
-
request = Net::HTTP::Get.new uri
|
145
|
-
begin
|
146
|
-
http.request request do |resp|
|
147
|
-
raise "Unable to connect to Streaming API" if resp.code != "200"
|
148
|
-
# Buffer for Shodan's bullshit.
|
149
|
-
raw_body = ""
|
150
|
-
resp.read_body do |chunk|
|
151
|
-
if /^\{"product":.*\}\}\n/.match(chunk)
|
152
|
-
begin
|
153
|
-
yield Oj.load(chunk)
|
154
|
-
rescue
|
155
|
-
# yolo
|
156
|
-
end
|
157
|
-
elsif /.*\}\}\n$/.match(chunk)
|
158
|
-
next if raw_body.empty?
|
159
|
-
raw_body << chunk
|
160
|
-
raw_body
|
161
|
-
elsif /^\{.*\b/.match(chunk)
|
162
|
-
raw_body << chunk
|
163
|
-
end
|
164
|
-
if m = /^\{"product":.*\}\}\n/.match(raw_body)
|
165
|
-
index = 0
|
166
|
-
while matched = m[index]
|
167
|
-
index += 1
|
168
|
-
raw_body = raw_body.gsub(/^\{"product":.*\}\}\n/, "")
|
169
|
-
begin
|
170
|
-
yield Oj.load(matched)
|
171
|
-
rescue
|
172
|
-
# yolo
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
176
|
-
end
|
177
|
-
end
|
178
|
-
ensure
|
179
|
-
http.finish
|
180
|
-
end
|
181
|
-
end
|
182
|
-
end
|
183
|
-
|
184
147
|
end
|
185
|
-
|
186
148
|
end
|
187
149
|
end
|