shodanz 1.0.6 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,27 +1,38 @@
1
- module Shodanz
1
+ require_relative 'utils.rb'
2
+
3
+ # frozen_string_literal: true
2
4
 
5
+ module Shodanz
3
6
  module API
4
- # The REST API provides methods to search Shodan, look up
5
- # hosts, get summary information on queries and a variety
7
+ # The REST API provides methods to search Shodan, look up
8
+ # hosts, get summary information on queries and a variety
6
9
  # of other utilities. This requires you to have an API key
7
10
  # which you can get from Shodan.
8
11
  #
9
12
  # @author Kent 'picat' Gruber
10
13
  class REST
14
+ include Shodanz::API::Utils
15
+
16
+ # @return [String]
11
17
  attr_accessor :key
12
18
 
13
19
  # The path to the REST API endpoint.
14
- URL = "https://api.shodan.io/"
20
+ URL = 'https://api.shodan.io/'
15
21
 
16
22
  # @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
17
23
  def initialize(key: ENV['SHODAN_API_KEY'])
18
- self.key = key
19
- warn "No key has been found or provided!" unless self.key?
24
+ @url = URL
25
+ @internet = Async::HTTP::Internet.new
26
+ self.key = key
27
+
28
+ warn 'No key has been found or provided!' unless key?
20
29
  end
21
30
 
22
31
  # Check if there's an API key.
23
32
  def key?
24
- return true if @key; false
33
+ return true if @key
34
+
35
+ false
25
36
  end
26
37
 
27
38
  # Returns all services that have been found on the given host IP.
@@ -31,92 +42,91 @@ module Shodanz
31
42
  # == Examples
32
43
  # # Typical usage.
33
44
  # rest_api.host("8.8.8.8")
34
- #
45
+ #
35
46
  # # All historical banners should be returned.
36
- # rest_api.host("8.8.8.8", history: true)
47
+ # rest_api.host("8.8.8.8", history: true)
37
48
  #
38
49
  # # Only return the list of ports and the general host information, no banners.
39
- # rest_api.host("8.8.8.8", minify: true)
50
+ # rest_api.host("8.8.8.8", minify: true)
40
51
  def host(ip, **params)
41
52
  get("shodan/host/#{ip}", params)
42
53
  end
43
54
 
44
- # This method behaves identical to "/shodan/host/search" with the only
45
- # difference that this method does not return any host results, it only
46
- # returns the total number of results that matched the query and any
47
- # facet information that was requested. As a result this method does
55
+ # This method behaves identical to "/shodan/host/search" with the only
56
+ # difference that this method does not return any host results, it only
57
+ # returns the total number of results that matched the query and any
58
+ # facet information that was requested. As a result this method does
48
59
  # not consume query credits.
49
60
  # == Examples
50
61
  # rest_api.host_count("apache")
51
62
  # rest_api.host_count("apache", country: "US")
52
63
  # rest_api.host_count("apache", country: "US", state: "MI")
53
64
  # rest_api.host_count("apache", country: "US", state: "MI", city: "Detroit")
54
- def host_count(query = "", facets: {}, **params)
65
+ def host_count(query = '', facets: {}, **params)
55
66
  params[:query] = query
56
67
  params = turn_into_query(params)
57
68
  facets = turn_into_facets(facets)
58
- get("shodan/host/count", params.merge(facets))
69
+ get('shodan/host/count', params.merge(facets))
59
70
  end
60
71
 
61
- # Search Shodan using the same query syntax as the website and use facets
72
+ # Search Shodan using the same query syntax as the website and use facets
62
73
  # to get summary information for different properties.
63
74
  # == Example
64
75
  # rest_api.host_search("apache", country: "US", facets: { city: "Detroit" }, page: 1, minify: false)
65
- def host_search(query = "", facets: {}, page: 1, minify: true, **params)
76
+ def host_search(query = '', facets: {}, page: 1, minify: true, **params)
66
77
  params[:query] = query
67
78
  params = turn_into_query(params)
68
79
  facets = turn_into_facets(facets)
69
80
  params[:page] = page
70
81
  params[:minify] = minify
71
- get("shodan/host/search", params.merge(facets))
82
+ get('shodan/host/search', params.merge(facets))
72
83
  end
73
84
 
74
- # This method lets you determine which filters are being used by
85
+ # This method lets you determine which filters are being used by
75
86
  # the query string and what parameters were provided to the filters.
76
- def host_search_tokens(query = "", **params)
87
+ def host_search_tokens(query = '', **params)
77
88
  params[:query] = query
78
89
  params = turn_into_query(params)
79
- get("shodan/host/search/tokens", params)
90
+ get('shodan/host/search/tokens', params)
80
91
  end
81
92
 
82
93
  # This method returns a list of port numbers that the crawlers are looking for.
83
94
  def ports
84
- get("shodan/ports")
95
+ get('shodan/ports')
85
96
  end
86
97
 
87
98
  # List all protocols that can be used when performing on-demand Internet scans via Shodan.
88
- def protocols
89
- get("shodan/protocols")
99
+ def protocols
100
+ get('shodan/protocols')
90
101
  end
91
102
 
92
103
  # Use this method to request Shodan to crawl a network.
93
104
  #
94
- # This method uses API scan credits: 1 IP consumes 1 scan credit. You
95
- # must have a paid API plan (either one-time payment or subscription)
105
+ # This method uses API scan credits: 1 IP consumes 1 scan credit. You
106
+ # must have a paid API plan (either one-time payment or subscription)
96
107
  # in order to use this method.
97
108
  #
98
109
  # IP, IPs or netblocks (in CIDR notation) that should get crawled.
99
110
  def scan(*ips)
100
- raise "Not enough scan credits!" unless self.info["scan_credits"] >= 1
101
- post("shodan/scan", ips: ips.join(","))
111
+ post('shodan/scan', ips: ips.join(','))
102
112
  end
103
113
 
104
114
  # Use this method to request Shodan to crawl the Internet for a specific port.
105
115
  #
106
- # This method is restricted to security researchers and companies with
107
- # a Shodan Data license. To apply for access to this method as a researcher,
108
- # please email jmath@shodan.io with information about your project.
116
+ # This method is restricted to security researchers and companies with
117
+ # a Shodan Data license. To apply for access to this method as a researcher,
118
+ # please email jmath@shodan.io with information about your project.
109
119
  # Access is restricted to prevent abuse.
110
120
  #
111
121
  # == Example
112
122
  # rest_api.crawl_for(port: 80, protocol: "http")
113
123
  def crawl_for(**params)
114
- params[:query] = ""
124
+ params[:query] = ''
115
125
  params = turn_into_query(params)
116
126
  post('shodan/scan/internet', params)
117
127
  end
118
128
 
119
- # Check the progress of a previously submitted scan request.
129
+ # Check the progress of a previously submitted scan request.
120
130
  def scan_status(id)
121
131
  get("shodan/scan/#{id}")
122
132
  end
@@ -130,7 +140,7 @@ module Shodanz
130
140
  def search_for_community_query(query, **params)
131
141
  params[:query] = query
132
142
  params = turn_into_query(params)
133
- get('shodan/query/search', params)
143
+ get('shodan/query/search', params)
134
144
  end
135
145
 
136
146
  # Use this method to obtain a list of popular tags for the saved search queries in Shodan.
@@ -147,16 +157,16 @@ module Shodanz
147
157
 
148
158
  # Look up the IP address for the provided list of hostnames.
149
159
  def resolve(*hostnames)
150
- get('dns/resolve', hostnames: hostnames.join(","))
160
+ get('dns/resolve', hostnames: hostnames.join(','))
151
161
  end
152
162
 
153
- # Look up the hostnames that have been defined for the
163
+ # Look up the hostnames that have been defined for the
154
164
  # given list of IP addresses.
155
165
  def reverse_lookup(*ips)
156
- get('dns/reverse', ips: ips.join(","))
166
+ get('dns/reverse', ips: ips.join(','))
157
167
  end
158
168
 
159
- # Shows the HTTP headers that your client sends when
169
+ # Shows the HTTP headers that your client sends when
160
170
  # connecting to a webserver.
161
171
  def http_headers
162
172
  get('tools/httpheaders')
@@ -167,7 +177,7 @@ module Shodanz
167
177
  get('tools/myip')
168
178
  end
169
179
 
170
- # Calculates a honeypot probability score ranging
180
+ # Calculates a honeypot probability score ranging
171
181
  # from 0 (not a honeypot) to 1.0 (is a honeypot).
172
182
  def honeypot_score(ip)
173
183
  get("labs/honeyscore/#{ip}")
@@ -177,46 +187,6 @@ module Shodanz
177
187
  def info
178
188
  get('api-info')
179
189
  end
180
-
181
- # Perform a direct GET HTTP request to the REST API.
182
- def get(path, **params)
183
- resp = Unirest.get "#{URL}#{path}?key=#{@key}", parameters: params
184
- if resp.code != 200
185
- raise resp.body['error'] if resp.body.key?('error')
186
- raise resp
187
- end
188
- resp.body
189
- end
190
-
191
- # Perform a direct POST HTTP request to the REST API.
192
- def post(path, **params)
193
- resp = Unirest.post "#{URL}#{path}?key=#{@key}", parameters: params
194
- if resp.code != 200
195
- raise resp.body['error'] if resp.body.key?('error')
196
- raise resp
197
- end
198
- resp.body
199
- end
200
-
201
- private
202
-
203
- def turn_into_query(params)
204
- filters = params.reject { |key, _| key == :query }
205
- filters.each do |key, value|
206
- params[:query] << " #{key}:#{value}"
207
- end
208
- params.select { |key, _| key == :query }
209
- end
210
-
211
- def turn_into_facets(facets)
212
- filters = facets.reject { |key, _| key == :facets }
213
- facets[:facets] = []
214
- filters.each do |key, value|
215
- facets[:facets] << "#{key}:#{value}"
216
- end
217
- facets[:facets] = facets[:facets].join(',')
218
- facets.select { |key, _| key == :facets }
219
- end
220
190
  end
221
191
  end
222
192
  end
@@ -1,38 +1,48 @@
1
- module Shodanz
1
+ require_relative 'utils.rb'
2
+
3
+ # frozen_string_literal: true
2
4
 
5
+ module Shodanz
3
6
  module API
4
- # The REST API provides methods to search Shodan, look up
5
- # hosts, get summary information on queries and a variety
7
+ # The REST API provides methods to search Shodan, look up
8
+ # hosts, get summary information on queries and a variety
6
9
  # of other utilities. This requires you to have an API key
7
10
  # which you can get from Shodan.
8
11
  #
9
- # Note: Only 1-5% of the data is currently provided to
10
- # subscription-based API plans. If your company is interested
11
- # in large-scale, real-time access to all of the Shodan data
12
+ # Note: Only 1-5% of the data is currently provided to
13
+ # subscription-based API plans. If your company is interested
14
+ # in large-scale, real-time access to all of the Shodan data
12
15
  # please contact us for pricing information (sales@shodan.io).
13
- #
16
+ #
14
17
  # @author Kent 'picat' Gruber
15
18
  class Streaming
19
+ include Shodanz::API::Utils
20
+
21
+ # @return [String]
16
22
  attr_accessor :key
17
23
 
18
- # The Streaming API is an HTTP-based service that returns
24
+ # The Streaming API is an HTTP-based service that returns
19
25
  # a real-time stream of data collected by Shodan.
20
- URL = "https://stream.shodan.io/"
26
+ URL = 'https://stream.shodan.io/'
21
27
 
22
28
  # @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
23
29
  def initialize(key: ENV['SHODAN_API_KEY'])
24
- self.key = key
25
- warn "No key has been found or provided!" unless self.key?
30
+ @url = URL
31
+ @internet = Async::HTTP::Internet.new
32
+ self.key = key
33
+
34
+ warn 'No key has been found or provided!' unless key?
26
35
  end
27
36
 
28
37
  # Check if there's an API key.
29
38
  def key?
30
39
  return true if @key; false
40
+
31
41
  end
32
42
 
33
- # This stream provides ALL of the data that Shodan collects.
34
- # Use this stream if you need access to everything and/ or want to
35
- # store your own Shodan database locally. If you only care about specific
43
+ # This stream provides ALL of the data that Shodan collects.
44
+ # Use this stream if you need access to everything and/ or want to
45
+ # store your own Shodan database locally. If you only care about specific
36
46
  # ports, please use the Ports stream.
37
47
  #
38
48
  # Sometimes data may be piped down stream that is weird to parse. You can choose
@@ -44,28 +54,28 @@ module Shodanz
44
54
  # puts data
45
55
  # end
46
56
  def banners(**params)
47
- slurp_stream("shodan/banners", params) do |data|
57
+ slurp_stream('shodan/banners', params) do |data|
48
58
  yield data
49
59
  end
50
60
  end
51
61
 
52
- # This stream provides a filtered, bandwidth-saving view of the Banners
62
+ # This stream provides a filtered, bandwidth-saving view of the Banners
53
63
  # stream in case you are only interested in devices located in certain ASNs.
54
64
  # == Example
55
- # api.banners_within_asns(3303, 32475) do |data|
65
+ # api.banners_within_asns(3303, 32475) do |data|
56
66
  # # do something with the banner hash
57
67
  # puts data
58
68
  # end
59
69
  def banners_within_asns(*asns, **params)
60
- slurp_stream("shodan/asn/#{asns.join(",")}", params) do |data|
70
+ slurp_stream("shodan/asn/#{asns.join(',')}", params) do |data|
61
71
  yield data
62
72
  end
63
73
  end
64
74
 
65
- # This stream provides a filtered, bandwidth-saving view of the Banners
75
+ # This stream provides a filtered, bandwidth-saving view of the Banners
66
76
  # stream in case you are only interested in devices located in a certain ASN.
67
77
  # == Example
68
- # api.banners_within_asn(3303) do |data|
78
+ # api.banners_within_asn(3303) do |data|
69
79
  # # do something with the banner hash
70
80
  # puts data
71
81
  # end
@@ -75,38 +85,38 @@ module Shodanz
75
85
  end
76
86
  end
77
87
 
78
- # Only returns banner data for the list of specified ports. This
79
- # stream provides a filtered, bandwidth-saving view of the Banners
88
+ # Only returns banner data for the list of specified ports. This
89
+ # stream provides a filtered, bandwidth-saving view of the Banners
80
90
  # stream in case you are only interested in a specific list of ports.
81
91
  # == Example
82
- # api.banners_within_countries("US","DE","JP") do |data|
92
+ # api.banners_within_countries("US","DE","JP") do |data|
83
93
  # # do something with the banner hash
84
94
  # puts data
85
95
  # end
86
96
  def banners_within_countries(*params)
87
- slurp_stream("shodan/countries/#{params.join(",")}") do |data|
97
+ slurp_stream("shodan/countries/#{params.join(',')}") do |data|
88
98
  yield data
89
99
  end
90
100
  end
91
101
 
92
- # Only returns banner data for the list of specified ports. This
93
- # stream provides a filtered, bandwidth-saving view of the
94
- # Banners stream in case you are only interested in a
102
+ # Only returns banner data for the list of specified ports. This
103
+ # stream provides a filtered, bandwidth-saving view of the
104
+ # Banners stream in case you are only interested in a
95
105
  # specific list of ports.
96
106
  # == Example
97
- # api.banners_on_port(80, 443) do |data|
107
+ # api.banners_on_port(80, 443) do |data|
98
108
  # # do something with the banner hash
99
109
  # puts data
100
110
  # end
101
111
  def banners_on_ports(*params)
102
- slurp_stream("shodan/ports/#{params.join(",")}") do |data|
112
+ slurp_stream("shodan/ports/#{params.join(',')}") do |data|
103
113
  yield data
104
114
  end
105
115
  end
106
116
 
107
- # Only returns banner data for a specific port. This
108
- # stream provides a filtered, bandwidth-saving view of the
109
- # Banners stream in case you are only interested in a
117
+ # Only returns banner data for a specific port. This
118
+ # stream provides a filtered, bandwidth-saving view of the
119
+ # Banners stream in case you are only interested in a
110
120
  # specific list of ports.
111
121
  # == Example
112
122
  # api.banners_on_port(80) do |banner|
@@ -120,10 +130,10 @@ module Shodanz
120
130
  end
121
131
 
122
132
  # Subscribe to banners discovered on all IP ranges described in the network alerts.
123
- # Use the REST API methods to create/ delete/ manage your network alerts and
133
+ # Use the REST API methods to create/ delete/ manage your network alerts and
124
134
  # use the Streaming API to subscribe to them.
125
135
  def alerts
126
- slurp_stream("alert") do |data|
136
+ slurp_stream('alert') do |data|
127
137
  yield data
128
138
  end
129
139
  end
@@ -134,54 +144,6 @@ module Shodanz
134
144
  yield data
135
145
  end
136
146
  end
137
-
138
- private
139
-
140
- # Perform the main function of consuming the streaming API.
141
- def slurp_stream(path, **params)
142
- uri = URI("#{URL}#{path}?key=#{@key}")
143
- Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
144
- request = Net::HTTP::Get.new uri
145
- begin
146
- http.request request do |resp|
147
- raise "Unable to connect to Streaming API" if resp.code != "200"
148
- # Buffer for Shodan's bullshit.
149
- raw_body = ""
150
- resp.read_body do |chunk|
151
- if /^\{"product":.*\}\}\n/.match(chunk)
152
- begin
153
- yield Oj.load(chunk)
154
- rescue
155
- # yolo
156
- end
157
- elsif /.*\}\}\n$/.match(chunk)
158
- next if raw_body.empty?
159
- raw_body << chunk
160
- raw_body
161
- elsif /^\{.*\b/.match(chunk)
162
- raw_body << chunk
163
- end
164
- if m = /^\{"product":.*\}\}\n/.match(raw_body)
165
- index = 0
166
- while matched = m[index]
167
- index += 1
168
- raw_body = raw_body.gsub(/^\{"product":.*\}\}\n/, "")
169
- begin
170
- yield Oj.load(matched)
171
- rescue
172
- # yolo
173
- end
174
- end
175
- end
176
- end
177
- end
178
- ensure
179
- http.finish
180
- end
181
- end
182
- end
183
-
184
147
  end
185
-
186
148
  end
187
149
  end