shodanz 1.0.6 → 2.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +33 -0
- data/README.md +83 -51
- data/examples/async_honeypot_detector.rb +21 -0
- data/examples/async_host_search_example.rb +29 -0
- data/examples/async_stream_example.rb +33 -0
- data/examples/streaming_banner_product_stats.rb +3 -0
- data/lib/shodanz.rb +9 -2
- data/lib/shodanz/apis/exploits.rb +20 -39
- data/lib/shodanz/apis/rest.rb +61 -91
- data/lib/shodanz/apis/streaming.rb +44 -82
- data/lib/shodanz/apis/utils.rb +201 -0
- data/lib/shodanz/client.rb +88 -1
- data/lib/shodanz/errors.rb +41 -0
- data/lib/shodanz/version.rb +3 -1
- data/shodanz.gemspec +21 -18
- metadata +71 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6def2a2b044dbd2e60e882a15d95d29e65c3319dd3265a9f133b72d58635f2aa
|
4
|
+
data.tar.gz: c9fb13c755a225b440107443b0a5d62ee67d3de3d4433285ec9cb1e23dce6c7c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a05b580ae377b72d7e3aedf613a780e9cf9c9b17a7d3358b798719e2cfeba5db47ac814b797872b1d85e6f81c24ac5031d0d7081dfddde48b861a9b5e354949a
|
7
|
+
data.tar.gz: 30c29c37cc7ab0e2728a18bb7ecd7fc402761b9ee23e70a7480711224dce7a4771f20fc371c1a47ee0c3db6671aee44a9477074f53aaf513289a268a0dbe26b5
|
@@ -0,0 +1,33 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches:
|
6
|
+
- master
|
7
|
+
pull_request:
|
8
|
+
branches:
|
9
|
+
- master
|
10
|
+
schedule:
|
11
|
+
- cron: "0 9 * * *"
|
12
|
+
|
13
|
+
jobs:
|
14
|
+
test:
|
15
|
+
if: |
|
16
|
+
github.actor == 'picatz' ||
|
17
|
+
github.actor == 'dependabot[bot]' ||
|
18
|
+
github.actor == 'dependabot-preview[bot]'
|
19
|
+
runs-on: ubuntu-latest
|
20
|
+
steps:
|
21
|
+
- uses: actions/checkout@v1
|
22
|
+
- name: Set up Ruby 2.7
|
23
|
+
uses: actions/setup-ruby@v1
|
24
|
+
with:
|
25
|
+
ruby-version: 2.7.x
|
26
|
+
- name: Rspec
|
27
|
+
env:
|
28
|
+
SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
|
29
|
+
run: |
|
30
|
+
gem install bundler
|
31
|
+
bundle install --jobs 4 --retry 3
|
32
|
+
bundle exec rspec
|
33
|
+
|
data/README.md
CHANGED
@@ -1,11 +1,14 @@
|
|
1
1
|
# Shodanz
|
2
2
|
|
3
|
-
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/shodanz.svg)](https://badge.fury.io/rb/shodanz)
|
4
|
+
[![Yard Docs](http://img.shields.io/badge/shodanz-docs-blue.svg)](https://www.rubydoc.info/gems/shodanz/)
|
5
|
+
![CI](https://github.com/picatz/shodanz/workflows/CI/badge.svg)
|
6
|
+
|
7
|
+
A modern, async Ruby [gem](https://rubygems.org/) for [Shodan](https://www.shodan.io/), the world's first search engine for Internet-connected devices.
|
4
8
|
|
5
9
|
## Installation
|
6
10
|
|
7
11
|
$ gem install shodanz
|
8
|
-
|
9
12
|
## Usage
|
10
13
|
|
11
14
|
```ruby
|
@@ -13,7 +16,36 @@ require "shodanz"
|
|
13
16
|
|
14
17
|
client = Shodanz.client.new(key: "YOUR_API_KEY")
|
15
18
|
```
|
16
|
-
> You can also set the `SHODAN_API_KEY` environment variable instead of passing the API key as an argument when creating a client.
|
19
|
+
> **NOTE:** You can also set the `SHODAN_API_KEY` environment variable instead of passing the API key as an argument when creating a client.
|
20
|
+
|
21
|
+
### Optional Async Support
|
22
|
+
|
23
|
+
Shodanz utilizes [async](https://github.com/socketry/async) to provide asyncronous IO. This doesn't break any existing scripts using Shodanz, but now offers even more flexibility to write more awesome things, like this asyncronous honeypot detector:
|
24
|
+
|
25
|
+
```ruby
|
26
|
+
require 'async'
|
27
|
+
require 'shodanz'
|
28
|
+
|
29
|
+
client = Shodanz.client.new
|
30
|
+
|
31
|
+
# Asynchronously stream banner info from shodan and check any
|
32
|
+
# IP addresses against the experimental honeypot scoring service.
|
33
|
+
client.streaming_api.banners do |banner|
|
34
|
+
if ip = banner['ip_str']
|
35
|
+
Async do
|
36
|
+
score = client.rest_api.honeypot_score(ip).wait
|
37
|
+
puts "#{ip} has a #{score * 100}% chance of being a honeypot"
|
38
|
+
rescue Shodanz::Errors::RateLimited
|
39
|
+
sleep rand
|
40
|
+
retry
|
41
|
+
rescue # any other errors
|
42
|
+
next
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
```
|
47
|
+
|
48
|
+
> **Note:** To run any Shodanz method asyncronously, simply wrap it in a `Async { ... }` block. To wait for any other async operation to finnish in the block, call `.wait` on it.
|
17
49
|
|
18
50
|
## REST API
|
19
51
|
|
@@ -28,9 +60,9 @@ Search'n for stuff, are 'ya?
|
|
28
60
|
Returns all services that have been found on the given host IP.
|
29
61
|
|
30
62
|
```ruby
|
31
|
-
client.
|
32
|
-
client.
|
33
|
-
client.
|
63
|
+
client.host("8.8.8.8") # Default
|
64
|
+
client.host("8.8.8.8", history: true) # All historical banners should be returned.
|
65
|
+
client.host("8.8.8.8", minify: true) # Only return the list of ports and the general host information, no banners.
|
34
66
|
```
|
35
67
|
|
36
68
|
#### Host Search
|
@@ -38,12 +70,12 @@ client.rest_api.host("8.8.8.8", minify: true) # Only return the list of ports a
|
|
38
70
|
Search Shodan using the same query syntax as the website and use facets to get summary information for different properties.
|
39
71
|
|
40
72
|
```ruby
|
41
|
-
client.
|
42
|
-
client.
|
43
|
-
client.
|
44
|
-
client.
|
45
|
-
client.
|
46
|
-
client.
|
73
|
+
client.host_search("mongodb")
|
74
|
+
client.host_search("nginx")
|
75
|
+
client.host_search("apache", after: "1/12/16")
|
76
|
+
client.host_search("ssh", port: 22, page: 1)
|
77
|
+
client.host_search("ssh", port: 22, page: 2)
|
78
|
+
client.host_search("ftp", port: 21, facets: { link: "Ethernet or modem" })
|
47
79
|
```
|
48
80
|
|
49
81
|
#### Search Shodan without Results
|
@@ -51,12 +83,12 @@ client.rest_api.host_search("ftp", port: 21, facets: { link: "Ethernet or modem"
|
|
51
83
|
This method behaves identical to `host_search` with the only difference that this method does not return any host results, it only returns the total number of results that matched the query and any facet information that was requested. As a result this method does not consume query credits.
|
52
84
|
|
53
85
|
```ruby
|
54
|
-
client.
|
55
|
-
client.
|
56
|
-
client.
|
57
|
-
client.
|
58
|
-
client.
|
59
|
-
client.
|
86
|
+
client.host_count("apache")
|
87
|
+
client.host_count("apache", country: "US")
|
88
|
+
client.host_count("apache", country: "US", state: "MI")
|
89
|
+
client.host_count("apache", country: "US", state: "MI", city: "Detroit")
|
90
|
+
client.host_count("nginx", facets: { country: 5 })
|
91
|
+
client.host_count("apache", facets: { country: 5 })
|
60
92
|
```
|
61
93
|
|
62
94
|
#### Scan Targets
|
@@ -64,7 +96,7 @@ client.rest_api.host_count("apache", facets: { country: 5 })
|
|
64
96
|
Use this method to request Shodan to crawl an IP or netblock.
|
65
97
|
|
66
98
|
```ruby
|
67
|
-
client.
|
99
|
+
client.scan("8.8.8.8")
|
68
100
|
```
|
69
101
|
|
70
102
|
#### Crawl Internet for Port
|
@@ -74,7 +106,7 @@ Use this method to request Shodan to crawl the Internet for a specific port.
|
|
74
106
|
This method is restricted to security researchers and companies with a Shodan Data license. To apply for access to this method as a researcher, please email `jmath@shodan.io` with information about your project. Access is restricted to prevent abuse.
|
75
107
|
|
76
108
|
```ruby
|
77
|
-
client.
|
109
|
+
client.crawl_for(port: 80, protocol: "http")
|
78
110
|
```
|
79
111
|
|
80
112
|
#### List Community Queries
|
@@ -82,12 +114,12 @@ client.rest_api.crawl_for(port: 80, protocol: "http")
|
|
82
114
|
Use this method to obtain a list of search queries that users have saved in Shodan.
|
83
115
|
|
84
116
|
```ruby
|
85
|
-
client.
|
86
|
-
client.
|
87
|
-
client.
|
88
|
-
client.
|
89
|
-
client.
|
90
|
-
client.
|
117
|
+
client.community_queries
|
118
|
+
client.community_queries(page: 2)
|
119
|
+
client.community_queries(sort: "votes")
|
120
|
+
client.community_queries(sort: "votes", page: 2)
|
121
|
+
client.community_queries(order: "asc")
|
122
|
+
client.community_queries(order: "desc")
|
91
123
|
```
|
92
124
|
|
93
125
|
#### Search Community Queries
|
@@ -95,8 +127,8 @@ client.rest_api.community_queries(order: "desc")
|
|
95
127
|
Use this method to search the directory of search queries that users have saved in Shodan.
|
96
128
|
|
97
129
|
```ruby
|
98
|
-
client.
|
99
|
-
client.
|
130
|
+
client.search_for_community_query("the best")
|
131
|
+
client.search_for_community_query("the best", page: 2)
|
100
132
|
```
|
101
133
|
|
102
134
|
#### Popular Community Query Tags
|
@@ -104,8 +136,8 @@ client.rest_api.search_for_community_query("the best", page: 2)
|
|
104
136
|
Use this method to obtain a list of popular tags for the saved search queries in Shodan.
|
105
137
|
|
106
138
|
```ruby
|
107
|
-
client.
|
108
|
-
client.
|
139
|
+
client.popular_query_tags
|
140
|
+
client.popular_query_tags(20)
|
109
141
|
```
|
110
142
|
|
111
143
|
#### Protocols
|
@@ -113,7 +145,7 @@ client.rest_api.popular_query_tags(20)
|
|
113
145
|
This method returns an object containing all the protocols that can be used when launching an Internet scan.
|
114
146
|
|
115
147
|
```ruby
|
116
|
-
client.
|
148
|
+
client.protocols
|
117
149
|
```
|
118
150
|
|
119
151
|
#### Ports
|
@@ -121,7 +153,7 @@ client.rest_api.protocols
|
|
121
153
|
This method returns a list of port numbers that the Shodan crawlers are looking for.
|
122
154
|
|
123
155
|
```ruby
|
124
|
-
client.
|
156
|
+
client.ports
|
125
157
|
```
|
126
158
|
|
127
159
|
#### Account Profile
|
@@ -129,7 +161,7 @@ client.rest_api.ports
|
|
129
161
|
Returns information about the Shodan account linked to this API key.
|
130
162
|
|
131
163
|
```ruby
|
132
|
-
client.
|
164
|
+
client.profile
|
133
165
|
```
|
134
166
|
|
135
167
|
#### DNS Lookup
|
@@ -137,8 +169,8 @@ client.rest_api.profile
|
|
137
169
|
Look up the IP address for the provided list of hostnames.
|
138
170
|
|
139
171
|
```ruby
|
140
|
-
client.
|
141
|
-
client.
|
172
|
+
client.resolve("google.com")
|
173
|
+
client.resolve("google.com", "bing.com")
|
142
174
|
```
|
143
175
|
|
144
176
|
#### Reverse DNS Lookup
|
@@ -146,8 +178,8 @@ client.rest_api.resolve("google.com", "bing.com")
|
|
146
178
|
Look up the hostnames that have been defined for the given list of IP addresses.
|
147
179
|
|
148
180
|
```ruby
|
149
|
-
client.
|
150
|
-
client.
|
181
|
+
client.reverse_lookup("74.125.227.230")
|
182
|
+
client.reverse_lookup("74.125.227.230", "204.79.197.200")
|
151
183
|
```
|
152
184
|
|
153
185
|
#### HTTP Headers
|
@@ -155,7 +187,7 @@ client.rest_api.reverse_lookup("74.125.227.230", "204.79.197.200")
|
|
155
187
|
Shows the HTTP headers that your client sends when connecting to a webserver.
|
156
188
|
|
157
189
|
```ruby
|
158
|
-
client.
|
190
|
+
client.http_headers
|
159
191
|
```
|
160
192
|
|
161
193
|
#### Your IP Address
|
@@ -163,7 +195,7 @@ client.rest_api.http_headers
|
|
163
195
|
Get your current IP address as seen from the Internet.
|
164
196
|
|
165
197
|
```ruby
|
166
|
-
client.
|
198
|
+
client.my_ip
|
167
199
|
```
|
168
200
|
|
169
201
|
#### Honeypot Score
|
@@ -171,13 +203,13 @@ client.rest_api.my_ip
|
|
171
203
|
Calculates a honeypot probability score ranging from 0 (not a honeypot) to 1.0 (is a honeypot).
|
172
204
|
|
173
205
|
```ruby
|
174
|
-
client.
|
206
|
+
client.honeypot_score('8.8.8.8')
|
175
207
|
```
|
176
208
|
|
177
209
|
#### API Plan Information
|
178
210
|
|
179
211
|
```ruby
|
180
|
-
client.
|
212
|
+
client.info
|
181
213
|
```
|
182
214
|
|
183
215
|
### Streaming API
|
@@ -188,7 +220,7 @@ The Streaming API is an HTTP-based service that returns a real-time stream of da
|
|
188
220
|
This stream provides ALL of the data that Shodan collects. Use this stream if you need access to everything and/ or want to store your own Shodan database locally. If you only care about specific ports, please use the Ports stream.
|
189
221
|
|
190
222
|
```ruby
|
191
|
-
client.
|
223
|
+
client.banners do |data|
|
192
224
|
# do something with banner data
|
193
225
|
puts data
|
194
226
|
end
|
@@ -199,7 +231,7 @@ end
|
|
199
231
|
This stream provides a filtered, bandwidth-saving view of the Banners stream in case you are only interested in devices located in certain ASNs.
|
200
232
|
|
201
233
|
```ruby
|
202
|
-
client.
|
234
|
+
client.banners_within_asns(3303, 32475) do |data|
|
203
235
|
# do something with banner data
|
204
236
|
puts data
|
205
237
|
end
|
@@ -210,18 +242,18 @@ end
|
|
210
242
|
This stream provides a filtered, bandwidth-saving view of the Banners stream in case you are only interested in devices located in certain countries.
|
211
243
|
|
212
244
|
```ruby
|
213
|
-
client.
|
245
|
+
client.banners_within_countries("DE", "US", "JP") do |data|
|
214
246
|
# do something with banner data
|
215
247
|
puts data
|
216
248
|
end
|
217
249
|
```
|
218
250
|
|
219
|
-
#### Banners Filtered by Ports
|
251
|
+
#### Banners Filtered by Ports
|
220
252
|
|
221
253
|
Only returns banner data for the list of specified ports. This stream provides a filtered, bandwidth-saving view of the Banners stream in case you are only interested in a specific list of ports.
|
222
254
|
|
223
255
|
```ruby
|
224
|
-
client.
|
256
|
+
client.banners_on_ports(21, 22, 80) do |data|
|
225
257
|
# do something with banner data
|
226
258
|
puts data
|
227
259
|
end
|
@@ -232,7 +264,7 @@ end
|
|
232
264
|
Subscribe to banners discovered on all IP ranges described in the network alerts.
|
233
265
|
|
234
266
|
```ruby
|
235
|
-
client.
|
267
|
+
client.alerts do |data|
|
236
268
|
# do something with banner data
|
237
269
|
puts data
|
238
270
|
end
|
@@ -243,7 +275,7 @@ end
|
|
243
275
|
Subscribe to banners discovered on the IP range defined in a specific network alert.
|
244
276
|
|
245
277
|
```ruby
|
246
|
-
client.
|
278
|
+
client.alert("HKVGAIRWD79Z7W2T") do |data|
|
247
279
|
# do something with banner data
|
248
280
|
puts data
|
249
281
|
end
|
@@ -258,8 +290,8 @@ The Exploits API provides access to several exploit/ vulnerability data sources.
|
|
258
290
|
Search across a variety of data sources for exploits and use facets to get summary information.
|
259
291
|
|
260
292
|
```ruby
|
261
|
-
client.exploits_api.search("python") # Search for
|
262
|
-
client.exploits_api.search(
|
293
|
+
client.exploits_api.search("python") # Search for python vulns.
|
294
|
+
client.exploits_api.search(port: 22) # Port number for the affected service if the exploit is remote.
|
263
295
|
client.exploits_api.search(type: "shellcode") # A category of exploit to search for.
|
264
296
|
client.exploits_api.search(osvdb: "100007") # Open Source Vulnerability Database ID for the exploit.
|
265
297
|
```
|
@@ -269,7 +301,7 @@ client.exploits_api.search(osvdb: "100007") # Open Source Vulnerability Dat
|
|
269
301
|
This method behaves identical to the Exploits API `search` method with the difference that it doesn't return any results.
|
270
302
|
|
271
303
|
```ruby
|
272
|
-
client.exploits_api.count("python") # Count
|
304
|
+
client.exploits_api.count("python") # Count python vulns.
|
273
305
|
client.exploits_api.count(port: 22) # Port number for the affected service if the exploit is remote.
|
274
306
|
client.exploits_api.count(type: "shellcode") # A category of exploit to search for.
|
275
307
|
client.exploits_api.count(osvdb: "100007") # Open Source Vulnerability Database ID for the exploit.
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
|
4
|
+
require 'async'
|
5
|
+
require 'shodanz'
|
6
|
+
|
7
|
+
client = Shodanz.client.new
|
8
|
+
|
9
|
+
client.streaming_api.banners do |banner|
|
10
|
+
if ip = banner['ip_str']
|
11
|
+
Async do
|
12
|
+
score = client.rest_api.honeypot_score(ip).wait
|
13
|
+
puts "#{ip} has a #{score * 100}% chance of being a honeypot"
|
14
|
+
rescue Shodanz::Errors::RateLimited
|
15
|
+
sleep rand
|
16
|
+
retry
|
17
|
+
rescue StandardError # any other errors
|
18
|
+
next
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
|
2
|
+
require 'async'
|
3
|
+
require 'shodanz'
|
4
|
+
|
5
|
+
client = Shodanz.client.new
|
6
|
+
|
7
|
+
webservers = ['apache', 'nginx', 'caddy', 'lighttpd', 'cherokee']
|
8
|
+
|
9
|
+
# we can use methods sequentially
|
10
|
+
started = Time.now.sec
|
11
|
+
webservers.each do |webserver|
|
12
|
+
# make HTTP request
|
13
|
+
client.rest_api.host_search(webserver)
|
14
|
+
# print webserver to STDOUT
|
15
|
+
puts webserver
|
16
|
+
end
|
17
|
+
puts "Sequential took #{Time.now.sec - started} seconds"
|
18
|
+
|
19
|
+
# we can also use methods asyncronously
|
20
|
+
started = Time.now.sec
|
21
|
+
Async do
|
22
|
+
webservers.each do |webserver|
|
23
|
+
# make HTTP request
|
24
|
+
client.rest_api.host_search(webserver)
|
25
|
+
# print webserver to STDOUT
|
26
|
+
puts webserver
|
27
|
+
end
|
28
|
+
end
|
29
|
+
puts "Asyncronous took #{Time.now.sec - started} seconds"
|
@@ -0,0 +1,33 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
|
2
|
+
require 'pry'
|
3
|
+
require 'async'
|
4
|
+
require 'shodanz'
|
5
|
+
|
6
|
+
client = Shodanz.client.new
|
7
|
+
|
8
|
+
stats = Hash.new(0)
|
9
|
+
|
10
|
+
ports = [21, 22, 80, 443]
|
11
|
+
services = ['ftp', 'ssh', 'http', 'https']
|
12
|
+
|
13
|
+
ports_with_service_names = ports.zip(services)
|
14
|
+
|
15
|
+
Async do
|
16
|
+
# collect banners for ports
|
17
|
+
ports_with_service_names.each do |port, service|
|
18
|
+
client.streaming_api.banners_on_port(port) do |banner|
|
19
|
+
if ip = banner['ip_str']
|
20
|
+
Async do
|
21
|
+
resp = client.rest_api.honeypot_score(ip).wait
|
22
|
+
binding.pry if resp.nil?
|
23
|
+
puts "#{ip} has a #{resp *100}% chance of being a honeypot"
|
24
|
+
rescue Shodanz::Errors::RateLimited
|
25
|
+
sleep 1
|
26
|
+
retry
|
27
|
+
rescue => error
|
28
|
+
binding.pry
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|