shodanz 1.0.6 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +85 -56
- data/examples/async_honeypot_detector.rb +19 -0
- data/examples/async_host_search_example.rb +29 -0
- data/examples/async_stream_example.rb +33 -0
- data/examples/streaming_banner_product_stats.rb +3 -0
- data/lib/shodanz.rb +9 -2
- data/lib/shodanz/apis/exploits.rb +12 -31
- data/lib/shodanz/apis/rest.rb +50 -80
- data/lib/shodanz/apis/streaming.rb +44 -82
- data/lib/shodanz/apis/utils.rb +175 -0
- data/lib/shodanz/client.rb +88 -1
- data/lib/shodanz/errors.rb +29 -0
- data/lib/shodanz/version.rb +3 -1
- data/shodanz.gemspec +21 -18
- metadata +60 -27
@@ -0,0 +1,175 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'utils.rb'
|
4
|
+
|
5
|
+
# fronzen_string_literal: true
|
6
|
+
|
7
|
+
module Shodanz
|
8
|
+
module API
|
9
|
+
# Utils provides simply get, post, and slurp_stream functionality
|
10
|
+
# to the client. Under the hood they support both async and non-async
|
11
|
+
# usage. You should basically never need to use these methods directly.
|
12
|
+
#
|
13
|
+
# @author Kent 'picat' Gruber
|
14
|
+
module Utils
|
15
|
+
# Perform a direct GET HTTP request to the REST API.
|
16
|
+
def get(path, **params)
|
17
|
+
return sync_get(path, params) unless Async::Task.current?
|
18
|
+
|
19
|
+
async_get(path, params)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Perform a direct POST HTTP request to the REST API.
|
23
|
+
def post(path, **params)
|
24
|
+
return sync_post(path, params) unless Async::Task.current?
|
25
|
+
|
26
|
+
async_post(path, params)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Perform the main function of consuming the streaming API.
|
30
|
+
def slurp_stream(path, **params)
|
31
|
+
if Async::Task.current?
|
32
|
+
async_slurp_stream(path, params) do |result|
|
33
|
+
yield result
|
34
|
+
end
|
35
|
+
else
|
36
|
+
sync_slurp_stream(path, params) do |result|
|
37
|
+
yield result
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def turn_into_query(params)
|
43
|
+
filters = params.reject { |key, _| key == :query }
|
44
|
+
filters.each do |key, value|
|
45
|
+
params[:query] << " #{key}:#{value}"
|
46
|
+
end
|
47
|
+
params.select { |key, _| key == :query }
|
48
|
+
end
|
49
|
+
|
50
|
+
def turn_into_facets(facets)
|
51
|
+
return {} if facets.nil?
|
52
|
+
|
53
|
+
filters = facets.reject { |key, _| key == :facets }
|
54
|
+
facets[:facets] = []
|
55
|
+
filters.each do |key, value|
|
56
|
+
facets[:facets] << "#{key}:#{value}"
|
57
|
+
end
|
58
|
+
facets[:facets] = facets[:facets].join(',')
|
59
|
+
facets.select { |key, _| key == :facets }
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
RATELIMIT = 'rate limit reached'
|
65
|
+
NOINFO = 'no information available'
|
66
|
+
NOQUERY = 'empty search query'
|
67
|
+
|
68
|
+
def handle_any_json_errors(json)
|
69
|
+
return json unless json.is_a?(Hash) && json.key?('error')
|
70
|
+
|
71
|
+
raise Shodanz::Errors::RateLimited if json['error'].casecmp(RATELIMIT) >= 0
|
72
|
+
raise Shodanz::Errors::NoInformation if json['error'].casecmp(NOINFO) >= 0
|
73
|
+
raise Shodanz::Errors::NoQuery if json['error'].casecmp(NOQUERY) >= 0
|
74
|
+
|
75
|
+
json
|
76
|
+
end
|
77
|
+
|
78
|
+
def getter(path, **params)
|
79
|
+
# param keys should all be strings
|
80
|
+
params = params.transform_keys(&:to_s)
|
81
|
+
# build up url string based on special params
|
82
|
+
url = "#{@url}#{path}?key=#{@key}"
|
83
|
+
# special params
|
84
|
+
%w[query ips hostnames].each do |param|
|
85
|
+
if (value = params.delete(param))
|
86
|
+
url += "&#{param}=#{value}"
|
87
|
+
end
|
88
|
+
end
|
89
|
+
resp = @internet.get(url)
|
90
|
+
|
91
|
+
# parse all lines in the response body as JSON
|
92
|
+
json = JSON.parse(resp.body.join)
|
93
|
+
|
94
|
+
handle_any_json_errors(json)
|
95
|
+
ensure
|
96
|
+
resp&.close
|
97
|
+
end
|
98
|
+
|
99
|
+
def poster(path, **params)
|
100
|
+
# param keys should all be strings
|
101
|
+
params = params.transform_keys(&:to_s)
|
102
|
+
# make POST request to server
|
103
|
+
resp = @internet.post("#{@url}#{path}?key=#{@key}", params)
|
104
|
+
|
105
|
+
# parse all lines in the response body as JSON
|
106
|
+
json = JSON.parse(resp.body.join)
|
107
|
+
|
108
|
+
handle_any_json_errors(json)
|
109
|
+
ensure
|
110
|
+
resp&.close
|
111
|
+
end
|
112
|
+
|
113
|
+
def slurper(path, **params)
|
114
|
+
# param keys should all be strings
|
115
|
+
params = params.transform_keys(&:to_s)
|
116
|
+
# check if limit
|
117
|
+
if (limit = params.delete('limit'))
|
118
|
+
counter = 0
|
119
|
+
end
|
120
|
+
# make GET request to server
|
121
|
+
resp = @internet.get("#{@url}#{path}?key=#{@key}", params)
|
122
|
+
# read body line-by-line
|
123
|
+
until resp.body.nil? || resp.body.empty?
|
124
|
+
resp.body.read.each_line do |line|
|
125
|
+
next if line.strip.empty?
|
126
|
+
|
127
|
+
yield JSON.parse(line)
|
128
|
+
if limit
|
129
|
+
counter += 1
|
130
|
+
resp.close if counter == limit
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
ensure
|
135
|
+
resp&.close
|
136
|
+
end
|
137
|
+
|
138
|
+
def async_get(path, **params)
|
139
|
+
Async::Task.current.async do
|
140
|
+
getter(path, params)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def sync_get(path, **params)
|
145
|
+
Async do
|
146
|
+
getter(path, params)
|
147
|
+
end.wait
|
148
|
+
end
|
149
|
+
|
150
|
+
def async_post(path, **params)
|
151
|
+
Async::Task.current.async do
|
152
|
+
poster(path, params)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def sync_post(path, **params)
|
157
|
+
Async do
|
158
|
+
poster(path, params)
|
159
|
+
end.wait
|
160
|
+
end
|
161
|
+
|
162
|
+
def async_slurp_stream(path, **params)
|
163
|
+
Async::Task.current.async do
|
164
|
+
slurper(path, params) { |data| yield data }
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def sync_slurp_stream(path, **params)
|
169
|
+
Async do
|
170
|
+
slurper(path, params) { |data| yield data }
|
171
|
+
end.wait
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
data/lib/shodanz/client.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Shodanz
|
2
4
|
# General client container class for all three
|
3
5
|
# of the available API endpoints in a
|
@@ -17,7 +19,8 @@ module Shodanz
|
|
17
19
|
# Optionally provide your Shodan API key, or the environment
|
18
20
|
# variable SHODAN_API_KEY will be used.
|
19
21
|
def initialize(key: ENV['SHODAN_API_KEY'])
|
20
|
-
raise
|
22
|
+
raise Shodanz::Errors::NoAPIKey if key.nil?
|
23
|
+
|
21
24
|
# pass the given API key to each of the underlying clients
|
22
25
|
#
|
23
26
|
# Note: you can optionally change these API keys later, if you
|
@@ -27,5 +30,89 @@ module Shodanz
|
|
27
30
|
@streaming_api = Shodanz.api.streaming.new(key: key)
|
28
31
|
@exploits_api = Shodanz.api.exploits.new(key: key)
|
29
32
|
end
|
33
|
+
|
34
|
+
def host(ip, **params)
|
35
|
+
rest_api.host(ip, **params)
|
36
|
+
end
|
37
|
+
|
38
|
+
def host_count(query = '', facets: {}, **params)
|
39
|
+
rest_api.host_count(query, facets: facets, **params)
|
40
|
+
end
|
41
|
+
|
42
|
+
def host_search(query = '', facets: {}, page: 1, minify: true, **params)
|
43
|
+
rest_api.host_search(query, facets: facets, page: page, minify: minify, **params)
|
44
|
+
end
|
45
|
+
|
46
|
+
def host_search_tokens(query = '', **params)
|
47
|
+
rest_api.host_search(query, params)
|
48
|
+
end
|
49
|
+
|
50
|
+
def ports
|
51
|
+
rest_api.ports
|
52
|
+
end
|
53
|
+
|
54
|
+
def protocols
|
55
|
+
rest_api.protocols
|
56
|
+
end
|
57
|
+
|
58
|
+
def scan(*ips)
|
59
|
+
rest_api.scan(ips)
|
60
|
+
end
|
61
|
+
|
62
|
+
def crawl_for(**params)
|
63
|
+
rest_api.scan(params)
|
64
|
+
end
|
65
|
+
|
66
|
+
def scan_status(id)
|
67
|
+
rest_api.scan_status(id)
|
68
|
+
end
|
69
|
+
|
70
|
+
def community_queries(**params)
|
71
|
+
rest_api.community_queries(params)
|
72
|
+
end
|
73
|
+
|
74
|
+
def search_for_community_query(query, **params)
|
75
|
+
rest_api.search_for_community_query(query, params)
|
76
|
+
end
|
77
|
+
|
78
|
+
def popular_query_tags(size = 10)
|
79
|
+
rest_api.popular_query_tags(size)
|
80
|
+
end
|
81
|
+
|
82
|
+
def profile
|
83
|
+
rest_api.profile
|
84
|
+
end
|
85
|
+
|
86
|
+
def resolve(*hostnames)
|
87
|
+
rest_api.resolve(hostnames)
|
88
|
+
end
|
89
|
+
|
90
|
+
def reverse_lookup(*ips)
|
91
|
+
rest_api.reverse_lookup(ips)
|
92
|
+
end
|
93
|
+
|
94
|
+
def http_headers
|
95
|
+
rest_api.http_headers
|
96
|
+
end
|
97
|
+
|
98
|
+
def my_ip
|
99
|
+
rest_api.my_ip
|
100
|
+
end
|
101
|
+
|
102
|
+
def honeypot_score(ip)
|
103
|
+
rest_api.honeypot_score(ip)
|
104
|
+
end
|
105
|
+
|
106
|
+
def info
|
107
|
+
rest_api.info
|
108
|
+
end
|
109
|
+
|
110
|
+
def exploit_search(query = '', page: 1, **params)
|
111
|
+
exploits_api.search(query, page: page, **params)
|
112
|
+
end
|
113
|
+
|
114
|
+
def exploit_count(query = '', page: 1, **params)
|
115
|
+
exploits_api.count(query, page: page, **params)
|
116
|
+
end
|
30
117
|
end
|
31
118
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Shodanz
|
4
|
+
module Errors
|
5
|
+
class RateLimited < StandardError
|
6
|
+
def initialize(msg = 'Request rate limit reached (1 request/ second). Please wait a second before trying again and slow down your API calls.')
|
7
|
+
super
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class NoInformation < StandardError
|
12
|
+
def initialize(msg = 'No information available.')
|
13
|
+
super
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class NoAPIKey < StandardError
|
18
|
+
def initialize(msg = 'No API key has been found or provided! ( setup your SHODAN_API_KEY environment varialbe )')
|
19
|
+
super
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class NoQuery < StandardError
|
24
|
+
def initialize(msg = 'Empty search query.')
|
25
|
+
super
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/shodanz/version.rb
CHANGED
data/shodanz.gemspec
CHANGED
@@ -1,29 +1,32 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require 'shodanz/version'
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
8
|
+
spec.name = 'shodanz'
|
8
9
|
spec.version = Shodanz::VERSION
|
9
10
|
spec.authors = ["Kent 'picatz' Gruber"]
|
10
|
-
spec.email = [
|
11
|
+
spec.email = ['kgruber1@emich.edu']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
-
spec.description =
|
14
|
-
spec.homepage =
|
15
|
-
spec.license =
|
13
|
+
spec.summary = "A modern, async Ruby gem for Shodan, the world's first search engine for Internet-connected devices."
|
14
|
+
spec.description = 'Featuring full support for the REST, Streaming and Exploits API'
|
15
|
+
spec.homepage = 'https://github.com/picatz/shodanz'
|
16
|
+
spec.license = 'MIT'
|
16
17
|
|
17
18
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
19
|
f.match(%r{^(test|spec|features)/})
|
19
20
|
end
|
20
|
-
spec.require_paths = [
|
21
|
-
|
22
|
-
spec.add_dependency
|
23
|
-
spec.add_dependency
|
24
|
-
|
25
|
-
|
26
|
-
spec.add_development_dependency
|
27
|
-
spec.add_development_dependency
|
28
|
-
spec.add_development_dependency
|
21
|
+
spec.require_paths = ['lib']
|
22
|
+
|
23
|
+
spec.add_dependency 'async', '~> 1.17.1'
|
24
|
+
spec.add_dependency 'async-http', '~> 0.38.1'
|
25
|
+
|
26
|
+
spec.add_development_dependency 'async-rspec', '~> 1.12.1'
|
27
|
+
spec.add_development_dependency 'bundler', '~> 1.17.2'
|
28
|
+
spec.add_development_dependency 'pry', '~> 0.12.2'
|
29
|
+
spec.add_development_dependency 'rake', '~> 12.3.2'
|
30
|
+
spec.add_development_dependency 'rb-readline', '~> 0.5.5'
|
31
|
+
spec.add_development_dependency 'rspec', '~> 3.8.0'
|
29
32
|
end
|
metadata
CHANGED
@@ -1,99 +1,127 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shodanz
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kent 'picatz' Gruber
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-05-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: async
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.1
|
19
|
+
version: 1.17.1
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.1
|
26
|
+
version: 1.17.1
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: async-http
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.38.1
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 0.38.1
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: async-rspec
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
48
|
-
type: :
|
47
|
+
version: 1.12.1
|
48
|
+
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 1.12.1
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: bundler
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.
|
61
|
+
version: 1.17.2
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.17.2
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.12.2
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
82
|
+
version: 0.12.2
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: rake
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
87
|
- - "~>"
|
74
88
|
- !ruby/object:Gem::Version
|
75
|
-
version: 12.3.
|
89
|
+
version: 12.3.2
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 12.3.2
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rb-readline
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.5.5
|
76
104
|
type: :development
|
77
105
|
prerelease: false
|
78
106
|
version_requirements: !ruby/object:Gem::Requirement
|
79
107
|
requirements:
|
80
108
|
- - "~>"
|
81
109
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
110
|
+
version: 0.5.5
|
83
111
|
- !ruby/object:Gem::Dependency
|
84
112
|
name: rspec
|
85
113
|
requirement: !ruby/object:Gem::Requirement
|
86
114
|
requirements:
|
87
115
|
- - "~>"
|
88
116
|
- !ruby/object:Gem::Version
|
89
|
-
version: 3.
|
117
|
+
version: 3.8.0
|
90
118
|
type: :development
|
91
119
|
prerelease: false
|
92
120
|
version_requirements: !ruby/object:Gem::Requirement
|
93
121
|
requirements:
|
94
122
|
- - "~>"
|
95
123
|
- !ruby/object:Gem::Version
|
96
|
-
version: 3.
|
124
|
+
version: 3.8.0
|
97
125
|
description: Featuring full support for the REST, Streaming and Exploits API
|
98
126
|
email:
|
99
127
|
- kgruber1@emich.edu
|
@@ -109,6 +137,9 @@ files:
|
|
109
137
|
- LICENSE.txt
|
110
138
|
- README.md
|
111
139
|
- Rakefile
|
140
|
+
- examples/async_honeypot_detector.rb
|
141
|
+
- examples/async_host_search_example.rb
|
142
|
+
- examples/async_stream_example.rb
|
112
143
|
- examples/debug.rb
|
113
144
|
- examples/streaming_banner_product_stats.rb
|
114
145
|
- examples/top_10_countries_running.rb
|
@@ -118,7 +149,9 @@ files:
|
|
118
149
|
- lib/shodanz/apis/exploits.rb
|
119
150
|
- lib/shodanz/apis/rest.rb
|
120
151
|
- lib/shodanz/apis/streaming.rb
|
152
|
+
- lib/shodanz/apis/utils.rb
|
121
153
|
- lib/shodanz/client.rb
|
154
|
+
- lib/shodanz/errors.rb
|
122
155
|
- lib/shodanz/version.rb
|
123
156
|
- shodanz.gemspec
|
124
157
|
homepage: https://github.com/picatz/shodanz
|
@@ -141,9 +174,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
141
174
|
version: '0'
|
142
175
|
requirements: []
|
143
176
|
rubyforge_project:
|
144
|
-
rubygems_version:
|
177
|
+
rubygems_version: 3.0.0.beta1
|
145
178
|
signing_key:
|
146
179
|
specification_version: 4
|
147
|
-
summary: A modern Ruby gem for Shodan, the world's first search engine for
|
148
|
-
devices.
|
180
|
+
summary: A modern, async Ruby gem for Shodan, the world's first search engine for
|
181
|
+
Internet-connected devices.
|
149
182
|
test_files: []
|