shodanz 1.0.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30b41a97e4733cbc6a78b283019eb135fe36114768f7e032b6bc6df9f317e051
-  data.tar.gz: 7d4bc7144c8dad6dd37b7df896fa8cef8248c132f640d81ea86ae5dc4d5631c5
+  metadata.gz: edfee7e174978325331ceb16d7d7f19a1cd7f7cdb6731269cf8f3bc9d9fdcb31
+  data.tar.gz: d89a612b5e3e90aa0ef9a288cf9650875f9a0031bf936d94b82d45852fb887ba
 SHA512:
-  metadata.gz: 5ac1a82fc5b214b7b0e9f9add4f381d1aa52618dfbdc0ff1b8005417ff93752f8e48b1e2b0ed146893bc6db9bc04d56c5df8b6c104d3207ca4df28d19e6ba1ca
-  data.tar.gz: fa645b5ce29c8fc267a6398127b8c2a0fb67937a67d38a92d686c669175d39fd53c9c55db3546d07d655fadd108914bcdf381710dda5b62a3d6c26e777b43248
+  metadata.gz: 3f9e3e19895688ad861517883165a0b06b965f89c28854d73bec521b7a0430ebd5394519aaf4fef5e5397c9c31520bff32eed9cbeed336f4c3932bf34b1c4f09
+  data.tar.gz: '09326a94bb100c9abe45715fc0430f4f8dedbfffbba359346b268d17b4c8523fbf3795c93cd054ed4b68af98e1166857385c8a6df5b66278d0fea56164bea6cc'

data/README.md

@@ -1,11 +1,10 @@
 # Shodanz
 
-A modern Ruby [gem](https://rubygems.org/) for [Shodan](https://www.shodan.io/), the world's first search engine for Internet-connected devices.
+A modern, async Ruby [gem](https://rubygems.org/) for [Shodan](https://www.shodan.io/), the world's first search engine for Internet-connected devices.
 
 ## Installation
 
     $ gem install shodanz
-
 ## Usage
 
 ```ruby
@@ -13,7 +12,37 @@ require "shodanz"
 
 client = Shodanz.client.new(key: "YOUR_API_KEY")
 ```
-> You can also set the `SHODAN_API_KEY` environment variable instead of passing the API key as an argument when creating a client.
+> **NOTE:** You can also set the `SHODAN_API_KEY` environment variable instead of passing the API key as an argument when creating a client.
+
+### Optional Async Support
+
+Shodanz utilizes [async](https://github.com/socketry/async) to provide asyncronous IO. This doesn't break any existing scripts using Shodanz, but now offers even more flexibility to write more awesome things, like this asyncronous honeypot detector:
+
+```ruby
+require 'async'
+require 'shodanz'
+
+client = Shodanz.client.new
+
+# Asyncronously stream live banner info from shodan
+# and check the IP addresses against the expierimental
+# honeypot scoring service to find potential honeypots.
+client.streaming_api.banners do |banner|
+  if ip = banner['ip_str']
+    Async do
+      score = client.rest_api.honeypot_score(ip).wait
+      puts "#{ip} has a #{score * 100}% chance of being a honeypot"
+    rescue Shodanz::Errors::RateLimited
+      sleep rand
+      retry
+    rescue # any other errors
+      next
+    end
+  end
+end
+```
+
+> **Note:** To run any Shodanz method asyncronously, simply wrap it in a `Async { ... }` block. To wait for any other async operation to finnish in the block, call `.wait` on it.
 
 ## REST API
 
@@ -28,9 +57,9 @@ Search'n for stuff, are 'ya?
 Returns all services that have been found on the given host IP.
 
 ```ruby
-client.rest_api.host("8.8.8.8")                # Default
-client.rest_api.host("8.8.8.8", history: true) # All historical banners should be returned.
-client.rest_api.host("8.8.8.8", minify: true)  # Only return the list of ports and the general host information, no banners. 
+client.host("8.8.8.8")                # Default
+client.host("8.8.8.8", history: true) # All historical banners should be returned.
+client.host("8.8.8.8", minify: true)  # Only return the list of ports and the general host information, no banners.
 ```
 
 #### Host Search
@@ -38,12 +67,12 @@ client.rest_api.host("8.8.8.8", minify: true)  # Only return the list of ports a
 Search Shodan using the same query syntax as the website and use facets to get summary information for different properties.
 
 ```ruby
-client.rest_api.host_search("mongodb")
-client.rest_api.host_search("nginx")
-client.rest_api.host_search("apache", after: "1/12/16")
-client.rest_api.host_search("ssh", port: 22, page: 1)
-client.rest_api.host_search("ssh", port: 22, page: 2)
-client.rest_api.host_search("ftp", port: 21, facets: { link: "Ethernet or modem" })
+client.host_search("mongodb")
+client.host_search("nginx")
+client.host_search("apache", after: "1/12/16")
+client.host_search("ssh", port: 22, page: 1)
+client.host_search("ssh", port: 22, page: 2)
+client.host_search("ftp", port: 21, facets: { link: "Ethernet or modem" })
 ```
 
 #### Search Shodan without Results
@@ -51,12 +80,12 @@ client.rest_api.host_search("ftp", port: 21, facets: { link: "Ethernet or modem"
 This method behaves identical to `host_search` with the only difference that this method does not return any host results, it only returns the total number of results that matched the query and any facet information that was requested. As a result this method does not consume query credits.
 
 ```ruby
-client.rest_api.host_count("apache")
-client.rest_api.host_count("apache", country: "US")
-client.rest_api.host_count("apache", country: "US", state: "MI")
-client.rest_api.host_count("apache", country: "US", state: "MI", city: "Detroit") 
-client.rest_api.host_count("nginx",  facets: { country: 5 })
-client.rest_api.host_count("apache", facets: { country: 5 })
+client.host_count("apache")
+client.host_count("apache", country: "US")
+client.host_count("apache", country: "US", state: "MI")
+client.host_count("apache", country: "US", state: "MI", city: "Detroit")
+client.host_count("nginx",  facets: { country: 5 })
+client.host_count("apache", facets: { country: 5 })
 ```
 
 #### Scan Targets
@@ -64,7 +93,7 @@ client.rest_api.host_count("apache", facets: { country: 5 })
 Use this method to request Shodan to crawl an IP or netblock.
 
 ```ruby
-client.rest_api.scan("8.8.8.8")
+client.scan("8.8.8.8")
 ```
 
 #### Crawl Internet for Port
@@ -74,7 +103,7 @@ Use this method to request Shodan to crawl the Internet for a specific port.
 This method is restricted to security researchers and companies with a Shodan Data license. To apply for access to this method as a researcher, please email `jmath@shodan.io` with information about your project. Access is restricted to prevent abuse.
 
 ```ruby
-client.rest_api.crawl_for(port: 80, protocol: "http")
+client.crawl_for(port: 80, protocol: "http")
 ```
 
 #### List Community Queries
@@ -82,12 +111,12 @@ client.rest_api.crawl_for(port: 80, protocol: "http")
 Use this method to obtain a list of search queries that users have saved in Shodan.
 
 ```ruby
-client.rest_api.community_queries
-client.rest_api.community_queries(page: 2)
-client.rest_api.community_queries(sort: "votes")
-client.rest_api.community_queries(sort: "votes", page: 2)
-client.rest_api.community_queries(order: "asc")
-client.rest_api.community_queries(order: "desc")
+client.community_queries
+client.community_queries(page: 2)
+client.community_queries(sort: "votes")
+client.community_queries(sort: "votes", page: 2)
+client.community_queries(order: "asc")
+client.community_queries(order: "desc")
 ```
 
 #### Search Community Queries
@@ -95,8 +124,8 @@ client.rest_api.community_queries(order: "desc")
 Use this method to search the directory of search queries that users have saved in Shodan.
 
 ```ruby
-client.rest_api.search_for_community_query("the best")
-client.rest_api.search_for_community_query("the best", page: 2)
+client.search_for_community_query("the best")
+client.search_for_community_query("the best", page: 2)
 ```
 
 #### Popular Community Query Tags
@@ -104,8 +133,8 @@ client.rest_api.search_for_community_query("the best", page: 2)
 Use this method to obtain a list of popular tags for the saved search queries in Shodan.
 
 ```ruby
-client.rest_api.popular_query_tags
-client.rest_api.popular_query_tags(20)
+client.popular_query_tags
+client.popular_query_tags(20)
 ```
 
 #### Protocols
@@ -113,7 +142,7 @@ client.rest_api.popular_query_tags(20)
 This method returns an object containing all the protocols that can be used when launching an Internet scan.
 
 ```ruby
-client.rest_api.protocols
+client.protocols
 ```
 
 #### Ports
@@ -121,7 +150,7 @@ client.rest_api.protocols
 This method returns a list of port numbers that the Shodan crawlers are looking for.
 
 ```ruby
-client.rest_api.ports
+client.ports
 ```
 
 #### Account Profile
@@ -129,7 +158,7 @@ client.rest_api.ports
 Returns information about the Shodan account linked to this API key.
 
 ```ruby
-client.rest_api.profile
+client.profile
 ```
 
 #### DNS Lookup
@@ -137,8 +166,8 @@ client.rest_api.profile
 Look up the IP address for the provided list of hostnames.
 
 ```ruby
-client.rest_api.resolve("google.com")
-client.rest_api.resolve("google.com", "bing.com")
+client.resolve("google.com")
+client.resolve("google.com", "bing.com")
 ```
 
 #### Reverse DNS Lookup
@@ -146,8 +175,8 @@ client.rest_api.resolve("google.com", "bing.com")
 Look up the hostnames that have been defined for the given list of IP addresses.
 
 ```ruby
-client.rest_api.reverse_lookup("74.125.227.230")
-client.rest_api.reverse_lookup("74.125.227.230", "204.79.197.200")
+client.reverse_lookup("74.125.227.230")
+client.reverse_lookup("74.125.227.230", "204.79.197.200")
 ```
 
 #### HTTP Headers
@@ -155,7 +184,7 @@ client.rest_api.reverse_lookup("74.125.227.230", "204.79.197.200")
 Shows the HTTP headers that your client sends when connecting to a webserver.
 
 ```ruby
-client.rest_api.http_headers
+client.http_headers
 ```
 
 #### Your IP Address
@@ -163,7 +192,7 @@ client.rest_api.http_headers
 Get your current IP address as seen from the Internet.
 
 ```ruby
-client.rest_api.my_ip
+client.my_ip
 ```
 
 #### Honeypot Score
@@ -171,13 +200,13 @@ client.rest_api.my_ip
 Calculates a honeypot probability score ranging from 0 (not a honeypot) to 1.0 (is a honeypot).
 
 ```ruby
-client.rest_api.honeypot_score('8.8.8.8')
+client.honeypot_score('8.8.8.8')
 ```
 
 #### API Plan Information
 
 ```ruby
-client.rest_api.info
+client.info
 ```
 
 ### Streaming API
@@ -188,7 +217,7 @@ The Streaming API is an HTTP-based service that returns a real-time stream of da
 This stream provides ALL of the data that Shodan collects. Use this stream if you need access to everything and/ or want to store your own Shodan database locally. If you only care about specific ports, please use the Ports stream.
 
 ```ruby
-client.streaming_api.banners do |data|
+client.banners do |data|
   # do something with banner data
   puts data
 end
@@ -199,7 +228,7 @@ end
 This stream provides a filtered, bandwidth-saving view of the Banners stream in case you are only interested in devices located in certain ASNs.
 
 ```ruby
-client.streaming_api.banners_within_asns(3303, 32475) do |data|
+client.banners_within_asns(3303, 32475) do |data|
   # do something with banner data
   puts data
 end
@@ -210,18 +239,18 @@ end
 This stream provides a filtered, bandwidth-saving view of the Banners stream in case you are only interested in devices located in certain countries.
 
 ```ruby
-client.streaming_api.banners_within_countries("DE", "US", "JP") do |data|
+client.banners_within_countries("DE", "US", "JP") do |data|
   # do something with banner data
   puts data
 end
 ```
 
-#### Banners Filtered by Ports 
+#### Banners Filtered by Ports
 
 Only returns banner data for the list of specified ports. This stream provides a filtered, bandwidth-saving view of the Banners stream in case you are only interested in a specific list of ports.
 
 ```ruby
-client.streaming_api.banners_on_ports(21, 22, 80) do |data| 
+client.banners_on_ports(21, 22, 80) do |data|
   # do something with banner data
   puts data
 end
@@ -232,7 +261,7 @@ end
 Subscribe to banners discovered on all IP ranges described in the network alerts.
 
 ```ruby
-client.streaming_api.alerts do |data|
+client.alerts do |data|
   # do something with banner data
   puts data
 end
@@ -243,7 +272,7 @@ end
 Subscribe to banners discovered on the IP range defined in a specific network alert.
 
 ```ruby
-client.streaming_api.alert("HKVGAIRWD79Z7W2T") do |data|
+client.alert("HKVGAIRWD79Z7W2T") do |data|
   # do something with banner data
   puts data
 end
@@ -258,10 +287,10 @@ The Exploits API provides access to several exploit/ vulnerability data sources.
 Search across a variety of data sources for exploits and use facets to get summary information.
 
 ```ruby
-client.exploits_api.search("python")             # Search for Snek vulns.
-client.exploits_api.search(post: 22)             # Port number for the affected service if the exploit is remote.
-client.exploits_api.search(type: "shellcode")    # A category of exploit to search for.
-client.exploits_api.search(osvdb: "100007")      # Open Source Vulnerability Database ID for the exploit.
+client.search("python")             # Search for Snek vulns.
+client.search(post: 22)             # Port number for the affected service if the exploit is remote.
+client.search(type: "shellcode")    # A category of exploit to search for.
+client.search(osvdb: "100007")      # Open Source Vulnerability Database ID for the exploit.
 ```
 
 #### Count
@@ -269,10 +298,10 @@ client.exploits_api.search(osvdb: "100007")      # Open Source Vulnerability Dat
 This method behaves identical to the Exploits API `search` method with the difference that it doesn't return any results.
 
 ```ruby
-client.exploits_api.count("python")             # Count Snek vulns.
-client.exploits_api.count(port: 22)             # Port number for the affected service if the exploit is remote.
-client.exploits_api.count(type: "shellcode")    # A category of exploit to search for.
-client.exploits_api.count(osvdb: "100007")      # Open Source Vulnerability Database ID for the exploit.
+client.count("python")             # Count Snek vulns.
+client.count(port: 22)             # Port number for the affected service if the exploit is remote.
+client.count(type: "shellcode")    # A category of exploit to search for.
+client.count(osvdb: "100007")      # Open Source Vulnerability Database ID for the exploit.
 ```
 
 ## License

data/examples/async_honeypot_detector.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'async'
+require 'shodanz'
+
+client = Shodanz.client.new
+
+client.streaming_api.banners do |banner|
+  if ip = banner['ip_str']
+    Async do
+      score = client.rest_api.honeypot_score(ip).wait
+      puts "#{ip} has a #{score * 100}% chance of being a honeypot"
+    rescue Shodanz::Errors::RateLimited
+      sleep rand
+      retry
+    rescue # any other errors
+      next
+    end
+  end
+end

data/examples/async_host_search_example.rb

@@ -0,0 +1,29 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'async'
+require 'shodanz'
+
+client = Shodanz.client.new
+
+webservers = ['apache', 'nginx', 'caddy', 'lighttpd', 'cherokee']
+
+# we can use methods sequentially
+started = Time.now.sec
+webservers.each do |webserver|
+  # make HTTP request
+  client.rest_api.host_search(webserver)
+  # print webserver to STDOUT
+  puts webserver
+end
+puts "Sequential took #{Time.now.sec - started} seconds"
+
+# we can also use methods asyncronously
+started = Time.now.sec
+Async do
+  webservers.each do |webserver|
+    # make HTTP request
+    client.rest_api.host_search(webserver)
+    # print webserver to STDOUT
+    puts webserver
+  end
+end
+puts "Asyncronous took #{Time.now.sec - started} seconds"

data/examples/async_stream_example.rb

@@ -0,0 +1,33 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'pry'
+require 'async'
+require 'shodanz'
+
+client = Shodanz.client.new
+
+stats = Hash.new(0)
+
+ports    = [21, 22, 80, 443]
+services = ['ftp', 'ssh', 'http', 'https']
+
+ports_with_service_names = ports.zip(services)
+
+Async do 
+  # collect banners for ports
+  ports_with_service_names.each do |port, service|
+    client.streaming_api.banners_on_port(port) do |banner|
+      if ip = banner['ip_str']
+        Async do
+          resp = client.rest_api.honeypot_score(ip).wait
+          binding.pry if resp.nil?
+          puts "#{ip} has a #{resp *100}% chance of being a honeypot"
+        rescue Shodanz::Errors::RateLimited
+          sleep 1
+          retry
+        rescue => error
+          binding.pry
+        end
+      end
+    end
+  end
+end

data/examples/streaming_banner_product_stats.rb

@@ -15,5 +15,8 @@ stats = Hash.new(0)
 # collect banners 
 streaming_api.banners do |banner|
   product = banner['product']
+  
+  next if product.nil?
+
   puts "#{stats[product] += 1} #{product}"
 end

data/lib/shodanz.rb

@@ -1,9 +1,16 @@
-require 'unirest'
-require 'oj'
+# frozen_string_literal: true
+
+require 'json'
+require 'async'
+require 'async/http/internet'
 require 'shodanz/version'
+require 'shodanz/errors'
 require 'shodanz/api'
 require 'shodanz/client'
 
+# disable async logs by default
+Async.logger.level = 4
+
 # Shodanz is a modern Ruby gem for Shodan, the world's
 # first search engine for Internet-connected devices.
 module Shodanz

data/lib/shodanz/apis/exploits.rb

@@ -1,3 +1,7 @@
+require_relative 'utils.rb'
+
+# frozen_string_literal: true
+
 module Shodanz
   module API
     # The Exploits API provides access to several exploit
@@ -9,16 +13,21 @@ module Shodanz
     #
     # @author Kent 'picat' Gruber
     class Exploits
+      include Shodanz::API::Utils
+
       # @return [String]
       attr_accessor :key
 
       # The path to the REST API endpoint.
-      URL = 'https://exploits.shodan.io/api/'.freeze
+      URL = 'https://exploits.shodan.io/api/'
 
       # @param key [String] SHODAN API key, defaulted to
       # the *SHODAN_API_KEY* enviroment variable.
       def initialize(key: ENV['SHODAN_API_KEY'])
-        self.key = key
+        @url      = URL
+        @internet = Async::HTTP::Internet.new
+        self.key  = key
+
         warn 'No key has been found or provided!' unless key?
       end
 
@@ -26,6 +35,7 @@ module Shodanz
       # @return [String]
       def key?
         return true if @key
+
         false
       end
 
@@ -54,35 +64,6 @@ module Shodanz
         params[:page] = page
         get('count', params.merge(facets))
       end
-
-      # Perform a direct GET HTTP request to the REST API.
-      def get(path, **params)
-        resp = Unirest.get "#{URL}#{path}?key=#{@key}", parameters: params
-        if resp.code != 200 && resp.body.key?('error')
-          raise resp.body['error']
-        end
-        resp.body
-      end
-
-      private
-
-      def turn_into_query(params)
-        filters = params.reject { |key, _| key == :query }
-        filters.each do |key, value|
-          params[:query] << " #{key}:#{value}"
-        end
-        params.select { |key, _| key == :query }
-      end
-
-      def turn_into_facets(facets)
-        filters = facets.reject { |key, _| key == :facets }
-        facets[:facets] = []
-        filters.each do |key, value|
-          facets[:facets] << "#{key}:#{value}"
-        end
-        facets[:facets] = facets[:facets].join(',')
-        facets.select { |key, _| key == :facets }
-      end
     end
   end
 end