shodanz 1.0.6 → 2.0.4

data/lib/shodanz.rb

@@ -1,9 +1,16 @@
-require 'unirest'
-require 'oj'
+# frozen_string_literal: true
+
+require 'json'
+require 'async'
+require 'async/http/internet'
 require 'shodanz/version'
+require 'shodanz/errors'
 require 'shodanz/api'
 require 'shodanz/client'
 
+# disable async logs by default
+Async.logger.level = 4
+
 # Shodanz is a modern Ruby gem for Shodan, the world's
 # first search engine for Internet-connected devices.
 module Shodanz

data/lib/shodanz/apis/exploits.rb

@@ -1,3 +1,7 @@
+require_relative 'utils.rb'
+
+# frozen_string_literal: true
+
 module Shodanz
   module API
     # The Exploits API provides access to several exploit
@@ -9,16 +13,21 @@ module Shodanz
     #
     # @author Kent 'picat' Gruber
     class Exploits
+      include Shodanz::API::Utils
+
       # @return [String]
       attr_accessor :key
 
       # The path to the REST API endpoint.
-      URL = 'https://exploits.shodan.io/api/'.freeze
+      URL = 'https://exploits.shodan.io/'
 
       # @param key [String] SHODAN API key, defaulted to
       # the *SHODAN_API_KEY* enviroment variable.
       def initialize(key: ENV['SHODAN_API_KEY'])
-        self.key = key
+        @url      = URL
+        @client   = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse(@url))
+        self.key  = key
+
         warn 'No key has been found or provided!' unless key?
       end
 
@@ -26,6 +35,7 @@ module Shodanz
       # @return [String]
       def key?
         return true if @key
+
         false
       end
 
@@ -35,53 +45,24 @@ module Shodanz
       #   api.search("SQL", port: 443)
       #   api.search(port: 22)
       #   api.search(type: "dos")
-      def search(query = '', page: 1, **params)
+      def search(query = '', facets: {}, page: 1, **params)
         params[:query] = query
-        params = turn_into_query(params)
-        facets = turn_into_facets(facets)
+        params = turn_into_query(**params)
+        facets = turn_into_facets(**facets)
         params[:page] = page
-        get('search', params.merge(facets))
+        get('api/search', **params.merge(**facets))
       end
 
       # This method behaves identical to the "/search" method with
       # the difference that it doesn't return any results.
       # == Example
       #   api.count(type: "dos")
-      def count(query = '', page: 1, **params)
+      def count(query = '', facets: {}, page: 1, **params)
         params[:query] = query
-        params = turn_into_query(params)
-        facets = turn_into_facets(params)
+        params = turn_into_query(**params)
+        facets = turn_into_facets(**facets)
         params[:page] = page
-        get('count', params.merge(facets))
-      end
-
-      # Perform a direct GET HTTP request to the REST API.
-      def get(path, **params)
-        resp = Unirest.get "#{URL}#{path}?key=#{@key}", parameters: params
-        if resp.code != 200 && resp.body.key?('error')
-          raise resp.body['error']
-        end
-        resp.body
-      end
-
-      private
-
-      def turn_into_query(params)
-        filters = params.reject { |key, _| key == :query }
-        filters.each do |key, value|
-          params[:query] << " #{key}:#{value}"
-        end
-        params.select { |key, _| key == :query }
-      end
-
-      def turn_into_facets(facets)
-        filters = facets.reject { |key, _| key == :facets }
-        facets[:facets] = []
-        filters.each do |key, value|
-          facets[:facets] << "#{key}:#{value}"
-        end
-        facets[:facets] = facets[:facets].join(',')
-        facets.select { |key, _| key == :facets }
+        get('api/count', **params.merge(**facets))
       end
     end
   end

data/lib/shodanz/apis/rest.rb

@@ -1,27 +1,38 @@
-module Shodanz
+require_relative 'utils.rb'
+
+# frozen_string_literal: true
 
+module Shodanz
   module API
-    # The REST API provides methods to search Shodan, look up 
-    # hosts, get summary information on queries and a variety 
+    # The REST API provides methods to search Shodan, look up
+    # hosts, get summary information on queries and a variety
     # of other utilities. This requires you to have an API key
     # which you can get from Shodan.
     #
     # @author Kent 'picat' Gruber
     class REST
+      include Shodanz::API::Utils
+
+      # @return [String]
       attr_accessor :key
 
       # The path to the REST API endpoint.
-      URL = "https://api.shodan.io/"
+      URL = 'https://api.shodan.io/'
 
       # @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
       def initialize(key: ENV['SHODAN_API_KEY'])
-        self.key = key
-        warn "No key has been found or provided!" unless self.key?
+        @url      = URL
+        @client   = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse(@url))
+        self.key  = key
+
+        warn 'No key has been found or provided!' unless key?
       end
 
       # Check if there's an API key.
       def key?
-        return true if @key; false
+        return true if @key
+
+        false
       end
 
       # Returns all services that have been found on the given host IP.
@@ -31,113 +42,112 @@ module Shodanz
       # == Examples
       #   # Typical usage.
       #   rest_api.host("8.8.8.8")
-      # 
+      #
       #   # All historical banners should be returned.
-      #   rest_api.host("8.8.8.8", history: true) 
+      #   rest_api.host("8.8.8.8", history: true)
       #
       #   # Only return the list of ports and the general host information, no banners.
-      #   rest_api.host("8.8.8.8", minify: true)  
+      #   rest_api.host("8.8.8.8", minify: true)
       def host(ip, **params)
-        get("shodan/host/#{ip}", params)
+        get("shodan/host/#{ip}", **params)
       end
 
-      # This method behaves identical to "/shodan/host/search" with the only 
-      # difference that this method does not return any host results, it only 
-      # returns the total number of results that matched the query and any 
-      # facet information that was requested. As a result this method does 
+      # This method behaves identical to "/shodan/host/search" with the only
+      # difference that this method does not return any host results, it only
+      # returns the total number of results that matched the query and any
+      # facet information that was requested. As a result this method does
       # not consume query credits.
       # == Examples
       #   rest_api.host_count("apache")
       #   rest_api.host_count("apache", country: "US")
       #   rest_api.host_count("apache", country: "US", state: "MI")
       #   rest_api.host_count("apache", country: "US", state: "MI", city: "Detroit")
-      def host_count(query = "", facets: {}, **params)
+      def host_count(query = '', facets: {}, **params)
         params[:query] = query
-        params = turn_into_query(params)
-        facets = turn_into_facets(facets)
-        get("shodan/host/count", params.merge(facets))
+        params = turn_into_query(**params)
+        facets = turn_into_facets(**facets)
+        get('shodan/host/count', **params.merge(**facets))
       end
 
-      # Search Shodan using the same query syntax as the website and use facets 
+      # Search Shodan using the same query syntax as the website and use facets
       # to get summary information for different properties.
       # == Example
       #   rest_api.host_search("apache", country: "US", facets: { city: "Detroit" }, page: 1, minify: false)
-      def host_search(query = "", facets: {}, page: 1, minify: true, **params)
+      def host_search(query = '', facets: {}, page: 1, minify: true, **params)
         params[:query] = query
-        params = turn_into_query(params)
-        facets = turn_into_facets(facets)
+        params = turn_into_query(**params)
+        facets = turn_into_facets(**facets)
         params[:page] = page
         params[:minify] = minify
-        get("shodan/host/search", params.merge(facets))
+        get('shodan/host/search', **params.merge(**facets))
       end
 
-      # This method lets you determine which filters are being used by 
+      # This method lets you determine which filters are being used by
       # the query string and what parameters were provided to the filters.
-      def host_search_tokens(query = "", **params)
+      def host_search_tokens(query = '', **params)
         params[:query] = query
-        params = turn_into_query(params)
-        get("shodan/host/search/tokens", params)
+        params = turn_into_query(**params)
+        get('shodan/host/search/tokens', **params)
       end
 
       # This method returns a list of port numbers that the crawlers are looking for.
       def ports
-        get("shodan/ports")
+        get('shodan/ports')
       end
 
       # List all protocols that can be used when performing on-demand Internet scans via Shodan.
-      def protocols 
-        get("shodan/protocols")
+      def protocols
+        get('shodan/protocols')
       end
 
       # Use this method to request Shodan to crawl a network.
       #
-      # This method uses API scan credits: 1 IP consumes 1 scan credit. You 
-      # must have a paid API plan (either one-time payment or subscription) 
+      # This method uses API scan credits: 1 IP consumes 1 scan credit. You
+      # must have a paid API plan (either one-time payment or subscription)
       # in order to use this method.
       #
       # IP, IPs or netblocks (in CIDR notation) that should get crawled.
       def scan(*ips)
-        raise "Not enough scan credits!" unless self.info["scan_credits"] >= 1
-        post("shodan/scan", ips: ips.join(","))
+        post('shodan/scan', ips: ips.join(','))
       end
 
       # Use this method to request Shodan to crawl the Internet for a specific port.
       #
-      # This method is restricted to security researchers and companies with 
-      # a Shodan Data license. To apply for access to this method as a researcher, 
-      # please email jmath@shodan.io with information about your project. 
+      # This method is restricted to security researchers and companies with
+      # a Shodan Data license. To apply for access to this method as a researcher,
+      # please email jmath@shodan.io with information about your project.
       # Access is restricted to prevent abuse.
       #
       # == Example
       #   rest_api.crawl_for(port: 80, protocol: "http")
       def crawl_for(**params)
-        params[:query] = ""
-        params = turn_into_query(params)
-        post('shodan/scan/internet', params)
+        params[:query] = ''
+        params = turn_into_query(**params)
+        post('shodan/scan/internet', **params)
       end
 
-      # Check the progress of a previously submitted scan request. 
+      # Check the progress of a previously submitted scan request.
       def scan_status(id)
         get("shodan/scan/#{id}")
       end
 
       # Use this method to obtain a list of search queries that users have saved in Shodan.
       def community_queries(**params)
-        get('shodan/query', params)
+        get('shodan/query', **params)
       end
 
       # Use this method to search the directory of search queries that users have saved in Shodan.
       def search_for_community_query(query, **params)
         params[:query] = query
-        params = turn_into_query(params)
-        get('shodan/query/search', params) 
+        params = turn_into_query(**params)
+        get('shodan/query/search', **params)
       end
 
       # Use this method to obtain a list of popular tags for the saved search queries in Shodan.
       def popular_query_tags(size = 10)
         params = {}
         params[:size] = size
-        get('shodan/query/tags', params)
+        get('shodan/query/tags', **params)
       end
 
       # Returns information about the Shodan account linked to this API key.
@@ -147,16 +157,16 @@ module Shodanz
 
       # Look up the IP address for the provided list of hostnames.
       def resolve(*hostnames)
-        get('dns/resolve', hostnames: hostnames.join(","))
+        get('dns/resolve', hostnames: hostnames.join(','))
       end
 
-      # Look up the hostnames that have been defined for the 
+      # Look up the hostnames that have been defined for the
       # given list of IP addresses.
       def reverse_lookup(*ips)
-        get('dns/reverse', ips: ips.join(","))
+        get('dns/reverse', ips: ips.join(','))
       end
 
-      # Shows the HTTP headers that your client sends when 
+      # Shows the HTTP headers that your client sends when
       # connecting to a webserver.
       def http_headers
         get('tools/httpheaders')
@@ -167,7 +177,7 @@ module Shodanz
         get('tools/myip')
       end
 
-      # Calculates a honeypot probability score ranging 
+      # Calculates a honeypot probability score ranging
       # from 0 (not a honeypot) to 1.0 (is a honeypot).
       def honeypot_score(ip)
         get("labs/honeyscore/#{ip}")
@@ -177,46 +187,6 @@ module Shodanz
       def info
         get('api-info')
       end
-
-      # Perform a direct GET HTTP request to the REST API.
-      def get(path, **params)
-        resp = Unirest.get "#{URL}#{path}?key=#{@key}", parameters: params
-        if resp.code != 200
-          raise resp.body['error'] if resp.body.key?('error')
-          raise resp
-        end
-        resp.body
-      end
-
-      # Perform a direct POST HTTP request to the REST API.
-      def post(path, **params)
-        resp = Unirest.post "#{URL}#{path}?key=#{@key}", parameters: params
-        if resp.code != 200
-          raise resp.body['error'] if resp.body.key?('error')
-          raise resp
-        end
-        resp.body
-      end
-
-      private
-
-      def turn_into_query(params)
-        filters = params.reject { |key, _| key == :query }
-        filters.each do |key, value|
-          params[:query] << " #{key}:#{value}"
-        end
-        params.select { |key, _| key == :query }
-      end
-
-      def turn_into_facets(facets)
-        filters = facets.reject { |key, _| key == :facets }
-        facets[:facets] = []
-        filters.each do |key, value|
-          facets[:facets] << "#{key}:#{value}"
-        end
-        facets[:facets] = facets[:facets].join(',')
-        facets.select { |key, _| key == :facets }
-      end
     end
   end
 end

data/lib/shodanz/apis/streaming.rb

@@ -1,38 +1,48 @@
-module Shodanz
+require_relative 'utils.rb'
+
+# frozen_string_literal: true
 
+module Shodanz
   module API
-    # The REST API provides methods to search Shodan, look up 
-    # hosts, get summary information on queries and a variety 
+    # The REST API provides methods to search Shodan, look up
+    # hosts, get summary information on queries and a variety
     # of other utilities. This requires you to have an API key
     # which you can get from Shodan.
     #
-    # Note: Only 1-5% of the data is currently provided to 
-    # subscription-based API plans. If your company is interested 
-    # in large-scale, real-time access to all of the Shodan data 
+    # Note: Only 1-5% of the data is currently provided to
+    # subscription-based API plans. If your company is interested
+    # in large-scale, real-time access to all of the Shodan data
     # please contact us for pricing information (sales@shodan.io).
-    # 
+    #
     # @author Kent 'picat' Gruber
     class Streaming
+      include Shodanz::API::Utils
+
+      # @return [String]
       attr_accessor :key
 
-      # The Streaming API is an HTTP-based service that returns 
+      # The Streaming API is an HTTP-based service that returns
       # a real-time stream of data collected by Shodan.
-      URL = "https://stream.shodan.io/"
+      URL = 'https://stream.shodan.io/'
 
       # @param key [String] SHODAN API key, defaulted to the *SHODAN_API_KEY* enviroment variable.
       def initialize(key: ENV['SHODAN_API_KEY'])
-        self.key = key
-        warn "No key has been found or provided!" unless self.key?
+        @url      = URL
+        @client   = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse(@url))
+        self.key  = key
+
+        warn 'No key has been found or provided!' unless key?
       end
 
       # Check if there's an API key.
       def key?
         return true if @key; false
+
       end
 
-      # This stream provides ALL of the data that Shodan collects. 
-      # Use this stream if you need access to everything and/ or want to 
-      # store your own Shodan database locally. If you only care about specific 
+      # This stream provides ALL of the data that Shodan collects.
+      # Use this stream if you need access to everything and/ or want to
+      # store your own Shodan database locally. If you only care about specific
       # ports, please use the Ports stream.
       #
       # Sometimes data may be piped down stream that is weird to parse. You can choose
@@ -44,28 +54,28 @@ module Shodanz
       #     puts data
       #   end
       def banners(**params)
-        slurp_stream("shodan/banners", params) do |data|
+        slurp_stream('shodan/banners', **params) do |data|
           yield data
         end
       end
 
-      # This stream provides a filtered, bandwidth-saving view of the Banners 
+      # This stream provides a filtered, bandwidth-saving view of the Banners
       # stream in case you are only interested in devices located in certain ASNs.
       # == Example
-      #   api.banners_within_asns(3303, 32475) do |data| 
+      #   api.banners_within_asns(3303, 32475) do |data|
       #     # do something with the banner hash
       #     puts data
       #   end
       def banners_within_asns(*asns, **params)
-        slurp_stream("shodan/asn/#{asns.join(",")}", params) do |data|
+        slurp_stream("shodan/asn/#{asns.join(',')}", **params) do |data|
           yield data
         end
       end
 
-      # This stream provides a filtered, bandwidth-saving view of the Banners 
+      # This stream provides a filtered, bandwidth-saving view of the Banners
       # stream in case you are only interested in devices located in a certain ASN.
       # == Example
-      #   api.banners_within_asn(3303) do |data| 
+      #   api.banners_within_asn(3303) do |data|
       #     # do something with the banner hash
       #     puts data
       #   end
@@ -75,38 +85,38 @@ module Shodanz
         end
       end
 
-      # Only returns banner data for the list of specified ports. This 
-      # stream provides a filtered, bandwidth-saving view of the Banners 
+      # Only returns banner data for the list of specified ports. This
+      # stream provides a filtered, bandwidth-saving view of the Banners
       # stream in case you are only interested in a specific list of ports.
       # == Example
-      #   api.banners_within_countries("US","DE","JP") do |data| 
+      #   api.banners_within_countries("US","DE","JP") do |data|
       #     # do something with the banner hash
       #     puts data
       #   end
       def banners_within_countries(*params)
-        slurp_stream("shodan/countries/#{params.join(",")}") do |data|
+        slurp_stream("shodan/countries/#{params.join(',')}") do |data|
           yield data
         end
       end
 
-      # Only returns banner data for the list of specified ports. This 
-      # stream provides a filtered, bandwidth-saving view of the 
-      # Banners stream in case you are only interested in a 
+      # Only returns banner data for the list of specified ports. This
+      # stream provides a filtered, bandwidth-saving view of the
+      # Banners stream in case you are only interested in a
       # specific list of ports.
       # == Example
-      #   api.banners_on_port(80, 443) do |data| 
+      #   api.banners_on_port(80, 443) do |data|
       #     # do something with the banner hash
       #     puts data
       #   end
       def banners_on_ports(*params)
-        slurp_stream("shodan/ports/#{params.join(",")}") do |data|
+        slurp_stream("shodan/ports/#{params.join(',')}") do |data|
           yield data
         end
       end
 
-      # Only returns banner data for a specific port. This 
-      # stream provides a filtered, bandwidth-saving view of the 
-      # Banners stream in case you are only interested in a 
+      # Only returns banner data for a specific port. This
+      # stream provides a filtered, bandwidth-saving view of the
+      # Banners stream in case you are only interested in a
       # specific list of ports.
       # == Example
       #   api.banners_on_port(80) do |banner|
@@ -120,10 +130,10 @@ module Shodanz
       end
 
       # Subscribe to banners discovered on all IP ranges described in the network alerts.
-      # Use the REST API methods to create/ delete/ manage your network alerts and 
+      # Use the REST API methods to create/ delete/ manage your network alerts and
       # use the Streaming API to subscribe to them.
       def alerts
-        slurp_stream("alert") do |data|
+        slurp_stream('alert') do |data|
           yield data
         end
       end
@@ -134,54 +144,6 @@ module Shodanz
           yield data
         end
       end
-
-      private
-
-      # Perform the main function of consuming the streaming API. 
-      def slurp_stream(path, **params)
-        uri = URI("#{URL}#{path}?key=#{@key}")
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
-          request = Net::HTTP::Get.new uri
-          begin
-            http.request request do |resp|
-              raise "Unable to connect to Streaming API" if resp.code != "200"
-              # Buffer for Shodan's bullshit.
-              raw_body = ""
-              resp.read_body do |chunk| 
-                if /^\{"product":.*\}\}\n/.match(chunk)
-                  begin
-                    yield Oj.load(chunk)
-                  rescue
-                    # yolo
-                  end
-                elsif /.*\}\}\n$/.match(chunk)
-                  next if raw_body.empty?
-                  raw_body << chunk
-                  raw_body
-                elsif /^\{.*\b/.match(chunk) 
-                  raw_body << chunk
-                end
-                if m = /^\{"product":.*\}\}\n/.match(raw_body)
-                  index = 0
-                  while matched = m[index]
-                    index += 1
-                    raw_body = raw_body.gsub(/^\{"product":.*\}\}\n/, "")
-                    begin
-                      yield Oj.load(matched)
-                    rescue
-                      # yolo
-                    end
-                  end
-                end
-              end
-            end
-          ensure
-            http.finish
-          end
-        end
-      end
-
     end
-
   end
 end