shodanz 1.0.6 → 2.0.4

data/lib/shodanz/apis/utils.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require_relative 'utils.rb'
+
+# fronzen_string_literal: true
+
+module Shodanz
+  module API
+    # Utils provides simply get, post, and slurp_stream functionality
+    # to the client. Under the hood they support both async and non-async
+    # usage. You should basically never need to use these methods directly.
+    #
+    # @author Kent 'picat' Gruber
+    module Utils
+      # Perform a direct GET HTTP request to the REST API.
+      def get(path, **params)
+        return sync_get(path, **params) unless Async::Task.current?
+
+        async_get(path, **params)
+      end
+
+      # Perform a direct POST HTTP request to the REST API.
+      def post(path, body: nil, **params)
+        return sync_post(path, params: params, body: body) unless Async::Task.current?
+
+        async_post(path, params: params, body: body)
+      end
+
+      # Perform the main function of consuming the streaming API.
+      def slurp_stream(path, **params)
+        if Async::Task.current?
+          async_slurp_stream(path, **params) do |result|
+            yield result
+          end
+        else
+          sync_slurp_stream(path, **params) do |result|
+            yield result
+          end
+        end
+      end
+
+      def turn_into_query(**params)
+        filters = params.reject { |key, _| key == :query }
+        filters.each do |key, value|
+          params[:query] << " #{key}:#{value}"
+        end
+        params.select { |key, _| key == :query }
+      end
+
+      def turn_into_facets(**facets)
+        return {} if facets.nil?
+
+        filters = facets.reject { |key, _| key == :facets }
+        facets[:facets] = []
+        filters.each do |key, value|
+          facets[:facets] << "#{key}:#{value}"
+        end
+        facets[:facets] = facets[:facets].join(',')
+        facets.select { |key, _| key == :facets }
+      end
+
+      private
+
+      RATELIMIT    = 'rate limit reached'
+      NOINFO       = 'no information available'
+      NOQUERY      = 'empty search query'
+      ACCESSDENIED = 'access denied'
+      INVALIDKEY   = 'invalid API key'
+
+      def handle_any_json_errors(json)
+        if json.is_a?(Hash) && json.key?('error')
+          raise Shodanz::Errors::RateLimited if json['error'].casecmp(RATELIMIT) >= 0
+          raise Shodanz::Errors::NoInformation if json['error'].casecmp(NOINFO) >= 0
+          raise Shodanz::Errors::NoQuery if json['error'].casecmp(NOQUERY) >= 0
+          raise Shodanz::Errors::AccessDenied if json['error'].casecmp(ACCESSDENIED) >= 0
+          raise Shodanz::Errors::InvalidKey if json['error'].casecmp(INVALIDKEY) >= 0
+        end
+        return json
+      end
+
+      def getter(path, **params)
+        # param keys should all be strings
+        params = params.transform_keys(&:to_s)
+        # build up url string based on special params
+        url = "/#{path}?key=#{@key}"
+        # special params
+        params.each do |param,value|
+          next if value.is_a?(String) && value.empty?
+          value = URI.encode_www_form_component("#{value}")
+          url += "&#{param}=#{value}"
+        end
+
+        resp = @client.get(url)
+
+        if resp.success?
+          # parse all lines in the response body as JSON
+          json = JSON.parse(resp.body.join)
+
+          handle_any_json_errors(json)
+
+          return json
+        else
+          raise "Got response status #{resp.status}"
+        end
+      ensure
+        @client.pool.close
+        resp&.close
+      end
+
+      def poster(path, one_shot: false, params: nil, body: nil)
+        # param keys should all be strings
+        params = params.transform_keys(&:to_s)
+        # and the key param is constant
+        params["key"] = @key
+        # encode as a URL string
+        params = URI.encode_www_form(params)
+        # optional JSON body string
+        json_body = body.nil? ? nil : JSON.dump(body)
+        # build URL path
+        path = "/#{path}?#{params}" 
+
+        # make POST request to server
+        resp = @client.post(path, nil, json_body)
+
+        if resp.success?
+          json = JSON.parse(resp.body.join)
+
+          handle_any_json_errors(json)
+
+          return json
+        else
+          raise "Got response status #{resp.status}"
+        end
+      ensure
+        @client.pool.close
+        resp&.close
+      end
+
+      def slurper(path, **params)
+        # param keys should all be strings
+        params = params.transform_keys(&:to_s)
+        # check if limit
+        if (limit = params.delete('limit'))
+          counter = 0
+        end
+        # make GET request to server
+        resp = @client.get("/#{path}?key=#{@key}", params)
+        # read body line-by-line
+        until resp.body.nil? || resp.body.empty?
+          resp.body.read.each_line do |line|
+            next if line.strip.empty?
+
+            yield JSON.parse(line)
+            if limit
+              counter += 1
+              resp.close if counter == limit
+            end
+          end
+        end
+      ensure
+        resp&.close
+      end
+
+      def async_get(path, **params)
+        Async::Task.current.async do
+          getter(path, **params)
+        end
+      end
+
+      def sync_get(path, **params)
+        Async do
+          getter(path, **params)
+        end.wait
+      end
+
+      def async_post(path, params: nil, body: nil)
+        Async::Task.current.async do
+          poster(path, params: params, body: body)
+        end
+      end
+
+      def sync_post(path, params: nil, body: nil)
+        Async do
+          poster(path, params: params, body: body)
+        end.wait
+      end
+
+      def async_slurp_stream(path, **params)
+        Async::Task.current.async do
+          slurper(path, **params) { |data| yield data }
+        end
+      end
+
+      def sync_slurp_stream(path, **params)
+        Async do
+          slurper(path, **params) { |data| yield data }
+        end.wait
+      end
+    end
+  end
+end

data/lib/shodanz/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shodanz
   # General client container class for all three
   # of the available API endpoints in a
@@ -17,7 +19,8 @@ module Shodanz
     # Optionally provide your Shodan API key, or the environment
     # variable SHODAN_API_KEY will be used.
     def initialize(key: ENV['SHODAN_API_KEY'])
-      raise "No API key has been found or provided! ( setup your SHODAN_API_KEY environment varialbe )" if key.nil?
+      raise Shodanz::Errors::NoAPIKey if key.nil?
+
       # pass the given API key to each of the underlying clients
       #
       # Note: you can optionally change these API keys later, if you
@@ -27,5 +30,89 @@ module Shodanz
       @streaming_api = Shodanz.api.streaming.new(key: key)
       @exploits_api  = Shodanz.api.exploits.new(key: key)
     end
+
+    def host(ip, **params)
+      rest_api.host(ip, **params)
+    end
+
+    def host_count(query = '', facets: {}, **params)
+      rest_api.host_count(query, facets: facets, **params)
+    end
+
+    def host_search(query = '', facets: {}, page: 1, minify: true, **params)
+      rest_api.host_search(query, facets: facets, page: page, minify: minify, **params)
+    end
+
+    def host_search_tokens(query = '', **params)
+      rest_api.host_search(query, params)
+    end
+
+    def ports
+      rest_api.ports
+    end
+
+    def protocols
+      rest_api.protocols
+    end
+
+    def scan(*ips)
+      rest_api.scan(ips)
+    end
+
+    def crawl_for(**params)
+      rest_api.scan(params)
+    end
+
+    def scan_status(id)
+      rest_api.scan_status(id)
+    end
+
+    def community_queries(**params)
+      rest_api.community_queries(params)
+    end
+
+    def search_for_community_query(query, **params)
+      rest_api.search_for_community_query(query, params)
+    end
+
+    def popular_query_tags(size = 10)
+      rest_api.popular_query_tags(size)
+    end
+
+    def profile
+      rest_api.profile
+    end
+
+    def resolve(*hostnames)
+      rest_api.resolve(hostnames)
+    end
+
+    def reverse_lookup(*ips)
+      rest_api.reverse_lookup(ips)
+    end
+
+    def http_headers
+      rest_api.http_headers
+    end
+
+    def my_ip
+      rest_api.my_ip
+    end
+
+    def honeypot_score(ip)
+      rest_api.honeypot_score(ip)
+    end
+
+    def info
+      rest_api.info
+    end
+
+    def exploit_search(query = '', page: 1, **params)
+      exploits_api.search(query, page: page, **params)
+    end
+
+    def exploit_count(query = '', page: 1, **params)
+      exploits_api.count(query, page: page, **params)
+    end
   end
 end

data/lib/shodanz/errors.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Shodanz
+  module Errors
+    class RateLimited < StandardError
+      def initialize(msg = 'Request rate limit reached (1 request/ second). Please wait a second before trying again and slow down your API calls.')
+        super
+      end
+    end
+
+    class NoInformation < StandardError
+      def initialize(msg = 'No information available.')
+        super
+      end
+    end
+
+    class NoAPIKey < StandardError
+      def initialize(msg = 'No API key has been found or provided! ( setup your SHODAN_API_KEY environment variable )')
+        super
+      end
+    end
+
+    class NoQuery < StandardError
+      def initialize(msg = 'Empty search query.')
+        super
+      end
+    end
+
+    class AccessDenied < StandardError
+      def initialize(msg = 'Shodan subscription doesn\'t support action, check API permissions!')
+        super
+      end
+    end
+
+    class InvalidKey < StandardError
+      def initialize(msg = 'Invalid API key used, or none given!')
+        super
+      end
+    end
+  end
+end

data/lib/shodanz/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shodanz
-  VERSION = '1.0.6'.freeze
+  VERSION = '2.0.4'
 end

data/shodanz.gemspec

@@ -1,29 +1,32 @@
-# coding: utf-8
-lib = File.expand_path("../lib", __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "shodanz/version"
+require 'shodanz/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "shodanz"
+  spec.name          = 'shodanz'
   spec.version       = Shodanz::VERSION
   spec.authors       = ["Kent 'picatz' Gruber"]
-  spec.email         = ["kgruber1@emich.edu"]
+  spec.email         = ['kgruber1@emich.edu']
 
-  spec.summary       = %q{A modern Ruby gem for Shodan, the world's first search engine for Internet-connected devices.}
-  spec.description   = %q{Featuring full support for the REST, Streaming and Exploits API}
-  spec.homepage      = "https://github.com/picatz/shodanz"
-  spec.license       = "MIT"
+  spec.summary       = "A modern, async Ruby gem for Shodan, the world's first search engine for Internet-connected devices."
+  spec.description   = 'Featuring full support for the REST, Streaming and Exploits API'
+  spec.homepage      = 'https://github.com/picatz/shodanz'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.require_paths = ["lib"]
-  
-  spec.add_dependency "unirest", "1.1.2"
-  spec.add_dependency "json" # specifying version breaks things, what?
-  spec.add_dependency "oj", "3.5.0"
-  
-  spec.add_development_dependency "bundler", "~> 1.16.1"
-  spec.add_development_dependency "rake", "~> 12.3.1"
-  spec.add_development_dependency "rspec", "~> 3.7.0"
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'async-http', '>= 0.38.1', '< 0.53.0'
+  spec.add_dependency 'async', '>= 1.17.1', '< 1.28.0'
+
+  spec.add_development_dependency 'async-rspec', '~> 1.14.0'
+  spec.add_development_dependency 'bundler', '~> 2.1.2'
+  spec.add_development_dependency 'pry', '~> 0.13.0'
+  spec.add_development_dependency 'rake', '~> 13.0.0'
+  spec.add_development_dependency 'rb-readline', '~> 0.5.5'
+  spec.add_development_dependency 'rspec', '~> 3.10.0'
 end

metadata

@@ -1,99 +1,139 @@
 --- !ruby/object:Gem::Specification
 name: shodanz
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 2.0.4
 platform: ruby
 authors:
 - Kent 'picatz' Gruber
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-04 00:00:00.000000000 Z
+date: 2020-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: unirest
+  name: async-http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.38.1
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: 0.53.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.38.1
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: 0.53.0
 - !ruby/object:Gem::Dependency
-  name: json
+  name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.17.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.28.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.17.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.28.0
 - !ruby/object:Gem::Dependency
-  name: oj
+  name: async-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
-  type: :runtime
+        version: 1.14.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 1.14.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.16.1
+        version: 2.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.16.1
+        version: 0.13.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.1
+        version: 13.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+- !ruby/object:Gem::Dependency
+  name: rb-readline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.1
+        version: 0.5.5
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.7.0
+        version: 3.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.7.0
+        version: 3.10.0
 description: Featuring full support for the REST, Streaming and Exploits API
 email:
 - kgruber1@emich.edu
@@ -101,6 +141,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -109,6 +150,9 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/async_honeypot_detector.rb
+- examples/async_host_search_example.rb
+- examples/async_stream_example.rb
 - examples/debug.rb
 - examples/streaming_banner_product_stats.rb
 - examples/top_10_countries_running.rb
@@ -118,7 +162,9 @@ files:
 - lib/shodanz/apis/exploits.rb
 - lib/shodanz/apis/rest.rb
 - lib/shodanz/apis/streaming.rb
+- lib/shodanz/apis/utils.rb
 - lib/shodanz/client.rb
+- lib/shodanz/errors.rb
 - lib/shodanz/version.rb
 - shodanz.gemspec
 homepage: https://github.com/picatz/shodanz
@@ -140,10 +186,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
-summary: A modern Ruby gem for Shodan, the world's first search engine for Internet-connected
-  devices.
+summary: A modern, async Ruby gem for Shodan, the world's first search engine for
+  Internet-connected devices.
 test_files: []