shodan-ruby 0.1.0

data/lib/shodan/query.rb

@@ -0,0 +1,250 @@
+#
+# shodan-ruby - A Ruby interface to SHODAN, a computer search engine.
+#
+# Copyright (c) 2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+require 'shodan/extensions/uri'
+require 'shodan/countries'
+require 'shodan/has_pages'
+require 'shodan/page'
+require 'shodan/shodan'
+
+require 'net/http'
+require 'nokogiri'
+
+module Shodan
+  class Query
+
+    include HasPages
+
+    # Search URL
+    SEARCH_URL = 'http://shodan.surtri.com/'
+
+    # Results per page
+    RESULTS_PER_PAGE = 20
+
+    # Search query
+    attr_accessor :query
+
+    # Countries to search within
+    attr_accessor :countries
+
+    # Hostnames to search for
+    attr_accessor :hostnames
+
+    # CIDR Network blocks to search within
+    attr_accessor :networks
+
+    # Ports to search for
+    attr_accessor :ports
+
+    #
+    # Creates a new Query object.
+    #
+    # @param [Hash] options
+    #   Additional options.
+    #
+    # @option options [String] :query
+    #   The query expression.
+    #
+    # @option options [Array<String>] :countries
+    #   The Country Codes to search within.
+    #
+    # @option options [String] :country
+    #   A Country Code to search within.
+    #
+    # @option options [Array<String>] :hostnames
+    #   The host names to search for.
+    #
+    # @option options [String] :hostname
+    #   A host name to search for.
+    #
+    # @option options [Array<String>] :networks
+    #   The CIDR network blocks to search within.
+    #
+    # @option options [String] :network
+    #   A CIDR network blocks to search within.
+    #
+    # @option options [Array<Integer>] :ports
+    #   The ports to search for.
+    #
+    # @option options [Integer] :port
+    #   A port to search for.
+    #
+    # @yield [query]
+    #   If a block is given, it will be passed the newly created Query
+    #   object.
+    #
+    # @yieldparam [Query] query
+    #   The newly created Query object.
+    #
+    def initialize(options={},&block)
+      @agent = Shodan.web_agent
+      @query = options[:query]
+
+      @countries = []
+
+      if options[:countries]
+        @countries += options[:countries]
+      elsif options[:country]
+        @countries << option[:country]
+      end
+
+      @hostnames = []
+
+      if options[:hostnames]
+        @hostnames += options[:hostnames]
+      elsif options[:hostname]
+        @hostnames << options[:hostname]
+      end
+
+      @networks = []
+
+      if options[:networks]
+        @networks += options[:networks]
+      elsif options[:network]
+        @networks << options[:network]
+      end
+
+      @ports = []
+
+      if options[:ports]
+        @ports += options[:ports]
+      elsif options[:port]
+        @ports << options[:port]
+      end
+
+      block.call(self) if block
+    end
+
+    #
+    # Converts a given URL into a Query object.
+    #
+    # @param [URI::HTTP, String] url
+    #   The query URL.
+    #
+    # @return [Query]
+    #   The Query object created from the URL.
+    #
+    def self.from_url(url)
+      url = URI(url.to_s)
+
+      return self.new(
+        :query => url.query_params['q'].gsub('+',' ')
+      )
+    end
+
+    #
+    # The results per page.
+    #
+    # @return [Integer]
+    #   The resutls per page.
+    #
+    # @see RESULTS_PER_PAGE
+    #
+    def results_per_page
+      RESULTS_PER_PAGE
+    end
+
+    #
+    # The query expression from the query.
+    #
+    # @return [String]
+    #   The query expression.
+    #
+    def expression
+      expr = []
+
+      expr << @query if @query
+
+      expr += @countries.map { |code| "country:#{code}" }
+      expr += @hostnames.map { |host| "hostname:#{host}" }
+      expr += @networks.map { |net| "net:#{net}" }
+      expr += @ports.map { |port| "port:#{port}" }
+
+      return expr.join(' ')
+    end
+
+    #
+    # The search URL for the query.
+    #
+    # @return [URI::HTTP]
+    #   The search URL.
+    #
+    def search_url
+      url = URI(SEARCH_URL)
+
+      url.query_params['q'] = expression
+      return url
+    end
+
+    #
+    # The search URL for the query and given page index.
+    #
+    # @param [Integer] index
+    #   The page index to lookup.
+    #
+    # @retrun [URI::HTTP]
+    #   The search URL for the query and the specified page index.
+    #
+    def page_url(index)
+      url = search_url
+
+      unless index == 1
+        url.query_params['page'] = index
+      end
+
+      return url
+    end
+
+    #
+    # Requests a page at the given index.
+    #
+    # @param [Integer] index
+    #   The index of the page.
+    #
+    # @return [Page]
+    #   The page at the specified index.
+    #
+    def page(index)
+      Page.new do |new_page|
+        doc = @agent.get(page_url(index))
+
+        doc.search('#search/div.result').each do |result|
+          div = result.at('div')
+
+          ip = if (a = div.at('a'))
+                 a.inner_text
+               else
+                 div.children.first.inner_text
+               end
+
+          hostname = if (host_node = result.at('div/a:last'))
+                       host_node.inner_text
+                     end
+
+          date = result.at('div/span').inner_text.scan(/\d+\.\d+\.\d/).first
+          response = result.at('p').inner_text.strip
+
+          new_page << Host.new(ip,date,response,hostname)
+        end
+      end
+    end
+
+  end
+end

data/lib/shodan/shodan.rb

@@ -0,0 +1,216 @@
+#
+# shodan-ruby - A Ruby interface to SHODAN, a computer search engine.
+#
+# Copyright (c) 2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+require 'shodan/query'
+
+require 'mechanize'
+
+module Shodan
+  # Common proxy port.
+  COMMON_PROXY_PORT = 8080
+
+  #
+  # The proxy information used by {Query}.
+  #
+  # @return [Hash]
+  #   The proxy information.
+  #
+  def Shodan.proxy
+    @@shodan_proxy ||= {:host => nil, :port => COMMON_PROXY_PORT, :user => nil, :password => nil}
+  end
+
+  #
+  # Creates a HTTP URI based from the given proxy information.
+  #
+  # @param [Hash] proxy_info (Shodan.proxy)
+  #   The proxy information.
+  #
+  # @option proxy_info [String] :host
+  #   The host of the proxy.
+  #
+  # @option proxy_info [Integer] :port (COMMON_PROXY_PORT)
+  #   The port of the proxy.
+  #
+  # @option proxy_info [String] :user
+  #   The user name to authenticate as.
+  #
+  # @option proxy_info [String] :password
+  #   The password to authenticate with.
+  #
+  def Shodan.proxy_uri(proxy_info=Shodan.proxy)
+    if Shodan.proxy[:host]
+      return URI::HTTP.build(
+        :host => Shodan.proxy[:host],
+        :port => Shodan.proxy[:port],
+        :userinfo => "#{Shodan.proxy[:user]}:#{Shodan.proxy[:password]}",
+        :path => '/'
+      )
+    end
+  end
+  
+  #
+  # The supported Shodan User-Agent Aliases.
+  #
+  # @return [Hash{String => String}]
+  #   The User-Agent aliases and strings.
+  #
+  def Shodan.user_agent_aliases
+    WWW::Mechanize::AGENT_ALIASES
+  end
+
+  #
+  # The default Shodan User-Agent
+  #
+  # @return [String]
+  #   The default User-Agent string used by Shodan.
+  #
+  def Shodan.user_agent
+    @@shodan_user_agent ||= Shodan.user_agent_aliases['Windows IE 6']
+  end
+
+  #
+  # Sets the default Shodan User-Agent.
+  #
+  # @param [String] agent
+  #   The User-Agent string to use.
+  #
+  # @return [String]
+  #   The new User-Agent string.
+  #
+  def Shodan.user_agent=(agent)
+    @@shodan_user_agent = agent
+  end
+
+  #
+  # Sets the default Shodan User-Agent alias.
+  #
+  # @param [String] name
+  #   The alias of the User-Agent string to use.
+  #
+  # @return [String]
+  #   The new User-Agent string.
+  # 
+  def Shodan.user_agent_alias=(name)
+    @@shodan_user_agent = Shodan.user_agent_aliases[name.to_s]
+  end
+
+  #
+  # Creates a new WWW::Mechanize agent.
+  #
+  # @param [Hash] options
+  #   Additional options.
+  #
+  # @option options [String] :user_agent_alias
+  #   The User-Agent alias to use.
+  #
+  # @option options [String] :user_agent (Shodan.user_agent)
+  #   The User-Agent string to use.
+  #
+  # @option options [Hash] :proxy (Shodan.proxy)
+  #   The proxy information to use.
+  #
+  # @option :proxy [String] :host
+  #   The host of the proxy.
+  #
+  # @option :proxy [Integer] :port
+  #   The port of the proxy.
+  #
+  # @option :proxy [String] :user
+  #   The user to authenticate as.
+  #
+  # @option :proxy [String] :password
+  #   The password to authenticate with.
+  #
+  # @example Creating a new Mechanize agent.
+  #   Shodan.web_agent
+  #
+  # @example Creating a new Mechanize agent with a User-Agent alias.
+  #   Shodan.web_agent(:user_agent_alias => 'Linux Mozilla')
+  #
+  # @example Creating a new Mechanize agent with a User-Agent string.
+  #   Shodan.web_agent(:user_agent => 'Google Bot')
+  #
+  def Shodan.web_agent(options={},&block)
+    agent = WWW::Mechanize.new
+
+    if options[:user_agent_alias]
+      agent.user_agent_alias = options[:user_agent_alias]
+    elsif options[:user_agent]
+      agent.user_agent = options[:user_agent]
+    elsif Shodan.user_agent
+      agent.user_agent = Shodan.user_agent
+    end
+
+    proxy = (options[:proxy] || Shodan.proxy)
+    if proxy[:host]
+      agent.set_proxy(proxy[:host],proxy[:port],proxy[:user],proxy[:password])
+    end
+
+    block.call(agent) if block
+    return agent
+  end
+
+  #
+  # Creates a new Query object.
+  #
+  # @param [Hash] options
+  #   Additional options.
+  #
+  # @option options [String] :query
+  #   The query expression.
+  #
+  # @option options [Array<String>] :countries
+  #   The Country Codes to search within.
+  #
+  # @option options [String] :country
+  #   A Country Code to search within.
+  #
+  # @option options [Array<String>] :hostnames
+  #   The host names to search for.
+  #
+  # @option options [String] :hostname
+  #   A host name to search for.
+  #
+  # @option options [Array<String>] :networks
+  #   The CIDR network blocks to search within.
+  #
+  # @option options [String] :network
+  #   A CIDR network blocks to search within.
+  #
+  # @option options [Array<Integer>] :ports
+  #   The ports to search for.
+  #
+  # @option options [Integer] :port
+  #   A port to search for.
+  #
+  # @yield [query]
+  #   If a block is given, it will be passed the newly created Query
+  #   object.
+  #
+  # @yieldparam [Query] query
+  #   The newly created Query object.
+  #
+  # @return [Query]
+  #   The new Query object.
+  #
+  def Shodan.query(options={},&block)
+    Query.new(options,&block)
+  end
+end

data/lib/shodan/version.rb

@@ -0,0 +1,24 @@
+#
+# shodan-ruby - A Ruby interface to SHODAN, a computer search engine.
+#
+# Copyright (c) 2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+module Shodan
+  # shodanrb version
+  VERSION = '0.1.0'
+end

data/lib/shodan.rb

@@ -0,0 +1,23 @@
+#
+# shodan-ruby - A Ruby interface to SHODAN, a computer search engine.
+#
+# Copyright (c) 2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+require 'shodan/query'
+require 'shodan/shodan'
+require 'shodan/version'

data/spec/has_pages_examples.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+shared_examples_for "has Pages" do
+  it "should have a first page" do
+    @query.first_page.should_not be_nil
+  end
+
+  it "should allow indexed access" do
+    @query[1].should_not be_nil
+  end
+
+  it "should allow accessing multiple pages" do
+    pages = @query.pages(1..2)
+    pages.should_not be_nil
+    pages.length.should == 2
+  end
+end

data/spec/host_spec.rb

@@ -0,0 +1,115 @@
+require 'shodan/host'
+
+require 'spec_helper'
+
+describe Host do
+  describe "non-HTTP host" do
+    before(:all) do
+      @host = Host.new('127.0.0.1','16.11.2009','SSH-2.0-OpenSSH_4.2')
+    end
+
+    it "should parse the date" do
+      @host.date.class.should == Date
+    end
+
+    it "should not have a HTTP version" do
+      @host.http_version.should be_nil
+    end
+
+    it "should not have a HTTP status code" do
+      @host.http_code.should be_nil
+    end
+
+    it "should not have a HTTP status message" do
+      @host.http_status.should be_nil
+    end
+
+    it "should not have HTTP response headers" do
+      @host.http_headers.should == {}
+    end
+
+    it "should not have a HTTP response body" do
+      @host.http_body.should be_nil
+    end
+  end
+
+  describe "HTTP host" do
+    before(:all) do
+      @host = Host.new(
+        '127.0.0.1',
+        '16.11.2009',
+        %{
+HTTP/1.0 200 OK
+X-powered-by: PHP/5.3.0
+Transfer-encoding: chunked
+X-gpg-key: http://chrislea.com/gpgkey.txt http://chrislea.com/gpgkey-virb.txt
+Vary: Cookie
+X-ssh-key: http://chrislea.com/sshkey.txt
+Server: nginx/0.8.24
+Connection: keep-alive
+Date: Fri, 20 Nov 2009 00:52:24 GMT
+X-the-question: Quis custodiet ipsos custodes?
+Content-type: text/html; charset=UTF-8
+X-pingback: http://chrislea.com/xmlrpc.php
+
+<HTML>
+</HTML>
+      }.strip
+      )
+    end
+
+    it "should parse the date" do
+      @host.date.class.should == Date
+    end
+
+    it "should not have a HTTP version" do
+      @host.http_version.should == '1.0'
+    end
+
+    it "should not have a HTTP status code" do
+      @host.http_code.should == 200
+    end
+
+    it "should not have a HTTP status message" do
+      @host.http_status.should == 'OK'
+    end
+
+    it "should not have HTTP response headers" do
+      @host.http_headers.should == {
+        'X-powered-by' => 'PHP/5.3.0',
+        'Transfer-encoding' => 'chunked',
+        'X-gpg-key' => 'http://chrislea.com/gpgkey.txt http://chrislea.com/gpgkey-virb.txt',
+        'Vary' => 'Cookie',
+        'X-ssh-key' => 'http://chrislea.com/sshkey.txt',
+        'Server' => 'nginx/0.8.24',
+        'Connection' => 'keep-alive',
+        'Date' => 'Fri, 20 Nov 2009 00:52:24 GMT',
+        'X-the-question' => 'Quis custodiet ipsos custodes?',
+        'Content-type' => 'text/html; charset=UTF-8',
+        'X-pingback' => 'http://chrislea.com/xmlrpc.php'
+      }
+    end
+
+    it "should provide a server name" do
+      @host.server_name.should == 'nginx'
+    end
+
+    it "should provide a server version" do
+      @host.server_version.should == '0.8.24'
+    end
+
+    it "should provide transparent access to the HTTP response headers" do
+      @host.x_the_question.should == 'Quis custodiet ipsos custodes?'
+    end
+
+    it "should raise a NoMethodError when accessing missing headers" do
+      lambda {
+        @host.lol
+      }.should raise_error(NoMethodError)
+    end
+
+    it "should not have a HTTP response body" do
+      @host.http_body.should == "<HTML>\n</HTML>"
+    end
+  end
+end

data/spec/page_has_hosts_examples.rb

@@ -0,0 +1,31 @@
+require 'shodan/page'
+
+require 'spec_helper'
+
+shared_examples_for "Page has Hosts" do
+  it "should have hosts" do
+    @page.length.should_not == 0
+  end
+
+  it "should have the maximum amount of hosts per page" do
+    @page.length.should == @query.results_per_page
+  end
+
+  it "should have IP addresses" do
+    @page.ips.should_not be_empty
+  end
+
+  it "should have valid IP addresses" do
+    @page.each_ip do |ip|
+      ip =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
+    end
+  end
+
+  it "should have dates" do
+    @page.dates.should_not be_empty
+  end
+
+  it "should have responses" do
+    @page.responses.should_not be_empty
+  end
+end