esearchy 0.2.0

data/README.rdoc

@@ -0,0 +1,112 @@
+= Esearch
+
+== DESCRIPTION
+Esearchy is a small library capable of searching the internet for email addresses. Currently, the supported search methods are, but not limited to:
+
+* Search engines: 
+  * Google
+  * Bing
+  * Yahoo,
+  * AltaVista
+* Social Engines:
+  * LinkedIn
+  * Google Profiles ( Based on DigiNinja's idea http://www.digininja.org/projects/gpscan.php)
+  * Naymz
+  * Classmantes
+* Other Engines
+  * PGP servers
+  * Usenets
+  * GoogleGroups Search
+  * Spider
+  * LDAP
+
+
+But Searches do not stop there, ESearchy it also looks for emails inside: 
+
+* PDF 
+* DOC
+* DOCX
+* XLSX
+* PPTX
+* ODT
+* ODP
+* ODS
+* ODB
+* ASN
+* TXT 
+
+Once all the text is parsed within the file the emails are added to the list of found accounts.
+
+In order to parse Microsoft Word (.doc):
+* You Either need a windows Platform with Word installed.
+* Install AntiWord. ( http://www.winfield.demon.nl/ )
+* Or if non of the above is on the OS, we perform a raw search inside the file.
+
+NOTE: THIS IS STILL BEING DEVELOPED CODE IS SUBMITTED DAILY SO BE AWARE THAT CODE MIGHT NOT WORK PROPERLY AL THE TIME. IF SOMETHING GOES WRONG PLEASE RAISE AN ISSUE.  
+
+
+== SUPPORT:
+
+* http://github.com/FreedomCoder/esearchy/issues
+* Emails from github.
+
+== SYNOPSIS:
+
+For now, there are two main ways of performing a search:
+
+* Executable CLI command
+
+
+* Library
+
+For thouse who want to integrate this to their application you can use it in "the ruby way"
+
+== REQUIREMENTS:
+
+* ruby 1.8 or 1.9
+* cgi
+* pdf/reader
+* json
+* spidr
+* ldap
+* rubyzip ( Migrating to FreedomCoder-rubyzip 0.9.2 so it's 1.9 compatible)
+
+== INSTALL:
+* > sudo gem sources -a http://gems.github.com   (If you do not have the repository)
+* > sudo gem install FreedomCoder-esearchy
+
+== THANKS:
+
+* http://www.penetrationtests.com/ [Penetration Testing Directory]
+* http://www.mundoruby.com.ar/ [Ruby's News Aggregator]
+* http://www.digininja.org/ [DigiNinja's Website]
+* http://hexale.blogspot.com/ [8A's webite]
+* http://www.kalmbach.com.ar/ [/jk's webite]
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 - 2009:
+
+* {Matias P. Brutti}[http://www.freedomcoder.com.ar]
+
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/esearchy

@@ -0,0 +1,334 @@
+#!/usr/bin/env ruby -wKU 
+#  esearchy
+#  
+#  Created by FreedomCoder on 2009-10-24.
+#  Copyright 2009 FreedomCoder's Labs. All rights reserved.
+# 
+if  RUBY_PLATFORM =~ /mingw|mswin/
+  require 'Win32API'
+  class Wcol
+    gsh = Win32API.new("kernel32", "GetStdHandle", ['L'], 'L') 
+    @textAttr = Win32API.new("kernel32","SetConsoleTextAttribute", ['L','N'], 'I')
+    @h = gsh.call(-11)
+  
+    def self.color(col)
+      @textAttr.call(@h,col)
+    end
+    
+    def self.puts(color, text)
+      self.color(@h,color)
+      puts text
+      self.color(@h,7)
+    end
+  end
+end
+
+
+require 'rubygems'
+require 'getoptlong'
+require 'sqlite3'
+require 'prawn'
+require 'prawn/layout'
+
+require '../lib/esearchy.rb'
+
+ESearchy::log = true
+
+@yahoo_key  = nil
+@bing_key   = nil
+@maxhits    = nil
+@params     = {}
+@list       = []
+@output     = nil
+@email_engines = [:Google,:Yahoo,:Bing,:Altavista,:Usenet,:PGP,:Spider,:GoogleGroups]
+@people_engines = [:LinkedIn,:Naymz,:Classmates,:GoogleProfiles]
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-google', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-yahoo', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-bing', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-altavisa', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-linkedin', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-gprofiles', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-naymz', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-ggroups', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-pgp', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-usenet', GetoptLong::NO_ARGUMENT ],
+  [ '--disable-spider', GetoptLong::NO_ARGUMENT ],
+  [ '--query','-q', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--company','-c', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--website','-w', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--file','-f', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--filter','-p', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--output','-o', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--yahookey','-y', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--bingkey','-b', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--maxhits','-m', GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+opts.each do |opt, arg|
+  case opt
+    when '--help':
+      # BEGIN OF HELP
+      ESearchy::LOG.puts "\nHELP for Esearchy\n---------------------\n
+      --help, -h
+      \tWell I guess you know what this is for (To obtain this Help).\n
+      INPUT PARAMS:
+      --query, -q [@domain.com]
+      \t The domain name to search.\n
+      --company, -c [Company Inc]
+      \t The company name to search.\n
+      --website, -w [www.domain.com]
+      \t The website name to spider.\n
+      --yahookey, -y [key]
+      \t The Yahoo API Key .\n
+      --bingkey, -b [key]
+      \t The Bing API Key .\n
+      --filter, -p
+      \t The pattern to use to filter emails.(not fully implemented)\n
+      --file, -f [file_name] 
+      \tIf we need to search more than one domain we can provide a list.\n
+      --output, -o
+      \tThe output file name.\n\n
+      PLUGIN OPTIONS:
+      --disable-google
+      \t Disables Google searches.\n
+      --disable-yahoo
+      \t Disables Yahoo searches.\n
+      --disable-bing
+      \t Disables Bing searches.\n
+      --disable-linkedin
+      \t Disables LinkedIn searches.\n
+      --disable-gprogiles
+      \t Disables Google Profiles searches.\n
+      --disable-naymz
+      \t Disables Naymz searches.\n
+      --disable-ggroups
+      \t Disables Google Groups searches.\n
+      --disable-pgp
+      \t Disables PGP searches.\n
+      --disable-usenet
+      \t Disables Usenet searches.\n\n
+      Copyright 2009 - FreedomCoder\n"
+      #END OF HELP
+      exit(0)
+    when '--disable-google':
+      @email_engines.delete(:Google) 
+    when '--disable-yahoo':
+      @email_engines.delete(:Yahoo)
+    when '--disable-bing':
+      @email_engines.delete(:Bing)
+    when '--disable-altavisa':
+      @email_engines.delete(:Altavista)
+    when '--disable-spider':
+      @email_engines.delete(:Spider)
+    when '--disable-linkedin':
+      @people_engines.delete(:LinkedIn)
+    when '--disable-gprofiles':
+      @people_engines.delete(:GoogleProfiles)
+    when '--disable-naymz':
+      @people_engines.delete(:Naymz)
+    when '--disable-classmates':
+      @email_engines.delete(:Classmates)
+    when '--disable-ggroups':
+      @email_engines.delete(:GoogleGroups)
+    when '--disable-pgp':
+      @email_engines.delete(:PGP)
+    when '--disable-usenet':
+      @email_engines.delete(:Usenet)
+    when '--query':
+      @params[:query] = arg
+    when '--company':
+      @params[:company] = arg
+    when '--file':
+      if File.exists?(arg)
+        open(arg,'r').each_line do |line|
+          temp[:query],temp[:company],temp[:website] = line.split(',')
+          @list << temp
+        end
+      else
+        raise ArgumentError, "File not found"
+        exit(0)
+      end
+    when '--yahookey':
+      @yahoo_key = arg
+    when '--bingkey':
+      @bing_key = arg
+    when '--filter':
+      @pattern = arg
+    when '--output':
+      @output = Output.new arg
+    when '--maxhits':
+      @params[:maxhits] = arg.to_i
+    else
+      puts "Unknown command. Please try again"
+      exit(0)
+  end
+end
+
+
+def configure(maxhits = 500, yahoo_key = nil, bing_key = nil)
+  File.new(ENV['HOME'] + "/.esearchyrc", +w ) do |line|
+    line << "MAXHITS=" + maxhits
+    line << "YAHOOKEY=" + yahoo_key if @yahoo_key
+    line << "BINGKEY=" + bing_key if @bing_key
+  end
+end
+
+def read_conf
+  File.read((ENV['HOME'] + "/.esearchyrc").readlines.each do |line|
+    key, value = line.split("=")
+    case key
+    when "MAXHITS"
+      @params[:maxhits] ||= value
+    when "YAHOOKEY"
+      @yahoo_key ||= value
+    when "BINGKEY"
+      @bing_key = ||value
+    end
+  end
+end
+
+def dump(results)
+  # this is for session handling.
+end
+
+### PRINTING FINDING METHODS ###
+def print_(list)
+  list.each do |email|
+    unless @emails.include?(email)
+      case RUBY_PLATFORM
+      when /mingw|mswin/
+        print_windows
+      when /linux|darwin/
+        print_linux
+      end
+    end
+  end
+end
+  
+def print_linux(email)
+  if email.match(/#{@query.gsub("@","").split('.')[0]}/)
+    puts "\033[31m" + email + "\033\[0m"
+  else
+    puts "\033[32m" + email + "\033\[0m"
+  end
+end
+  
+def print_windows(email)
+  if email.match(/#{@query.gsub("@","").split('.')[0]}/)
+    Wcol::puts(12, email)
+  else
+    Wcol::puts(2, email)
+  end
+end
+
+### SAVING TO DISK ###
+class Output
+
+  def initialize(name)
+    @output = name
+  end
+  
+  def save(data)
+    case @output
+    when /pdf/
+      save_pdf
+    when /csv/
+      save_csv
+    when /sqlite/
+      save_sqlite
+    end
+  end
+  
+  private 
+  def save_csv(data)
+    out = File.new(name, "w")
+    out << "EMAILS/PERSON, TYPE, CLASS, MATCH\n"
+    data.each do |r|
+      out << r.each { |x| x
+    end
+    
+  end
+  
+  def save_pdf(data)
+    Prawn::Document.generate(name) do  
+      table data, 
+        :position => :center, 
+        :headers => ["Email/Person", "Type", "Class", "Match"],
+        :header_color => "0046f9",
+        :row_colors => :pdf_writer, #["ffffff","ffff00"],
+        :font_size => 10,
+        :vertical_padding => 2,
+        :horizontal_padding => 5
+  end
+  
+  def save_sqlite(data)
+    @db = SQLite3::Database.new(file)
+    @db.execute("CREATE TABLE IF NOT EXISTS results (
+      id integer primary key asc, 
+      object text, 
+      type char, 
+      class text,
+      match char);")
+      
+    @results.each do |r| 
+      @db.execute("INTERT INTO results (object,type,class,match) 
+                   VALUES (#{r[0].to_s},#{r[1]},#{r[2]},#{r[3]});")
+    end
+    @db.commit        
+  end
+end
+
+def execute(p)
+  if p[:query]
+    search = ESearchy::Search.new(p)
+
+    search.start do |s|
+      s.Emails(@email_engines) do |e|
+        e.Yahoo.appid= @yahoo_key if @yahoo_key
+        e.Bing.appid= @bing_key if @bing_key 
+        e.search do |e| 
+          dump(e.results)
+          print_(e.emails.uniq)
+        end
+        e.docs do |e| 
+          dump(e.results)
+          print_(e.emails.uniq)
+        end
+      end
+    
+      s.People(@people_engines) do |p|
+        p.search { |p| dump(p.results) }
+      end
+    end
+    @output.save(search.results)
+    puts "-------RESULTS--------"
+    print_ search.emails
+    print_ search.people
+  end
+end
+
+puts "DISCLOSURE: This is just an example tool ESearchy is more and more a piece 
+of code intended to work as a Library and you should create your own little.rb file :)"
+puts "------------------------------------------------------------------------"
+puts "REMINDER:" 
+puts "- if you want to use GoogleProfiles, LinkedIn, Classmates or Naymz, you will need to use the --company (-c) <company_name> option"
+puts "- If you want to spider a website you need to use the --website (-w) <URL> option"
+
+
+unless File.exist?(ENV['HOME'] + "/.esearchyrc")
+  configure(@params[:maxhits],@yahoo_key,@bing_key)
+else
+  read_conf
+end
+
+unless @list.empty?
+  @list.each {|p| execute(p)}
+end
+
+unless @params.empty?
+  execute(@params)
+end
+puts "Happy Hacking :)\nGood Bye.\n"

data/lib/esearchy/LocalEngines/directory.rb

@@ -0,0 +1,16 @@
+module ESearchy
+  module LocalEngines
+    class Directory
+      def initialize(dir)
+        @documents = Queue.new
+        @emails = []
+      end
+      
+      def search
+        files = Dir["#{@dir}/**/*.*"]
+        files.select {|x| /.pdf$|.doc$|.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$|.txt$|.rtf$|.ans$|.csv$|.xml|.json$|.html$/i}.each { |f| @documents.push(f) }
+      end
+      
+    end
+  end
+end

data/lib/esearchy/OtherEngines/googlegroups.rb

@@ -0,0 +1,27 @@
+module ESearchy  
+  module OtherEngines
+    class GoogleGroups < ESearchy::GenericEngine
+      ENGINE = "groups.google.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 1
+      
+      def search 
+        @querypath = "/groups/search?&safe=off&num=100&q=" +  @query  + "&btnG=Search&start="
+        super
+      end
+            
+      def parse( html )
+        hits = html.scan(/<\/b> of about <b>(.*)<\/b> for /)
+        if hits.empty? or hits == nil
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits[0][0].gsub(",","").to_i)
+        end
+        super html.scan(/<div class=g align="left"><a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" target=""/)
+      end
+    end
+  end
+end
+
+  

data/lib/esearchy/OtherEngines/ldap.rb

@@ -0,0 +1,44 @@
+module ESearchy
+  module OtherEngines
+    class Ldap
+      
+      def initialize(host = nil, port = nil, base = nil, 
+                     scope = nil, filter = nil, attr = nil)
+        @HOST    = host
+        @PORT    = port || LDAP::LDAP_PORT
+        @base    = base || "dc=localhost,dc=#{host}"
+        @scope   = scope || 2
+        @filter  = filter || '(objectclass=person)'
+        @attrs   = attr || ['sn', 'cn'] 
+      end
+      attr_accessor :HOST, :PORT, :SSLPORT, :base, :scope, :filter, :attrs
+      
+      def search(bind, &block)
+        connect(bind)
+        begin
+          block.call(self) if block_given?
+          @conn.search(base, scope, filter, attrs, block)
+        rescue LDAP::ResultError
+          @conn.perror("search")        
+        end
+        @conn.perror("search")
+        close
+      end  
+      
+      private
+      
+      def connect(bind)
+        begin
+          @conn = LDAP::Conn.new(@HOST, @PORT)
+          @conn.bind(bind) # i.e. 'cn=root, dc=localhost, dc=localdomain','secret'
+        rescue LDAP::Error
+          @conn.perror("bind")
+        end
+      end
+      
+      def close
+        @conn.unbind
+      end
+    end
+  end
+end

data/lib/esearchy/OtherEngines/pgp.rb

@@ -0,0 +1,22 @@
+module ESearchy
+  module OtherEngines
+    class PGP < ESearchy::GenericEngine
+      ENGINE = "pgp.mit.edu"
+      PORT  = 11371
+      NUM = 0 # Do not really ned it :)
+      TYPE = 1
+      
+      def search 
+        @querypath = "/pks/lookup?search=" + @query
+        get ENGINE, PORT, @querypath, {'User-Agent' => UserAgent::fetch } do |r|
+          D "Searching #{self.class}"
+          crawler(r.body)
+        end
+      end
+
+      def parse( html )
+        super html.scan(/href=["|']([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)["|']/)
+      end
+    end
+  end
+end

data/lib/esearchy/OtherEngines/spider.rb

@@ -0,0 +1,43 @@
+require 'spidr'
+require 'json'
+
+module ESearchy  
+  module OtherEngines
+    class Spider < ESearchy::GenericEngine
+      ENGINE = "" #Do not really need any of them. 
+      PORT  = 0
+      NUM = 0
+      TYPE = 1 
+      
+      def search 
+        Spidr.site(website()) do |spider|
+          spider.every_page do |page|
+            D page.url
+            crawler(page.body)
+            parse(page.body)
+          end
+        end
+      end
+      
+      def website
+        begin
+          ESearchy::Search.website || @website
+        rescue
+          raise ESearchyMissingWebsite, "Mssing website url Object.website=(value)"
+        end
+      end
+      
+      def website=(v)
+        @website=v
+      end
+            
+      def parse( html )
+        array = html.scan(/href=["|']([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)["|']/).map! do |r|
+          r[0].match(/http:\/\/|https:\/\/|ftp:\/\//) ? r : [website() + r[0]]
+        end
+        super array
+      end
+      
+    end
+  end
+end

data/lib/esearchy/OtherEngines/usenet.rb

@@ -0,0 +1,22 @@
+module ESearchy
+  module OtherEngines
+    class Usenet
+      ENGINE = "usenet-addresses.mit.edu"
+      PORT  = 80
+      NUM = 0 # Do not really ned it :)
+      TYPE = 1
+      
+      def search
+        @querypath = "/cgi-bin/udb?T=" + @query + "&G=&S=&N=&O=&M=500"
+        get ENGINE, PORT, @querypath, {'User-Agent' => UserAgent::fetch } do |r|
+          D "Searching #{self.class}"
+          crawler(r.body)
+        end
+      end
+
+      def parse( html )
+        super html.scan(/href=["|']([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)["|']/)
+      end
+    end
+  end
+end

data/lib/esearchy/SearchEngines/altavista.rb

@@ -0,0 +1,25 @@
+module ESearchy  
+  module SearchEngines
+    class Altavista < ESearchy::GenericEngine
+      ENGINE = "www.altavista.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 1
+      
+      def search 
+        @querypath = "/web/results?itag=ody&kgs=0&kls=0&nbq=50&q=" + @query + "&stq="
+        super
+      end
+            
+      def parse( html )
+        hits = html.scan(/AltaVista found (.*) results<\/A>/) 
+        if hits.empty? or hits == nil
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits[0][0].gsub(',','').to_i)
+        end
+        super html.scan(/<a class='res' href='([a-zA-Z0-9:\/\/.&?%=\-_+]*)'>/)
+      end
+    end
+  end
+end

data/lib/esearchy/SearchEngines/bing.rb

@@ -0,0 +1,32 @@
+module ESearchy  
+  module SearchEngines
+    class Bing < ESearchy::GenericEngine
+      ENGINE = "api.search.live.net"
+      PORT  = 80
+      NUM = 50
+      TYPE = 1
+      
+      def search
+        @querypath = "/json.aspx?AppId=" + @appid + "&query=" + @query +
+                     "&Sources=Web&Web.Count=50&Web.Offset=" or 
+                     raise ESearchyMissingAppID, "Missing AppID <Class.appid=>"
+        super
+      end
+      
+      def appid=(value)
+        @appid = value
+      end
+      
+      def parse(json)
+        doc = JSON.parse(json)
+        hits = doc["SearchResponse"]["Web"]["Total"].to_i 
+        if hits == nil or hits == 0
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits)
+        end
+        super doc["SearchResponse"]["Web"]["Results"]
+      end
+    end
+  end
+end