FreedomCoder-esearchy 0.0.19 → 0.1

data/README.rdoc

@@ -1,20 +1,52 @@
 = Esearch
 
 == DESCRIPTION
-Esearchy is a small library capable of searching the internet for email addresses. Currently, the supported search methods are engines such as Google, Bing, Yahoo, PGP servers, GoogleGroups, LinkedIn, etc , but I intend to add many more.
+Esearchy is a small library capable of searching the internet for email addresses. Currently, the supported search methods are, but not limited to:
+
+* Search engines: 
+  * Google
+  * Bing
+  * Yahoo,
+  * AltaVista
+* Social Networks:
+  * LinkedIn
+  * Google Profiles ( Based on DigiNinja's idea http://www.digininja.org/files/gpscan.rb )
+  * Naymz
+* PGP servers
+* Usenets
+* GoogleGroups Search
+
+
+But Searches do not stop there, ESearchy it also looks for emails inside: 
+
+* PDF 
+* DOC
+* DOCX
+* XLSX
+* PPTX
+* ODT
+* ODP
+* ODS
+* ODB
+* ASN
+* TXT 
+
+Once all the text is parsed within the file the emails are added to the list of found accounts.
 
-Also, the library searches inside .pdf, .docx, .xlsx, .pptx, asn and .txt files for emails addresses and adds them to the list of found accounts. Finally, we have support for .docs files but for now only in Windows Platforms.
 In order to parse Microsoft Word (.doc):
 * You Either need a windows Platform with Word installed.
 * Install AntiWord. ( http://www.winfield.demon.nl/ )
+* Or if non of the above is on the OS, we perform a raw search inside the file.
 
 NOTE: THIS IS STILL BEING DEVELOPED CODE IS SUBMITTED DAILY SO BE AWARE THAT CODE MIGHT NOT WORK PROPERLY AL THE TIME. IF SOMETHING GOES WRONG PLEASE RAISE AN ISSUE.  
 
-In order to work Bing and Yahoo need an appid, for which you will have to create one for each and place them in fles so the library will be able to work properly.
+In order to work Bing and Yahoo need an appid. This API Keys could be added once in below mentioned files so the library will be able to work properly, and or passed as parameter each time we execute a search.
 * data/yahoo.key
 * data/bing.key
 
-Soon, users should be able to also pass them as parameters.
+Furthermore, in order to use the LinkedIn People Search we need to pass two parameters. 
+* company name ( with Object.company_name )
+* User/password for LinkedIn. (with Object.linkedin_credentials) For this we can use ESearchy.BUGMENOT which will search a linkedin user in bugmenot.com
 
 == SUPPORT:
 
@@ -80,7 +112,7 @@ Not short of that now, we also have the possibility of choosing between a Librar
 * cgi
 * pdf/reader
 * json
-* rubyzip
+* rubyzip ( Migrating to FreedomCoder-rubyzip 0.9.2 so it's 1.9 compatible)
 
 == INSTALL:
 

data/lib/esearchy.rb

@@ -1,16 +1,25 @@
+require 'resolv'
 local_path = "#{File.dirname(__FILE__) + '/esearchy/'}"
-%w{google googlegroups bing yahoo pgp keys 
-   linkedin logger bugmenot}.each { |lib| require local_path + lib } 
+%w{keys logger bugmenot}.each { |lib| require local_path + lib } 
+%w{google bing yahoo altavista }.each { |lib| require local_path + "SearchEngines/"+lib }
+%w{linkedin googleprofiles naymz}.each { |lib| require local_path + "SocialNetworks/"+lib }
+%w{googlegroups pgp usenet}.each { |lib| require local_path + "OtherEngines/"+lib }
 
 class ESearchy
-  #Constants 
   
+  #Constants 
   LIBRARY = 1
   APP = 2
-  
   LOG = Logger.new(1, $stdout)
   BUGMENOT = BMN::fetch_user("linkedin.com")
-  
+  DEFAULT_ENGINES = [:Google, :Bing, :Yahoo, :PGP, :LinkedIn, 
+                     :GoogleGroups, :Altavista, :Usenet, :GoogleProfiles, :Naymz]
+  case RUBY_PLATFORM 
+  when /mingw|mswin/
+    TEMP = "C:\\WINDOWS\\Temp\\"
+  else
+    TEMP = "/tmp/"
+  end
   #End Constants
   
   def log_type=(value)
@@ -21,12 +30,6 @@ class ESearchy
     ESearchy::LOG.file = value
   end
   
-  def eng(arr)
-    hsh = {}; arr.each {|e| hsh[e] = instance_eval "#{e}"}; hsh
-  end
-  
-  DEFAULT_ENGINES = [:Google, :Bing, :Yahoo, :PGP, :LinkedIn]
-  
   def initialize(options={}, &block)
     @query = options[:query]
     @depth_search = options[:depth] || true
@@ -36,14 +39,26 @@ class ESearchy
                                      :Bing => Bing, 
                                      :Yahoo => Yahoo,
                                      :PGP => PGP, 
-                                     :LinkedIn => LinkedIn }
-                                     
+                                     :LinkedIn => LinkedIn,
+                                     :GoogleGroups => GoogleGroups,
+                                     :Altavista => Altavista,
+                                     :Usenet => Usenet,
+                                     :GoogleProfiles => GoogleProfiles,
+                                     :Naymz => Naymz }
     @engines.each {|n,e| @engines[n] = e.new(@maxhits)}
     @threads = Array.new
     block.call(self) if block_given?
   end
+  
+  #Attributes
   attr_accessor :engines, :query, :threads, :depth_search
   attr_reader :maxhits
+
+  def self.create(query=nil, &block)
+    self.new :query => query do |search|
+      block.call(search) if block_given?
+    end
+  end
   
   def search(query=nil)
     @engines.each do |n,e|
@@ -53,7 +68,7 @@ class ESearchy
       LOG.puts "+--- Finishing Search for #{n} ---+\n"
     end
   end
-  
+  # retrieve emails
   def emails
     emails = []
     @engines.each do |n,e|
@@ -62,12 +77,30 @@ class ESearchy
     emails
   end
   
+  def people
+    people = []
+    [:LinkedIn, :GoogleProfiles].each do |e| 
+      people.concat(@engines[e].people) if @engines[e]
+    end
+    people.uniq!
+  end
+  
+  ## Filter methods ##
   def clean(&block)
     emails.each do |e|
       e.delete_if block.call
     end
   end
   
+  def filter(regex)
+    emails.each.select { |email| email =~ regex }
+  end
+  
+  def filter_by_score(score)
+    emails.each.select { |email| score >= calculate_score(emails) }
+  end
+    
+  ## Option methods ##
   def maxhits=(value)
     @engines.each do |n,e|
       e.maxhits = value
@@ -82,27 +115,39 @@ class ESearchy
     @engines[:Bing].appid = value
   end
   
-  def linkedin_credentials(*args)
+  def linkedin_credentials
+    return @engines[:LinkedIn].username, @engines[:LinkedIn].password
+  end
+  
+  def linkedin_credentials=(*args)
     if args.size == 2
       @engines[:LinkedIn].username = args[0]
       @engines[:LinkedIn].password = args[1]
       return true
-    elsif args.size ==1
+    elsif args.size == 1
       @engines[:LinkedIn].username = args[0][0]
       @engines[:LinkedIn].password = args[0][1]
       return true
     end
     false
   end
-  alias_method :linkedin_credentials=, :linkedin_credentials
   
-  def company_name(company)
-    @engines[:LinkedIn].company_name = company
+  def company_name
+    (@engines[:LinkedIn] || 
+     @engines[:GoogleProfiles] || 
+     @engines[:Naymz]).company_name || nil
+  end
+  
+  def company_name=(company)
+    [:LinkedIn, :GoogleProfiles, :Naymz].each do |e| 
+      @engines[e].company_name = company if @engines[e]
+    end
+    
   end
-  alias_method :company_name=, :company_name
   
   def search_engine(key, value)
-    if [:Google, :Bing, :Yahoo, :PGP, :LinkedIn, :GoogleGroups].include?(key)
+    if [:Google, :Bing, :Yahoo, :PGP, :LinkedIn, 
+        :GoogleGroups, :AltaVisa, :Usenet, :GoogleProfiles, Naymz].include?(key)
       if value == true 
         unless @engines[key]
           @engines[key] = instance_eval "#{key}.new(@maxhits)"
@@ -115,31 +160,52 @@ class ESearchy
     end
   end
   
-  %w{Google Bing Yahoo PGP LinkedIn GoogleGroups}.each do |engine|
+  %w{Google Bing Yahoo PGP LinkedIn GoogleGroups 
+     Altavista Usenet GoogleProfiles Naymz}.each do |engine|
     class_eval "
       def search_#{engine}=(value)
         search_engine :#{engine}, value
       end"
   end
-  
-  def save_to_file(file)
+  ## Saving methods
+  def save_to_file(file, list=nil)
     open(file,"a") do |f|
-      emails.each { |e| f << e + "\n" }
+      list ? list.each { |e| f << e + "\n" } : emails.each { |e| f << e + "\n" }
     end
   end
   
-  def filter(regex)
-    emails.each.select { |email| email =~ regex }
+  def save_to_sqlite(file)
+    # TODO save to sqlite3
+    # table esearchy with fields (id, Domain, email, score)
   end
 
-  def self.create(query=nil, &block)
-    self.new :query => query do |search|
-      block.call(search) if block_given?
-    end
+  ## checking methods ##
+  
+  def verify_email!(arg = emails)
+    # TODO
+    # Connect to mail server if possible verify else 
+    # return 0 for false  2 for true or 1 for error.
+    # VRFY & EXPN & 'RCPT TO:
+    return false
+  end
+  
+  def verify_domain!(e)
+    Resolv::DNS.open.getresources(e.split('@')[-1],Resolv::DNS::Resource::IN::MX) > 0 ? true : false
   end
   
   private
   
+  def eng(arr)
+    hsh = {}; arr.each {|e| hsh[e] = instance_eval "#{e}"}; hsh
+  end
+  
+  def calculate_score(email)
+    score = 0.0
+    score = score + 0.2 if email =~ /#{@query}/
+    score = score + 0.3 if verify_domain!(email)  
+    score = 1.0 if verify_email!(email)
+  end
+  
   def depth_search?
     @depth_search
   end

data/lib/esearchy/{googlegroups.rb → OtherEngines/googlegroups.rb}

@@ -1,6 +1,6 @@
 %w{rubygems cgi net/http}.each { |lib| require lib }
-local_path = "#{File.dirname(__FILE__)}/"
-%w{searchy keys useragent}.each {|lib| require local_path + lib}
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
 
 class GoogleGroups
   include Searchy
@@ -13,6 +13,7 @@ class GoogleGroups
     @r_pdfs = Queue.new
     @r_txts = Queue.new
     @r_officexs = Queue.new
+    @emails = []
     @lock = Mutex.new
     @threads = []
   end
@@ -26,19 +27,19 @@ class GoogleGroups
         request = Net::HTTP::Get.new( "/groups/search?&safe=off&num=100" + 
                                        "&q=" + CGI.escape(query) + 
                                        "&btnG=Search&start=#{@start}", 
-                                       {'Cookie' => UserAgent::fetch})
+                                       {'User-Agent' => UserAgent::fetch})
         response = http.request(request)
         case response
         when Net::HTTPSuccess, Net::HTTPRedirection
           parse(response.body)
           @start = @start + 100
           if @totalhits > @start
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
             search_emails(response.body)
             sleep(4)
             search(query)
           else
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
             search_emails(response.body)
           end
         else
@@ -48,6 +49,7 @@ class GoogleGroups
     rescue Net::HTTPFatalError
       ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
     end
+    @start = 0
   end
   
   def parse(html)

data/lib/esearchy/{pgp.rb → OtherEngines/pgp.rb}

@@ -1,5 +1,5 @@
 %w{cgi net/http}.each { |lib| require lib }
-local_path = "#{File.dirname(__FILE__)}/"
+local_path = "#{File.dirname(__FILE__)}/../"
 %w{searchy useragent}.each {|lib| require local_path + lib}
 
 class PGP
@@ -10,6 +10,7 @@ class PGP
     @emails = []
     @lock = Mutex.new
   end
+  attr_accessor :emails
   
   def search(query)
     @query = query
@@ -17,7 +18,7 @@ class PGP
     begin
       http.start do |http|
         request = Net::HTTP::Get.new( "/pks/lookup?search=#{@query}",
-                                      {'Cookie' => UserAgent::fetch})
+                                      {'User-Agent' => UserAgent::fetch})
         response = http.request(request)
         case response
         when Net::HTTPSuccess, Net::HTTPRedirection
@@ -31,13 +32,4 @@ class PGP
       ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
     end
   end
-  
-  def emails
-    @totalhits == 0 ? emails : emails[0..@totalhits]
-  end
-  
-  def emails=(value)
-    emails = value
-  end
-  
 end

data/lib/esearchy/OtherEngines/usenet.rb

@@ -0,0 +1,35 @@
+%w{cgi net/http}.each { |lib| require lib }
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
+
+class Usenet
+  include Searchy
+  
+  def initialize(maxhits=0)
+    @totalhits = maxhits
+    @emails = []
+    @lock = Mutex.new
+  end
+  attr_accessor :emails
+  
+  def search(query)
+    @query = query
+    http = Net::HTTP.new("usenet-addresses.mit.edu",80)
+    begin
+      http.start do |http|
+        request = Net::HTTP::Get.new( "/cgi-bin/udb?T=#{@query}&G=&S=&N=&O=&M=500",
+                                      {'User-Agent' => UserAgent::fetch})
+        response = http.request(request)
+        case response
+        when Net::HTTPSuccess, Net::HTTPRedirection
+          ESearchy::LOG.puts "Searching #{self.class}"
+          search_emails(response.body)
+        else
+          return response.error!
+        end
+      end
+    rescue Net::HTTPFatalError
+      ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
+    end
+  end
+end

data/lib/esearchy/SearchEngines/altavista.rb

@@ -0,0 +1,71 @@
+%w{rubygems cgi net/http}.each { |lib| require lib }
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
+
+class Altavista
+  include Searchy
+  
+  def initialize(maxhits = 0, start = 0)
+    @start = start
+    @totalhits = maxhits
+    @emails = []
+    @r_urls = Queue.new
+    @r_docs = Queue.new
+    @r_pdfs = Queue.new
+    @r_txts = Queue.new
+    @r_officexs = Queue.new
+    @lock = Mutex.new
+    @threads = []
+  end
+  attr_accessor :emails
+  
+  def search(query)
+    @query = query
+    http = Net::HTTP.new("www.altavista.com",80)
+    begin
+      http.start do |http|
+        request = Net::HTTP::Get.new( "/web/results?itag=ody&kgs=0&kls=0&nbq=50" + 
+                                       "&q=" + CGI.escape(query) + 
+                                       "&stq=#{@start}", 
+                                       {'User-Agent' => UserAgent::fetch})
+        response = http.request(request)
+        case response
+        when Net::HTTPSuccess, Net::HTTPRedirection
+          parse(response.body)
+          @start = @start + 100
+          if @totalhits > @start
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}"
+            search_emails(response.body.gsub(/<b>|<\/b>/,""))
+            sleep(4)
+            search(query)
+          else
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}"
+            search_emails(response.body.gsub(/<b>|<\/b>/,""))
+          end
+        else
+          return response.error!
+        end
+      end
+    rescue Net::HTTPFatalError
+      ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
+    end
+  end
+  
+  def parse(html)
+    @totalhits= html.scan(/AltaVista found (.*) results<\/A>/)[0][0].gsub(',','').to_i if @totalhits == 0
+    html.scan(/<a class='res' href='([a-zA-Z0-9:\/\/.&?%=\-_+]*)'>/).each do |result|
+      case result[0]
+      when /.pdf$/i
+        @r_pdfs << CGI.unescape(result[0])
+      when /.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$/i
+        @r_officexs << CGI.unescape(result[0])
+      when /.doc$/i
+        @r_docs << CGI.unescape(result[0])
+      when /.txt$|.rtf$|ans$/i
+        @r_txts << CGI.unescape(result[0])
+      else
+        @r_urls << CGI.unescape(result[0])
+      end
+    end
+  end
+end

data/lib/esearchy/{bing.rb → SearchEngines/bing.rb}

@@ -1,6 +1,6 @@
 %w{rubygems json cgi net/http}.each { |lib| require lib }
-local_path = "#{File.dirname(__FILE__)}/"
-%w{searchy keys useragent}.each {|lib| require local_path + lib}
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
 
 class Bing
   include Searchy
@@ -28,19 +28,19 @@ class Bing
         request = Net::HTTP::Get.new("/json.aspx" + "?Appid="+ @appid + 
                                     "&query=" + CGI.escape(query) + 
                                     "&sources=web&web.count=50&web.offset=#{@start}",
-                                    {'Cookie' => UserAgent::fetch})
+                                    {'User-Agent' => UserAgent::fetch})
         response = http.request(request)
         case response
         when Net::HTTPSuccess, Net::HTTPRedirection
           parse(response.body)
           @start = @start + 50
           if @totalhits > @start
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}"
             search_emails(response.body)
             sleep(4)
             search(query)
           else
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}"
             search_emails(response.body)
           end
         else

data/lib/esearchy/{google.rb → SearchEngines/google.rb}

@@ -1,6 +1,6 @@
 %w{rubygems cgi net/http}.each { |lib| require lib }
-local_path = "#{File.dirname(__FILE__)}/"
-%w{searchy keys useragent}.each {|lib| require local_path + lib}
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
 
 class Google
   include Searchy
@@ -27,20 +27,20 @@ class Google
         request = Net::HTTP::Get.new( "/cse?&safe=off&num=100&site=" + 
                                        "&q=" + CGI.escape(query) + 
                                        "&btnG=Search&start=#{@start}", 
-                                       {'Cookie' => UserAgent::fetch})
+                                       {'User-Agent' => UserAgent::fetch})
         response = http.request(request)
         case response
         when Net::HTTPSuccess, Net::HTTPRedirection
           parse(response.body)
           @start = @start + 100
           if @totalhits > @start
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
-            search_emails(response.body)
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
+            search_emails(response.body.gsub(/<em>|<\/em>/,""))
             sleep(4)
             search(query)
           else
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
-            search_emails(response.body)
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
+            search_emails(response.body.gsub(/<em>|<\/em>/,""))
           end
         else
           return response.error!

data/lib/esearchy/{yahoo.rb → SearchEngines/yahoo.rb}

@@ -1,6 +1,6 @@
 %w{rubygems json cgi net/http}.each { |lib| require lib }
-local_path = "#{File.dirname(__FILE__)}/"
-%w{searchy keys useragent}.each {|lib| require local_path + lib}
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path  + lib}
 
 class Yahoo
   include Searchy
@@ -28,19 +28,19 @@ class Yahoo
         request = Net::HTTP::Get.new("/ysearch/web/v1/" + CGI.escape(query) + 
                                      "?appid="+ @appid + 
                                      "&format=json&count=50"+ 
-                                     "&start=#{@start}", {'Cookie' => UserAgent::fetch} )
+                                     "&start=#{@start}", {'User-Agent' => UserAgent::fetch} )
         response = http.request(request)
         case response
         when Net::HTTPSuccess, Net::HTTPRedirection
           parse(response.body)
           @start = @start + 50
           if @totalhits > @start
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}"
             search_emails(response.body)
             sleep(4)
             search(@query)
           else
-            ESearchy::LOG.puts "Searching in URL: #{self.class} up to point #{@start}"
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-50} to #{@start}"
             search_emails(response.body)
           end
         else
@@ -50,6 +50,7 @@ class Yahoo
     rescue Net::HTTPFatalError
       ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
     end
+    @start = 0
   end
 
   def parse(json)

data/lib/esearchy/SocialNetworks/googleprofiles.rb

@@ -0,0 +1,96 @@
+#
+# Big Thanks go to DigiNinja at digininja.org for telling me about this Google Hack.
+#
+%w{rubygems cgi net/http}.each { |lib| require lib }
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
+
+class GoogleProfiles
+  include Searchy
+  
+  def initialize(maxhits = 0, start = 0)
+    @start = start
+    @totalhits = maxhits
+    @emails = []
+    @people = []
+    @company_name = nil
+    @r_urls = Queue.new
+    @r_docs = Queue.new
+    @r_pdfs = Queue.new
+    @r_txts = Queue.new
+    @r_officexs = Queue.new
+    @lock = Mutex.new
+    @threads = []
+  end
+  attr_accessor :emails, :company_name, :people
+  
+  def search(query)
+    @query = query
+    http = Net::HTTP.new("www.google.com",80)
+    begin
+      http.start do |http|
+        request = Net::HTTP::Get.new( "/cse?q=site:www.google.com+intitle:%22Google+" +
+                                      "Profile%22+%22Companies+I%27ve+worked+for%22+%22at+" + 
+                                      CGI.escape(@company_name) + 
+                                      "%22&hl=en&cof=&num=100&filter=0" +   
+                                      "&safe=off&start=#{@start}",
+                                       {'User-Agent' => UserAgent::fetch})
+        response = http.request(request)
+        case response
+        when Net::HTTPSuccess, Net::HTTPRedirection
+          parse(response.body)
+          @start = @start + 100
+          if @totalhits > @start
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
+            parse(response.body)
+            search_emails(response.body.gsub(/<em>|<\/em>/,""))
+            sleep(4)
+            search(query)
+          else
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
+            parse(response.body)
+            search_emails(response.body.gsub(/<em>|<\/em>/,""))
+          end
+        else
+          return response.error!
+        end
+      end
+    rescue Net::HTTPFatalError
+      ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
+    end
+    @start = 0
+  end
+  
+  def search_person(name,last)
+    email = []
+    # Search Yahoo
+    y = Yahoo.new(50)
+    y.search("first:\"#{name}\" last:\"#{last}\"")
+    emails.concat(y.emails).uniq!
+    # Search Google
+    #g = Google.new(50)
+    #g.search("#{name} #{last}")
+    #emails.concat(g.emails).uniq!
+    return emails
+  end
+  
+  def enquire_person(profile)
+    # TO DO: parse profile to obtain more information
+  end
+  
+  def parse(html)
+    @totalhits= html.scan(/<\/b> of[ about | ]\
+<b>(.*)<\/b> from/)[0][0].gsub(",","").to_i if @totalhits == 0
+    html.scan(/<h2 class=r><a href="([0-9A-Z\
+a-z:\\\/?&=@+%.;"'()_-]+)" class=l>([\w\s]*) -/).each do |profile|
+      @domain = @query.match(/@/) ? @query : ("@" + @query)
+      name,last = profile[1].split(" ") 
+      @people << [name,last]
+      @emails << "#{name.split(' ')[0]}.#{last.split(' ')[0]}#{@domain}"
+      @emails << "#{name[0,1]}#{last.split(' ')[0]}#{@domain}"
+      #@emails.concat(fix(search_person(name,last)))
+      @emails.uniq!
+      print_emails(@emails)
+    end
+  end
+end

data/lib/esearchy/{linkedin.rb → SocialNetworks/linkedin.rb}

@@ -1,6 +1,6 @@
 %w{rubygems cgi net/http net/https}.each { |lib| require lib }
-local_path = "#{File.dirname(__FILE__)}/"
-%w{searchy yahoo google useragent}.each {|lib| require local_path + lib}
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
 
 class LinkedIn
   include Searchy

data/lib/esearchy/SocialNetworks/naymz.rb

@@ -0,0 +1,96 @@
+# http://www.google.com/cse?q=site:naymz.com%20%2B%20%22@%20Boeing%22
+%w{rubygems cgi net/http}.each { |lib| require lib }
+local_path = "#{File.dirname(__FILE__)}/../"
+%w{searchy useragent}.each {|lib| require local_path + lib}
+
+class Naymz
+  include Searchy
+  
+  def initialize(maxhits = 0, start = 0)
+    @start = start
+    @totalhits = maxhits
+    @emails = []
+    @people = []
+    @company_name = nil
+    @r_urls = Queue.new
+    @r_docs = Queue.new
+    @r_pdfs = Queue.new
+    @r_txts = Queue.new
+    @r_officexs = Queue.new
+    @lock = Mutex.new
+    @threads = []
+  end
+  attr_accessor :emails, :company_name, :people
+  
+  def search(query)
+    @query = query
+    http = Net::HTTP.new("www.google.com",80)
+    begin
+      http.start do |http|
+        request = Net::HTTP::Get.new( "/cse?q=site:naymz.com%20%2B%20%22@%20" + 
+                                      CGI.escape(@company_name) + 
+                                      "%22&hl=en&cof=&num=100&filter=0" +   
+                                      "&safe=off&start=#{@start}",
+                                       {'User-Agent' => UserAgent::fetch})
+        response = http.request(request)
+        case response
+        when Net::HTTPSuccess, Net::HTTPRedirection
+          parse(response.body)
+          @start = @start + 100
+          if @totalhits > @start
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
+            parse(response.body)
+            search_emails(response.body.gsub(/<em>|<\/em>/,""))
+            sleep(4)
+            search(query)
+          else
+            ESearchy::LOG.puts "Searching #{self.class} from #{@start-100} to #{@start}"
+            parse(response.body)
+            search_emails(response.body.gsub(/<em>|<\/em>/,""))
+          end
+        else
+          return response.error!
+        end
+      end
+    rescue Net::HTTPFatalError
+      ESearchy::LOG.puts "Error: Something went wrong with the HTTP request"
+    end
+    @start = 0
+  end
+  
+  def search_person(name,last)
+    email = []
+    # Search Yahoo
+    y = Yahoo.new(50)
+    y.search("first:\"#{name}\" last:\"#{last}\"")
+    emails.concat(y.emails).uniq!
+    # Search Google
+    #g = Google.new(50)
+    #g.search("#{name} #{last}")
+    #emails.concat(g.emails).uniq!
+    return emails
+  end
+  
+  def enquire_person(profile)
+    # TO DO: parse profile to obtain more information
+  end
+  
+  def parse(html)
+    @totalhits= html.scan(/<\/b> of[ about | ]\
+<b>(.*)<\/b> from/)[0][0].gsub(",","").to_i if @totalhits == 0
+    html.scan(/<h2 class=r><a href="([0-9A-Z\
+a-z:\\\/?&=@+%.;"'()_-]+)" class=l>([\w\s]*) -/).each do |profile|
+      @domain = @query.match(/@/) ? @query : ("@" + @query)
+      person = profile[1].split(" ").delete_if do 
+        |x| x =~ /mr.|mr|ms.|ms|phd.|dr.|dr|phd|phd./i
+      end
+      name,last = person.size > 2 ? [person[0],person[-1]] : person
+      @people << person
+      @emails << "#{name.split(' ')[0]}.#{last.split(' ')[0]}#{@domain}"
+      @emails << "#{name[0,1]}#{last.split(' ')[0]}#{@domain}"
+      #@emails.concat(fix(search_person(name,last)))
+      @emails.uniq!
+      print_emails(@emails)
+    end
+  end
+end

data/lib/esearchy/searchy.rb

@@ -10,32 +10,15 @@ if RUBY_PLATFORM =~ /mingw|mswin/
 end
 
 module Searchy
-  case RUBY_PLATFORM 
-  when /mingw|mswin/
-    TEMP = "C:\\WINDOWS\\Temp\\"
-  else
-    TEMP = "/tmp/"
-  end    
   
   def search_emails(string)
-    string = string.gsub("<em>","") if self.class == Google
-    # OLD regex list = string.scan(/[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*@\
-    #           (?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/)
-#    list = string.scan(/[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*_at_\
-#(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\
-#[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]\
-#*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|[a-z0-9!#$&'*+=?^_`{|}~-]+\
-#(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\
-#[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*\s@\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+\
-#[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\sdot\s[a-z0-9!#$&'*+=?\^_`\
-#{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\sdot\s)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/)
-    list = string.scan(/[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*_at_\
+    list = string.scan(/[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*_at_\
 (?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
-[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]\
+[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]\
 *[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+\
-(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
-[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?\^_`{|}~-]+)*\s@\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+\
-[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\sdot\s[a-z0-9!#$&'*+=?\^_`\
+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
+[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*\s@\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+\
+[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\sdot\s[a-z0-9!#$&'*+=?^_`\
 {|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\sdot\s)+[a-z](?:[a-z-]*[a-z])??/)
     @lock.synchronize do
       print_emails(list)
@@ -55,7 +38,7 @@ module Searchy
             response = http.request(request)
             case response
             when Net::HTTPSuccess, Net::HTTPRedirection
-              name = Searchy::TEMP + "#{hash_url(web.to_s)}.pdf"
+              name = ESearchy::TEMP + "#{hash_url(web.to_s)}.pdf"
               open(name, "wb") do |file|
                 file.write(response.body)
               end

data/lib/esearchy/useragent.rb

@@ -3,11 +3,6 @@ class UserAgent
     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Acoo Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Acoo Browser; InfoPath.2; .NET CLR 2.0.50727; Alexa Toolbar)",
-    "amaya/9.52 libwww/5.4.0",
-    "amaya/11.1 libwww/5.4.0",
-    "Amiga-AWeb/3.5.07 beta",
-    "AmigaVoyager/3.4.4 (MorphOS/PPC native)",
-    "AmigaVoyager/2.95 (compatible; MC680x0; AmigaOS)",
     "Mozilla/4.0 (compatible; MSIE 7.0; AOL 7.0; Windows NT 5.1; FunWebProducts)",
     "Mozilla/4.0 (compatible; MSIE 6.0; AOL 8.0; Windows NT 5.1; SV1)",
     "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.0; Windows NT 5.1; .NET CLR 1.1.4322; Zango 10.1.181.0)",
@@ -36,11 +31,6 @@ class UserAgent
     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Crazy Browser 1.0.5; .NET CLR 1.1.4322; InfoPath.1)",
     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer 1.5.0; .NET CLR 1.0.3705)",
     "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Demeter/1.0.9 Safari/125",
-    "Dillo/0.8.5",
-    "Dillo/2.0",
-    "Doris/1.15 [en] (Symbian)",
-    "ELinks/0.13.GIT (textmode; Linux 2.6.22-2-686 i686; 148x68-3)",
-    "ELinks/0.9.3 (textmode; Linux 2.6.11 i686; 79x24)",
     "Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.1.12) Gecko/20080208 (Debian-1.8.1.12-2) Epiphany/2.20",
     "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.3) Gecko/20041007 Epiphany/1.4.7",
     "Mozilla/5.0 (Windows; U; Win95; en-US; rv:1.5) Gecko/20031007 Firebird/0.7",
@@ -67,12 +57,10 @@ class UserAgent
     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; GreenBrowser)",
     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; GreenBrowser)",
     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; GreenBrowser)",
-    "HotJava/1.1.2 FCS",
     "Mozilla/3.0 (x86 [cs] Windows NT 5.1; Sun)",
     "Mozilla/5.1 (X11; U; Linux i686; en-US; rv:1.8.0.3) Gecko/20060425 SUSE/1.5.0.3-7 Hv3/alpha",
     "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SIMBAR={CFBFDAEA-F21E-4D6E-A9B0-E100A69B860F}; Hydra Browser; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30)",
     "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Hydra Browser; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
-    "IBrowse/2.3 (AmigaOS 3.9)",
     "Mozilla/5.0 (compatible; IBrowse 3.0; AmigaOS4.0)",
     "Mozilla/4.5 (compatible; iCab 2.9.1; Macintosh; U; PPC)",
     "iCab/3.0.2 (Macintosh; U; PPC Mac OS X)",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: FreedomCoder-esearchy
 version: !ruby/object:Gem::Version 
-  version: 0.0.19
+  version: "0.1"
 platform: ruby
 authors: 
 - Matias P. Brutti
@@ -33,14 +33,14 @@ dependencies:
         version: 1.1.6
     version: 
 - !ruby/object:Gem::Dependency 
-  name: rubyzip
+  name: FreedomCoder-rubyzip
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.9.1
+        version: 0.9.3
     version: 
 description: 
 email: matiasbrutti@gmail.com
@@ -51,28 +51,35 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
-- esearchy.rb
 - bin
 - bin/esearchy
 - data
 - data/bing.key
 - data/yahoo.key
 - lib
-- lib/esearchy.rb
+- esearchy.rb
 - lib/esearchy
-- lib/esearchy/bing.rb
-- lib/esearchy/google.rb
-- lib/esearchy/googlegroups.rb
+- lib/esearchy.rb
+- lib/esearchy/bugmenot.rb
 - lib/esearchy/keys.rb
-- lib/esearchy/linkedin.rb
+- lib/esearchy/logger.rb
+- lib/esearchy/OtherEngines
+- lib/esearchy/OtherEngines/pgp.rb
+- lib/esearchy/OtherEngines/googlegroups.rb
+- lib/esearchy/OtherEngines/usenet.rb
 - lib/esearchy/pdf2txt.rb
-- lib/esearchy/pgp.rb
+- lib/esearchy/SearchEngines/
+- lib/esearchy/SearchEngines/bing.rb
+- lib/esearchy/SearchEngines/google.rb
+- lib/esearchy/SearchEngines/yahoo.rb
+- lib/esearchy/SearchEngines/altavista.rb
 - lib/esearchy/searchy.rb
-- lib/esearchy/yahoo.rb
+- lib/esearchy/SocialNetworks/
+- lib/esearchy/SocialNetworks/googleprofiles.rb
+- lib/esearchy/SocialNetworks/linkedin.rb
+- lib/esearchy/SocialNetworks/naymz.rb
 - lib/esearchy/wcol.rb
 - lib/esearchy/useragent.rb
-- lib/esearchy/logger.rb
-- lib/esearchy/bugmenot.rb
 - README.rdoc
 has_rdoc: true
 homepage: http://freedomcoder.com.ar/esearchy