esearchy 0.2.0.7 → 0.2.0.8

data/bin/esearchy

@@ -31,11 +31,11 @@ require 'getoptlong'
 require 'esearchy'
 
 ESearchy::log = true
-
 @yahoo_key  = nil
 @bing_key   = nil
 @maxhits    = nil
 @docs       = true
+@profiling  = false
 @params     = {}
 @list       = []
 @output     = nil
@@ -64,6 +64,7 @@ opts = GetoptLong.new(
   [ '--enable-pgp', GetoptLong::NO_ARGUMENT ],
   [ '--enable-usenet', GetoptLong::NO_ARGUMENT ],
   [ '--enable-spider', GetoptLong::NO_ARGUMENT ],
+  [ '--profiling', GetoptLong::NO_ARGUMENT ],
   [ '--disable-google', GetoptLong::NO_ARGUMENT ],
   [ '--disable-yahoo', GetoptLong::NO_ARGUMENT ],
   [ '--disable-bing', GetoptLong::NO_ARGUMENT ],
@@ -148,7 +149,7 @@ def print_(list)
           print_linux(item)
         end
       when Array
-        puts item[0].join " " + "-->" + item[1]
+        puts item.join " "
       end
     end
   end
@@ -210,6 +211,10 @@ class Output
     end
   end
   
+  def save_html(data)
+    
+  end
+  
   def save_sqlite(data)
     require 'sqlite3'
     @db = SQLite3::Database.new(@output)
@@ -257,10 +262,22 @@ def execute(p)
         end
       end
     end
+    
+    if @profiling
+      puts "Gathering Profile data from people"
+      puts "----------------------------------\n"
+      res = ESearchy::Profiling.new(search.people.uniq)
+      res.search
+      search.people.concat(res.people)
+      search.results.concat(res.results)
+    end
+    
     @output.save(search.results) if @output
-    puts "-------FINAL RESULTS--------"
+    puts "\n-------==< FINAL RESULTS >==--------"
     print_ search.emails.uniq
-    print_ search.people.uniq
+    search.people.uniq.each do |person, profile|
+      puts person.join(" ") + " -> " + profile
+    end
   end
 end
 
@@ -326,6 +343,8 @@ opts.each do |opt, arg|
       puts "\t Enables PGP searches.\n"
       puts "--enable-usenet"
       puts "\t Enables Usenet searches.\n"
+      puts "--profiling"
+      puts "\t Enables People's profiling.\n"
       puts "--disable-docs"
       puts "\t Disables searches inside docs.\n"
       puts "--disable-google"
@@ -358,11 +377,12 @@ opts.each do |opt, arg|
       #END OF HELP
       exit(0)
     when '--enable-all' then
-      @people_engines = [:LinkedIn, :Naymz, :Classmates, :GoogleProfiles, 
+      @people_engines = [:LinkedIn, :Naymz, :Classmates, :GoogleProfiles,
                          :Spoke, :JigSaw, :Ziggs, :Plaxo]
       @email_engines = [:Google, :Bing, :Yahoo, :Altavista, :PGP, :Spider ,:Usenet, :GoogleGroups ]
     when '--enable-people' then
-      @people_engines = [:LinkedIn, :Naymz, :Classmates, :GoogleProfiles, :Spoke]
+      @people_engines = [:LinkedIn, :Naymz, :Classmates, :GoogleProfiles, 
+                         :Spoke, :JigSaw, :Ziggs, :Plaxo]
     when '--enable-emails' then
       @email_engines = [:Google, :Bing, :Yahoo, :Altavista, :PGP, :Spider ,:Usenet, :GoogleGroups ]
     when '--enable-google' then
@@ -398,43 +418,45 @@ opts.each do |opt, arg|
     when '--enable-usenet' then
       @email_engines << :Usenet
     when '--disable-people' then
-       @people_engines = []
-     when '--disable-emails' then
-       @email_engines = []
-     when '--disable-google' then
-       @email_engines.delete(:Google) 
-     when '--disable-yahoo' then
-       @email_engines.delete(:Yahoo)
-     when '--disable-bing' then
-       @email_engines.delete(:Bing)
-     when '--disable-altavisa' then
-       @email_engines.delete(:Altavista)
-     when '--disable-spider' then
-       @email_engines.delete(:Spider)
-     when '--disable-linkedin' then
-       @people_engines.delete(:LinkedIn)
-     when '--disable-gprofiles' then
-       @people_engines.delete(:GoogleProfiles)
-     when '--disable-naymz' then
-       @people_engines.delete(:Naymz)
-     when '--disable-classmates' then
-       @email_engines.delete(:Classmates)
-     when '--disable-ggroups' then
-       @email_engines.delete(:GoogleGroups)
-     when '--disable-spoke' then
-       @people_engines.delete(:Spoke)
-     when '--disable-jigsaw' then
-       @people_engines.delete(:JigSaw)
-     when '--disable-ziggs' then
-       @people_engines.delete(:Ziggs)
-     when '--disable-plaxo' then
-       @people_engines.delete(:Plaxo)
-     when '--disable-pgp' then
-       @email_engines.delete(:PGP)
-     when '--disable-usenet' then
-       @email_engines.delete(:Usenet)
-     when '--disable-docs' then
-       @docs = false
+      @people_engines = []
+    when '--disable-emails' then
+      @email_engines = []
+    when '--disable-google' then
+      @email_engines.delete(:Google) 
+    when '--disable-yahoo' then
+      @email_engines.delete(:Yahoo)
+    when '--disable-bing' then
+      @email_engines.delete(:Bing)
+    when '--disable-altavisa' then
+      @email_engines.delete(:Altavista)
+    when '--disable-spider' then
+      @email_engines.delete(:Spider)
+    when '--disable-linkedin' then
+      @people_engines.delete(:LinkedIn)
+    when '--disable-gprofiles' then
+      @people_engines.delete(:GoogleProfiles)
+    when '--disable-naymz' then
+      @people_engines.delete(:Naymz)
+    when '--disable-classmates' then
+      @email_engines.delete(:Classmates)
+    when '--disable-ggroups' then
+      @email_engines.delete(:GoogleGroups)
+    when '--disable-spoke' then
+      @people_engines.delete(:Spoke)
+    when '--disable-jigsaw' then
+      @people_engines.delete(:JigSaw)
+    when '--disable-ziggs' then
+      @people_engines.delete(:Ziggs)
+    when '--disable-plaxo' then
+      @people_engines.delete(:Plaxo)
+    when '--disable-pgp' then
+      @email_engines.delete(:PGP)
+    when '--disable-usenet' then
+      @email_engines.delete(:Usenet)
+    when '--disable-docs' then
+      @docs = false
+    when '--profiling' then
+      @profiling = true
     when '--query' then
       @params[:query] = arg
     when '--company' then

data/lib/esearchy.rb

@@ -5,6 +5,7 @@ require 'cgi'
 require 'json'
 require 'digest/sha2'
 require 'zip/zip'
+require 'uri'
 require 'zip/zipfilesystem'
 require 'pdf/reader'
 if RUBY_PLATFORM =~ /mingw|mswin/
@@ -20,5 +21,6 @@ require 'esearchy/socialengines'
 require 'esearchy/localengines'
 require 'esearchy/bugmenot'
 require 'esearchy/docs'
+require 'esearchy/profiling'
 require 'esearchy/useragent'
 require 'esearchy/esearchy'

data/lib/esearchy/SocialEngines/classmates.rb

@@ -23,10 +23,10 @@ module ESearchy
       
       def crawl_people(html)
         html.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) \|/).each do |profile|
-          profile = profile[0].to_s
-          person = profile[1].split(" ") 
-          @people << [ p,  profile ]
-          @results << [person, "P", profile,  self.class.to_s.upcase, "N"]
+          pf = profile[0].to_s
+          p = profile[1].split(" ") 
+          @people << [ p, pf ]
+          @results << [p, "P", pf,  self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/googleprofiles.rb

@@ -26,10 +26,10 @@ module ESearchy
       
       def crawl_people(text)
         text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
-          profile = profile[0].to_s
-          person = profile[1].split(" ") 
-          @people << [ p,  profile ]
-          @results << [person, "P",profile, self.class.to_s.upcase, "N"]
+          pf = profile[0].to_s
+          p = profile[1].split(" ") 
+          @people << [ p, pf ]
+          @results << [p, "P",profile, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/jigsaw.rb

@@ -26,10 +26,10 @@ module ESearchy
       
       def crawl_people(text)
         text.scan(/<a href="(http\:\/\/www.jigsaw.com\/scid[0-9A-Za-z\/?&=@+%.;'_-]+\.xhtml)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*),/).each do |profile|
-          profile = profile[0].to_s
+          pf = profile[0].to_s
           p = profile[1].split(" ")
-          @people << [ p,  profile ]
-          @results << [p, "P", profile, self.class.to_s.upcase, "N"]
+          @people << [ p, pf ]
+          @results << [p, "P", pf, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/linkedin.rb

@@ -25,10 +25,10 @@ module ESearchy
       
       def crawl_people(text)
         text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
-          profile = profile[0].to_s
+          pf = profile[0].to_s
           p = profile[1].split(" ")
-          @people << [ p,  profile ]
-          @results << [p, "P", profile, self.class.to_s.upcase, "N"]
+          @people << [ p, pf ]
+          @results << [p, "P", pf, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/naymz.rb

@@ -24,12 +24,12 @@ module ESearchy
       
       def crawl_people(html)
         html.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
-          profile = profile[0].to_s
-          person = profile[1].split(" ").delete_if do 
+          pf = profile[0].to_s
+          p = profile[1].split(" ").delete_if do 
             |x| x =~ /mr.|mr|ms.|ms|phd.|dr.|dr|phd|phd./i
           end
-          @people << [ p,  profile ]
-          @results << [person, "P", profile, self.class.to_s.upcase, "N"]
+          @people << [ p, pf ]
+          @results << [p, "P", pf, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/plaxo.rb

@@ -26,10 +26,10 @@ module ESearchy
       
       def crawl_people(text)
         text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*)&#39;/).each do |profile|
-          profile = profile[0].to_s
+          pf = profile[0].to_s
           p = profile[1].split(" ")
-          @people << [ p,  profile ]
-          @results << [p, "P", profile, self.class.to_s.upcase, "N"]
+          @people << [ p, pf ]
+          @results << [p, "P", pf, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/spoke.rb

@@ -22,13 +22,13 @@ module ESearchy
           @totalhits = totalhits(hits[0][0].gsub(",","").to_i)
         end
       end
-      
+            
       def crawl_people(text)
         text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*),/).each do |profile|
-          profile = profile[0].to_s
+          pf = profile[0].to_s
           p = profile[1].split(" ")
-          @people << [ p,  profile ]
-          @results << [p, "P", profile, self.class.to_s.upcase, "N"]
+          @people << [ p, pf ]
+          @results << [p, "P", pf, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/SocialEngines/ziggs.rb

@@ -26,10 +26,10 @@ module ESearchy
       
       def crawl_people(text)
         text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
-          profile = profile[0].to_s
+          pf = profile[0].to_s
           p = profile[1].split(" ")
-          @people << [ p,  profile ]
-          @results << [p, "P", profile, self.class.to_s.upcase, "N"]
+          @people << [ p, pf ]
+          @results << [p, "P", pf, self.class.to_s.upcase, "N"]
         end
       end
     end

data/lib/esearchy/esearchy.rb

@@ -3,7 +3,7 @@ def D m
 end
 
 module ESearchy
-  VERSION = "0.2.0.7"
+  VERSION = "0.2.0.8"
   @@log = false
   
   def self.log

data/lib/esearchy/genericengine.rb

@@ -64,7 +64,7 @@ module ESearchy
         D "Error: Something went wrong :("
       end
     end
-    
+        
     def header
       begin
         return self.class::HEADER

data/lib/esearchy/profiling.rb

@@ -0,0 +1,87 @@
+module ESearchy  
+  class Profiling
+    def initialize(people)
+      @peo = people.clone
+      @people = []
+      @results = []
+    end
+    attr_accessor :people, :results
+    
+    def search
+      @peo.each { |person, profile| crawl(person, profile) }
+    end
+    
+    private
+    def get_profile(uri_str, limit = 10)
+      begin
+        # You should choose better exception.
+        raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+        response = Net::HTTP.get_response(URI.parse(uri_str))
+        case response
+        when Net::HTTPSuccess     then response.body
+        when Net::HTTPRedirection then get_profile(response['location'], limit - 1)
+        else
+          response.error!
+        end
+      rescue
+        return nil
+      end
+    end
+    
+    def crawl(person, profile)
+      text = get_profile(profile)
+      if text
+        case profile
+        when /spoke.com/ then
+          D "Crawling #{person.join(" ")}'s profile for co-workers:" 
+          cw = text.scan(/<a class="personLinkTag" href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)">([\w\s]*)<\/a>/)
+          add_persons(cw, person, "http://www.spoke.com")
+        when /classmate.com/ then
+          return nil
+        when /google.com/ then
+          D "Crawling #{person.join(" ")}'s Google profile for other Social Networks"
+          text.scan(/<div class="link"><a class="url" href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" rel="me">([\w\s]*)<\/a>/).each do |prof|
+            url = prof[0]
+            network = prof[1]
+            D "\t-#{network.strip} : #{url.strip}"
+          end
+        when /jigsaw.com/ then
+          D "Crawling #{person.join(" ")}'s JigSaw profile for co-workers:"
+          cw = text.scan(/<li><p style="margin-top: 15px"><a href='([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)'>([\w\s]*)<\/a>/)
+          add_persons(cw, person, "http://www.jigsaw.com")
+        when /linkedin.com/ then
+          return nil
+        when /naymz.com/ then
+          D "Crawling #{person.join(" ")}'s Google profile for other Social Networks"
+          text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" rel="external">[\n\s]*([\w\s]*)\n/).each do |prof|
+            url = prof[0]
+            network = prof[1]
+            D "\t-#{network.strip} : #{url.strip} "
+          end
+        when /plaxo.com/ then
+          D "Crawling #{person.join(" ")}'s Plaxo profile for other Social Networks:"
+          text.scan(/rel="me nofollow" title="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)">([\w\s]*)<\/a><\/div><\/td>/).each do |prof|
+            url = prof[0]
+            network = url.scan(/:\/\/(.*)\./)[0][0]
+            username = prof[1]
+            D "\t-#{network} : #{username} : #{url} "
+          end
+        when /ziggs.com/ then
+          return nil
+        end
+      else
+        D "Something went wrong Crawling #{person.join(" ")}'s Profile\n"
+      end
+    end
+    
+    def add_persons(cowork, per,  url)
+      cowork.uniq.each do |profile|
+        pf = url + profile[0].to_s
+        p = profile[1].split(" ")
+        D "\t-#{p.join(" ")} -> #{pf}"
+        @people << [ p, pf ]
+        @results << [p, "P", pf, per.to_s.upcase, "N"]
+      end
+    end
+  end
+end

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: esearchy
 version: !ruby/object:Gem::Version 
+  hash: 79
   prerelease: false
   segments: 
   - 0
   - 2
   - 0
-  - 7
-  version: 0.2.0.7
+  - 8
+  version: 0.2.0.8
 platform: ruby
 authors: 
 - Matias P. Brutti
@@ -15,16 +16,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-22 00:00:00 -03:00
+date: 2010-05-17 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: pdf-reader
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 9
         segments: 
         - 0
         - 7
@@ -36,9 +39,11 @@ dependencies:
   name: json
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 1
         segments: 
         - 1
         - 1
@@ -50,9 +55,11 @@ dependencies:
   name: FreedomCoder-rubyzip
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 61
         segments: 
         - 0
         - 9
@@ -64,9 +71,11 @@ dependencies:
   name: spidr
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 21
         segments: 
         - 0
         - 2
@@ -95,6 +104,7 @@ files:
 - lib/esearchy/OtherEngines/spider.rb
 - lib/esearchy/OtherEngines/usenet.rb
 - lib/esearchy/otherengines.rb
+- lib/esearchy/profiling.rb
 - lib/esearchy/SearchEngines/altavista.rb
 - lib/esearchy/SearchEngines/bing.rb
 - lib/esearchy/SearchEngines/google.rb
@@ -112,6 +122,7 @@ files:
 - lib/esearchy/useragent.rb
 - lib/esearchy.rb
 - README.rdoc
+- bin/esearchy
 has_rdoc: true
 homepage: http://freedomcoder.com.ar/esearchy
 licenses: []
@@ -122,23 +133,27 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A library to search for emails in search engines