esearchy 0.2.0

data/lib/esearchy/SearchEngines/google.rb

@@ -0,0 +1,30 @@
+module ESearchy  
+  module SearchEngines
+    class Google < ESearchy::GenericEngine
+      ENGINE = "www.google.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 1
+      
+      def search 
+        @querypath = "/cse?&safe=off&num=100&site=&q=" +  @query  + "&btnG=Search&start="
+        super
+      end
+            
+      def parse( html )
+        hits = html.scan(/<\/b> of [\w\s]*<b>(.*)<\/b> for /)
+        if hits.empty? or hits == nil
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits[0][0].gsub(",","").to_i)
+        end
+        super html.scan(/<div class=g><span class="b w xsm">\[([A-Z]+)\]<\/span> \
+<h2 class=r><a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)"|<h2 class=r><a href="\
+([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)"/)
+      end
+    end
+  end
+end
+
+  
+  

data/lib/esearchy/SearchEngines/yahoo.rb

@@ -0,0 +1,32 @@
+module ESearchy  
+  module SearchEngines
+    class Yahoo < ESearchy::GenericEngine
+      ENGINE = "boss.yahooapis.com"
+      PORT  = 80
+      NUM = 50
+      TYPE = 1
+      
+      def search
+        @querypath = "/ysearch/web/v1/" + @query + 
+                     "?appid="+ @appid + "&format=json&count=50" or 
+                      raise ESearchyMissingAppID, "Missing AppID <Class.appid=>"
+        super
+      end
+      
+      def appid=(value)
+        @appid = value
+      end
+      
+      def parse(json)
+        doc = JSON.parse(json) 
+        hits = doc["ysearchresponse"]["totalhits"].to_i 
+        if hits == nil or hits == 0
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits)
+        end
+        super doc["ysearchresponse"]["resultset_web"]
+      end
+    end
+  end
+end

data/lib/esearchy/SocialEngines/classmates.rb

@@ -0,0 +1,33 @@
+module ESearchy  
+  module SocialEngines
+    class Classmates < ESearchy::GenericEngine
+      ENGINE = "www.google.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 2
+      
+      def search 
+        @querypath = "/cse?q=site%3Awww.classmates.com+%22work+at+" +  CGI.escape(@company)  +   
+                     "%22&hl=en&cof=&num=100&filter=0&safe=off&start=" or raise ESearchyMissingCompany, "Mssing website url Object.company=(value)"
+        super
+      end
+            
+      def parse( html )
+        hits = html.scan(/<\/b> of[ about | ]<b>(.*)<\/b> from/)
+        if hits.empty? or hits == nil
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits[0][0].gsub(",","").to_i)
+        end
+      end
+      
+      def crawl_people(html)
+        html.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) \|/).each do |profile|
+          name,last = profile[1].split(" ") 
+          @people << [name,last] 
+          @results << [[name,last], "P", self.class.to_s.upcase, "N"]
+        end
+      end
+    end
+  end
+end

data/lib/esearchy/SocialEngines/googleprofiles.rb

@@ -0,0 +1,36 @@
+module ESearchy  
+  module SocialEngines
+    class GoogleProfiles < ESearchy::GenericEngine
+      ENGINE = "www.google.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 2
+      
+      def search 
+        @querypath = "/cse?q=site:www.google.com+intitle:%22Google+" +
+                     "Profile%22+%22Companies+I%27ve+worked+for%22+%22at+" +  
+                     CGI.escape(@company) + "%22&hl=en&cof=&num=100&filter=0&safe=off&start=" or
+                     raise ESearchyMissingCompany, "Mssing website url Object.company=(value)"
+        super
+      end
+            
+      def parse( html )
+        #Results <b>1</b> - <b>8</b> of <b>8</b> from <b>www.google.com</b>
+        hits = html.scan(/<\/b> of <b>(.*)<\/b> from /)
+        if hits.empty? or hits == nil
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits[0][0].gsub(",","").to_i) unless @was_here
+        end
+      end
+      
+      def crawl_people(text)
+        text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
+          name,last = profile[1].split(" ") 
+          @people << [name,last]
+          @results << [[name,last], "P", self.class.to_s.upcase, "N"]
+        end
+      end
+    end
+  end
+end

data/lib/esearchy/SocialEngines/linkedin.rb

@@ -0,0 +1,35 @@
+module ESearchy  
+  module SocialEngines
+    class LinkedIn < ESearchy::GenericEngine
+      ENGINE = "www.google.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 2
+      
+      def search 
+        @querypath = "/cse?q=site%3Awww.linkedin.com/in+%22at+" +  
+                     CGI.escape(@company) + "%22&hl=en&cof=&num=100&filter=0&safe=off&start=" or
+                     raise ESearchyMissingCompany, "Mssing website url Object.company=(value)"
+        super
+      end
+            
+      def parse( html )
+        #Results <b>1</b> - <b>8</b> of <b>8</b> from <b>www.google.com</b>
+        hits = html.scan(/<\/b> of [\w\s]*<b>(.*)<\/b> from /)
+        if hits.empty? or hits == nil
+          @totalhits = 0
+        else
+          @totalhits = totalhits(hits[0][0].gsub(",","").to_i)
+        end
+      end
+      
+      def crawl_people(text)
+        text.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
+          name,last = profile[1].split(" ")
+          @people << [name,last]
+          @results << [[name,last], "P", self.class.to_s.upcase, "N"]
+        end
+      end
+    end
+  end
+end

data/lib/esearchy/SocialEngines/linkedinfull.rb

@@ -0,0 +1,100 @@
+require 'net/https'
+module ESearchy  
+  module SocialEngines
+    class LinkedIn < ESearchy::GenericEngine
+      ENGINE = "www.linkedin.com"
+      PORT  = 80
+      NUM = 1
+      TYPE = 2
+      
+      def search 
+        @querypath = "/search?search=&currentCompany=co&company=" + CGI.escape(@company) +         
+         "&proposalType=Y&newnessType=Y&pplSearchOrigin=MDYS&searchLocationType=Y&page_num=" or 
+          raise ESearchyMissingCompany, "Mssing website url Object.company=(value)"
+        super
+      end
+            
+      def parse( html )
+        p html
+        p html.scan(/<p class="summary">[\n\s]+<strong>(.*)<\/strong> results/)#.gsub(/,|./,"")
+        #unless @was_here
+        #  @totalhits= totalhits html.scan(/<p class="summary">[\n\s]+<strong>(.*)<\/strong> results/)[0][0].to_i
+        #end 
+      end
+            
+      def credentials=(c)
+        @user = c[0].to_s
+        @pwd = c[1].to_s
+        LinkedIn.const_set :HEADER, login
+        self.start=(1)
+      end
+      
+      def maxhits=(v)
+        super v/10
+      end
+      
+      private
+      def crawl_people(html)
+        list = html.scan(/title="View profile">[\n\s]+<span class="given-name">(.*)<\/span>\
+[\n\s]+<span class="family-name">(.*)<\/span>/)
+        @people.concat(list).uniq!
+        list.each { |p| @results << [p, "P", self.class.to_s.upcase, "N"] }
+      end
+      
+      def login
+        begin
+          get ENGINE, PORT, "/secure/login?trk=hb_signin", {'User-Agent' => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.5) Gecko/20091102"} do |r|
+            @l_headers = r.to_hash
+            @l_headers.each {|k,v| @l_headers[k] = v.to_s}
+            @csrfToken = r.body.scan(/<input type="hidden" name="csrfToken" value="ajax:(.*)">/)[0][0]
+            puts "------------------------------------------------------------------------------------"
+            puts "------------------------------------------------------------------------------------"
+            p @l_headers
+            p @csrfToken
+            puts "------------------------------------------------------------------------------------"
+            puts "------------------------------------------------------------------------------------"
+          end
+          http = Net::HTTP.new(ENGINE,443)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.start do |http|
+            body = "csrfToken=ajax:#{@csrfToken}" +
+                   "session_key=#{@user}" +
+                   "&session_password=#{@pwd}" +
+                   "&session_login=Sign+In&session_login=&session_rikey="
+                   
+                  @l_headers['Host'] = "www.linkedin.com"
+                  @l_headers['User-Agent'] = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5"
+                  @l_headers['Accept'] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+                  @l_headers['Accept-Language'] = "en-us,en;q=0.5"
+                  @l_headers['Accept-Charset'] = "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
+                  @l_headers['Keep-Alive'] = "300"
+                  @l_headers['Connection'] = "keep-alive"
+                  @l_headers['Referer'] = "https://www.linkedin.com/secure/login?trk=hb_signin"
+                  @l_headers['Cookie'] = "JSESSIONID=\"ajax:5367441617418183976\"; visit=G; bcookie=\"v=1&8231965c-b4b7-48f2-8349-76514ba89b69\"; lang=\"v=2&lang=en&c=\"; NSC_MC_QH_MFP=e242089229a3; __utma=226841088.2037160969.1259078198.1259078198.1259078198.1; __utmb=226841088.2.10.1259078198; __utmc=226841088; __utmz=226841088.1259078198.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=226841088.user; leo_auth_token=\"GST:9_t6crYtB4AWStfoqhWQ6LYPKakWfHk_dotQyAHagiRX1HlEvqVt5-:1259081816:56d4aecb2e985d7f8a30d74e758f261ea8b92065\"; NSC_MC_WT_YUSL_IUUQ=e2420f8429a0"
+                  @l_headers['Content-Type'] = "application/x-www-form-urlencoded"
+                  @l_headers['Content-Length'] = body.size.to_s
+             
+            request = Net::HTTP::Post.new("/secure/login", @l_headers)
+            request.body = CGI.escape(body)
+            response = http.request(request)
+            case response
+            when Net::HTTPSuccess, Net::HTTPRedirection
+              puts "------------------------------------------------------------------------------"
+              puts "------------------------------------------------------------------------------"
+              p response.to_hash
+              p response.body
+              puts "-----------------------------------------------------------------------------"
+              puts "-----------------------------------------------------------------------------"
+              return {'Cookie' => response['Set-Cookie'], 'User-Agent' => UserAgent::fetch} 
+            else
+              return response.error!
+            end
+          end
+        rescue Net::HTTPFatalError
+          D "Error: Something went wrong while login to LinkedIn.\n\t${$@}"
+        end
+      end
+    end
+  end
+end

data/lib/esearchy/SocialEngines/naymz.rb

@@ -0,0 +1,36 @@
+module ESearchy  
+  module SocialEngines
+    class Naymz < ESearchy::GenericEngine
+      ENGINE = "www.google.com"
+      PORT  = 80
+      NUM = 100
+      TYPE = 2
+      
+      def search 
+        @querypath = "/cse?q=site:naymz.com%20%2B%20%22@%20" +  CGI.escape(@company)  +     
+                     "%22&hl=en&cof=&num=100&filter=0&safe=off&start="
+        super
+      end
+            
+      def parse( html )
+        #</b> of about <b>760</b> from <b>
+        hits = html.scan(/<\/b> of about <b>(.*)<\/b> from/)
+        if hits.empty? or hits == nil
+           @totalhits = 0
+         else
+           @totalhits= totalhits hits[0][0].gsub(",","").to_i unless @was_here
+         end
+      end
+      
+      def crawl_people(html)
+        html.scan(/<a href="([0-9A-Za-z:\\\/?&=@+%.;"'()_-]+)" class=l[\sonmousedown="return clk(this.href,'','','res','\d','')"]*>([\w\s]*) -/).each do |profile|
+          person = profile[1].split(" ").delete_if do 
+            |x| x =~ /mr.|mr|ms.|ms|phd.|dr.|dr|phd|phd./i
+          end
+          @people << person
+          @results << [person, "P", self.class.to_s.upcase, "N"]
+        end
+      end
+    end
+  end
+end

data/lib/esearchy/bugmenot.rb

@@ -0,0 +1,26 @@
+require 'base64'
+
+module ESearchy
+  class Bugmenot  
+    def self.fetch(domain = "www.linkedin.com")
+      begin
+        url = Net::HTTP.get URI.parse("http://www.bugmenot.com/view/#{domain}")
+        key = ( url.scan(/var key =(.*);/)[0][0].to_i + 112 ) / 12
+    
+        user, pass = url.scan(/tr><th>Username <\/th><td><script>d\('(.*)'\);<\/script><\/td><\/tr>
+  [\n\s]+<tr><th>Password <\/th><td><script>d\('(.*)'\);<\/script><\/td><\/tr>/)[0]
+        user = decode(user,key)
+        pass = decode(pass,key)
+        return [user, pass]
+      rescue
+        return [nil,nil]
+      end
+    end
+    
+    private
+    def decode(input, offset)
+      # thanks tlrobinson @ github
+      input.unpack("m*")[0][4..-1].unpack("C*").map{|c| c - offset }.pack("C*")
+    end
+  end
+end

data/lib/esearchy/docs.rb

@@ -0,0 +1,267 @@
+module ESearchy
+  class PageTextReceiver
+     attr_accessor :content
+
+     def initialize
+       @content = []
+     end
+
+     # Called when page parsing starts
+     def begin_page(arg = nil)
+       @content << ""
+     end
+
+     # record text that is drawn on the page
+     def show_text(string, *params)
+       @content.last << string.strip
+     end
+
+     # there's a few text callbacks, so make sure we process them all
+     alias :super_show_text :show_text
+     alias :move_to_next_line_and_show_text :show_text
+     alias :set_spacing_next_line_show_text :show_text
+
+     # this final text callback takes slightly different arguments
+     def show_text_with_positioning(*params)
+       params = params.first
+       params.each { |str| show_text(str) if str.kind_of?(String)}
+     end
+  end
+  
+  class Docs
+    case RUBY_PLATFORM 
+    when /mingw|mswin/
+      TEMP = "C:\\WINDOWS\\Temp\\"
+    else
+      TEMP = "/tmp/"
+    end
+    attr_reader :documents, :emails, :results
+    
+    def initialize(doc=nil, size = 10485760)
+      case doc
+      when Array
+        @@documents = Queue.new
+        self.merge doc
+      else
+        @@documents = doc || Queue.new
+      end
+      @size = size
+      @emails = []
+      @results = []
+      @lock = Mutex.new
+    end
+  
+    ## Class methods
+    def self.search(doc)
+      self.new(doc)
+      search(doc)
+    end
+    
+    def merge(array)
+      array.each {|a| push(a) }
+    end
+        
+    def self.push(doc)
+      push(doc)
+    end
+    
+    def push(doc)
+      @@documents.push(doc)
+    end
+  
+   def local_search
+     threads = []
+     while @documents.size >=1
+       threads << Thread.new do
+         doc = @@documents.pop
+         detect_type(doc.split(".")[-1], doc)
+       end
+       threads.each {|t| t.join } if @threads != nil
+     end
+   end
+  
+    def search
+      threads = []
+      while @@documents.size >=1
+        threads << Thread.new do
+          document = @@documents.pop
+          url = document[0].gsub(' ','+')
+          format = document[1]
+          if data = download(url)
+            name = save_to_disk(url, format, data)
+            detect_type(format,name)
+            remove_from_disk(name)
+          end
+        end
+        threads.each {|t| t.join } if threads != nil
+      end
+    end
+  
+    private
+  
+    def detect_type(format,name)
+      case format
+      when /.pdf/
+        pdf(name)
+      when /.doc/
+        doc(name)
+      when /txt|rtf|ans/
+        plain(name)
+      when /.docx|.xlsx|.pptx|.odt|.odp|.ods|.odb/
+        xml(name)
+      else
+        D "Error: Not currently parsing #{format}"
+      end
+    end
+  
+    def download(doc)
+      web = URI.parse(doc)
+      begin
+        http = Net::HTTP.new(web.host,80)
+        http.start do |http|
+          request = Net::HTTP::Head.new("#{web.path}#{web.query}")
+          response = http.request(request)
+          if response.content_length < @size
+            D "Downloading document: #{web.to_s}\n"
+            request = Net::HTTP::Get.new("#{web.path}#{web.query}")
+            response = http.request(request)
+            case response
+            when Net::HTTPSuccess, Net::HTTPRedirection
+              return response.body
+            else
+              return response.error!
+            end
+          else
+            D "Debug: Skipping #{web.to_s}. bigger than 10MB.\n"
+          end
+        end
+      rescue Net::HTTPFatalError
+        D "Error: HTTPFatalError - Unable to download.\n"
+      rescue Net::HTTPServerException
+        D "Error: Not longer there. 404 Not Found.\n"
+      rescue
+        D "Error: < .. SocketError .. >\n"
+      end
+      nil
+    end
+
+    def save_to_disk(url, format, data)
+      name = TEMP + "#{hash_url(url)}" + format
+      open(name, "wb") { |file| file.write(data) }
+      name
+    end
+    
+    def remove_from_disk(name)
+      `rm "#{name}"`
+    end
+    
+    def hash_url(url)
+      Digest::SHA2.hexdigest("#{Time.now.to_f}--#{url}")
+    end
+    
+    def pdf(name)
+      begin
+        receiver = PageTextReceiver.new
+        pdf = PDF::Reader.file(name, receiver)
+        search_emails(receiver.content.inspect)
+      rescue PDF::Reader::UnsupportedFeatureError
+        D "Error: Encrypted PDF - Unable to parse.\n"
+      rescue PDF::Reader::MalformedPDFError
+        D "Error: Malformed PDF - Unable to parse.\n"
+      rescue
+        D "Error: Unknown - Unable to parse.\n"
+      end
+    end
+
+    def doc(name)
+      if RUBY_PLATFORM =~ /mingw|mswin/
+        begin
+          word(name)
+        rescue
+          antiword(name)
+        end
+      elsif RUBY_PLATFORM =~ /linux|darwin/
+        begin
+          antiword(name)
+        rescue
+          D "Error: Unable to parse .doc"
+        end
+      else
+        D "Error: Platform not supported."
+      end
+    end
+
+    def word(name)
+      word = WIN32OLE.new('word.application')
+      word.documents.open(name)
+      word.selection.wholestory
+      search_emails(word.selection.text.chomp)
+      word.activedocument.close( false )
+      word.quit
+    end
+
+    def antiword(name)
+      case RUBY_PLATFORM
+      when /mingw|mswin/
+        if File.exists?("C:\\antiword\\antiword.exe")
+          search_emails(`C:\\antiword\\antiword.exe "#{name}" -f -s`) 
+        end
+      when /linux|darwin/
+        if File.exists?("/usr/bin/antiword") or 
+           File.exists?("/usr/local/bin/antiword") or 
+           File.exists?("/opt/local/bin/antiword")
+          search_emails(`antiword "#{name}" -f -s`) 
+        end
+      else
+         # This G h e t t o but, for now it works on emails 
+         # that do not contain Capital letters:)
+         D "Debug: Using the Ghetto way."
+         search_emails(File.open(name).readlines[0..19].to_s)
+      end
+    end
+
+    def plain(name)
+      search_emails(File.open(name).readlines.to_s)
+    end
+
+    def xml(name)
+      begin
+        Zip::ZipFile.open(name) do |zip|
+          text = z.entries.each { |e| zip.file.read(e.name) if e.name =~ /.xml$/}
+          search_emails(text)
+        end
+      rescue
+        D "Error: Unable to parse .#{name.scan(/\..[a-z]*$/)}\n"
+      end
+    end
+    
+    def search_emails(text)
+      list = text.scan(/[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*_at_\
+(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
+[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]\
+*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+\
+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
+[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*\s@\s(?:[a-z0-9](?:[a-z0-9-]*\
+[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\sdot\s[a-z0-9!#$&'*+=?^_`\
+{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\sdot\s)+[a-z](?:[a-z-]*[a-z])??/i)
+      @lock.synchronize do
+        #print_(list)
+        c_list = fix(list)
+        @emails.concat(c_list).uniq!
+        c_list.zip do |e| 
+          @results << [e[0], "E", self.class.to_s.upcase, 
+           e[0].match(/#{CGI.unescape(ESearchy::Search.query).gsub("@","").split('.')[0]}/) ? "T" : "F"]
+        end
+      end
+    end
+
+    def fix(list)
+      list.each do |e|
+        e.gsub!(" at ","@")
+        e.gsub!("_at_","@")
+        e.gsub!(" dot ",".")
+        e.gsub!(/[+0-9]{0,3}[0-9()]{3,5}[-]{0,1}[0-9]{3,4}[-]{0,1}[0-9]{3,5}/,"")
+      end
+    end
+  end
+end