esearchy 0.2.0

data/lib/esearchy/esearchy.rb

@@ -0,0 +1,195 @@
+def D m
+  puts m if ESearchy::log
+end
+
+module ESearchy
+  
+  @@log = false
+  #BUGMENOT = ESearchy::Bugmenot::fetch("linkedin.com")
+  
+  def self.log
+    @@log
+  end
+  
+  def self.log=(v)
+    @@log=v
+  end
+  
+  
+  class ESearchyMissingAppID < StandardError; end
+  class ESearchyMissingQuery < StandardError; end
+  class ESearchyMissingWebsite < StandardError; end
+  class ESearchyMissingCompany < StandardError; end
+  
+  class Search
+    
+    EMAIL_ENGINES = {
+      :Google         => ESearchy::SearchEngines::Google, 
+      :Bing           => ESearchy::SearchEngines::Bing, 
+      :Yahoo          => ESearchy::SearchEngines::Yahoo, 
+      :Altavista      => ESearchy::SearchEngines::Altavista,
+      :PGP            => ESearchy::OtherEngines::PGP, 
+      :Spider         => ESearchy::OtherEngines::Spider,
+      :Usenet         => ESearchy::OtherEngines::Usenet,
+      :GoogleGroups   => ESearchy::OtherEngines::GoogleGroups  
+    }
+
+    PEOPLE_ENGINES = {
+      :LinkedIn       => ESearchy::SocialEngines::LinkedIn, 
+      :GoogleProfiles => ESearchy::SocialEngines::GoogleProfiles, 
+      :Naymz          => ESearchy::SocialEngines::Naymz,
+      :Classmates     => ESearchy::SocialEngines::Classmates
+      }
+      
+    def initialize(args, &block)    
+      @@query = args[:query] || nil
+      @@company = args[:company] || nil
+      @@maxhits = args[:maxhits] || nil
+      @@start_at = args[:start_at] || nil
+      @@website = args[:website] || nil
+      ESearchy.log = args[:log] if args[:log]
+      $emails = []
+      $people = []
+      $results = []
+      block.call(self) if block_given?
+    end
+    
+    def self.query
+      @@query
+    end
+    
+    def self.company
+      @@company
+    end
+    
+    def self.website
+      @@website
+    end
+    
+    def self.maxhits
+      @@maxhits
+    end
+    
+    def maxhits
+      @@maxhits
+    end
+    
+    def start(&block)
+      block.call(self)
+    end
+    
+    def emails
+      $emails
+    end
+    
+    def people
+      $people
+    end
+    
+    def results
+      $results
+    end
+    
+    def Emails(*args, &block)
+      Emails.new(*args, &block)
+    end
+    
+    def People(*args, &block)
+      People.new(*args, &block)
+    end
+    
+    module MetaType
+      def results
+        $results
+      end
+      
+      def docs(&block)
+        @engines.each_key {|e| @documents.concat(@engines[e].documents) }
+        res = ESearchy::Docs.new(@documents)
+        res.search
+        $emails.concat(res.emails)
+        $results.concat(res.results)
+        block.call(res) if block_given?
+      end
+      
+      def method_missing(name, *args)
+        @engines[name.to_sym]
+      end
+      
+      def maxhits=(v)
+        @engines.each_key {|e| @engines[e].maxhits=v}
+      end
+
+      def [](v)
+        @engine[v]
+      end
+      
+      private
+      def save_results(e)
+        $results.concat(@engines[e].results)
+      end
+    end
+    
+    class Emails
+      include MetaType
+      
+      def initialize(*args, &block)
+        @engines={}
+        @documents = []
+        args.each do |e|
+          @engines[e] = ESearchy::Search::EMAIL_ENGINES[e].new(Search.query)
+        end
+        self.maxhits=Search.maxhits if Search.maxhits
+        block.call(self) if block_given?
+      end
+      attr_reader :emails
+      
+      def search(&block)
+        @engines.each_key do |e| 
+          @engines[e].search
+          save_emails(e)
+          save_results(e)
+          block.call(@engines[e]) if block_given?
+        end
+        nil
+      end
+      
+      private
+      def save_emails(e)
+        $emails.concat(@engines[e].emails)
+      end
+    end
+    
+    class People
+      include MetaType
+      
+      def initialize(*args, &block)
+        @engines={}
+        args.each do |e|
+          @engines[e] = ESearchy::Search::PEOPLE_ENGINES[e].new(Search.company)
+        end
+        self.maxhits=Search.maxhits if Search.maxhits
+        block.call(self) if block_given?
+      end
+      
+      def people
+        $people
+      end
+      
+      def search(&block)
+        @engines.each_key do |e| 
+          @engines[e].search
+          save_people(e)
+          save_results(e)
+          block.call(self) if block_given?
+        end
+        nil
+      end
+      
+      private
+      def save_people(e)
+        $people.concat(@engines[e].people)
+      end
+    end
+  end
+end

data/lib/esearchy/genericengine.rb

@@ -0,0 +1,153 @@
+module ESearchy  
+  class GenericEngine
+    
+    def initialize(query, start = 0, stop = nil, &block)
+      @query = CGI.escape(query) or raise ESearchyMissingQuery
+      @company = ESearchy::Search.company || ""
+      @start = start
+      @totalhits = stop
+      @documents = []
+      self.class::TYPE < 2 ? @emails = [] : @people = []
+      @results = []
+      block.call(self) if block_given?
+    end
+    
+    attr_reader :documents
+    attr_reader :results
+    attr_reader :emails
+    attr_reader :people
+    
+    
+    def search
+      get self.class::ENGINE, self.class::PORT, 
+          @querypath + @start.to_s, header() do |r|
+        parse(r.body)
+        D "Searching #{self.class} from #{@start} to #{calculate_top()}.\n"
+        crawler(r.body.gsub(/<em>|<\/em>/,"").gsub(/<b>|<\/b>/,"")) unless @totalhits == 0
+        @start = @start + self.class::NUM
+        sleep(4) and search if @totalhits > @start
+      end
+    end
+    
+    def start=(v)
+      @start=v
+    end
+        
+    def maxhits=(v)
+      @totalhits=v
+    end
+    
+    def company=(v)
+      @company=v
+    end
+    
+    private
+    
+    def get(url, port, querystring = "/", headers = {}, &block)
+      http = Net::HTTP.new(url,port)
+      begin
+        http.start do |http|
+          request = Net::HTTP::Get.new(querystring, headers)
+          response = http.request(request)
+          case response
+          when Net::HTTPSuccess, Net::HTTPRedirection
+            block.call(response)
+          else
+            return response.error!
+          end
+        end
+      rescue Net::HTTPFatalError
+        D "Error: Something went wrong with the HTTP request"
+      end
+    end
+    
+    def header
+      begin
+        return self.class::HEADER
+      rescue
+        return {'User-Agent' => UserAgent::fetch}
+      end
+    end
+    
+    def calculate_top
+      (@start+self.class::NUM) > @totalhits ? @totalhits : @start+self.class::NUM
+    end
+    
+    def totalhits(realhits)
+      @totalhits > realhits ? realhits : @totalhits
+    end
+    
+    def parse(object)
+      case object
+      when Array
+        parse_html object
+      when Json
+        parse_json object
+      end
+    end
+    
+    def parse_html ( array )
+      array.each do |a|
+        case a[0]
+        when /(PDF|DOC|XLS|PPT|TXT)/
+          @documents << [a[1],"."+$1.to_s.downcase]
+        when nil
+          if a[2] =~ /(.pdf$|.doc$|.docx$|.xlsx$|.pptx$|.odt$|.odp$\
+|.ods$|.odb$|.txt$|.rtf$|.ans$|.csv$)/i
+            @documents << [a[2],$1.to_s.downcase]
+          end
+        when /(.pdf$|.doc$|.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$|.txt$|.rtf$|.ans$|.csv$)/i
+          @documents << [CGI.unescape(a[2] || ""),$1.to_s.downcase]
+        else
+          #D "I do not parse this doc's \"#{a}\" filetype yet:)"
+        end
+      end
+    end
+    
+    def parse_json ( json )
+      json.each do |j|
+        case j["url"]
+        when /(.pdf$|.doc$|.docx$|.xlsx$|.pptx$|.odt$|.odp$|.ods$|.odb$|.txt$|.rtf$|.ans$|.csv$)/i
+          @documents << [j["url"],$1.to_s.downcase]
+        else
+          @urls << [j["url"],$1.to_s.downcase]
+        end
+      end
+    end
+    
+    def crawler(text)
+      self.class::TYPE < 2 ? crawl_emails(text) : crawl_people(text)
+    end
+    
+    def crawl_emails(text)
+      list = text.scan(/[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*_at_\
+(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
+[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]\
+*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+\
+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|\
+[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\.[a-z0-9!#$&'*+=?^_`{|}~-]+)*\s@\s(?:[a-z0-9](?:[a-z0-9-]*\
+[a-z0-9])?\.)+[a-z](?:[a-z-]*[a-z])?|[a-z0-9!#$&'*+=?^_`{|}~-]+(?:\sdot\s[a-z0-9!#$&'*+=?^_`\
+{|}~-]+)*\sat\s(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\sdot\s)+[a-z](?:[a-z-]*[a-z])??/i)
+      #print_(list)
+      c_list = fix(list)
+      @emails.concat(c_list).uniq!
+      c_list.zip do |e| 
+        @results << [e[0], "E", self.class.to_s.upcase, 
+                     e[0].match(/#{CGI.unescape(@query).gsub("@","").split('.')[0]}/) ? "T" : "F"]
+      end
+    end
+    
+    def fix(list)
+      list.each do |e|
+        e.gsub!(" at ","@")
+        e.gsub!("_at_","@")
+        e.gsub!(" dot ",".")
+        e.gsub!(/[+0-9]{0,3}[0-9()]{3,5}[-]{0,1}[0-9]{3,4}[-]{0,1}[0-9]{3,5}/,"")
+      end
+    end
+    
+    def crawl_people(text)
+      raise "This is just a container"
+    end 
+  end
+end

data/lib/esearchy/localengines.rb

@@ -0,0 +1 @@
+require 'esearchy/localengines/directory'

data/lib/esearchy/otherengines.rb

@@ -0,0 +1,5 @@
+require 'esearchy/OtherEngines/pgp'
+require 'esearchy/OtherEngines/usenet'
+require 'esearchy/OtherEngines/spider'
+require 'esearchy/OtherEngines/googlegroups'
+require 'esearchy/OtherEngines/ldap'

data/lib/esearchy/searchengines.rb

@@ -0,0 +1,4 @@
+require 'esearchy/searchengines/google'
+require 'esearchy/searchengines/bing'
+require 'esearchy/searchengines/yahoo'
+require 'esearchy/searchengines/altavista'

data/lib/esearchy/socialengines.rb

@@ -0,0 +1,4 @@
+require 'esearchy/socialengines/classmates'
+require 'esearchy/socialengines/googleprofiles'
+require 'esearchy/socialengines/linkedin'
+require 'esearchy/socialengines/naymz'

data/lib/esearchy/useragent.rb

@@ -0,0 +1,188 @@
+class UserAgent
+  @USER_AGENTS=[
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Acoo Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Acoo Browser; InfoPath.2; .NET CLR 2.0.50727; Alexa Toolbar)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 7.0; Windows NT 5.1; FunWebProducts)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; AOL 8.0; Windows NT 5.1; SV1)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.0; Windows NT 5.1; .NET CLR 1.1.4322; Zango 10.1.181.0)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; AOL 6.0; Windows NT 5.1)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
+    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
+    "Mozilla/5.0 (X11; U; Linux; C -) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.5",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; JyxoToolbar1.0;  Embedded Web Browser from: http://bsalsa.com/; Avant Browser; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 1.1.4322)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; GTB5; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Avant Browser)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Avant Browser; Avant Browser; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)",
+    "Mozilla/5.0 (X11; 78; CentOS; US-en) AppleWebKit/527+ (KHTML, like Gecko) Bolt/0.862 Version/3.0 Safari/523.15",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.2) Gecko/20040825 Camino/0.8.1",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X Mach-O; en; rv:1.8.1.12) Gecko/20080206 Camino/1.5.5",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030111 Chimera/0.6",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.10) Gecko/20070228 Camino/1.0.4",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en; rv:1.8.1.4pre) Gecko/20070511 Camino/1.6pre",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418.9 (KHTML, like Gecko, Safari) Safari/419.3 Cheshire/1.0.ALPHA",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/419 (KHTML, like Gecko, Safari/419.3) Cheshire/1.0.ALPHA",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.36 Safari/525.19",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042815 Firefox/3.0.10 CometBird/3.0.10",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2009011615 Firefox/3.0.5 CometBird/3.0.5",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Crazy Browser 3.0.0 Beta2)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; Crazy Browser 2.0.1)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Crazy Browser 1.0.5; .NET CLR 1.1.4322; InfoPath.1)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer 1.5.0; .NET CLR 1.0.3705)",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Demeter/1.0.9 Safari/125",
+    "Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.1.12) Gecko/20080208 (Debian-1.8.1.12-2) Epiphany/2.20",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.3) Gecko/20041007 Epiphany/1.4.7",
+    "Mozilla/5.0 (Windows; U; Win95; en-US; rv:1.5) Gecko/20031007 Firebird/0.7",
+    "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.5) Gecko/20031007 Firebird/0.7",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; ko; rv:1.9.1b2) Gecko/20081201 Firefox/3.1b2",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; cs; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8",
+    "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.9) Gecko/20050711 Firefox/1.0.5",
+    "Mozilla/5.0 (X11; U; SunOS sun4u; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5",
+    "Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.8.0.5) Gecko/20060819 Firefox/1.5.0.5",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1b3) Gecko/20090305 Firefox/3.1b3 GTB5",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.12) Gecko/20080214 Firefox/2.0.0.12",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.0; es-ES; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.9) Gecko/20071113 BonEcho/2.0.0.9",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061026 BonEcho/2.0",
+    "Mozilla/5.0 (BeOS; U; Haiku BePC; en-US; rv:1.8.1.21pre) Gecko/20090227 BonEcho/2.0.0.21pre",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko/2009033017 GranParadiso/3.0.8",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1b3pre) Gecko/20090109 Shiretoko/3.1b3pre",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1b4pre) Gecko/20090311 Shiretoko/3.1b4pre",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.1) Gecko/20060314 Flock/0.5.13.2",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.2) Gecko/2008092122 Firefox/3.0.2 Flock/2.0b3",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Fluid/0.9.4 Safari/525.13",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12) Gecko/20050929 Galeon/1.3.21",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko/20090327 Galeon/2.0.7",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; GreenBrowser)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; GreenBrowser)",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; GreenBrowser)",
+    "Mozilla/3.0 (x86 [cs] Windows NT 5.1; Sun)",
+    "Mozilla/5.1 (X11; U; Linux i686; en-US; rv:1.8.0.3) Gecko/20060425 SUSE/1.5.0.3-7 Hv3/alpha",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SIMBAR={CFBFDAEA-F21E-4D6E-A9B0-E100A69B860F}; Hydra Browser; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Hydra Browser; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
+    "Mozilla/5.0 (compatible; IBrowse 3.0; AmigaOS4.0)",
+    "Mozilla/4.5 (compatible; iCab 2.9.1; Macintosh; U; PPC)",
+    "iCab/3.0.2 (Macintosh; U; PPC Mac OS X)",
+    "iCab/4.0 (Macintosh; U; Intel Mac OS X)",
+    "Mozilla/5.0 (Java 1.6.0_01; Windows XP 5.1 x86; en) ICEbrowser/v6_1_2",
+    "ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.9) Gecko/20071030 Iceape/1.1.6 (Debian-1.1.6-3)",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.8) Gecko/20071008 Iceape/1.1.5 (Ubuntu-1.1.5-1ubuntu0.7.10)",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.11) Gecko/20071203 IceCat/2.0.0.11-g1",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092921 IceCat/3.0.3-g1",
+    "Mozilla/5.0 (X11; U; Linux i686; de; rv:1.9.0.5) Gecko/2008122011 Iceweasel/3.0.5 (Debian-3.0.5-1)",
+    "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.1) Gecko/20061205 Iceweasel/2.0.0.1 (Debian-2.0.0.1+dfsg-4)",
+    "Mozilla/5.0 (X11; U; Linux i686; it; rv:1.9.0.5) Gecko/2008122011 Iceweasel/3.0.5 (Debian-3.0.5-1)",
+    "Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
+    "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1; .NET CLR 3.0.04506.30)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0; .NET CLR 2.0.50727)",
+    "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT;)",
+    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; GTB5; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.2; OfficeLiveConnector.1.3; OfficeLivePatch.0.0)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iRider 2.21.1108; FDM)",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/528.7 (KHTML, like Gecko) Iron/1.0.155.0 Safari/528.7",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Iron/0.2.152.0 Safari/12081672.525",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/528.5 (KHTML, like Gecko) Iron/0.4.155.0 Safari/528.5",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.8.1.19) Gecko/20081217 K-Meleon/1.5.2",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060706 K-Meleon/1.0",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.6) Gecko/20060731 K-Ninja/2.0.2",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
+    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman2.0)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; KKMAN3.2)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman3.0)",
+    "Mozilla/5.0 (compatible; Konqueror/2.2.1; Linux)",
+    "Mozilla/5.0 (compatible; Konqueror/3.5; SunOS)",
+    "Mozilla/5.0 (compatible; Konqueror/4.1; OpenBSD) KHTML/4.1.4 (like Gecko)",
+    "Mozilla/5.0 (compatible; Konqueror/3.1-rc5; i686 Linux; 20020712)",
+    "Links (0.96; Linux 2.4.20-18.7 i586)",
+    "Links (0.98; Win32; 80x25)",
+    "Links (2.1pre18; Linux 2.4.31 i686; 100x37)",
+    "Links (2.1; Linux 2.6.18-gentoo-r6 x86_64; 80x24)",
+    "Links (2.2; Linux 2.6.25-gentoo-r9 sparc64; 166x52)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Linux 2.6.26-1-amd64) Lobo/0.98.3",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows XP 5.1) Lobo/0.98.4",
+    "Mozilla/4.0 (compatible; Lotus-Notes/6.0; Windows-NT)",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1b3pre) Gecko/2008 Lunascape/4.9.9.98",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528+ (KHTML, like Gecko, Safari/528.0) Lunascape/5.0.2.0",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; Lunascape 2.1.3)",
+    "Lynx/2.8.6rel.4 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.6.3",
+    "Lynx/2.8.3dev.6 libwww-FM/2.14",
+    "Lynx/2.8.5dev.16 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.7a",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MyIE2)",
+    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; MAXTHON 2.0)",
+    "Midori/0.1.5 (X11; Linux; U; en-gb) WebKit/532+",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.0.1) Gecko/20020919",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.12) Gecko/20050915",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.2.1; MultiZilla v1.1.32 final) Gecko/20021130",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4; MultiZilla v1.5.0.0f) Gecko/20030624",
+    "NCSA_Mosaic/2.0 (Windows 3.1)",
+    "NCSA_Mosaic/3.0 (Windows 95)",
+    "NCSA_Mosaic/2.6 (X11; SunOS 4.1.3 sun4m)",
+    "Mozilla/3.01 (compatible; Netbox/3.5 R92; Linux 2.2)",
+    "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; NetCaptor 6.5.0RC1)",
+    "Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)",
+    "Mozilla/5.0 (Windows; U; Win 9x 4.90; de-DE; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.12) Gecko/20080219 Firefox/2.0.0.12 Navigator/9.0.0.6",
+    "Mozilla/4.08 [en] (WinNT; U ;Nav)",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02",
+    "Mozilla/3.0 (Win95; I)",
+    "Mozilla/4.51 [en] (Win98; U)",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1",
+    "NetSurf/2.0 (RISC OS; armv3l)",
+    "NetSurf/1.2 (Linux; i686)",
+    "Mozilla/4.7 (compatible; OffByOne; Windows 2000)",
+    "Mozilla/4.7 (compatible; OffByOne; Windows 98)",
+    "OmniWeb/2.7-beta-3 OWF/1.0",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/420+ (KHTML, like Gecko, Safari) OmniWeb/v595",
+    "Mozilla/4.5 (compatible; OmniWeb/4.1.1-v424.6; Mac_PowerPC)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.10 [en]",
+    "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.2.15 Version/10.00",
+    "Opera/5.11 (Windows 98; U) [en]",
+    "Opera/9.51 (Macintosh; Intel Mac OS X; U; en)",
+    "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 4.0) Opera 6.01 [en]",
+    "Opera/9.02 (Windows XP; U; ru)",
+    "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98) Opera 5.12 [en]",
+    "Opera/9.70 (Linux i686 ; U; en) Presto/2.2.1",
+    "Opera/7.03 (Windows NT 5.0; U) [en]",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.24",
+    "Opera/6.0 (Windows 2000; U) [fr]",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009030821 Firefox/3.0.7 Orca/1.1 build 2",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009022300 Firefox/3.0.6 Orca/1.1 build 1",
+    "Mozilla/1.10 [en] (Compatible; RISC OS 3.70; Oregano 1.10)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; PhaseOut-www.phaseout.net)",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4a) Gecko/20030411 Phoenix/0.5",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021029 Phoenix/0.4",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; cs-CZ) AppleWebKit/527+ (KHTML, like Gecko)  QtWeb Internet Browser/2.5 http://www.QtWeb.net",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/527+ (KHTML, like Gecko) QtWeb Internet Browser/1.2 http://www.QtWeb.net",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/527+ (KHTML, like Gecko) QtWeb Internet Browser/1.7 http://www.QtWeb.net",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_6; it-it) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.1; cs-CZ) AppleWebKit/523.15 (KHTML, like Gecko) Version/3.0 Safari/523.15",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.7",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fi-fi) AppleWebKit/420+ (KHTML, like Gecko) Safari/419.3",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6",
+    "Mozilla/5.0 (X11; U; Linux i686; rv:1.9.1a2pre) Gecko/20080824052448 SeaMonkey/2.0a1pre",
+    "Mozilla/5.0 (Windows; U; Win 9x 4.90; en-GB; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4",
+    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1b3pre) Gecko/20081208 SeaMonkey/2.0a3pre",
+    "Mozilla/5.0 (BeOS; U; BeOS BePC; en-US; rv:1.9a1) Gecko/20060702 SeaMonkey/1.5a",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1b3pre) Gecko/20081202 SeaMonkey/2.0a2",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.13) Gecko/20080313 SeaMonkey/1.1.9",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; ja-jp) AppleWebKit/419 (KHTML, like Gecko) Shiira/1.2.3 Safari/125",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/417.9 (KHTML, like Gecko, Safari) Shiira/1.1",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; fr) AppleWebKit/418.9.1 (KHTML, like Gecko) Shiira Safari/125",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Sleipnir/2.8.1",
+    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 2.0.50727) Sleipnir/2.8.4",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Stainless/0.4 Safari/525.20.1",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/528.16 (KHTML, like Gecko) Stainless/0.5.3 Safari/525.20.1",
+    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.18 (KHTML, like Gecko) Sunrise/1.7.4 like Safari/4525.22",
+    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.7 (KHTML, like Gecko) SunriseBrowser/0.853",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.10pre) Gecko/2009041814 Firefox/3.0.10pre (Swiftfox)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 1.1.4322; TheWorld)"]
+  def self.fetch
+    @USER_AGENTS[rand(@USER_AGENTS.size)]
+  end
+end

data/lib/esearchy.rb

@@ -0,0 +1,24 @@
+#EXTERNAL REQUIRES
+require 'rubygems'
+require 'net/http'
+require 'cgi'
+require 'json'
+require 'digest/sha2'
+require 'zip/zip'
+require 'zip/zipfilesystem'
+require 'pdf/reader'
+if RUBY_PLATFORM =~ /mingw|mswin/
+ require 'win32ole'
+end
+require 'ldap' # gem install ruby-ldap
+
+#ESEARCHY REQUIRES
+require 'esearchy/genericengine'
+require 'esearchy/searchengines'
+require 'esearchy/otherengines'
+require 'esearchy/socialengines'
+require 'esearchy/localengines'
+require 'esearchy/bugmenot'
+require 'esearchy/docs'
+require 'esearchy/useragent'
+require 'esearchy/esearchy'