bots 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: be4c3337063af2d514fd1016c94d66385fa84dbde27dbfa5174d5b0908c9efb1
+  data.tar.gz: 13ef3dccd336269fdb1960f243c8b7dd08abf782ba89cfbf935c4e802a29ca45
+SHA512:
+  metadata.gz: 354704a255a80def04f1993d3a73d5b358fedfeeda387278f47ab995ed10d2afced94fe175a15bc8dd7c55b1209798598b55a5f2d2071424f11613091cd87813
+  data.tar.gz: c1fae508d2b55269f0039524ee4e87714d7cc7244922faa83ac5b6c374abc29cdaab34355059ff92e5d6a0469be44900ac628fcfe55da80def3993549a2bcf27

data/lib/base.rb

@@ -0,0 +1,36 @@
+module BlackStack
+    module Bots
+        class Bot
+            attr_accessor :ip # ip address of proxy
+            attr_accessor :user # user of proxy
+            attr_accessor :password # password of proxy
+            attr_accessor :ports # array of ports
+            attr_accessor :port_index # index of the port
+
+            def initialize(h)
+                # array of numbers from 4000 to 4249
+                unless h[:proxy].nil?
+                    self.ip = h[:proxy][:ip]
+                    self.user = h[:proxy][:user]
+                    self.password = h[:proxy][:password]
+                    self.ports = (h[:proxy][:port_from]..h[:proxy][:port_to]).to_a
+                end
+                self.port_index = -1
+            end # initialize
+
+            # return true if the bot is using a proxy
+            def proxy?
+                !self.ip.nil? 
+            end
+        end # Bot
+
+        class MechanizeBot < BlackStack::Bots::Bot
+            attr_accessor :agent # mechanize agent
+        end # MechanizeBot
+
+        class SeleniumBot < BlackStack::Bots::Bot
+            attr_accessor :driver # selenium driver
+        end # MechanizeBot
+
+    end # Bots
+end # BlackStack

data/lib/bots.rb

@@ -0,0 +1,9 @@
+require 'mechanize'
+require 'selenium-webdriver'
+require 'simple_cloud_logging'
+require 'colorize'
+require 'csv'
+
+require_relative './base'
+require_relative './google'
+require_relative './indeed'

data/lib/google.rb

@@ -0,0 +1,80 @@
+module BlackStack
+    module Bots
+        class Google < BlackStack::Bots::MechanizeBot
+            def search(query)
+                ret = []
+                # initialize mechanize agent
+                self.agent = Mechanize.new
+                # set a proxy with user and password
+                self.port_index += 1
+                self.port_index = 0 if self.port_index >= self.ports.length
+                self.agent.set_proxy(self.ip, self.ports[self.port_index], self.user, self.password) if self.proxy?
+                # grab the page
+                page = agent.get('http://www.google.com/')
+                google_form = page.form('f')
+                google_form.q = query
+                page = agent.submit(google_form, google_form.buttons.first)
+                # iterate divs with class starting with 'g '
+                page.search('h3').each do |h3|
+                    # get the class of the div
+                    title = h3.text.strip
+                    # get the link inside the div
+                    a = h3.parent.parent.parent
+                    href = a['href']
+                    descr = a.parent.parent.css('/div').last.text.strip
+                    # get the value of the paremter with name param1 from the querystring using URI
+                    uri = URI.parse(href)
+                    params = CGI.parse(uri.query)
+                    url = params['q'].first
+                    # add to the list array of results
+                    ret << { :title=>title, :url=>url, :description=>descr }
+                end
+                # destroy mechanize agent
+                self.agent.shutdown
+                # return
+                ret
+            end # search
+        end # Google
+=begin
+        class GoogleEnrichment < BlackStack::Bots::Google
+
+            # get an array of domains that may be the domain of the company
+            def possible_domains_for_company(company_name)
+                search = "\"#{company_name}\" home page"
+                self.search(search).map { |r| 
+                    # get domain from url using URI, and removing www., and downcasing.
+                    URI.parse(r[:url]).host.gsub(/^www\./, '').downcase
+                }        
+            end # possible_domains_for_company
+
+            # find email from fname, lname and cname
+            def find_email(fname, lname, cname)
+                domains = self.possible_domains_for_company(cname)
+                if domains.size > 0
+                    domains.each { |domain|
+                        # array of possible emails
+                        emails = []
+                        #emails << "#{fname}@#{domain}"
+                        #emails << "#{lname}@#{domain}"
+                        emails << "#{fname}#{lname}@#{domain}"
+                        emails << "#{fname}.#{lname}@#{domain}"
+                        emails << "#{fname}_#{lname}@#{domain}"
+                        emails << "#{fname[0]}#{lname}@#{domain}"
+                        # iterate array of possible emails
+                        emails.each { |email|
+                            # search for that email
+                            search = "\"#{email}\""
+                            results = self.search(search)
+                            # find results with the exact email in the description
+                            return email if results.select { |result| result[:description].downcase =~ /\b#{email.downcase}\b/ }
+                        }
+                    }
+                end
+                return nil      
+            end # find_email
+
+        end # GoogleEnrichment
+=end
+    end # Bots
+end # BlackStack
+

data/lib/indeed.rb

@@ -0,0 +1,52 @@
+require_relative './base'
+module BlackStack
+    module Bots
+        class Indeed < BlackStack::Bots::SeleniumBot
+
+            def results(url, page=1)
+                ret = []
+                # launch a chrome browser with selenium
+                driver = Selenium::WebDriver.for :chrome
+                browser = driver.browser
+                # TODO: set a proxy with user and password
+                driver.get url
+                # get the ul list with class .jobsearch-ResultsList
+                ul = driver.find_element(:class=>'jobsearch-ResultsList')
+                # scroll to the bottom
+                driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
+                # iterate li elements
+                i = 0
+                ul.find_elements('css', 'li').each { |li|
+                    h = {}
+                    i += 1
+                    links = li.find_elements('css', 'a.jcs-JobTitle')
+                    if links.size == 1
+                        link = li.find_element('css', 'a.jcs-JobTitle')
+                        h[:title] = link.text
+                        h[:url] = link.attribute('href')
+                    
+                        o = li.find_elements('css','span.companyName').first
+                        h[:company] = o ? o.text : ''
+
+                        o = li.find_elements('css','div.companyLocation').first
+                        h[:location] = o ? o.text : ''
+                        
+                        o = li.find_elements('css','div.salary-snippet-container').first
+                        h[:salary] = o ? o.text : ''
+                        
+                        o = li.find_elements('css','span.date').first
+                        h[:posted] = o ? o.text.gsub("Posted\nPosted", '').strip : ''
+                        
+                        h[:snippets] = li.find_elements('css','div.job-snippet > ul > li').map { |li| li.text }
+                        
+                        ret << h
+                    end
+                }
+                # destroy selenium browser
+                driver.quit
+                # return
+                ret
+            end # results
+        end # Indeed
+    end # Bots
+end # BlackStack

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: bots
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Leandro Daniel Sardi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-08-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: simple_cloud_logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.7
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.7
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.7
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.5
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.10.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.10.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.10.0
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+description: Ruby gem for scraping information from the public web.
+email: leandro@connectionsphere.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/base.rb
+- lib/bots.rb
+- lib/google.rb
+- lib/indeed.rb
+homepage: https://rubygems.org/gems/bots
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key: 
+specification_version: 4
+summary: Ruby gem for scraping information from the public web.
+test_files: []