indeed_scraper2022 0.1.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bcc9484e0c35971209ee1419b9ff7d07d912831eedf889d10789c2ab4bf0863
-  data.tar.gz: c3be84be0f499ed92794a8de0c3109cc99d3ba73f6aef74f9426a12b0a250e10
+  metadata.gz: 972f811430fae59121e39c9c4752b64fc43b37165a52dcec8c3eac42cf1e4555
+  data.tar.gz: 85e987eb264b098b4c892e2e05d2ab082e3b8968fff2bdd519552e889f014f9d
 SHA512:
-  metadata.gz: 9e6d64b754bbd242f6d038f385602ed4dc5fe2d8112da12709a673a494419a159c9afe2ef4fa6bc95cccd160f9bceb22b03783108ffec213ed96442372092f81
-  data.tar.gz: eae4c65e2cd7784aff9c3db394fa29e44344e61a5275a31790a1fc21c57df34204585aa5e9e3db8af28a251cf2366675d4d9027b9a6578456dd9f64744842caa
+  metadata.gz: e7d3c2a13e315383248c557806dd0184d8831f46a2e314816395f62fcb886ba2e38a3e1f2deb180ceb33b614cf0b7be8a13379028fc56982e388948575bdb02c
+  data.tar.gz: 6ca4792d260c43b22fcee5ead8928525df17f1db112c49ac4cc7d7a5c9b29a8f483b94c18cd39bace1d4a4ee553ff83254a7fb445d76abf06c1705d19bac455c

data/lib/indeed_scraper2022.rb

@@ -2,23 +2,31 @@
 
 # file: indeed_scraper2022.rb
 
-require 'mechanize'
+require 'ferrumwizard'
 require 'nokorexi'
+require 'yaml'
 
 # Given the nature of changes to jobsearch websites,
 # don't rely upon this gem working in the near future.
 
 
+
 class IndeedScraper2022Err < Exception
 end
 
 class IndeedScraper2022
 
-  def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '', debug: false)
+  attr_reader :browser
+
+  def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '',
+                 headless: true, cookies: nil, debug: false)
 
     @debug = debug
     @url_base, @q, @location = url, q, location
-    @results = search
+    @headless, @cookies = headless, cookies
+
+    fw = FerrumWizard.new( headless: @headless, cookies: @cookies, debug: @debug)
+    @browser = fw.browser
 
   end
 
@@ -28,6 +36,101 @@ class IndeedScraper2022
     @results
   end
 
+  def search(q: @q, location: @location, start: nil)
+
+    url = @url_base
+    url += 'start=' + start if start
+
+    @browser.goto(url)
+    #@browser.network.wait_for_idle
+    puts 'sleeping for 4 seconds' if @debug
+    sleep 4
+
+    if q.length > 1 then
+
+      input = @browser.at_xpath("//input[@name='q']")
+
+      # select any existing text and overwrite it
+      input.focus.type(:home); sleep 0.2
+      input.focus.type(:shift, :end); sleep 0.2
+      input.focus.type(q); sleep 0.2
+    end
+
+    if location.length > 1 then
+
+      input2 = @browser.at_xpath("//input[@name='l']")
+
+      # select any existing text and overwrite it
+      input2.focus.type(:home); sleep 0.2
+      input2.focus.type(:shift, :end); sleep 0.2
+      input2.focus.type(location); sleep 0.2
+
+    end
+
+    button = @browser.at_xpath("//button[@type='submit']")
+    button.click
+    #@browser.network.wait_for_idle
+    puts 'sleeping for 2 seconds' if @debug
+    sleep 2
+
+    doc2 = Nokogiri::XML(@browser.body)
+
+    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
+    puts 'a2: ' + a2.length.inspect if @debug
+
+    @a2 = a2.map {|x| Rexle.new x.to_s }
+
+    @results = @a2.map do |doc|
+
+      div = doc.element("a[@class='desktop']/div[@class='slider"  \
+          "_container']/div[@class='slider_list']/div[@class='sl"  \
+          "ider_item']/div[@class='job_seen_beacon']")
+      td = div.element("table[@class='jobCard_mainContent']/tbo"  \
+          "dy/tr/td[@class='resultContent']")
+
+      # job title (e.g. Software Developer)
+      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
+          "class='jobTitle-color-purple']/span")&.text
+      puts 'jobtitle: ' + jobtitle.inspect if @debug
+
+      salary = td.element("div[@class='metadataContainer']/"  \
+          "div[@class='salary-snippet-container']/div[@class='sa"  \
+          "lary-snippet']/span")&.text
+
+      puts 'salary: ' + salary.inspect if @debug
+      div1 = td.element("div[@class='companyInfo']")
+
+      # company name (e.g. Coda Octopus Products Ltd)
+      company_name = div1.element("span[@class='companyName']")&.text
+
+      # company location (e.g. Edinburgh)
+      location = div1.element("div[@class='companyLocation']")&.text
+      tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
+
+      div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
+          "v[@class='result-footer']")
+
+      # job (e.g. Our products are primarily written in C#, using...)
+      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+
+      # visually (e.g. Posted 14 days ago)
+      dateposted =  div3.element("span[@class='date']")&.texts
+      date = (Date.today - dateposted.first.to_i).to_s if dateposted
+
+      {
+        link:  @url_base.sub(/\/[^\/]+$/,'') \
+          + doc.root.attributes[:href].gsub(/&amp;/,'&'),
+        title: jobtitle,
+        salary: salary,
+        company: company_name,
+        location: location,
+        jobsnippet: jobsnippet,
+        date: date
+      }
+
+    end
+  end
+
   def page(n)
 
     if n < 1 or n > @results.length then
@@ -64,16 +167,18 @@ class IndeedScraper2022
         "ickyContainer-companyrating']/div/div[@class='icl-u-x"  \
         "s-mr--xs']")[1]
     clink = div3.element('//a')
-    company = cname ? cname.text : clink.text
+    company = cname.text ? cname.text : clink.text
     companylink = clink.attributes[:href] if clink
 
+    salary = div1.element("//span[@class='attribute_snippet']")&.text
+    type = div1.element("//span[@class='jobsearch-JobMetadataHeader-item']")&.texts&.last
     div5 = div3.xpath("div/div")
     location, worklocation = div5.map(&:text).compact
 
     # icl (e.g. Full-time, Permanent)
     jobtype = div1.element("div/div/div[@class='jobsearch-J"  \
         "obMetadataHeader-item']/span[@class='icl-u-xs-mt--xs']")
-    jobtype = jobtype.texts.join if jobtype
+    jobtype = jobtype&.texts.join if jobtype
 
     # jobsearch (e.g. Urgently needed)
     jobnote1 = e0.element("//div[@class='jobsearch-DesktopTag"  \
@@ -81,98 +186,87 @@ class IndeedScraper2022
         "h-DesktopTag-text']")&.text
 
     # jobsearch (e.g. 10 days ago)
-    datepost = e0.element("//div[@class='jobsearch-JobTab-con"  \
-        "tent']/div[@class='jobsearch-JobMetadataFooter']/div")&.text
+    days = e0.element("//div[@class='jobsearch-JobTab-con"  \
+        "tent']/div[@class='jobsearch-JobMetadataFooter']/div[2]")&.text
+    d = Date.today - days.to_i
+    datepost = d.strftime("%Y-%m-%d")
+
 
     jobdesc = e0.element("//div[@class='icl-u-xs-mt--md']/div[@cl"  \
         "ass='jobsearch-jobDescriptionText']").xml
 
     {
       title: jobtitle,
+      type: type,
       company: company,
       companylink: companylink,
       location: location,
+      salary: salary,
       worklocation: worklocation,
       note: jobnote1,
-      date: (Date.today - datepost.to_i).to_s,
+      date: datepost,
       desc: jobdesc
     }
 
   end
 
-  def search(q='', location='')
 
-    a = Mechanize.new
-
-    page = a.get(@url_base)
-    form = page.forms.first
-    form.fields[0].value = @q
-    form.fields[1].value = @location
-    pg = form.submit
+end
 
-    doc2 = Nokogiri::XML(pg.body)
+class IS22Plus < IndeedScraper2022
 
-    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
-    puts 'a2: ' + a2.length.inspect if @debug
+  def initialize(q: '', location: '', headless: true, cookies: nil, debug: false)
+    super(q: q, location: location, headless: headless, cookies: cookies,
+          debug: debug)
+  end
 
-    @a2 = a2.map {|x| Rexle.new x.to_s }
+  def archive(filepath='/tmp/indeed')
 
-    @a2.map do |doc|
+    return unless @results
 
-      div = doc.element("a[@class='desktop']/div[@class='slider"  \
-          "_container']/div[@class='slider_list']/div[@class='sl"  \
-          "ider_item']/div[@class='job_seen_beacon']")
-      td = div.element("table[@class='jobCard_mainContent']/tbo"  \
-          "dy/tr/td[@class='resultContent']")
+    FileUtils.mkdir_p filepath
 
-      # job title (e.g. Software Developer)
-      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
-          "class='jobTitle-color-purple']/span")&.text
-      puts 'jobtitle: ' + jobtitle.inspect if @debug
+    idxfile = File.join(filepath, 'index.yml')
 
-      salary = td.element("div[@class='metadataContainer']/"  \
-          "div[@class='salary-snippet-container']/div[@class='sa"  \
-          "lary-snippet']/span")&.text
+    index = if File.exists? idxfile then
+      YAML.load(File.read(idxfile))
+    else
+      {}
+    end
 
-      puts 'salary: ' + salary.inspect if @debug
-      div1 = td.element("div[@class='companyInfo']")
+    @results.each.with_index do |item, i|
 
-      # company name (e.g. Coda Octopus Products Ltd)
-      company_name = div1.element("span[@class='companyName']")&.text
+      puts 'saving ' + item[:title] if @debug
+      puts 'link: ' + item[:link].inspect
+      links = RXFReader.reveal(item[:link])
+      puts 'links: ' + links.inspect
 
-      # company location (e.g. Edinburgh)
-      location = div1.element("div[@class='companyLocation']")&.text
-      tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
+      url = links.last
+      id = url[/(?<=\?jk=)[^&]+/]
 
-      div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
-          "v[@class='result-footer']")
+      if index[id.to_sym] then
+        next
+      else
 
-      # job (e.g. Our products are primarily written in C#, using...)
-      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+        File.write File.join(filepath, 'j' + id + '.txt'), page(i+1)
 
-      # visually (e.g. Posted 14 days ago)
-      dateposted =  div3.element("span[@class='date']").texts
-      date = (Date.today - dateposted.first.to_i).to_s
+        h = {
+          link: url[/^[^&]+/],
+          title: item[:title].to_s,
+          salary: item[:salary].to_s,
+          company: item[:company].to_s.strip,
+          location: item[:location].to_s,
+          jobsnippet: item[:jobsnippet],
+          date: item[:date]
+        }
 
-      {
-        link:  @url_base.sub(/\/[^\/]+$/,'') \
-          + doc.root.attributes[:href].gsub(/&amp;/,'&'),
-        title: jobtitle,
-        salary: salary,
-        company: company_name,
-        location: location,
-        jobsnippet: jobsnippet,
-        date: date
-      }
+        index[id.to_sym] = h
+      end
 
     end
-  end
-end
 
-class IS22Plus < IndeedScraper2022
+    File.write idxfile, index.to_yaml
 
-  def initialize(q: '', location: '', debug: false)
-    super(q: q, location: location, debug: debug)
   end
 
   def list()
@@ -183,4 +277,5 @@ class IS22Plus < IndeedScraper2022
 
   end
 
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.3.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-03-22 00:00:00.000000000 Z
+date: 2022-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi
@@ -58,25 +58,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.0
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: ferrumwizard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.4
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.4
+        version: 0.2.2
 description: 
 email: digital.robertson@gmail.com
 executables: []