indeed_scraper2022 0.1.2 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19d85a0b62308f9e6c53eda22a9b26fb559e62d9528597fde6d89bb117f0deb0
-  data.tar.gz: 43aea0e9efb6d6ed45c1efe4e480d8422939d20f42cd5b8514f8733b6f3ebd03
+  metadata.gz: 5e33dfd54667ecc9f8b7985aa07af403be8d95729ce68e8a40d8c985d57bd4e1
+  data.tar.gz: a2c041ec8103b6afac3a422e7b73bc82c89fd7f8d955240439a29ec0347c8a5f
 SHA512:
-  metadata.gz: bf0592ffdbaf9dbba2e84be8dcbdbf18c1d8fcb25e0e33eacbbb786decb0f8d18240371149ef9850bbde890e5b64893660f2ad2e0d149cb5e40e900802d82bb4
-  data.tar.gz: 60e00efcef2f86e6e64c8f02d45e943ab5cdc1090b3d361f3d6e305b4389694a6061618da693dee1d81b11dbe729f02786c06e71562b10d581b88503189f676a
+  metadata.gz: 8e640cb8262a057bb588b501ee1122a59e6e239e2a5988dd0566ffffb814a2fef763c36fdeae1ba5dc4e6f819ca145374058bd62373ce776df1e393057a49fc0
+  data.tar.gz: 0a6bfe0ef2b685d5711a95704cee3fa67d58eb7c9d0f149c872f9c23b0cc489382ab1327b101f7accf982f7ea1f1a6d56dc20ae528ef1fd4d6105c9ef93067da

data/lib/indeed_scraper2022.rb

@@ -2,20 +2,29 @@
 
 # file: indeed_scraper2022.rb
 
-require 'mechanize'
+require 'ferrumwizard'
 require 'nokorexi'
 
 # Given the nature of changes to jobsearch websites,
 # don't rely upon this gem working in the near future.
 
 
+class IndeedScraper2022Err < Exception
+end
+
 class IndeedScraper2022
 
-  def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '', debug: false)
+  attr_reader :browser
+
+  def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '',
+                 headless: true, cookies: nil, debug: false)
 
     @debug = debug
     @url_base, @q, @location = url, q, location
-    @results = search
+    @headless, @cookies = headless, cookies
+
+    fw = FerrumWizard.new( headless: @headless, cookies: @cookies, debug: @debug)
+    @browser = fw.browser
 
   end
 
@@ -25,7 +34,107 @@ class IndeedScraper2022
     @results
   end
 
+  def search(q: @q, location: @location, start: nil)
+
+    url = @url_base
+    url += 'start=' + start if start
+
+    @browser.goto(url)
+    #@browser.network.wait_for_idle
+    puts 'sleeping for 4 seconds' if @debug
+    sleep 4
+
+    if q.length > 1 then
+
+      input = @browser.at_xpath("//input[@name='q']")
+
+      # select any existing text and overwrite it
+      input.focus.type(:home); sleep 0.2
+      input.focus.type(:shift, :end); sleep 0.2
+      input.focus.type(q); sleep 0.2
+    end
+
+    if location.length > 1 then
+
+      input2 = @browser.at_xpath("//input[@name='l']")
+
+      # select any existing text and overwrite it
+      input2.focus.type(:home); sleep 0.2
+      input2.focus.type(:shift, :end); sleep 0.2
+      input2.focus.type(location); sleep 0.2
+
+    end
+
+    button = @browser.at_xpath("//button[@type='submit']")
+    button.click
+    #@browser.network.wait_for_idle
+    puts 'sleeping for 2 seconds' if @debug
+    sleep 2
+
+    doc2 = Nokogiri::XML(@browser.body)
+
+    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
+    puts 'a2: ' + a2.length.inspect if @debug
+
+    @a2 = a2.map {|x| Rexle.new x.to_s }
+
+    @results = @a2.map do |doc|
+
+      div = doc.element("a[@class='desktop']/div[@class='slider"  \
+          "_container']/div[@class='slider_list']/div[@class='sl"  \
+          "ider_item']/div[@class='job_seen_beacon']")
+      td = div.element("table[@class='jobCard_mainContent']/tbo"  \
+          "dy/tr/td[@class='resultContent']")
+
+      # job title (e.g. Software Developer)
+      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
+          "class='jobTitle-color-purple']/span")&.text
+      puts 'jobtitle: ' + jobtitle.inspect if @debug
+
+      salary = td.element("div[@class='metadataContainer']/"  \
+          "div[@class='salary-snippet-container']/div[@class='sa"  \
+          "lary-snippet']/span")&.text
+
+      puts 'salary: ' + salary.inspect if @debug
+      div1 = td.element("div[@class='companyInfo']")
+
+      # company name (e.g. Coda Octopus Products Ltd)
+      company_name = div1.element("span[@class='companyName']")&.text
+
+      # company location (e.g. Edinburgh)
+      location = div1.element("div[@class='companyLocation']")&.text
+      tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
+
+      div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
+          "v[@class='result-footer']")
+
+      # job (e.g. Our products are primarily written in C#, using...)
+      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+
+      # visually (e.g. Posted 14 days ago)
+      dateposted =  div3.element("span[@class='date']")&.texts
+      date = (Date.today - dateposted.first.to_i).to_s if dateposted
+
+      {
+        link:  @url_base.sub(/\/[^\/]+$/,'') \
+          + doc.root.attributes[:href].gsub(/&amp;/,'&'),
+        title: jobtitle,
+        salary: salary,
+        company: company_name,
+        location: location,
+        jobsnippet: jobsnippet,
+        date: date
+      }
+
+    end
+  end
+
   def page(n)
+
+    if n < 1 or n > @results.length then
+      raise IndeedScraper2022Err, 'Invalid page no.'
+    end
+
     url = @results[n-1][:link]
     fetchjob(url)
   end
@@ -56,16 +165,18 @@ class IndeedScraper2022
         "ickyContainer-companyrating']/div/div[@class='icl-u-x"  \
         "s-mr--xs']")[1]
     clink = div3.element('//a')
-    company = cname ? cname.text : clink.text
+    company = cname.text ? cname.text : clink.text
     companylink = clink.attributes[:href] if clink
 
+    salary = div1.element("//span[@class='attribute_snippet']")&.text
+    type = div1.element("//span[@class='jobsearch-JobMetadataHeader-item']")&.texts&.last
     div5 = div3.xpath("div/div")
     location, worklocation = div5.map(&:text).compact
 
     # icl (e.g. Full-time, Permanent)
     jobtype = div1.element("div/div/div[@class='jobsearch-J"  \
         "obMetadataHeader-item']/span[@class='icl-u-xs-mt--xs']")
-    jobtype = jobtype.texts.join if jobtype
+    jobtype = jobtype&.texts.join if jobtype
 
     # jobsearch (e.g. Urgently needed)
     jobnote1 = e0.element("//div[@class='jobsearch-DesktopTag"  \
@@ -73,106 +184,57 @@ class IndeedScraper2022
         "h-DesktopTag-text']")&.text
 
     # jobsearch (e.g. 10 days ago)
-    datepost = e0.element("//div[@class='jobsearch-JobTab-con"  \
-        "tent']/div[@class='jobsearch-JobMetadataFooter']/div")&.text
+    days = e0.element("//div[@class='jobsearch-JobTab-con"  \
+        "tent']/div[@class='jobsearch-JobMetadataFooter']/div[2]")&.text
+    d = Date.today - days.to_i
+    datepost = d.strftime("%Y-%m-%d")
+
 
     jobdesc = e0.element("//div[@class='icl-u-xs-mt--md']/div[@cl"  \
         "ass='jobsearch-jobDescriptionText']").xml
 
     {
       title: jobtitle,
+      type: type,
       company: company,
       companylink: companylink,
       location: location,
+      salary: salary,
       worklocation: worklocation,
       note: jobnote1,
-      date: (Date.today - datepost.to_i).to_s,
+      date: datepost,
       desc: jobdesc
     }
 
   end
 
-  def search(q='', location='')
-
-    a = Mechanize.new
-
-    page = a.get(@url_base)
-    form = page.forms.first
-    form.fields[0].value = @q
-    form.fields[1].value = @location
-    pg = form.submit
-
-    doc2 = Nokogiri::XML(pg.body)
-
-    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
-    puts 'a2: ' + a2.length.inspect if @debug
-
-    @a2 = a2.map {|x| Rexle.new x.to_s }
-
-    @a2.map do |doc|
-
-      div = doc.element("a[@class='desktop']/div[@class='slider"  \
-          "_container']/div[@class='slider_list']/div[@class='sl"  \
-          "ider_item']/div[@class='job_seen_beacon']")
-      td = div.element("table[@class='jobCard_mainContent']/tbo"  \
-          "dy/tr/td[@class='resultContent']")
-
-      # job title (e.g. Software Developer)
-      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
-          "class='jobTitle-color-purple']/span")&.text
-      puts 'jobtitle: ' + jobtitle.inspect if @debug
-
-      salary = td.element("div[@class='metadataContainer']/"  \
-          "div[@class='salary-snippet-container']/div[@class='sa"  \
-          "lary-snippet']/span")&.text
 
-      puts 'salary: ' + salary.inspect if @debug
-      div1 = td.element("div[@class='companyInfo']")
-
-      # company name (e.g. Coda Octopus Products Ltd)
-      company_name = div1.element("span[@class='companyName']")&.text
-
-      # company location (e.g. Edinburgh)
-      location = div1.element("div[@class='companyLocation']")&.text
-      tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
+end
 
-      div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
-          "v[@class='result-footer']")
+class IS22Plus < IndeedScraper2022
 
-      # job (e.g. Our products are primarily written in C#, using...)
-      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+  def initialize(q: '', location: '', headless: true, cookies: nil, debug: false)
+    super(q: q, location: location, headless: headless, cookies: cookies,
+          debug: debug)
+  end
 
-      # visually (e.g. Posted 14 days ago)
-      dateposted =  div3.element("span[@class='date']").texts
-      date = (Date.today - dateposted.first.to_i).to_s
+  def archive()
 
-      {
-        link:  @url_base.sub(/\/[^\/]+$/,'') \
-          + doc.root.attributes[:href].gsub(/&amp;/,'&'),
-        title: jobtitle,
-        salary: salary,
-        company: company_name,
-        location: location,
-        jobsnippet: jobsnippet,
-        date: date
-      }
+    return unless @results
 
+    1.upto(@results.length).each do |n|
+      page(n)
     end
-  end
-end
-
-class IS22Plus < IndeedScraper2022
 
-  def initialize(q: '', location: '', debug: false)
-    super(q: q, location: location, debug: debug)
   end
 
   def list()
 
     @results.map.with_index do |x,i|
-      "%2d. %s" % [i,x[:title]]
+      "%2d. %s" % [i+1,x[:title]]
     end.join("\n")
 
   end
 
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.1
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-01-25 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi
@@ -43,42 +43,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.5
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.5
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: ferrumwizard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.4
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.4
+        version: 0.2.2
 description: 
-email: digital.robertson@Gmail.com
+email: digital.robertson@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -103,8 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.10
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Attempts to scrape the indeed.com jobsearch results (1 page).