indeed_scraper2022 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bcc9484e0c35971209ee1419b9ff7d07d912831eedf889d10789c2ab4bf0863
-  data.tar.gz: c3be84be0f499ed92794a8de0c3109cc99d3ba73f6aef74f9426a12b0a250e10
+  metadata.gz: 07f5323381b5751c470454f6f4c3ba6dced6f1424e054b85360a49d814d662ba
+  data.tar.gz: 3d25353b9f8a0543944cac82ef6dc91adf7d3e83444f3c6ef469f15cbba8a3d8
 SHA512:
-  metadata.gz: 9e6d64b754bbd242f6d038f385602ed4dc5fe2d8112da12709a673a494419a159c9afe2ef4fa6bc95cccd160f9bceb22b03783108ffec213ed96442372092f81
-  data.tar.gz: eae4c65e2cd7784aff9c3db394fa29e44344e61a5275a31790a1fc21c57df34204585aa5e9e3db8af28a251cf2366675d4d9027b9a6578456dd9f64744842caa
+  metadata.gz: 5e13ae04b46bfa3eb15aab8d0aff388d8caec591c413493db591c37da099d2bcd5ba340a72137d4aa7d374652b68bc1d037b86fe4cc2ed2ae5b0a56c5202f00b
+  data.tar.gz: ca14ae99251aabbcaee08a3bb6f240742ed1fab0f438496dc742ef39a10abb13e310b2d6a93bc472f5e1b3e45cfd8956d6a62f803b1d3a152054cf4e1ae35402

data/lib/indeed_scraper2022.rb

@@ -2,7 +2,7 @@
 
 # file: indeed_scraper2022.rb
 
-require 'mechanize'
+require 'ferrumwizard'
 require 'nokorexi'
 
 # Given the nature of changes to jobsearch websites,
@@ -14,11 +14,13 @@ end
 
 class IndeedScraper2022
 
-  def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '', debug: false)
+  def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '',
+                 headless: true, cookies: nil, debug: false)
 
     @debug = debug
     @url_base, @q, @location = url, q, location
-    @results = search
+    @headless, @cookies = headless, cookies
+    @results = search(q: @q, location: @location)
 
   end
 
@@ -28,6 +30,87 @@ class IndeedScraper2022
     @results
   end
 
+  def search(q: @q, location: @location, start: nil)
+
+    fw = FerrumWizard.new( headless: @headless, cookies: @cookies, debug: @debug)
+
+    url = @url_base
+    url += 'start=' + start if start
+
+    browser = fw.browser
+    browser.goto(url)
+
+    if q.length > 1 then
+      input = browser.at_xpath("//input[@name='q']")
+      input.focus.type(q)
+    end
+
+    if location.length > 1 then
+      input2 = browser.at_xpath("//input[@name='l']")
+      input2.focus.type(location)
+    end
+
+    button = browser.at_xpath("//button[@type='submit']")
+    button.click
+
+    doc2 = Nokogiri::XML(browser.body)
+
+    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
+    puts 'a2: ' + a2.length.inspect if @debug
+
+    @a2 = a2.map {|x| Rexle.new x.to_s }
+
+    @a2.map do |doc|
+
+      div = doc.element("a[@class='desktop']/div[@class='slider"  \
+          "_container']/div[@class='slider_list']/div[@class='sl"  \
+          "ider_item']/div[@class='job_seen_beacon']")
+      td = div.element("table[@class='jobCard_mainContent']/tbo"  \
+          "dy/tr/td[@class='resultContent']")
+
+      # job title (e.g. Software Developer)
+      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
+          "class='jobTitle-color-purple']/span")&.text
+      puts 'jobtitle: ' + jobtitle.inspect if @debug
+
+      salary = td.element("div[@class='metadataContainer']/"  \
+          "div[@class='salary-snippet-container']/div[@class='sa"  \
+          "lary-snippet']/span")&.text
+
+      puts 'salary: ' + salary.inspect if @debug
+      div1 = td.element("div[@class='companyInfo']")
+
+      # company name (e.g. Coda Octopus Products Ltd)
+      company_name = div1.element("span[@class='companyName']")&.text
+
+      # company location (e.g. Edinburgh)
+      location = div1.element("div[@class='companyLocation']")&.text
+      tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
+
+      div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
+          "v[@class='result-footer']")
+
+      # job (e.g. Our products are primarily written in C#, using...)
+      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+
+      # visually (e.g. Posted 14 days ago)
+      dateposted =  div3.element("span[@class='date']")&.texts
+      date = (Date.today - dateposted.first.to_i).to_s if dateposted
+
+      {
+        link:  @url_base.sub(/\/[^\/]+$/,'') \
+          + doc.root.attributes[:href].gsub(/&/,'&'),
+        title: jobtitle,
+        salary: salary,
+        company: company_name,
+        location: location,
+        jobsnippet: jobsnippet,
+        date: date
+      }
+
+    end
+  end
+
   def page(n)
 
     if n < 1 or n > @results.length then
@@ -64,16 +147,18 @@ class IndeedScraper2022
         "ickyContainer-companyrating']/div/div[@class='icl-u-x"  \
         "s-mr--xs']")[1]
     clink = div3.element('//a')
-    company = cname ? cname.text : clink.text
+    company = cname.text ? cname.text : clink.text
     companylink = clink.attributes[:href] if clink
 
+    salary = div1.element("//span[@class='attribute_snippet']")&.text
+    type = div1.element("//span[@class='jobsearch-JobMetadataHeader-item']")&.texts&.last
     div5 = div3.xpath("div/div")
     location, worklocation = div5.map(&:text).compact
 
     # icl (e.g. Full-time, Permanent)
     jobtype = div1.element("div/div/div[@class='jobsearch-J"  \
         "obMetadataHeader-item']/span[@class='icl-u-xs-mt--xs']")
-    jobtype = jobtype.texts.join if jobtype
+    jobtype = jobtype&.texts.join if jobtype
 
     # jobsearch (e.g. Urgently needed)
     jobnote1 = e0.element("//div[@class='jobsearch-DesktopTag"  \
@@ -81,98 +166,46 @@ class IndeedScraper2022
         "h-DesktopTag-text']")&.text
 
     # jobsearch (e.g. 10 days ago)
-    datepost = e0.element("//div[@class='jobsearch-JobTab-con"  \
-        "tent']/div[@class='jobsearch-JobMetadataFooter']/div")&.text
+    days = e0.element("//div[@class='jobsearch-JobTab-con"  \
+        "tent']/div[@class='jobsearch-JobMetadataFooter']/div[2]")&.text
+    d = Date.today - days.to_i
+    datepost = d.strftime("%Y-%m-%d")
+
 
     jobdesc = e0.element("//div[@class='icl-u-xs-mt--md']/div[@cl"  \
         "ass='jobsearch-jobDescriptionText']").xml
 
     {
       title: jobtitle,
+      type: type,
       company: company,
       companylink: companylink,
       location: location,
+      salary: salary,
       worklocation: worklocation,
       note: jobnote1,
-      date: (Date.today - datepost.to_i).to_s,
+      date: datepost,
       desc: jobdesc
     }
 
   end
 
-  def search(q='', location='')
-
-    a = Mechanize.new
-
-    page = a.get(@url_base)
-    form = page.forms.first
-    form.fields[0].value = @q
-    form.fields[1].value = @location
-    pg = form.submit
-
-    doc2 = Nokogiri::XML(pg.body)
-
-    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
-    puts 'a2: ' + a2.length.inspect if @debug
-
-    @a2 = a2.map {|x| Rexle.new x.to_s }
-
-    @a2.map do |doc|
-
-      div = doc.element("a[@class='desktop']/div[@class='slider"  \
-          "_container']/div[@class='slider_list']/div[@class='sl"  \
-          "ider_item']/div[@class='job_seen_beacon']")
-      td = div.element("table[@class='jobCard_mainContent']/tbo"  \
-          "dy/tr/td[@class='resultContent']")
-
-      # job title (e.g. Software Developer)
-      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
-          "class='jobTitle-color-purple']/span")&.text
-      puts 'jobtitle: ' + jobtitle.inspect if @debug
-
-      salary = td.element("div[@class='metadataContainer']/"  \
-          "div[@class='salary-snippet-container']/div[@class='sa"  \
-          "lary-snippet']/span")&.text
-
-      puts 'salary: ' + salary.inspect if @debug
-      div1 = td.element("div[@class='companyInfo']")
 
-      # company name (e.g. Coda Octopus Products Ltd)
-      company_name = div1.element("span[@class='companyName']")&.text
-
-      # company location (e.g. Edinburgh)
-      location = div1.element("div[@class='companyLocation']")&.text
-      tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
-
-      div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
-          "v[@class='result-footer']")
+end
 
-      # job (e.g. Our products are primarily written in C#, using...)
-      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+class IS22Plus < IndeedScraper2022
 
-      # visually (e.g. Posted 14 days ago)
-      dateposted =  div3.element("span[@class='date']").texts
-      date = (Date.today - dateposted.first.to_i).to_s
+  def initialize(q: '', location: '', headless: true, cookies: nil, debug: false)
+    super(q: q, location: location, headless: headless, cookies: cookies,
+          debug: debug)
+  end
 
-      {
-        link:  @url_base.sub(/\/[^\/]+$/,'') \
-          + doc.root.attributes[:href].gsub(/&amp;/,'&'),
-        title: jobtitle,
-        salary: salary,
-        company: company_name,
-        location: location,
-        jobsnippet: jobsnippet,
-        date: date
-      }
+  def archive()
 
+    1.upto(15).each do |n|
+      page(n)
     end
-  end
-end
 
-class IS22Plus < IndeedScraper2022
-
-  def initialize(q: '', location: '', debug: false)
-    super(q: q, location: location, debug: debug)
   end
 
   def list()
@@ -183,4 +216,5 @@ class IS22Plus < IndeedScraper2022
 
   end
 
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-03-22 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi
@@ -58,25 +58,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.0
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: ferrumwizard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.4
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.4
+        version: 0.2.2
 description: 
 email: digital.robertson@gmail.com
 executables: []