indeed_scraper2022 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0455b8d92dab8642f01c53ce43afc11eb24fbff801cce8df3306fd31eebeb223
-  data.tar.gz: eb8d24662b095fb0049209534ddfb1f0805454787105e091dc15eb7ca363d1b1
+  metadata.gz: 6bcc9484e0c35971209ee1419b9ff7d07d912831eedf889d10789c2ab4bf0863
+  data.tar.gz: c3be84be0f499ed92794a8de0c3109cc99d3ba73f6aef74f9426a12b0a250e10
 SHA512:
-  metadata.gz: 342f6534ef851e12e78bf8e190ea98b437a6503c26510938dcb0021a30033c6be5ae819560a46ab0c11a4b42f833238517bbf9779c35c11e671d3a81ed8eedc9
-  data.tar.gz: 74be894713e2572bef1777cfafe099cc875d258fad6d741703a7cf00d22863c44cf02a877aa5106d73182d17408a16fc16f6b9538f4507c602e5588062a12334
+  metadata.gz: 9e6d64b754bbd242f6d038f385602ed4dc5fe2d8112da12709a673a494419a159c9afe2ef4fa6bc95cccd160f9bceb22b03783108ffec213ed96442372092f81
+  data.tar.gz: eae4c65e2cd7784aff9c3db394fa29e44344e61a5275a31790a1fc21c57df34204585aa5e9e3db8af28a251cf2366675d4d9027b9a6578456dd9f64744842caa

data/lib/indeed_scraper2022.rb

@@ -9,6 +9,9 @@ require 'nokorexi'
 # don't rely upon this gem working in the near future.
 
 
+class IndeedScraper2022Err < Exception
+end
+
 class IndeedScraper2022
 
   def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '', debug: false)
@@ -25,17 +28,78 @@ class IndeedScraper2022
     @results
   end
 
-  def page()
-  end
+  def page(n)
 
-  # used for debugging
-  #
-  def a2()
-    @a2
+    if n < 1 or n > @results.length then
+      raise IndeedScraper2022Err, 'Invalid page no.'
+    end
+
+    url = @results[n-1][:link]
+    fetchjob(url)
   end
 
   private
 
+  def fetchjob(url)
+
+    doc = Nokorexi.new(url).to_doc
+    e0 = doc.element("html/body/div/div/div/div/div/div/div/div")
+
+    #div = e0.element("//div[@class='jobsearch-JobComponent']")
+    div1 = e0.element("//div[@class='jobsearch-DesktopStickyContainer']")
+    div2 = div1.element("div")
+
+    # jobsearch (e.g. Full Stack Website Developer (Wordpress))
+    jobtitle = div2.element("div[@class='jobsearch-JobInfoHead"  \
+        "er-title-container']/h1[@class='jobsearch-JobInfoHead"  \
+        "er-title']")&.text
+
+    div3 = div2.element("div[@class='jobsearch-CompanyInfoCon"  \
+        "tainer']/div[@class='jobsearch-CompanyInfoWithoutHead"  \
+        "erImage']/div/div[@class='jobsearch-DesktopStickyCont"  \
+        "ainer-subtitle']")
+
+    # icl (e.g. Lyles Sutherland)
+    cname = div3.xpath("div[@class='jobsearch-DesktopSt"  \
+        "ickyContainer-companyrating']/div/div[@class='icl-u-x"  \
+        "s-mr--xs']")[1]
+    clink = div3.element('//a')
+    company = cname ? cname.text : clink.text
+    companylink = clink.attributes[:href] if clink
+
+    div5 = div3.xpath("div/div")
+    location, worklocation = div5.map(&:text).compact
+
+    # icl (e.g. Full-time, Permanent)
+    jobtype = div1.element("div/div/div[@class='jobsearch-J"  \
+        "obMetadataHeader-item']/span[@class='icl-u-xs-mt--xs']")
+    jobtype = jobtype.texts.join if jobtype
+
+    # jobsearch (e.g. Urgently needed)
+    jobnote1 = e0.element("//div[@class='jobsearch-DesktopTag"  \
+        "']/div[@class='urgently-hiring']/div[@class='jobsearc"  \
+        "h-DesktopTag-text']")&.text
+
+    # jobsearch (e.g. 10 days ago)
+    datepost = e0.element("//div[@class='jobsearch-JobTab-con"  \
+        "tent']/div[@class='jobsearch-JobMetadataFooter']/div")&.text
+
+    jobdesc = e0.element("//div[@class='icl-u-xs-mt--md']/div[@cl"  \
+        "ass='jobsearch-jobDescriptionText']").xml
+
+    {
+      title: jobtitle,
+      company: company,
+      companylink: companylink,
+      location: location,
+      worklocation: worklocation,
+      note: jobnote1,
+      date: (Date.today - datepost.to_i).to_s,
+      desc: jobdesc
+    }
+
+  end
+
   def search(q='', location='')
 
     a = Mechanize.new
@@ -47,6 +111,7 @@ class IndeedScraper2022
     pg = form.submit
 
     doc2 = Nokogiri::XML(pg.body)
+
     a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
     puts 'a2: ' + a2.length.inspect if @debug
 
@@ -62,34 +127,36 @@ class IndeedScraper2022
 
       # job title (e.g. Software Developer)
       jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
-          "class='jobTitle-color-purple']/span").text
+          "class='jobTitle-color-purple']/span")&.text
       puts 'jobtitle: ' + jobtitle.inspect if @debug
 
       salary = td.element("div[@class='metadataContainer']/"  \
           "div[@class='salary-snippet-container']/div[@class='sa"  \
-          "lary-snippet']/span")
-      salary = salary.text if salary
+          "lary-snippet']/span")&.text
+
       puts 'salary: ' + salary.inspect if @debug
       div1 = td.element("div[@class='companyInfo']")
 
       # company name (e.g. Coda Octopus Products Ltd)
-      company_name = div1.element("span[@class='companyName']").text
+      company_name = div1.element("span[@class='companyName']")&.text
 
       # company location (e.g. Edinburgh)
-      location = div1.element("div[@class='companyLocation']").text
+      location = div1.element("div[@class='companyLocation']")&.text
       tbody = div.element("table[@class='jobCardShelfContainer']/tbody")
 
       div3 = tbody.element("tr[@class='underShelfFooter']/td/di"  \
           "v[@class='result-footer']")
 
       # job (e.g. Our products are primarily written in C#, using...)
-      jobsnippet = div3.element("div[@class='job-snippet']/ul/li").text
+      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
 
       # visually (e.g. Posted 14 days ago)
       dateposted =  div3.element("span[@class='date']").texts
       date = (Date.today - dateposted.first.to_i).to_s
 
       {
+        link:  @url_base.sub(/\/[^\/]+$/,'') \
+          + doc.root.attributes[:href].gsub(/&amp;/,'&'),
         title: jobtitle,
         salary: salary,
         company: company_name,
@@ -101,3 +168,19 @@ class IndeedScraper2022
     end
   end
 end
+
+class IS22Plus < IndeedScraper2022
+
+  def initialize(q: '', location: '', debug: false)
+    super(q: q, location: location, debug: debug)
+  end
+
+  def list()
+
+    @results.map.with_index do |x,i|
+      "%2d. %s" % [i+1,x[:title]]
+    end.join("\n")
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-01-24 00:00:00.000000000 Z
+date: 2022-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi
@@ -43,20 +43,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.5
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.5
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +78,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 2.8.4
 description: 
-email: james@jamesrobertson.eu
+email: digital.robertson@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -96,15 +96,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.10
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: Attempts to scrape the indeed.com jobsearch results (1 page).