indeed_scraper2022 0.4.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f98a83b7ed582d1b2973882701833688aec2d6d2bd132241a26c01a32915f93
-  data.tar.gz: dc5c34a5af19cdffbd244e15416914e91c8a06f365f0fff28bcd537a30ec468e
+  metadata.gz: e9d1593bc7531fd77592f8765ccac9011f47980f0fb1f0d7e9c89513a319b85e
+  data.tar.gz: ac88d94ce2271fdc4561f43ff7218baed73db60867506c20bc34522fd357cb89
 SHA512:
-  metadata.gz: 4ce40e021339f6b1c24faed495ebbd4b257200f62d1466ffa54cd05e654ccef29b23dbfcf4af64b9426f4d2dbde6bb778f6b920a7656af8289e3f81a269ba54a
-  data.tar.gz: 634325fed61c7888b08fd72bfc47f4d64f98f5514110169152389e49940f15a080c02f62811aae2b5f34c6a604c17d2512d2219dba0f4473dc1034900bdb7ec6
+  metadata.gz: 133e26d46ca8bae7b08aff57a7451c4cf23f507f4cb4e3875f3b3ca320c60e14f747952f1b893a8016cfd3dfb907440597864158e0cf9a5dc92a833d98855aff
+  data.tar.gz: 0c5d883b0b9a1582a2b83c8a64d47e420b7c0f79f2e6f8b1e409439fd171539adca26f10c66b57db19f9d5238c3017649c527d0e2f8fe405b3aa73c4cf276e8f

data/lib/indeed_scraper2022.rb

@@ -5,6 +5,7 @@
 require 'ferrumwizard'
 require 'nokorexi'
 require 'yaml'
+require 'reveal_url22'
 
 # Given the nature of changes to jobsearch websites,
 # don't rely upon this gem working in the near future.
@@ -44,9 +45,10 @@ class IndeedScraper2022
   end
 
   def search(q: @q, location: @location, start: nil)
-
+    puts 'inside search' if @debug
     url = @url_base
     url += 'start=' + start if start
+    puts 'url: ' + url.inspect if @debug
 
     @browser.goto(url)
     #@browser.network.wait_for_idle
@@ -81,34 +83,52 @@ class IndeedScraper2022
     sleep 2
 
     doc2 = Nokogiri::XML(@browser.body)
+    File.write '/tmp/body.txt', doc2.to_s if @debug
 
-    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
+    a2 = doc2.root.xpath  "//li/div[div/div/div/div/table/tbody/tr/td/div/h2/a]"
     puts 'a2: ' + a2.length.inspect if @debug
 
     @a2 = a2.map {|x| Rexle.new x.to_s }
 
     @results = @a2.map do |doc|
 
-      div = doc.element("a[@class='desktop']/div[@class='slider"  \
+      div = doc.element("div[@class='cardOutline']/div[@class='slider"  \
           "_container']/div[@class='slider_list']/div[@class='sl"  \
           "ider_item']/div[@class='job_seen_beacon']")
+
       td = div.element("table[@class='jobCard_mainContent']/tbo"  \
           "dy/tr/td[@class='resultContent']")
 
       # job title (e.g. Software Developer)
-      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
-          "class='jobTitle-color-purple']/span")&.text
+      job = td.element("div[@class='tapItem-gutter']/h2[@"  \
+          "class='jobTitle-color-purple']/a")
+      href = job.attributes[:href]
+      jobtitle = job.element("span")&.text
+
       puts 'jobtitle: ' + jobtitle.inspect if @debug
 
-      salary = td.element("div[@class='metadataContainer']/"  \
-          "div[@class='salary-snippet-container']/div[@class='sa"  \
-          "lary-snippet']/span")&.text
+      sal = td.element("div[@class='metadataContainer']/"  \
+          "div[@class='salary-snippet-container']")
+
+      salary = if sal then
+        sal_e = sal.element("div[@class='attribute_snippet']")
+        if sal_e then
+          sal_e.texts[0]
+        else
+          sal_e2 = sal.element("div[@class='salary-snippet']/span")
+          sal_e2 ? sal_e2.text : ''
+        end
+      else
+        ''
+      end
 
       puts 'salary: ' + salary.inspect if @debug
       div1 = td.element("div[@class='companyInfo']")
 
       # company name (e.g. Coda Octopus Products Ltd)
-      company_name = div1.element("span[@class='companyName']")&.text
+      coname = div1.element("span[@class='companyName']")
+      puts 'coname: ' + coname.text.inspect if @debug
+      company_name = coname.text.to_s.strip.length > 1 ? coname.text : coname.element('a').text
 
       # company location (e.g. Edinburgh)
       location = div1.element("div[@class='companyLocation']")&.text
@@ -118,7 +138,12 @@ class IndeedScraper2022
           "v[@class='result-footer']")
 
       # job (e.g. Our products are primarily written in C#, using...)
-      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+      advert_items = div3.xpath("div[@class='job-snippet']/ul/li/text()")
+      jobsnippet = if advert_items.any? then
+        advert_items.join("\n")
+      else
+        div3.element("div[@class='job-snippet']").text
+      end
 
       # visually (e.g. Posted 14 days ago)
       dateposted =  div3.element("span[@class='date']")&.texts
@@ -126,7 +151,7 @@ class IndeedScraper2022
 
       {
         link:  @url_base.sub(/\/[^\/]+$/,'') \
-          + doc.root.attributes[:href].gsub(/&/,'&'),
+          + href.gsub(/&/,'&'),
         title: jobtitle,
         salary: salary,
         company: company_name,
@@ -237,7 +262,7 @@ class IS22Plus < IndeedScraper2022
 
   def initialize(q: '', location: '', headless: true, cookies: nil, debug: false)
     super(q: q, location: location, headless: headless, cookies: cookies,
-          debug: debug)
+          debug: true)
   end
 
   # note: The most efficient method to accumulate vacancy articles is to
@@ -263,10 +288,8 @@ class IS22Plus < IndeedScraper2022
 
       puts 'saving ' + item[:title] if @debug
       puts 'link: ' + item[:link].inspect
-      links = RXFReader.reveal(item[:link])
-      puts 'links: ' + links.inspect if @debug
-
-      url = links.last
+      url = URL.reveal(item[:link])
+      item[:link] = url
       puts 'url: ' + url.inspect if @debug
       id = url[/(?<=jk=)[^&]+/]
 
@@ -288,7 +311,7 @@ class IS22Plus < IndeedScraper2022
           salary: item[:salary].to_s,
           company: item[:company].to_s.strip,
           location: item[:location].to_s,
-          jobsnippet: item[:jobsnippet],
+          jobsnippet: item[:jobsnippet].to_s,
           date: item[:date],
           added: Time.now.strftime("%Y-%m-%d")
         }
@@ -324,8 +347,6 @@ class IS22Archive
 
   def initialize(filepath='/tmp/indeed', debug: false)
 
-    @debug = debug
-
     FileUtils.mkdir_p filepath
     @idxfile = File.join(filepath, 'index.yml')
 
@@ -339,12 +360,13 @@ class IS22Archive
 
   def list()
 
-    @index.map.with_index do |x,i|
+    @index.to_a.reverse.map.with_index do |x,i|
 
       id, h = x
 
       puts 'h: ' + h.inspect if @debug
-      "%2d. %s: %s" % [i+1, Date.parse(h[:added]).strftime("%d %b"), h[:title]]
+      co = h[:company].length > 1 ? " (%s)" % h[:company] : ''
+      "%2d. %s: %s%s" % [i+1, Date.parse(h[:added]).strftime("%d %b"), h[:title], co]
 
     end.join("\n")
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.1
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-04-14 00:00:00.000000000 Z
+date: 2022-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi
@@ -77,6 +77,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.1
+- !ruby/object:Gem::Dependency
+  name: reveal_url22
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: 
 email: digital.robertson@gmail.com
 executables: []