indeed_scraper2022 0.4.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88f80a06ef0ab435c144d3b4ec53f1c98f2da7c427224c31dbef44f62fdafee3
-  data.tar.gz: d0f549053bb225e7c8ebb2492715c6c470f689e191e9ae747e8f97317a61c02c
+  metadata.gz: 1cf994e7ea8c7fd89ad96e55d2d3f2d3908b8c565e3f07bad5056e4d63b90b65
+  data.tar.gz: eff54d2290a32a6a78cd31646ceef2736d329b40ea327f72f7a5eb4594dd3fe2
 SHA512:
-  metadata.gz: 90d23c6c35a87cdcf763dc15a072f35ede166aafe2f1f5eed9294de28e916cdaca3eb043a034fb50750c9444af9e5042dc50d01390816efcdde360d2d01c4e55
-  data.tar.gz: 30153c9c5aafdb5e89d56632e223edfe0196a71c4d9294cc3f963f2039238cd27b836ac9ac42b49cad27988ada868d0d002246699399f7d175216b97689d46ed
+  metadata.gz: 4da62ce0099ee02985981f8f166e371a4b05388998c94b5b5d3e33637e7dc649cc7aefbbeb19e002d777634b0bef9222563aad552c89d575757633c7764b5eb9
+  data.tar.gz: dc4a0bcdb86bef4925880c95c5b558de5e1c4cff0756e134f8502be97e0dc2a16e4e38fd7217db6b9994a02c2384956c90ec7a66913b8baaf5fbdfe92d46b322

data/lib/indeed_scraper2022.rb

@@ -5,6 +5,7 @@
 require 'ferrumwizard'
 require 'nokorexi'
 require 'yaml'
+require 'reveal_url22'
 
 # Given the nature of changes to jobsearch websites,
 # don't rely upon this gem working in the near future.
@@ -44,9 +45,10 @@ class IndeedScraper2022
   end
 
   def search(q: @q, location: @location, start: nil)
-
+    puts 'inside search' if @debug
     url = @url_base
     url += 'start=' + start if start
+    puts 'url: ' + url.inspect if @debug
 
     @browser.goto(url)
     #@browser.network.wait_for_idle
@@ -81,28 +83,44 @@ class IndeedScraper2022
     sleep 2
 
     doc2 = Nokogiri::XML(@browser.body)
+    File.write '/tmp/body.txt', doc2.to_s if @debug
 
-    a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
+    a2 = doc2.root.xpath  "//li/div[div/div/div/div/table/tbody/tr/td/div/h2/a]"
     puts 'a2: ' + a2.length.inspect if @debug
 
     @a2 = a2.map {|x| Rexle.new x.to_s }
 
     @results = @a2.map do |doc|
 
-      div = doc.element("a[@class='desktop']/div[@class='slider"  \
+      div = doc.element("div[@class='cardOutline']/div[@class='slider"  \
           "_container']/div[@class='slider_list']/div[@class='sl"  \
           "ider_item']/div[@class='job_seen_beacon']")
+
       td = div.element("table[@class='jobCard_mainContent']/tbo"  \
           "dy/tr/td[@class='resultContent']")
 
       # job title (e.g. Software Developer)
-      jobtitle = td.element("div[@class='tapItem-gutter']/h2[@"  \
-          "class='jobTitle-color-purple']/span")&.text
+      job = td.element("div[@class='tapItem-gutter']/h2[@"  \
+          "class='jobTitle-color-purple']/a")
+      href = job.attributes[:href]
+      jobtitle = job.element("span")&.text
+
       puts 'jobtitle: ' + jobtitle.inspect if @debug
 
-      salary = td.element("div[@class='metadataContainer']/"  \
-          "div[@class='salary-snippet-container']/div[@class='sa"  \
-          "lary-snippet']/span")&.text
+      sal = td.element("div[@class='metadataContainer']/"  \
+          "div[@class='salary-snippet-container']")
+
+      salary = if sal then
+        sal_e = sal.element("div[@class='attribute_snippet']")
+        if sal_e then
+          sal_e.texts[0]
+        else
+          sal_e2 = sal.element("div[@class='salary-snippet']/span")
+          sal_e2 ? sal_e2.text : ''
+        end
+      else
+        ''
+      end
 
       puts 'salary: ' + salary.inspect if @debug
       div1 = td.element("div[@class='companyInfo']")
@@ -120,7 +138,12 @@ class IndeedScraper2022
           "v[@class='result-footer']")
 
       # job (e.g. Our products are primarily written in C#, using...)
-      jobsnippet = div3.xpath("div[@class='job-snippet']/ul/li/text()").join("\n")
+      advert_items = div3.xpath("div[@class='job-snippet']/ul/li/text()")
+      jobsnippet = if advert_items.any? then
+        advert_items.join("\n")
+      else
+        div3.element("div[@class='job-snippet']").text
+      end
 
       # visually (e.g. Posted 14 days ago)
       dateposted =  div3.element("span[@class='date']")&.texts
@@ -128,7 +151,7 @@ class IndeedScraper2022
 
       {
         link:  @url_base.sub(/\/[^\/]+$/,'') \
-          + doc.root.attributes[:href].gsub(/&/,'&'),
+          + href.gsub(/&/,'&'),
         title: jobtitle,
         salary: salary,
         company: company_name,
@@ -265,10 +288,8 @@ class IS22Plus < IndeedScraper2022
 
       puts 'saving ' + item[:title] if @debug
       puts 'link: ' + item[:link].inspect
-      links = RXFReader.reveal(item[:link])
-      puts 'links: ' + links.inspect if @debug
-
-      url = links.last
+      url = URL.reveal(item[:link])
+      item[:link] = url
       puts 'url: ' + url.inspect if @debug
       id = url[/(?<=jk=)[^&]+/]
 
@@ -290,7 +311,7 @@ class IS22Plus < IndeedScraper2022
           salary: item[:salary].to_s,
           company: item[:company].to_s.strip,
           location: item[:location].to_s,
-          jobsnippet: item[:jobsnippet],
+          jobsnippet: item[:jobsnippet].to_s,
           date: item[:date],
           added: Time.now.strftime("%Y-%m-%d")
         }
@@ -321,18 +342,17 @@ end
 
 
 class IS22Archive
+  include RXFReadWriteModule
 
   attr_reader :index
 
   def initialize(filepath='/tmp/indeed', debug: false)
 
-    @debug = debug
-
-    FileUtils.mkdir_p filepath
+    FileX.mkdir_p filepath
     @idxfile = File.join(filepath, 'index.yml')
 
-    @index = if File.exists? @idxfile then
-      YAML.load(File.read(@idxfile))
+    @index = if FileX.exists? @idxfile then
+      YAML.load(FileX.read(@idxfile))
     else
       {}
     end
@@ -353,4 +373,67 @@ class IS22Archive
 
   end
 
+  def to_html()
+
+    rows = latest().map do |h|
+
+      puts 'h: ' + h.inspect if @debug
+      co = h[:company].length > 1 ? " (%s)" % h[:company] : ''
+      "* %s: [%s](%s)%s" % [h[:added].strftime("%d %b"), h[:title], h[:link], co]
+
+    end.join("\n")
+
+
+    md = '# Indeed.com: Latest jobs
+
+' + rows
+
+    RDiscount.new(md).to_html
+
+  end
+
+  def to_form(action: '')
+
+    rows = latest().map.with_index do |h, i|
+
+      co = h[:company].length > 1 ? " (%s)" % h[:company] : ''
+
+      "<input type='checkbox' id='#{h[:jobid]}' name='#{h[:jobid]}' value='#{h[:title]}'/>
+      <label for='j#{i}'>#{h[:added].strftime("%d %b")}: #{h[:title] + ' ' + co}</label><br/>
+      "
+
+    end.join("\n")
+
+
+    return "<form action='#{action}'>#{rows}" +
+        "<input type='submit' value='submit'/></form>"
+
+  end
+
+  def filter(a)
+
+    dx = Dynarex.new
+    a2 = latest().select {|h| a.include? h[:jobid] }
+    dx.import a2
+
+    return dx
+  end
+
+  private
+
+  def latest()
+
+    a = @index.to_a.map do |id, h|
+      h[:jobid] = id
+      h[:added] = Date.parse(h[:added]) if h[:added].is_a? String
+      h
+    end
+
+    a.select do |x|
+      x[:added] >= (Date.today - 7)
+    end.reverse
+
+  end
+
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.2
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-04-16 00:00:00.000000000 Z
+date: 2022-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi
@@ -77,6 +77,46 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.1
+- !ruby/object:Gem::Dependency
+  name: url_reveal22
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: dynarex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.11
 description: 
 email: digital.robertson@gmail.com
 executables: []