indeed_scraper2022 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07f5323381b5751c470454f6f4c3ba6dced6f1424e054b85360a49d814d662ba
-  data.tar.gz: 3d25353b9f8a0543944cac82ef6dc91adf7d3e83444f3c6ef469f15cbba8a3d8
+  metadata.gz: 5e33dfd54667ecc9f8b7985aa07af403be8d95729ce68e8a40d8c985d57bd4e1
+  data.tar.gz: a2c041ec8103b6afac3a422e7b73bc82c89fd7f8d955240439a29ec0347c8a5f
 SHA512:
-  metadata.gz: 5e13ae04b46bfa3eb15aab8d0aff388d8caec591c413493db591c37da099d2bcd5ba340a72137d4aa7d374652b68bc1d037b86fe4cc2ed2ae5b0a56c5202f00b
-  data.tar.gz: ca14ae99251aabbcaee08a3bb6f240742ed1fab0f438496dc742ef39a10abb13e310b2d6a93bc472f5e1b3e45cfd8956d6a62f803b1d3a152054cf4e1ae35402
+  metadata.gz: 8e640cb8262a057bb588b501ee1122a59e6e239e2a5988dd0566ffffb814a2fef763c36fdeae1ba5dc4e6f819ca145374058bd62373ce776df1e393057a49fc0
+  data.tar.gz: 0a6bfe0ef2b685d5711a95704cee3fa67d58eb7c9d0f149c872f9c23b0cc489382ab1327b101f7accf982f7ea1f1a6d56dc20ae528ef1fd4d6105c9ef93067da

data/lib/indeed_scraper2022.rb

@@ -14,13 +14,17 @@ end
 
 class IndeedScraper2022
 
+  attr_reader :browser
+
   def initialize(url='https://uk.indeed.com/?r=us', q: '', location: '',
                  headless: true, cookies: nil, debug: false)
 
     @debug = debug
     @url_base, @q, @location = url, q, location
     @headless, @cookies = headless, cookies
-    @results = search(q: @q, location: @location)
+
+    fw = FerrumWizard.new( headless: @headless, cookies: @cookies, debug: @debug)
+    @browser = fw.browser
 
   end
 
@@ -32,35 +36,49 @@ class IndeedScraper2022
 
   def search(q: @q, location: @location, start: nil)
 
-    fw = FerrumWizard.new( headless: @headless, cookies: @cookies, debug: @debug)
-
     url = @url_base
     url += 'start=' + start if start
 
-    browser = fw.browser
-    browser.goto(url)
+    @browser.goto(url)
+    #@browser.network.wait_for_idle
+    puts 'sleeping for 4 seconds' if @debug
+    sleep 4
 
     if q.length > 1 then
-      input = browser.at_xpath("//input[@name='q']")
-      input.focus.type(q)
+
+      input = @browser.at_xpath("//input[@name='q']")
+
+      # select any existing text and overwrite it
+      input.focus.type(:home); sleep 0.2
+      input.focus.type(:shift, :end); sleep 0.2
+      input.focus.type(q); sleep 0.2
     end
 
     if location.length > 1 then
-      input2 = browser.at_xpath("//input[@name='l']")
-      input2.focus.type(location)
+
+      input2 = @browser.at_xpath("//input[@name='l']")
+
+      # select any existing text and overwrite it
+      input2.focus.type(:home); sleep 0.2
+      input2.focus.type(:shift, :end); sleep 0.2
+      input2.focus.type(location); sleep 0.2
+
     end
 
-    button = browser.at_xpath("//button[@type='submit']")
+    button = @browser.at_xpath("//button[@type='submit']")
     button.click
+    #@browser.network.wait_for_idle
+    puts 'sleeping for 2 seconds' if @debug
+    sleep 2
 
-    doc2 = Nokogiri::XML(browser.body)
+    doc2 = Nokogiri::XML(@browser.body)
 
     a2 = doc2.xpath  "//a[div/div/div/div/table/tbody/tr/td/div]"
     puts 'a2: ' + a2.length.inspect if @debug
 
     @a2 = a2.map {|x| Rexle.new x.to_s }
 
-    @a2.map do |doc|
+    @results = @a2.map do |doc|
 
       div = doc.element("a[@class='desktop']/div[@class='slider"  \
           "_container']/div[@class='slider_list']/div[@class='sl"  \
@@ -202,7 +220,9 @@ class IS22Plus < IndeedScraper2022
 
   def archive()
 
-    1.upto(15).each do |n|
+    return unless @results
+
+    1.upto(@results.length).each do |n|
       page(n)
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - James Robertson