indeed_scraper2022 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e33dfd54667ecc9f8b7985aa07af403be8d95729ce68e8a40d8c985d57bd4e1
-  data.tar.gz: a2c041ec8103b6afac3a422e7b73bc82c89fd7f8d955240439a29ec0347c8a5f
+  metadata.gz: 972f811430fae59121e39c9c4752b64fc43b37165a52dcec8c3eac42cf1e4555
+  data.tar.gz: 85e987eb264b098b4c892e2e05d2ab082e3b8968fff2bdd519552e889f014f9d
 SHA512:
-  metadata.gz: 8e640cb8262a057bb588b501ee1122a59e6e239e2a5988dd0566ffffb814a2fef763c36fdeae1ba5dc4e6f819ca145374058bd62373ce776df1e393057a49fc0
-  data.tar.gz: 0a6bfe0ef2b685d5711a95704cee3fa67d58eb7c9d0f149c872f9c23b0cc489382ab1327b101f7accf982f7ea1f1a6d56dc20ae528ef1fd4d6105c9ef93067da
+  metadata.gz: e7d3c2a13e315383248c557806dd0184d8831f46a2e314816395f62fcb886ba2e38a3e1f2deb180ceb33b614cf0b7be8a13379028fc56982e388948575bdb02c
+  data.tar.gz: 6ca4792d260c43b22fcee5ead8928525df17f1db112c49ac4cc7d7a5c9b29a8f483b94c18cd39bace1d4a4ee553ff83254a7fb445d76abf06c1705d19bac455c

data/lib/indeed_scraper2022.rb

@@ -4,11 +4,13 @@
 
 require 'ferrumwizard'
 require 'nokorexi'
+require 'yaml'
 
 # Given the nature of changes to jobsearch websites,
 # don't rely upon this gem working in the near future.
 
 
+
 class IndeedScraper2022Err < Exception
 end
 
@@ -218,14 +220,53 @@ class IS22Plus < IndeedScraper2022
           debug: debug)
   end
 
-  def archive()
+  def archive(filepath='/tmp/indeed')
 
     return unless @results
 
-    1.upto(@results.length).each do |n|
-      page(n)
+    FileUtils.mkdir_p filepath
+
+    idxfile = File.join(filepath, 'index.yml')
+
+    index = if File.exists? idxfile then
+      YAML.load(File.read(idxfile))
+    else
+      {}
     end
 
+    @results.each.with_index do |item, i|
+
+      puts 'saving ' + item[:title] if @debug
+      puts 'link: ' + item[:link].inspect
+      links = RXFReader.reveal(item[:link])
+      puts 'links: ' + links.inspect
+
+      url = links.last
+      id = url[/(?<=\?jk=)[^&]+/]
+
+      if index[id.to_sym] then
+        next
+      else
+
+        File.write File.join(filepath, 'j' + id + '.txt'), page(i+1)
+
+        h = {
+          link: url[/^[^&]+/],
+          title: item[:title].to_s,
+          salary: item[:salary].to_s,
+          company: item[:company].to_s.strip,
+          location: item[:location].to_s,
+          jobsnippet: item[:jobsnippet],
+          date: item[:date]
+        }
+
+        index[id.to_sym] = h
+      end
+
+    end
+
+    File.write idxfile, index.to_yaml
+
   end
 
   def list()

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indeed_scraper2022
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,7 +35,7 @@ cert_chain:
   YLGLkwmbiUHX5dRU3RwOwOrZiyvND5BIj7S6dZ6jYHe0I727apgQNc3swTz5mW6I
   SW/2zInu2bkj/meWm5eBoWHT
   -----END CERTIFICATE-----
-date: 2022-03-30 00:00:00.000000000 Z
+date: 2022-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokorexi