myuniversaljobsmatch 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 837c02494a5a6a08e58fea53557868f787c6f224
-  data.tar.gz: c1276943fb43a93ba5fa08d031d34e5656734d4e
+SHA256:
+  metadata.gz: 526c9daa5c4db413642d12ab424cabcb7ed88efd3c3fbde667380fe371ed57d2
+  data.tar.gz: 50c8a05605223a4b34ac3ea8f3d5be74b1e75a882e91238d682960ecd4cfaac8
 SHA512:
-  metadata.gz: f689e903a43f8f50ae6b166c722a289e1b8f75866e3df996a9d4539c58ef3997294c3606bfaf2ffb4763b73a98c0d2d29c80a6b3e827af11c8b033df3737b905
-  data.tar.gz: 38297b8114aab9237578449a56175d66cdda4178373dea0cf0ea568069871eb0d70e35e9b1536857c6b79cafdd9c2d85dcb6e3b09465cc3609123303f3aa3a58
+  metadata.gz: ff01c7f3c147f4ce0d528b04dc2c951df7b3426515e8e97a64225d0c579fbe8969f976f996fe9f3aabd90826e859a3fb7d70117fff8c9caf1dba0ea04f227892
+  data.tar.gz: 6a78bfac7569f00f0dff35696e56c73ef95611a75abb126abf666dfc8acdb3f5567d9999ab5f7b612dd5896432857d4f97d6d436589af787c4361f6eeafff28a

data/lib/myuniversaljobsmatch.rb

@@ -1,16 +1,31 @@
 #!/usr/bin/env ruby
 
+# file: myuniversaljobsmatch.rb
+
+require 'chronic'
 require 'nokorexi'
 require 'dynarex'
 
-# file: myuniversaljobsmatch.rb
+module StringToDate
+  
+  refine String do
+    
+    def to_date()
+      Chronic.parse(self, :endian_precedence => :little).to_date
+    end
+    
+  end
+  
+end
 
 class MyUniversalJobsMatch
+  using StringToDate
 
   def initialize(filepath: '')
 
     @filepath = filepath
-    @url_base = 'https://jobsearch.direct.gov.uk/'
+
+    @url_base = 'https://findajob.dwp.gov.uk/'
     
     @dx = Dynarex.new 'ujm[title,tags]/item(job_id, title, ' + \
         'description, posting_date, company, location, industries, job_type)'
@@ -20,51 +35,76 @@ class MyUniversalJobsMatch
   def dynarex()
     @dx
   end
-
-  def search(title: '', where: '')
-
-    url = @url_base + "jobsearch/powersearch.aspx?qt=2&rad=20&tm=-1&" + \
-                                                "where=#{where}&tjt=#{title}"
-    doc = Nokorexi.new(url).to_doc
+  
+  # options:
+  #      results_per_page: 10, 25, 50
+  #      sort_by: date, highest_salary, lowest_salary
+  #      hours: any, full_time, part_time
+  #      contract_type: any, permanent, temporary, contract, apprenticeship
+
+  def search(title: '', where: '', results_per_page: nil, sort_by: nil, 
+             hours: nil, contract_type: nil)
     
-    table = doc.root.at_css('.JSresults')
-
-    a = table.xpath('tr/td').map do |row|
-
-      # get the date
-      #date = row.element('td/span/text()')
-      date = row.element('div/div[2]/span/text()')
-
-      #link = row.element('td[3]/a')
-      link = row.element('div[3]/div[2]/a')
-
-      url = link.attributes[:href]
-      jobid = url[/JobID=(\d+)/,1]
-      title = link.text
-
-      #company = row.element('td[4]/span/text()')
-      company = row.element('div[4]/div[2]/span/text()')
-      #location = row.element('td[5]/span/text()')
-      location = row.element('div[5]/div[2]/span/text()')
-
-      [jobid, date, title, url, company, location]
+    params = {
+      adv: 1,
+      q: title,
+      w: where
+    }
+    
+    params[:pp] = results_per_page if results_per_page
+    params[:cty] = contract_type if contract_type
+    params[:cti] = hours if hours
+    
+    case sort_by.to_sym
+    when 'date'
+      params[:sb] = :date
+      params[:sd] = :down
+    when :highest_salary
+      params[:sb] = :salary
+      params[:sd] = :down
+    when :lowest_salary
+      params[:sb] = :salary      
+      params[:sd] = :up
+    end
+        
+    url = @url_base + 'search/?' + params.map {|x| x.join('=') }.join('&')
+    doc = Nokorexi.new(url).to_doc    
+    
+    rows = doc.root.xpath('//div[@class="search-result"]')
+    
+    a = rows.map do |row|
+
+      items  = row.xpath('ul/li')
+      joburl = row.element('h3/a/@href').to_s
+      jobtitle = row.element('h3/a/text()')
+      jobid = joburl[/\d+$/]
+      jobref = title[/^\d+/].to_s
+
+      date = items[0].text.to_date
+      company = items[1].text('strong')
+      location = items[1].element('span/text()')
+      salary = items[2].text('strong') if items[2]
+      desc = row.text('p').strip
+
+      [jobid, jobref, date, jobtitle, joburl, company, location, 
+       salary, desc]
       
     end
 
 
     dx = Dynarex.new('vacancies[title, desc, date, time, tags, xslt]/' + \
-             'vacancy(job_id, date, title, url, company, location, created_at)')
+             'vacancy(job_id, job_ref, date, title, url, company, location, salary, desc)')
 
-    dx.title = "Universal Jobmatch jobs - Search results for '#{title}'"
-    dx.desc = "generated from web scrape of jobsearch." + \
-                                                "direct.gov.uk; source: " + url
+    dx.title = "Find a job - Search results for '#{title}'"
+    dx.desc = "generated from web scrape of Find a job." + \
+                                                "findajob.dwp.gov.uk/; source: " + url
     dx.tags = 'jobs vacancies jobmatch ' + title.split.first
-    dx.date = Time.now.strftime("%Y-%b-%s")
+    dx.date = Time.now.strftime("%Y-%b-%d")
     dx.time = Time.now.strftime("%H:%M")
 
     a.each do |row|
-      dx.create Hash[%i(job_id date title url company location created_at).\
-                                                                      zip(row)]
+      dx.create Hash[(%i(job_id job_ref date title url company) + \
+                      %i(location salary desc)).zip(row)]
     end
 
     return dx
@@ -72,29 +112,28 @@ class MyUniversalJobsMatch
   
   def query(id)
 
-    url = @url_base + 'GetJob.aspx?JobID=' + id
-    buffer = RXFHelper.read(url).first
-
-    doc = Nokorexi.new(buffer.gsub(/<br\s*\/?>/i,"\n")).to_doc
-    content = doc.root.at_css '.jobViewContent'
-    title = content.element('h2[2]/text()')
-    description = content.element('div')
-
-    a = doc.root.at_css('.jobViewSummary').xpath('dl/*')
+    doc = Nokorexi.new(@url_base + 'details/' + id).to_doc    
     
-    a2 = []
-    while a.length > 0 do
-
-      if a.first.name == 'dt' then
-        a2 << [a.shift.text, '']
-      elsif
-        a2[-1][-1] << a.shift.text + "\n"
-      end
-    end
+    title = doc.root.text('head/title')
+    
+    rows = doc.root.xpath('//table[1]/tbody/tr')
     
-    a2.concat [['title', title], ['description', description.content]]
+    h = rows.map do |tr|
+      
+      [
+        tr.text('th').downcase.rstrip[0..-2].gsub(/ +/,'_').to_sym, 
+        tr.text('td').to_s
+      ]
+      
+    end.to_h
 
-    Hash[a2.map{|k,v| [k.downcase.gsub(' ','_').to_sym,v]}]
+    h[:description] = doc.root.element('//div[@itemprop="description"]').xml\
+        .gsub(/<br *\/> */,"\n").gsub(/<\/?[^\>]+\/?>/,'').strip    
+    
+    h[:posting_date] = h[:posting_date].to_date
+    h[:closing_date] = h[:closing_date].to_date
+    
+    {title: title}.merge(h)
     
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: myuniversaljobsmatch
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -10,68 +10,93 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDXjCCAkagAwIBAgIBATANBgkqhkiG9w0BAQUFADAsMSowKAYDVQQDDCFnZW1t
-  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMTgwMzE0MTYwNDU5WhcN
-  MTkwMzE0MTYwNDU5WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
-  cnRzb24vREM9ZXUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwvxO6
-  /Q9Y1cYxWKKg4PMVCbguM5aDUfvyCr5w20V7tqAsp5MF7El3y/8KeOQlsh7J9o/T
-  SshCULIy/tSthsvdA97s0+AAkzt1emEfNIqR85wx+Hrrq4D+AQ9q/GlT8V3AFJKI
-  AiZIjM1ESakVJA5waXrAYh2A8Ayp3UJwUb8hv62kmrmnIktR0b0OyAeKEUaI3ghb
-  N1iHrWv3QjGmyARTvMhCq0pxo1y5LdHPpq0VSNdopjLSlhsj//1tZ/iaqwhv1TBC
-  67ZvGS4+XnCEqPusgMwIdw5fLtKSd/i/sx5KraQZUcnZAqxWlrGKUTCaYH/A9lM4
-  5WY4C1vEmtUvlxWBAgMBAAGjgYowgYcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
-  HQYDVR0OBBYEFAopjAg4lTdn3ImcdT1OW4cNyOX4MCYGA1UdEQQfMB2BG2dlbW1h
-  c3RlckBqYW1lc3JvYmVydHNvbi5ldTAmBgNVHRIEHzAdgRtnZW1tYXN0ZXJAamFt
-  ZXNyb2JlcnRzb24uZXUwDQYJKoZIhvcNAQEFBQADggEBAIVf3GQn8WYBPII/v6Iw
-  cQflYsQ7wVL+UPY7dDjJur7sPr0u1DZaoniCu2dIAX32XNkYKxHbwOmn/bDP/oi/
-  JtwEKxpTmaAk8SMnr1vxK6E0XtKsJbcYpG38Guql3tv2N5AnGUsab6iqXy2nNZ9X
-  lav0CKlK3EoLsO2GaWOvkIAGud/PcvBadlCcAha+UL5Uh2E70BWgK+6CRj5+DvR7
-  cmxlnHavH4ed7VTk0JkPmqTwxJnaKaztnY+UbUfTbRtUHkd46azAuipUuNecMn3y
-  UU8JFnL4NzGxTWzYwPOUlB4AT+rPKOw1qGeuy62M7UrpTKGgeNSvA1NPFgkOtWLO
-  DMA=
+  MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjAwMTE5MTYxMzA0WhcN
+  MjEwMTE4MTYxMzA0WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCmy6lP
+  Cp884sO5pUZxsR9L5pJ4U3GUKmqgEG5AupSMe6HSm9pZi+f0cF+fNMF53Ac27elI
+  +twgMKKIrWc8HXiy3K+9tMSTzU0/uJGK7Y2iIZ1QHUMNy9S4VdglohHnF21IGFtj
+  j3Nq+xLCEvxcZoUjbgk563GZKgd5XJYwzyb4q+erGrlasaQ9A+ZakAWT9vCXnaGO
+  wSe8Wx7KrvKk3UCvtH0tXDt8T8pwzN38qKPU1lpdDZQi63jqh8+qW2Lyu1KCWrDf
+  rj7ef8JC4WBbUbPABvFefjGmLMMYueKpOjCxMWeKFZz5Q83KcJMewkY/8sGLplyB
+  f+yKLPa5Mi8NnbWcevxmyss+/1z/W/dW+Dl5/3d/c1xXfi7nAjLfjGF8qQ9E82DM
+  7Z3mp5gSGCxQBzNg50+ytE0hIzg2Hh+HkjMm/wt4nm7m4rmdY2FPvY1h7VKEbQtN
+  PUHF5MrVBIC9HCFVaIxpcO7ObK47PytfH3CfhPLHEd5ZU8Wo/WyYHiO+CQECAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU+mA09dUS
+  2VF8+toUk7mApgYy+lAwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
+  BgkqhkiG9w0BAQsFAAOCAYEADZRsfL/QTHTM1VPNIJcgGFULZD7SaxAwcS25TOXs
+  fwQI62ZGWyoJIhSgmOKFJh+04pzy5s1SygS6GTTfbFV3ES3TAorH4HabUF27CF9i
+  7sCjvT4w/DZC1ue6P2qLdRhhXoqW+2ssGKvAB1VfIEQMA0LQU0IEUN5c+wG3r//J
+  g/nBG0lJna47dC6+MQo3tVpBOdxH7pMtTIvAxXUjjJdB9xrvzs/4NZ+k7nGb3uNF
+  NLXCpLx5+93WdJMAuyv88wvGChod/PWWDgzZP+/Zms7vvoTpmjMBm2xaRrgZFr+P
+  NFCMdBli9Mc0iv9lL5V3C+F0gqwmSs0tIFCr67XxPPbA27uReKc8XHDaVOaU8zJ9
+  3L7/hZqxmix8fqzeNWxLIT9itNGNgiqUyeEwHgKJTj9B3aXs2RVaFIfRNRn4BA5d
+  LFRA1HdKZrhZiH0YWMH+NWhB4jh0blOUN/hE0m8hL/vNySv6AUD63CDSgaYE2A4C
+  L6XR3+hIK/fqVrE8fRag8DUp
   -----END CERTIFICATE-----
-date: 2018-03-14 00:00:00.000000000 Z
+date: 2020-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: dynarex
+  name: chronic
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0.10'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.30
+        version: 0.10.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0.10'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.7.30
+        version: 0.10.2
 - !ruby/object:Gem::Dependency
-  name: nokorexi
+  name: dynarex
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 1.8.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.21
+- !ruby/object:Gem::Dependency
+  name: nokorexi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 description: 
 email: james@jamesrobertson.eu
 executables: []
@@ -98,9 +123,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
-summary: A web scraper which searches for job adverts on jobsearch.direct.gov.uk
+summary: A web scraper which searches for job adverts on findajob.dwp.gov.uk
 test_files: []