jobs_crawler 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bf276967ec890fe47ae3556b931865531dce5eb427a78387d804745939d88c7
-  data.tar.gz: 5ff554b07156269e2c09663f98f39dbb1fd91cce060e7fa153bde29686a9c329
+  metadata.gz: '0911fc283f16a8ffac616ba37ab85b8cc6dc546db997923e8c8389ace1217b6f'
+  data.tar.gz: 2a7479d3999a5f61b957c2b2295c5d2b60f625ba67eba8a1c048773cda515d20
 SHA512:
-  metadata.gz: 1dd42c07d06b1a2c8132da264974ac55682be7d2c0e342987786f8a6a22952a0e3545e05719a0bf6ced18ece7ee1cbf61ec178e96d5937207f5e331acc812dcc
-  data.tar.gz: 193672e516faa7680bce476cd614b7d0c6f2020b15abf63cb154af0b5aab7383fa3a5bd6333485474277a28ae55f7605760ea8ad251d806de9cd4931b7a37720
+  metadata.gz: 80aa00dccd2f024104ea007f3ebc132ea3d25d42684634e3a0876fa2c95434988c426ba9a0704992cda5f99d4e812c7a2f3fa42e5baa0094543e3f9ad0562807
+  data.tar.gz: 8586fcfc2dc87ac888daeeb44f89d7b95db79ed80357cc5b2e75d2225213bc8cea217e8cd62aa740110b51fcef8d7a969c03a30a4ed451279742e193ad8f4c6f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    jobs_crawler (0.1.5)
+    jobs_crawler (0.1.6)
       wombat
 
 GEM

data/lib/jobs_crawler/robots/atoo_job.rb

@@ -1,29 +1,14 @@
-module JobsCrawler
-  class Robots::AtooJob
-    attr_reader :url
-
-    def initialize(url)
-      @url    = url
-      @engine = Mechanize.new
-    end
-
-    def crawl
-      set_html
-      to_json
-    end
-
-    private
-
-    def set_html
-      @html = Nokogiri::HTML(body)
-    end
-
+module JobsCrawler::Robots
+  class AtooJob < Base
     def to_json
       {
         date_de_publication: date_publication,
         description:         description,
       }
     end
+
+    private
+
     def description
       @html.css('#main > article > div.main-section.jobs-detail-3 > div:nth-child(2) > div > div > div.section-content.col-lg-8.col-md-8.col-sm-12.col-xs-12 > div.row > div.col-lg-8.col-md-8.col-sm-12.col-xs-12.col-xs-12 > div').text
     end
@@ -31,9 +16,5 @@ module JobsCrawler
     def date_publication
       @html.css('#main > article > div.main-section.jobs-detail-3 > div:nth-child(1) > div > div > div > div > div > div > div.cs-text > ul > li:nth-child(1) > span').text
     end
-
-    def body
-      @engine.get(url).body
-    end
   end
 end

data/lib/jobs_crawler/robots/base.rb

@@ -0,0 +1,33 @@
+module JobsCrawler::Robots
+  class Base
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+      @engine = Mechanize.new
+    end
+
+    def crawl
+      set_html
+      to_json
+    end
+
+    def extract_content(css_selector)
+      @html.css(css_selector).text
+    end
+
+    def to_json
+      raise NotImplemetedError, 'You need to provide a concrete implemetatioen'
+    end
+
+    def set_html
+      @html = Nokogiri::HTML(body)
+    end
+
+    private
+
+    def body
+      @engine.get(url).body
+    end
+  end
+end

data/lib/jobs_crawler/robots/emploi_senegal.rb

@@ -1,13 +1,22 @@
-module JobsCrawler
-  class Robots::EmploiSenegal
-    include Wombat::Crawler
-
-    base_url 'https://www.emploisenegal.com'
-    path     '/recherche-jobs-senegal'
-    ¬
-    links 'css=.search-results .job-description-wrapper', :iterator do
-      url({ xpath: ".//@data-href" })
-      titre({ xpath: ".//text()" })
+module JobsCrawler::Robots
+  class EmploiSenegal < Base
+
+    def to_json
+     {
+       date_de_publication: date_de_publication,
+       description:         description,
+     }
+    end
+
+    private
+
+    def description
+      extract_content('#job-ad-details-261761 > div > div')
     end
+
+    def date_de_publication
+      extract_content('#node-261761 > div > div.job-ad-publication-date')
+    end
+
   end
 end

data/lib/jobs_crawler/robots/pages_jaunes_senegal.rb

@@ -1,13 +1,20 @@
-module JobsCrawler
-  class Robots::PagesJaunesSenegal
-    include Wombat::Crawler
+module JobsCrawler::Robots
+  class PagesJaunesSenegal < Base
+    def to_json
+      {
+        date_de_publication: date_de_publication,
+        description:         description,
+      }
+    end
+
+    private
+
+    def date_de_publication
+      I18n.l Date.today, format: :long
+    end
 
-    base_url 'https://www.emploisenegal.com'
-    path     '/recherche-jobs-senegal'
-    ¬
-    links 'css=.search-results .job-description-wrapper', :iterator do
-      url({ xpath: ".//@data-href" })
-      titre({ xpath: ".//text()" })
+    def description
+      extract_content('#contenu > div:nth-child(1) > table')
     end
   end
 end

data/lib/jobs_crawler/robots/senjob.rb

@@ -1,9 +1,20 @@
 module JobsCrawler::Robots
-  class Senjob
-    include Wombat::Crawler
+  class Senjob < Base
+    def to_json
+      {
+        date_de_publication: date_de_publication,
+        description:         description
+      }
+    end
 
-    description "css=#articlebi .preview"
-    reference   "xpath=//html/body/div[3]/table/tbody/tr[5]/td[2]"
-    deadline    "css=body > div:nth-child(16) > table > tbody > tr:nth-child(5) > td:nth-child(2)"
+    private
+
+    def date_de_publication
+      @html.xpath('//*[@id="tablesOffres"]/tbody/tr[1]/td').text
+    end
+
+    def description
+      extract_content('#tablesOffres > tbody > tr:nth-child(1) > td')
+    end
   end
 end

data/lib/jobs_crawler/robots.rb

@@ -1,4 +1,5 @@
 module JobsCrawler::Robots
+  require 'jobs_crawler/robots/base'
   require 'jobs_crawler/robots/senjob'
   require 'jobs_crawler/robots/atoo_job'
   require 'jobs_crawler/robots/emploi_senegal'
@@ -8,5 +9,13 @@ module JobsCrawler::Robots
      def crawl_atoo_job(url)
        JobsCrawler::Robots::AtooJob.new(url).crawl
      end
+
+     def crawl_emploi_senegal(url)
+       JobsCrawler::Robots::EmploiSenegal.new(url).crawl
+     end
+
+     def crawl_pages_jaunes_senegal(url)
+       JobsCrawler::Robots::EmploiSenegal.new(url).crawl
+     end
   end
 end

data/lib/jobs_crawler/version.rb

@@ -1,3 +1,3 @@
 module JobsCrawler
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jobs_crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Pathe
@@ -94,6 +94,7 @@ files:
 - lib/jobs_crawler/indexers/senjob.rb
 - lib/jobs_crawler/robots.rb
 - lib/jobs_crawler/robots/atoo_job.rb
+- lib/jobs_crawler/robots/base.rb
 - lib/jobs_crawler/robots/emploi_senegal.rb
 - lib/jobs_crawler/robots/pages_jaunes_senegal.rb
 - lib/jobs_crawler/robots/senjob.rb