remote_job_scraper 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f12102d94abaccc92c800bb21d459fddce99380
-  data.tar.gz: 2af9bb36f83a766ab3ce90b3a9473b4cb1166683
+  metadata.gz: efcddd08a1843eaa0509f9bdd853ae47476df9db
+  data.tar.gz: 7bb7b71b57b3d8b6b1b8255320092849e7ff7b97
 SHA512:
-  metadata.gz: af613f9bff539e13a3585d80801e1983792e6d2125e0322526c411ee3a58ad8eb9ea955e88b07847e51a151e2eb072a7e4697b2141111ccadf5de047ad81bf46
-  data.tar.gz: a49fb2f1e599c3df9d49aa4c5be63d3930b43873c11a33465ad447fdc0fe7c7f33030c1c1f7d73fae0faf49ef7fa2fe5205262a9186c8260eec468265486a0fe
+  metadata.gz: 03bab4b51e95b76ab697d58577843c56b5b8ce0a2beaac21946d96e11babbe21ec0d410a46dfaf400e540cee70ac7546e084f624dd2b7d7dfe5dcf74be4dd966
+  data.tar.gz: 3e1f3570fe6766b5b20a8acd5cd1f01768da398d9a7de759caf2efa5a0c459c6641f3f3fb81919a6b47cec0df102de7d3e0b5944eacc9664ef6af97c57945724

data/.gitignore

@@ -11,3 +11,4 @@ data/
 # rspec failure tracking
 .rspec_status
 .DS_Store
+.byebug_history

data/README.md

@@ -40,6 +40,12 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Debugging
+```
+require 'byebug'
+byebug
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/rafaltrojanowski/remote_job_scraper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/lib/remote_job_scraper.rb

@@ -6,6 +6,7 @@ require 'sites/we_work_remotely'
 require 'sites/remote_ok'
 require 'sites/jobs_rails42'
 require 'sites/github_remote_jobs'
+require 'sites/elixir_companies'
 
 require 'support/offer_parser'
 require 'support/user_agent'

data/lib/remote_job_scraper/cli.rb

@@ -5,14 +5,22 @@ module RemoteJobScraper
 
     AVAILABLE_SITES = %w(we_work_remotely remote_ok 42jobs_rails)
 
-    desc 'collect_companies',
+    desc 'collect_remote_companies',
       "Retrieves remote companies
-       [Example]: remote_job_scraper collect_companies
+       [Example]: remote_job_scraper collect_remote_companies
       "
-    def collect_companies
+    def collect_remote_companies
       Sites::GithubRemoteJobs.new.collect_companies
     end
 
+    desc 'collect_elixir_companies',
+      "Retrieves Elixir companies
+       [Example]: remote_job_scraper collect_elixir_companies
+      "
+    def collect_elixir_companies
+      Sites::ElixirCompanies.new.collect_companies
+    end
+
     desc 'collect_jobs LIMIT DELAY',
       "Retrieves data from #{AVAILABLE_SITES.join(', ')}.
        [Example]: remote_job_scraper collect_jobs 10 9.0..10.0

data/lib/remote_job_scraper/version.rb

@@ -1,3 +1,3 @@
 module RemoteJobScraper
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/sites/elixir_companies.rb

@@ -0,0 +1,87 @@
+require_relative 'base'
+
+module Sites
+  class ElixirCompanies < Base
+
+    HOST = 'https://elixir-companies.com'.freeze
+    PATH = '/en/browse'.freeze
+    JOB_ITEM_SELECTOR = 'div.company.box'.freeze
+    STORE_DIR = 'data/elixir_companies'.freeze
+
+    def initialize
+      @url = "#{self.class::HOST}#{self.class::PATH}"
+      @current_time = Time.now
+      @timestamp = @current_time.strftime("%Y%m%d%H%M%S")
+      @doc = nil
+      @total_pages = 26
+      @rows_count = 0
+      @jobs_count = get_jobs_count
+    end
+
+    def collect_companies(limit: nil)
+      FileUtils.mkdir_p STORE_DIR
+
+      (1..@total_pages).each do |page|
+        process_page(page: page, limit: limit)
+      end
+    end
+
+    def companies_count
+      @rows_count
+    end
+
+    private
+
+    def process_page(page:, limit:)
+      current_page = "#{@url}?page=#{page}"
+      doc = Nokogiri::HTML(open_page(current_page))
+      puts "[Info] Getting the data from #{current_page}"
+
+      CSV.open(filepath, 'ab') do |csv|
+        doc.css(JOB_ITEM_SELECTOR).each do |company_box|
+          return if limit == @rows_count
+          csv << get_row(company_box)
+          @rows_count += 1
+        end
+      end
+
+      puts "[Done] Collected #{@jobs_count} job offers from #{url}. Data stored in: #{filepath}." if page == @total_pages
+    end
+
+    def get_row(company_box)
+      company_title = company_box.css('div.content p.title').text
+      company_info  = company_box.css('div.content.company-info p')
+
+      # A bit ugly way to get a data between span elements
+      array = company_info.text.split("\n").select do |element|
+        element =~ /[a-zA-Z]/
+      end.map!(&:strip).delete_if do |element|
+        element == "GitHub" || element == "Add a job"
+      end
+
+      has_blog = array[2] && (array[2].include?("/") || array[2].include?("blog"))
+
+      industry = array[0]
+      company_website = array[1]
+      is_hiring = company_box["class"].include?("has-ribbon")
+
+      if has_blog
+        blog = array[2]
+        location = array[4]
+      else
+        blog = nil
+        location = array[2]
+      end
+
+      row = [company_title, industry, company_website, blog, location]
+      hiring = is_hiring ? "Hiring!" : nil
+      row.push hiring
+    end
+
+    def get_jobs_count
+      jobs_count = 16 * @total_pages # roughly - first page has 14 items
+      puts "[Info] There are #{jobs_count} remote jobs on [ElixirCompanies]."
+      jobs_count
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remote_job_scraper
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Rafał Trojanowski
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-07 00:00:00.000000000 Z
+date: 2019-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -166,6 +166,7 @@ files:
 - lib/remote_job_scraper/configuration.rb
 - lib/remote_job_scraper/version.rb
 - lib/sites/base.rb
+- lib/sites/elixir_companies.rb
 - lib/sites/elixir_radar.rb
 - lib/sites/github_remote_jobs.rb
 - lib/sites/jobs_rails42.rb