rubyscraper 0.3.0 → 0.9.0

data/lib/rubyscraper/processor.rb

@@ -0,0 +1,47 @@
+require 'json'
+require 'rubyscraper/paginator'
+require 'rubyscraper/summary_scraper'
+require 'rubyscraper/sub_page_scraper'
+
+class Processor
+  attr_reader :sites, :record_limit, :single_site, :scrape_delay
+
+  def initialize(config_file, single_site, record_limit, scrape_delay)
+    @scrape_file   = config_file
+    @scrape_config = JSON.parse(File.read(@scrape_file))
+    @sites         = @scrape_config
+    @single_site   = single_site
+    @record_limit  = record_limit
+    @scrape_delay  = scrape_delay
+  end
+
+  def call
+    !single_site.empty? ? scrape_single_site : scrape_all_sites
+  end
+
+  private
+
+  def scrape_single_site
+    site = sites.select { |s| s["name"] == single_site }.first
+    scrape_site(site)
+  end
+
+  def scrape_all_sites
+    sites.inject [] do |all_results, site|
+      all_results += scrape_site(site)
+    end
+  end
+
+  def scrape_site(site)
+    paginator = Paginator.new(site, record_limit)
+    paginator.define_pagination_params
+
+    results = SummaryScraper.new(site, paginator.add_on, paginator.steps).call
+    results = SubPageScraper.new(site, results, scrape_delay).call if has_sub_pages?(site)
+    results
+  end
+
+  def has_sub_pages?(site)
+    site["summary"]["has_sub_pages"] == "true"
+  end
+end

data/lib/rubyscraper/sub_page_scraper.rb

@@ -0,0 +1,53 @@
+require 'capybara'
+require 'capybara/poltergeist'
+
+class SubPageScraper
+  attr_reader :site, :listings, :delay
+  include Capybara::DSL
+
+  def initialize(site, listings, delay)
+    @site     = site
+    @listings = listings
+    @delay    = delay
+
+    Capybara.register_driver :poltergeist do |app|
+      Capybara::Poltergeist::Driver.new(app, js_errors: false)
+    end
+    Capybara.default_driver = :poltergeist
+  end
+
+  def call
+    puts "Pulling #{@listings.count} listings from #{@site["name"]}:"
+    listings = @listings.inject [] do |results, listing|
+      sleep delay
+      listing = pull_sub_page_data(site, listing)
+      listing = listing_cleanup(listing)
+      results << listing
+    end; puts "\n"; listings
+  end
+
+  def pull_sub_page_data(site, listing)
+    visit listing["url"]
+    site["sub_page"]["fields"].each do |field|
+      if field["method"] == "all"
+        if has_css?(field["path"])
+          values = all(field["path"]).map do |elem|
+            elem.send(field["loop_collect"])
+          end
+          listing[field["field"]] = values.join(field["join"])
+        end
+      else
+        if has_css?(field["path"])
+          listing[field["field"]] = 
+            send(field["method"].to_sym,field["path"]).text
+        end
+      end
+    end; print "."; listing
+  end
+
+  def listing_cleanup(listing)
+    # Remove 'Headquarters: ' from weworkremotely jobs
+    listing["location"].slice!("Headquarter: ") if !listing["location"].to_s.empty?
+    listing
+  end
+end

data/lib/rubyscraper/summary_scraper.rb

@@ -0,0 +1,65 @@
+require 'capybara'
+require 'capybara/poltergeist'
+
+class SummaryScraper
+  attr_reader :site, :pagination_addon, :pagination_pages
+  include Capybara::DSL
+
+  def initialize(site, pagination_addon, pagination_pages)
+    @site             = site
+    @pagination_addon = pagination_addon
+    @pagination_pages = pagination_pages
+
+    Capybara.register_driver :poltergeist do |app|
+      Capybara::Poltergeist::Driver.new(app, js_errors: false)
+    end
+    Capybara.default_driver = :poltergeist
+  end
+
+  def call
+    pagination_pages.inject [] do |results, page|
+      results += get_summaries(page)
+    end
+  end
+
+  private
+
+  def get_summaries(page_no)
+    visit page_url(page_no)
+
+    all(site["summary"]["loop"]).inject [] do |results, listing|
+      record = pull_summary_data(site, listing)
+      record = listing_cleanup(site, record)
+      results << record
+    end
+  end
+
+  def page_url(page)
+    site["base_url"] + site["summary"]["url"] + pagination_addon + page.to_s
+  end
+  
+  def pull_summary_data(site, record)
+    output = Hash.new
+    site["summary"]["fields"].each do |field|
+      if field["attr"]
+        if record.has_css?(field["path"])
+          output[field["field"]] = 
+            record.send(field["method"].to_sym, field["path"])[field["attr"]]
+        end
+      else
+        if record.has_css?(field["path"])
+          output[field["field"]] = 
+            record.send(field["method"].to_sym, field["path"]).text
+        end
+      end
+    end; output
+  end
+
+  def listing_cleanup(site, listing)
+    # Add base url if not present
+    unless listing["url"].match(/^http/)
+      listing["url"] = "#{site["base_url"]}#{listing["url"]}"
+    end
+    listing
+  end
+end

data/lib/rubyscraper/version.rb

@@ -1,3 +1,3 @@
 class RubyScraper
-  VERSION = "0.3.0"
+  VERSION = "0.9.0"
 end

data/rubyscraper.gemspec

@@ -13,13 +13,12 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/ndwhtlssthr/rubyscraper'
   s.executables << 'rubyscraper'
 
-  s.add_dependency "capybara"
-  s.add_dependency "poltergeist"
-  s.add_dependency "rest-client"
-  s.add_dependency "slop"
+  s.add_dependency "capybara", "~> 2.4"
+  s.add_dependency "poltergeist", "~> 1.6"
+  s.add_dependency "rest-client", "~> 1.8"
 
   s.add_development_dependency "bundler", "~> 1.9"
   s.add_development_dependency "rake", "~> 10.0"
-  s.add_development_dependency 'rspec', '~> 3.0'
-  s.add_development_dependency 'pry'
+  s.add_development_dependency "rspec", "~> 3.0"
+  s.add_development_dependency "pry", "~> 0.10"
 end

data/spec/paginator_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+describe Paginator do
+  it 'returns defaults if not paginated' do
+    json = '{"summary":{
+      "paginated":"false"
+    }}'
+    site = JSON.parse(json)
+
+    paginator = Paginator.new(site, :all)
+    paginator.define_pagination_params
+    expect(paginator.add_on).to eq ""
+    expect(paginator.steps).to eq [""]
+  end
+
+  it 'returns the correct add_on with pagination' do
+    json = '{"summary":{
+      "paginated":"true",
+      "pagination":{
+        "format":"&pg=NUM",
+        "start":"1",
+        "scale":"1",
+        "records_per_page":"25"
+      }
+    }}'
+    site = JSON.parse(json)
+
+    paginator = Paginator.new(site, 50)
+    paginator.define_pagination_params
+    expect(paginator.add_on).to eq "&pg=NUM"
+  end
+
+  it 'returns the correct pages when given record limit' do
+    json = '{"summary":{
+      "paginated":"true",
+      "pagination":{
+        "format":"&pg=NUM",
+        "start":"1",
+        "scale":"1",
+        "records_per_page":"25"
+      }
+    }}'
+    site = JSON.parse(json)
+
+    paginator = Paginator.new(site, 50)
+    paginator.define_pagination_params
+    expect(paginator.steps).to eq [1, 2]
+  end
+
+  it 'adds an additional page if pages wrap to next page' do
+    json = '{"summary":{
+      "paginated":"true",
+      "pagination":{
+        "format":"&pg=NUM",
+        "start":"1",
+        "scale":"1",
+        "records_per_page":"25"
+      }
+    }}'
+    site = JSON.parse(json)
+
+    paginator = Paginator.new(site, 58)
+    paginator.define_pagination_params
+    expect(paginator.steps).to eq [1, 2, 3]
+  end
+
+  it 'can handle a starting of 0' do
+    json = '{"summary":{
+      "paginated":"true",
+      "pagination":{
+        "format":"&pg=NUM",
+        "start":"0",
+        "scale":"10",
+        "records_per_page":"10"
+      }
+    }}'
+    site = JSON.parse(json)
+
+    paginator = Paginator.new(site, 32)
+    paginator.define_pagination_params
+    expect(paginator.steps).to eq [0, 10, 20, 30]
+  end
+end

data/spec/rubyscraper_spec.rb

@@ -1,11 +1,7 @@
 require 'spec_helper'
 
-describe Rubyscraper do
+describe RubyScraper do
   it 'has a version number' do
-    expect(Rubyscraper::VERSION).not_to be nil
-  end
-
-  it 'does something useful' do
-    expect(false).to eq(true)
+    expect(RubyScraper::VERSION).not_to be nil
   end
 end

data/spec/spec_helper.rb

@@ -1,2 +1,5 @@
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'rubyscraper'
+require 'rubyscraper/paginator'
+require 'rubyscraper/summary_scraper'
+require 'rubyscraper/sub_page_scraper'

data/spec/sub_page_scraper_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe SubPageScraper do
+  it 'can pull record subfields from a list of existing jobs' do
+    jobs = [{"url" => "http://careers.stackoverflow.com/jobs/84266/software-developer-qhr-technologies"},
+            {"url" => "http://careers.stackoverflow.com/jobs/81592/service-engineer-bloomberg-lp"}]
+    json = '{
+      "sub_page":{
+        "fields":[
+          {
+            "field":"company",
+            "method":"find",
+            "path":"a.employer"
+          },
+          {
+            "field":"location",
+            "method":"find",
+            "path":"span.location"
+          },
+          {
+            "field":"description",
+            "method":"all",
+            "path":"div.description p",
+            "loop_collect":"text",
+            "join":"\n"
+          },
+          {
+            "field":"tags",
+            "method":"all",
+            "path":"div.tags a.post-tag",
+            "loop_collect":"text",
+            "join":", "
+          }
+        ]
+      }
+    }'
+    site = JSON.parse(json)
+
+    scraper = SubPageScraper.new(site, jobs)
+    results = scraper.call
+    expect(results.length).to eq 2
+    expect(results.first["company"]).to be_a String
+    expect(results.first["company"]).to_not be_empty
+    expect(results.first["location"]).to be_a String
+    expect(results.first["location"]).to_not be_empty
+    expect(results.first["description"]).to be_a String
+    expect(results.first["description"]).to_not be_empty
+    expect(results.first["tags"]).to be_a String
+    expect(results.first["tags"]).to_not be_empty
+  end
+end

data/spec/summary_scraper_spec.rb

@@ -0,0 +1,125 @@
+require 'spec_helper'
+
+describe SummaryScraper do
+  it 'can pull records from first page of paginated site' do
+    paginator = OpenStruct.new(add_on: "&pg=", pages: [1])
+    json = '{
+      "base_url":"http://www.careers.stackoverflow.com",
+      "summary":{
+        "url":"/jobs/tag/ruby?sort=p",
+        "loop":".listResults .-item",
+        "fields":[
+          {
+            "field":"position",
+            "method":"find",
+            "path":"h3.-title a"
+          },
+          {
+            "field":"url",
+            "method":"find",
+            "path":"h3.-title a",
+            "attr":"href"
+          },
+          {
+            "field":"posting_date",
+            "method":"first",
+            "path":"p._muted"
+          }
+        ]
+      }
+    }'
+    site = JSON.parse(json)
+
+    scraper = SummaryScraper.new(site, paginator.add_on, paginator.pages)
+    results = scraper.call
+    expect(results.length).to eq 25
+    expect(results.first["position"]).to be_a String
+    expect(results.first["position"]).to_not be_empty
+    expect(results.first["url"]).to be_a String
+    expect(results.first["url"]).to match(/^http/)
+    expect(results.first["posting_date"]).to be_a String
+    expect(results.first["posting_date"]).to_not be_empty
+  end
+
+  it 'can pull records from multiple pages of paginated site' do
+    paginator = OpenStruct.new(add_on: "&pg=", pages: [1, 2])
+    json = '{
+      "base_url":"http://www.careers.stackoverflow.com",
+      "summary":{
+        "url":"/jobs/tag/ruby?sort=p",
+        "loop":".listResults .-item",
+        "fields":[
+          {
+            "field":"position",
+            "method":"find",
+            "path":"h3.-title a"
+          },
+          {
+            "field":"url",
+            "method":"find",
+            "path":"h3.-title a",
+            "attr":"href"
+          },
+          {
+            "field":"posting_date",
+            "method":"first",
+            "path":"p._muted"
+          }
+        ]
+      }
+    }'
+    site = JSON.parse(json)
+
+    scraper = SummaryScraper.new(site, paginator.add_on, paginator.pages)
+    results = scraper.call
+    expect(results.length).to be > 26
+  end
+
+  it 'can pull records from non-paginated site' do
+    paginator = OpenStruct.new(add_on: "", pages: [""])
+    json = '{
+      "base_url":"https://weworkremotely.com",
+      "summary":{
+        "url":"/categories/2/jobs",
+        "has_sub_pages":"false",
+        "loop":"section.jobs ul li",
+        "fields":[
+          {
+            "field":"position",
+            "method":"find",
+            "path":"span.title"
+          },
+          {
+            "field":"company",
+            "method":"find",
+            "path":"span.company"
+          },
+          {
+            "field":"url",
+            "method":"find",
+            "path":"a",
+            "attr":"href"
+          },
+          {
+            "field":"posting_date",
+            "method":"find",
+            "path":"span.date"
+          }
+        ]
+      }
+    }'
+    site = JSON.parse(json)
+
+    scraper = SummaryScraper.new(site, paginator.add_on, paginator.pages)
+    results = scraper.call
+    expect(results.length).to be > 1
+    expect(results.first["position"]).to be_a String
+    expect(results.first["position"]).to_not be_empty
+    expect(results.first["company"]).to be_a String
+    expect(results.first["company"]).to_not be_empty
+    expect(results.first["url"]).to be_a String
+    expect(results.first["url"]).to match(/^http/)
+    expect(results.first["posting_date"]).to be_a String
+    expect(results.first["posting_date"]).to_not be_empty
+  end
+end