rubyscraper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6f4cfba8b1442632b6c54f30886d254ff25fbbd3
+  data.tar.gz: 4bc8431e8294d5900819d29735f69650e93448da
+SHA512:
+  metadata.gz: 73ff93065f3079602dfcc58e35e08686ce4fff2d693b7f80c63e8cd37f011826156731b7d8a42a84472232c4a0c5b18dcb8efde1e7a7ede2ded37c431d22f95f
+  data.tar.gz: 04e30fe957b8d95a25cb0620539eca3b8956eb1c5638c8bd038aa6b461f4868555889b033201b350dc651f5faff7e2e38c2fac86ea98ce4bf13ac76de21834c5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rubyscraper.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    rubyscraper (0.1.0)
+      capybara
+      poltergeist
+      rest-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    capybara (2.4.4)
+      mime-types (>= 1.16)
+      nokogiri (>= 1.3.3)
+      rack (>= 1.0.0)
+      rack-test (>= 0.5.4)
+      xpath (~> 2.0)
+    cliver (0.3.2)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mime-types (2.4.3)
+    mini_portile (0.6.2)
+    multi_json (1.11.0)
+    netrc (0.10.3)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    poltergeist (1.6.0)
+      capybara (~> 2.1)
+      cliver (~> 0.3.1)
+      multi_json (~> 1.0)
+      websocket-driver (>= 0.2.0)
+    rack (1.6.0)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rake (10.4.2)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    websocket-driver (0.5.4)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.2)
+    xpath (2.0.0)
+      nokogiri (~> 1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.9)
+  rake (~> 10.0)
+  rubyscraper!

data/README.md

@@ -0,0 +1,39 @@
+# RubyScraper
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/rubyscraper`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rubyscraper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rubyscraper
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/rubyscraper/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rubyscraper"

data/bin/rubyscraper

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'rubyscraper/binary'
+RubyScraper::Binary.call(ARGV, $stdout, $stderr)
+

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/assets/search-terms.txt

@@ -0,0 +1,5 @@
+ruby
+ruby+on+rails
+javascript
+junior
+full-stack

data/lib/rubyscraper.rb

@@ -0,0 +1,90 @@
+require 'capybara'
+require 'capybara/poltergeist'
+require 'rest-client'
+require 'rubyscraper/version'
+
+class RubyScraper
+  include Capybara::DSL
+
+  def initialize(endpoint)
+    Capybara.register_driver :poltergeist do |app|
+      Capybara::Poltergeist::Driver.new(app, js_errors: false)
+    end
+    Capybara.default_driver = :poltergeist
+    @jobs = []
+    @posted_jobs = 0
+    @endpoint = endpoint
+    @search_terms_file = File.expand_path('../assets/search-terms.txt', __FILE__)
+    @search_terms = []
+    File.foreach(@search_terms_file) { |x| @search_terms << x.strip }
+  end
+
+  def scrape
+    get_summaries
+    get_bodies
+    send_to_server
+    return @jobs.length, @posted_jobs
+  end
+
+  def get_summaries
+    @search_terms.each do |term|
+      visit "http://careers.stackoverflow.com/jobs?searchTerm=#{term}&sort=p"
+      (1..2).to_a.each do |page|
+        visit "http://careers.stackoverflow.com/jobs?searchTerm=ruby&sort=p&pg=#{page}"
+        all(".listResults .-item").each do |listing|
+          position = listing.find("h3.-title a").text
+          url = listing.find("h3.-title a")["href"]
+          posting_date = listing.first("p._muted").text
+
+          @jobs << { position: position, url: url, posting_date: posting_date }
+        end
+      end
+      puts "Pulled #{term} job summaries."
+    end
+  end
+
+  def get_bodies
+    @jobs.each_with_index do |job, i|
+      puts "Job #{i+1} pulled."
+      sleep 1
+      visit "http://careers.stackoverflow.com#{job[:url]}"
+      if has_css?("a.employer")
+        job[:company] = find("a.employer").text
+      end
+      if has_css?("span.location")
+        job[:location] = find("span.location").text
+      end
+      #job[:description] = first("div.description p")
+      description = all("div.description p").map do |p|
+        p.text
+      end
+      job[:description] = description.join("\n")
+      tags = all("div.tags a.post-tag").map do |tag|
+        tag.text
+      end
+      job[:tags] = tags
+    end
+  end
+
+  def send_to_server
+    @jobs.each_with_index do |job, i|
+      new_job = {
+        position: job[:position],
+        location: job[:location],
+        description: job[:description],
+        source: "http://careers.stackoverflow.com#{job[:url]}"
+      }
+      RestClient.post(@endpoint, job: new_job){ |response, request, result, &block|
+        case response.code
+        when 201
+          @posted_jobs += 1
+          puts "Job saved."
+        when 302
+          puts "Job already exists."
+        else
+          puts "Bad request."
+        end
+      }
+    end
+  end
+end

data/lib/rubyscraper/binary.rb

@@ -0,0 +1,17 @@
+require 'rubyscraper'
+
+class RubyScraper
+  class Binary
+    def self.call(argv, outstream, errstream)
+      outstream.puts "StackOverflow Job Scraper"
+      outstream.puts "---------------------------------------------"
+      outstream.puts "Started scraping..."
+      endpoint = argv.first
+      outstream.puts "Sending post requests to #{endpoint}"
+      jobs_scraped, jobs_saved = RubyScraper.new(endpoint).scrape
+      outstream.puts "Scraped #{jobs_scraped} jobs, succesfully posted #{jobs_saved} jobs."
+      outstream.puts "---------------------------------------------"
+      outstream.puts "Completed!"
+    end
+  end
+end

data/lib/rubyscraper/version.rb

@@ -0,0 +1,3 @@
+class RubyScraper
+  VERSION = "0.1.0"
+end

data/rubyscraper.gemspec

@@ -0,0 +1,24 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'rubyscraper/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'rubyscraper'
+  s.version     = RubyScraper::VERSION
+  s.licenses    = ['MIT']
+  s.summary     = "Scrapes the sites..."
+  s.description = "Scrapes job sites for job details and sends post request to server."
+  s.authors     = ["Nathan Owsiany"]
+  s.email       = 'nowsiany@gmail.com'
+  s.files       = Dir["**/*"].select { |f| File.file? f  } - Dir['*.gem']
+  s.homepage    = 'https://github.com/ndwhtlssthr/rubyscraper'
+  s.executables << 'rubyscraper'
+
+  s.add_dependency "capybara"
+  s.add_dependency "poltergeist"
+  s.add_dependency "rest-client"
+
+  s.add_development_dependency "bundler", "~> 1.9"
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'pry'
+end

data/spec/rubyscraper_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe Rubyscraper do
+  it 'has a version number' do
+    expect(Rubyscraper::VERSION).not_to be nil
+  end
+
+  it 'does something useful' do
+    expect(false).to eq(true)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'rubyscraper'

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: rubyscraper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nathan Owsiany
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Scrapes job sites for job details and sends post request to server.
+email: nowsiany@gmail.com
+executables:
+- rubyscraper
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/rubyscraper
+- bin/setup
+- lib/assets/search-terms.txt
+- lib/rubyscraper.rb
+- lib/rubyscraper/binary.rb
+- lib/rubyscraper/version.rb
+- rubyscraper.gemspec
+- spec/rubyscraper_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/ndwhtlssthr/rubyscraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Scrapes the sites...
+test_files: []