tf2r 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 143e9f53a0a365e8bf98ac186a8b81acc7a3f5a0
+  data.tar.gz: f417eab178af88208690c7f257389ae84bf49cf3
+SHA512:
+  metadata.gz: 3030e85ac97ab0a5726f8b7aa4f5b0e03bfacd540fcdc581d3f92428432580c50ad9cb9c7975031e68f0fd24e35816258283ca4329bc32697e1c4dffd7082776
+  data.tar.gz: 1cf2a0eeae95a3d7b83e248039e706ccbdb96c1cd59421550433a5c5848eff185abc65b5afece44c0254ab1781239485d36b2bf08860b5de73d02a3a22e17976

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.ruby-gemset

@@ -0,0 +1 @@
+tf2r_scraper

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tf2r.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    tf2r (0.0.1)
+      mechanize (~> 2.7)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    domain_name (0.5.19)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    method_source (0.8.2)
+    mime-types (2.3)
+    mini_portile (0.6.0)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    ntlm-http (0.1.1)
+    pry (0.10.0)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.3.2)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.3)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.3)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.3)
+    slop (3.6.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  pry (~> 0.10)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  tf2r!

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014, Justin Kim <yulli@yulli.org>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# TF2R [![Code Climate](https://codeclimate.com/github/justinkim/tf2r/badges/gpa.svg)](https://codeclimate.com/github/justinkim/tf2r)
+
+This gem provides a `TF2R::Scraper` that has the ability to scrape various pages on [TF2R](http://tf2r.com) into usable data.
+
+Yes, this gem is [semantically versioned](http://semver.org/)!
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'tf2r'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install tf2r
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/tf2r/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/tf2r/scraper.rb

@@ -0,0 +1,195 @@
+module TF2R
+  class Scraper
+    def initialize(options)
+      @mech = Mechanize.new { |mech|
+        mech.user_agent = options[:user_agent] || "TF2R::Scraper #{VERSION}"
+      }
+
+      @mech.cookie_jar.load(options[:cookies_txt], :cookiestxt) if options[:cookies_txt]
+    end
+
+    def fetch(url)
+      @mech.get(url)
+    end
+  end
+end
+
+__END__
+
+# This is the old Scraper from NervyPipe.
+class Scraper
+  def initialize(cookies_txt_path)
+    @cookies_txt_path = cookies_txt_path
+
+    @main = Mechanize.new { |agent|
+      # the User-Agent field in headers
+      agent.user_agent = 'Jenna Bot'
+    }
+
+    auth_cookies(@main)
+  end
+
+  def auth_cookies(mech)
+    # Before anything, load our auth cookies into the cookie jar
+    # This requires a Netscape-style cookies.txt to be in the working dir
+    #
+    # cookies.txt must include at least a valid "session" cookie from tf2r.com
+    mech.cookie_jar.load_cookiestxt(@cookies_txt_path)
+  end
+
+  # Simply return the Mechanize::Page for a url
+  def fetch(url)
+    @main.get(url)
+  end
+
+  def run(type)
+    case type
+    when :raffle
+      scrape_raffle(@main.get 'http://tf2r.com/kblf84f.html')
+    when :user
+      scrape_user(@main.get 'http://tf2r.com/user/76561198061719848.html')
+    when :main
+      scrape_main_page
+    when :ranks
+      scrape_ranks
+    end
+  end
+
+  def scrape_main_page
+    page = @main.get('http://tf2r.com/raffles.html')
+
+    # This regex matches all Mechanize::Page::Links on the main raffles page that are actual raffles
+    raffle_mech_links = page.links_with(href: /tf2r\.com\/k/)
+
+    # an array of strings, which are raffle links
+    raffle_links = raffle_mech_links.map { |x| x.uri.to_s }
+
+    # the array should have raffles from bottom-to-top, old-to-new
+    raffle_links.reverse!
+  end
+
+  def scrape_raffle_for_user(page)
+    # This is an array of all things Reag was nice enough to class "raffle_infomation"
+    # Reag made a typo, so the class really is "raffle_infomation"
+    raffle_infos = page.parser.css('.raffle_infomation')
+
+    # User information
+    steam_id = raffle_infos[2].css('a')[0].attributes['href'].text.split('/')[-1].split('.')[0].to_i
+    username = raffle_infos[2].css('a').text
+    avatar_link = raffle_infos[1].css('a')[0].css('img')[0].attributes['src'].text
+
+    # posrept will be nil if the Scraper's user has already voted on a user's rep in the raffle
+    posrepa = raffle_infos.css('.upvb').text.split
+    posrepa.delete('+')
+    posrep = posrepa[-1].to_i.to_s
+
+    negrepa = raffle_infos.css('.downvb').text.split
+    negrepa.delete('+')
+    negrep = negrepa[-1].to_i.to_s
+
+    colour = raffle_infos[2].css('a')[0].attributes['style'].value.split('#')[-1].split(';')[0].downcase.chomp
+    # The creator of the raffle, using above
+    userhash = {steam_id: steam_id, username: username, avatar_link: avatar_link, posrep: posrep, negrep: negrep, colour: colour}
+  end
+
+  def scrape_raffle_for_raffle(page)
+    # This is an array of all things Reag was nice enough to class "raffle_infomation"
+    # Reag made a typo, so the class really is "raffle_infomation"
+    raffle_infos = page.parser.css('.raffle_infomation')
+
+    # Raffle information
+    uri = page.uri # is a URI:HTTP
+    path = uri.path # is "/welcome.html" for "http://tf2r.com/welcome.html"
+    link_snippet = path.split('/')[1].split('.html')[0] # is 'kabc123' for 'http://tf2r.com/kabc123.html'
+
+    title = raffle_infos[0].text.split('Title: ')[-1]
+    # Lots of info in a single table
+    raffle_tds = raffle_infos[3].css('td')
+    description = raffle_tds[1].text
+
+    start_time_string = raffle_tds[9].text
+    start_time = DateTime.strptime(start_time_string, '%a, %d %b %Y %H:%M:%S %z').to_time
+    end_time_string = raffle_tds[11].text
+    end_time = DateTime.strptime(end_time_string, '%a, %d %b %Y %H:%M:%S %z').to_time
+
+    win_chance_pre_round = raffle_tds[5].text.to_f / 100 # also #winc
+    win_chance = win_chance_pre_round.round(5)
+
+    entries = raffle_tds[7].text # also #entry
+    # Entries looks like "42/123", as "current/max"
+    # Split by slash, multiple assignment to array with elements mapped to integers
+    # Equivalent to a = b[0].to_i; c = b[1].to_i
+    current_entries, max_entries = entries.split('/').map { |x| x.to_i }
+
+    is_done = end_time <= Time.now || current_entries == max_entries || page.parser.css('.welcome_font').text.include?('No winners') || page.parser.css('.welcome_font').text.include?('Winner(s):')
+
+    rafflehash = {link_snippet: link_snippet, title: title, description: description, start_time: start_time, end_time: end_time,
+      win_chance: win_chance, current_entries: current_entries, max_entries: max_entries, is_done: is_done}
+  end
+
+  def scrape_raffle_for_participants(page)
+    participants = []
+    participant_divs = page.parser.css('.pentry')
+    participant_divs.each do |participant|
+      steam_id = participant.css('a')[-1].attributes['href'].text.split('/')[-1].split('.')[0].to_i
+      username = participant.text
+      colour = participant.css('a')[-1].attributes['style'].text.split('#')[-1].split(';')[0].downcase.chomp
+
+      participants << {steam_id: steam_id, username: username, colour: colour}
+    end
+
+    participants.uniq.reverse
+  end
+
+  def scrape_raffle(page, portions = :all)
+    userhash, rafflehash, participants = {}, {}, []
+
+    case portions
+    when :core
+      userhash = scrape_raffle_for_user(page)
+      rafflehash = scrape_raffle_for_raffle(page)
+    when :participants
+      participants = scrape_raffle_for_participants(page)
+    else
+      userhash = scrape_raffle_for_user(page)
+      rafflehash = scrape_raffle_for_raffle(page)
+      participants = scrape_raffle_for_participants(page)
+    end
+
+    [userhash, rafflehash, participants]
+  end
+
+  def scrape_user(user_page)
+    if user_page.parser.css('.profile_info').empty?
+      username, avatar_link, posrep, negrep, colour = nil, nil, nil, nil, nil
+      steam_id = user_page.uri.path.split('/')[-1].split('.')[0].to_i
+    else
+      pp user_page.parser.css('.profile_info')
+      raffle_infos = user_page.parser.css('.raffle_infomation') # sic
+
+      steam_id = user_page.uri.path.split('/')[-1].split('.')[0].to_i
+      username = user_page.parser.title.split('TF2R Item Raffles - ')[-1]
+      avatar_link = raffle_infos[0].css('img')[0].attributes['src'].text
+
+      posrep = raffle_infos.css('.upvb').text.to_i.to_s
+      negrep = raffle_infos.css('.downvb').text.to_i.to_s
+
+      colour = raffle_infos[1].css('a')[0].attributes['style'].value.split('#')[-1].split(';')[0].downcase.chomp
+    end
+
+    userhash = {steam_id: steam_id, username: username, avatar_link: avatar_link, posrep: posrep, negrep: negrep, colour: colour}
+  end
+
+  def scrape_ranks
+    # This scrapes the info page for the various ranks that exist
+    page = @main.get('http://tf2r.com/info.html')
+
+    ranks_div = page.parser.css('#ranks')
+    divs = ranks_div.css('div')
+    rank_divs = []
+    divs.each { |div|
+      rank_divs << div unless div.attributes['style'].nil? || !(div.attributes['style'].value.include? 'color')
+    }
+    colours = rank_divs.map {|div| div.attributes['style'].value.split('color:#')[-1].split(';')[0].downcase.chomp }
+  end
+end

data/lib/tf2r/version.rb

@@ -0,0 +1,3 @@
+module TF2R
+  VERSION = "0.0.1"
+end

data/lib/tf2r.rb

@@ -0,0 +1,9 @@
+require 'mechanize'
+require 'pry'
+
+require 'tf2r/scraper'
+require 'tf2r/version'
+
+module TF2R
+  # Your code goes here...
+end

data/spec/scraper_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe TF2R::Scraper do
+  let(:scraper) { TF2R::Scraper.new({}) }
+
+  it 'is instantiable' do
+    expect{
+      TF2R::Scraper.new({})
+    }.not_to raise_error
+  end
+
+  describe '#new' do
+    context 'no options are given' do
+      it 'creates an agent with default user agent if none is specified' do
+        scraper = TF2R::Scraper.new({})
+        expect(scraper.instance_variable_get(:@mech).agent.user_agent).to eql("TF2R::Scraper #{TF2R::VERSION}")
+      end
+    end
+
+    context 'a user agent is given' do
+      it 'creates an agent with the specified user agent' do
+        custom = 'Custom User Agent'
+        scraper = TF2R::Scraper.new({user_agent: custom})
+        expect(scraper.instance_variable_get(:@mech).agent.user_agent).to eql(custom)
+      end
+    end
+
+    context 'a cookies.txt file is given' do
+      it 'stores the given cookie' do
+        test_filename = File.join(File.dirname(__FILE__), 'test_cookies.txt')
+        scraper = TF2R::Scraper.new({cookies_txt: File.new(test_filename)})
+        expect(scraper.instance_variable_get(:@mech).cookie_jar.cookies[0].value).to eq("example_value")
+      end
+    end
+  end
+
+  describe '#fetch' do
+    it 'returns a Mechanize::Page' do
+      expect(scraper.fetch('http://google.com')).to be_a(Mechanize::Page)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'tf2r'
+
+RSpec.configure do |config|
+  config.order = 'random'
+end

data/spec/test_cookies.txt

@@ -0,0 +1,2 @@
+# This is an example cookies.txt file used for testing.
+example.com	FALSE	/	FALSE	2147483647	example_key	example_value

data/tf2r.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tf2r/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "tf2r"
+  spec.version       = TF2R::VERSION
+  spec.authors       = ["Justin Kim"]
+  spec.email         = ["yulli@yulli.org"]
+  spec.summary       = %q{A utility for use with TF2R.}
+  spec.description   = %q{This gem provides a utility for performing various tasks with tf2r.com.}
+  spec.homepage      = "https://github.com/justinkim/tf2r"
+  spec.license       = "ISC"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry", "~> 0.10"
+
+  spec.add_runtime_dependency "mechanize", "~> 2.7"
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: tf2r
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Justin Kim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+description: This gem provides a utility for performing various tasks with tf2r.com.
+email:
+- yulli@yulli.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/tf2r.rb
+- lib/tf2r/scraper.rb
+- lib/tf2r/version.rb
+- spec/scraper_spec.rb
+- spec/spec_helper.rb
+- spec/test_cookies.txt
+- tf2r.gemspec
+homepage: https://github.com/justinkim/tf2r
+licenses:
+- ISC
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: A utility for use with TF2R.
+test_files:
+- spec/scraper_spec.rb
+- spec/spec_helper.rb
+- spec/test_cookies.txt