closest_weightlifting_gem 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 786b4111c162920f2b4dabd1b55b3e67ba6a211c
-  data.tar.gz: 4501d815a84546ec4463c58184c63d26d17e6c74
+  metadata.gz: c14f13107e2795302c5a22106b1bdc3818cde5ed
+  data.tar.gz: dff447ea0e068ebe463fb1987b410e9febe74d62
 SHA512:
-  metadata.gz: 67b2d1669bc805e84370a9ce7f9d381afb73dbfdfe83c589c2b312aab0c08ed3cf2e7ba4e9fba92f4b2e7aaf5c837bf0dd2dae0daa1903f8c6521e641198053b
-  data.tar.gz: 0b517cd6bdbcb91225f35f89ac523ee31834e2795d53422f3ca5115f7886560fcbe5cd6ceabd0d06c22522815d73cf23d0aed5860a16840dde8981814c86843d
+  metadata.gz: 5d3d695c3a5b3dc5c2a5b3b4b7beb7eacc2094f7ac986e783edd6483f6779fd752ad03a98917df59122482bad93286b59fd03336f45ad288261becb730c0392f
+  data.tar.gz: f21d93c40f5c4cf12be6d52eef04304ef1149f2708eabae1963785e545aa008014857b01b4d9f749b22beafd5b9172bba9bf5502a68d7e11015dc08b69527eeb

data/README.md

@@ -2,13 +2,6 @@
 
 This application scrapes the [USA Weightlifting's Find a Club ](http://www.teamusa.org/usa-weightlifting/clubs-lwc/find-a-club) feature and builds a Gym object for each club, which includes some basic biographical information.
 
-TODO:
----
-* Implement proper options from CLI#show_gym
-  * show_gym page from #find_by_names menu should take it back to those search results instead of the state results
-* Geocode gym addresses
-* Implement user location input to find nearest gyms
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -25,15 +18,24 @@ Or install it yourself as:
 
     $ gem install closest_weightlifting_gem
 
-## Usage
-
+## Development
 
+To interact with the command line interface, clone the repo locally and change directories into the new folder.   Then you can run `bundle install` or `bin/setup` from the terminal prompt to install dependencies that aren't already on your system. Then, call the full program by typing `bin/closest-weightlifting-gem` into the prompt.  Note:  It scrapes everything up front so that I can query against the entire set of 1168 gyms (as of June 11 2016).
 
-## Development
+You can also run `bin/console` for an interactive prompt that will allow you to experiment. From within this console, you can call `basic_setup` and it will scrape all the gyms in Florida, Illinois, and New York to add some data into the system.
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+TODO:
+---
+* DRY up Scraper class
+* Hook up to ActiveRecord
+* Geocode gym addresses
+* Implement user location input to find nearest gyms
+* Map
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+Done:
+---
+* Implement proper options from CLI#show_gym
+* show_gym page from #find_by_names menu should take it back to those search results instead of the state results
 
 ## Contributing
 
@@ -43,4 +45,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/zacsco
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/bin/console

@@ -2,24 +2,15 @@
 
 require "bundler/setup"
 require "closest_weightlifting_gem"
+require 'pry'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require 'pry'
-
-def reload!
-  load './lib/closest_weightlifting_gem.rb'
-end
-
 def basic_setup
   ["FL","IL","NY"].each do |state|
     ClosestWeightliftingGem::Scraper.scrape_state_page(state)
   end
 end
 
-Pry.start
+Pry.start

data/closest_weightlifting_gem.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "pry", "~> 0.10.3"
+  spec.add_development_dependency "pry", "~> 0.10.4"
 
   spec.add_dependency "nokogiri", "~> 1.6.7.2"
   spec.add_dependency "require_all", "~> 1.3.3"

data/lib/closest_weightlifting_gem/cli.rb

@@ -14,50 +14,12 @@ class ClosestWeightliftingGem::CLI
     ClosestWeightliftingGem::Scraper.scrape_main
   end
 
-  # Not implemented currently, but plan to introduce when I figure out geocoder
-  #
-  # def address_input
-  #   puts <<-DOC.gsub /^\s*/, ''
-  #     Please enter your address below and I'll find
-  #     the closest weightlifting gym. (ex. 123 Main St. New York, NY)
-  #   DOC
-
-  #   address = gets.chomp
-  # end
-
   def main_menu
     puts "What do you want to do now?"
 
     options(__method__)
   end
 
-  def show_gyms(gyms)
-    width = 80
-    puts line
-    puts "OK, your search yielded #{gyms.size} gyms!\n"
-    puts "Here's the list: "
-    puts line
-    puts "     Gym Name".ljust(width/2)+"       City"
-
-    gyms.each_with_index do |gym, i|
-      puts "#{(i+1).to_s.rjust(3)}. #{gym.name.ljust(width/2)}  #{gym.city}, #{gym.state}"
-    end
-
-    options(__method__, gyms)
-  end
-  
-  def show_gym(gym)
-    ClosestWeightliftingGem::Scraper.scrape_attributes(gym) if !gym.coach
-
-    puts "Name: #{gym.name}"
-    puts "Address: #{gym.full_address}"
-    puts "Director: #{gym.director}"
-    puts "Coach: #{gym.coach}"
-    puts "Website: #{gym.website}"
-
-    options(__method__, gym)
-  end
-
   def options(menu, data=nil)
     menu = menu.to_s
 
@@ -84,31 +46,75 @@ class ClosestWeightliftingGem::CLI
       when "show_gym"
         puts line
         puts <<-DOC.gsub(/^\s*/,'')
-          1. Open Website
-          2. Back to State Results
-          3. Main menu
-          4. Exit
+          1. Open in Google Maps
+          2. Back to Previous Search
+          2. Main menu
+          3. Exit
         DOC
 
         process_gym(data)
       else
-        puts "WTF"
+        puts "What was that??"
+    end
+  end
+
+  # Not implemented currently, but plan to introduce when I figure out geocoder
+  #
+  def address_input
+    puts <<-DOC.gsub /^\s*/, ''
+      Please enter your address below and I'll find
+      the closest weightlifting gym. (ex. 123 Main St. New York, NY)
+    DOC
+
+    address = gets.chomp
+  end
+
+  def show_gyms(gyms)
+    width = 80
+    puts "="*width
+    puts "OK, your search yielded #{gyms.size} gyms!\n"
+    puts "Here's the list: "
+    puts "="*width
+    puts "     Gym Name".ljust(width/2)+"       City"
+
+    gyms.each_with_index do |gym, i|
+      puts "#{(i+1).to_s.rjust(3)}. #{gym.name.ljust(width/2)}  #{gym.city}, #{gym.state}"
+    end
+
+    options(__method__, gyms)
+  end
+
+  def show_gym(gym)
+    puts "Name: #{gym.name}"
+    puts "Address: #{gym.full_address}"
+    puts "Director: #{gym.director}"
+    puts "Phone: #{gym.phone}"
+
+    options(__method__, gym)
+  end
+
+  def open_in_google_maps(gym)
+    if gym.full_address
+      system("open", "http://maps.google.com/?q=#{gym.full_address}")
+    else
+      puts "Sorry, that gym doesn't have an address listed.\n\n\n\n"
+      show_gym(gym)
     end
   end
 
   def open_in_browser(gym)
     if gym.website == "none"
-      puts "Sorry, that gym doesn't have a website. :("
+      puts "Sorry, that gym doesn't have a website. :(\n\n\n"
     elsif !gym.website.include?("http")
       system("open", "http://#{gym.website}")
-    else 
+    else
       system("open", "#{gym.website}")
     end
   end
 
   def process_main
     input = nil
-    
+
     while input != "exit"
       input = gets.strip.downcase
 
@@ -123,12 +129,8 @@ class ClosestWeightliftingGem::CLI
           show_gyms(ClosestWeightliftingGem::Gym.find_by_name(search_term))
         when "3"
           good_bye
-
-          exit
         when "exit"
           good_bye
-
-          exit
         else
           puts "I didn't understand that, please try again."
       end
@@ -137,14 +139,15 @@ class ClosestWeightliftingGem::CLI
 
   def process_gyms(gyms)
     input = nil
-    
+
     until input == "exit"
       input = gets.strip.downcase
+
       case input
         when "1"
           puts "Select a gym by the number:"
           gym_num = gets.chomp.to_i - 1
-          
+
           show_gym(gyms[gym_num])
         when "2"
           show_gyms(gyms)
@@ -152,12 +155,8 @@ class ClosestWeightliftingGem::CLI
           main_menu
         when "4"
           good_bye
-
-          exit
         when "exit"
           good_bye
-
-          exit
         else
           puts "I didn't understand that, please try again."
       end
@@ -166,26 +165,22 @@ class ClosestWeightliftingGem::CLI
 
   def process_gym(gym)
     input = nil
-    
+
     until input == "exit"
       input = gets.strip.downcase
+
       case input
         when "1"
-          open_in_browser(gym)
-
-          options("show_gym", gym)
+          open_in_google_maps(gym)
+          process_gym(gym)
         when "2"
-          show_gyms(ClosestWeightliftingGem::Gym.find_by_state(gym.state))
+          show_gyms(ClosestWeightliftingGem::Gym.find_by_last_search)
         when "3"
           main_menu
         when "4"
          good_bye
-
-         exit
         when "exit"
           good_bye
-
-          exit
         else
           puts "I didn't understand that, please try again."
       end
@@ -199,5 +194,7 @@ class ClosestWeightliftingGem::CLI
 
   def good_bye
     puts "Adios friend.  Hope you come back to lift weights again!"
+
+    exit
   end
-end
+end

data/lib/closest_weightlifting_gem/gym.rb

@@ -1,3 +1,5 @@
+require 'pry'
+
 class ClosestWeightliftingGem::Gym
   attr_accessor :name,
                 :full_address,
@@ -7,8 +9,6 @@ class ClosestWeightliftingGem::Gym
                 :zipcode,
                 :phone,
                 :director,
-                :coach,
-                :usaw_url,
                 :website
 
   @@all = []
@@ -25,7 +25,11 @@ class ClosestWeightliftingGem::Gym
   end
 
   def full_address
-    "#{street} #{city}, #{state} #{zipcode}"
+    if street.nil?
+      nil
+    else
+      "#{street}, #{city}, #{state} #{zipcode}"
+    end
   end
 
   def self.all
@@ -45,26 +49,29 @@ class ClosestWeightliftingGem::Gym
   end
 
   def self.weird_gyms
-    self.all.find_all do |g| 
-      g.phone == nil || g.street == nil || g.name == nil 
+    self.all.find_all do |g|
+      g.phone.nil? || g.street.nil? || g.name.nil? || g.city.nil?
     end
   end
 
   def self.find_by_name(input)
     self.searches << { method: __method__, value: input }
 
-    select_few = self.all.select do |gym|
-      gym.name.upcase.include?(input.upcase)
-    end
+    self.all.find_all { |gym| gym.name.upcase.include?(input.upcase) }
   end
 
   def self.find_by_state(state)
     self.searches << { method: __method__, value: state }
-    
+
     self.all.find_all { |gym| gym.state == state.upcase }
   end
 
+  def self.find_by_last_search
+    last_search = @@searches.last
+    self.send(last_search[:method], last_search[:value])
+  end
+
   def self.count_by_state
     self.all.inject(Hash.new(0)) { |total, gym| total[gym.state] += 1 ; total }
   end
-end
+end

data/lib/closest_weightlifting_gem/scraper.rb

@@ -1,73 +1,126 @@
-class ClosestWeightliftingGem::Scraper
+require 'pry'
+require 'net/http'
+require 'json'
+require 'uri'
 
+class ClosestWeightliftingGem::Scraper
   BASE_URL = "https://webpoint.usaweightlifting.org/wp15/Companies/"
 
+  def self.get_state_abbreviations(index)
+    index.search("select#CompanyState").children[2..-1].collect { |child| child.attr("value") }
+  end
+
   def self.scrape_main
     puts "Fetching index..."
-    index = Nokogiri::HTML(open("#{BASE_URL}/Clubs.wp?frm=t"))
+    index = Nokogiri::HTML(open("#{BASE_URL}/Clubs.wp?frm=t&RF=Zp%2CST"))
 
     get_state_abbreviations(index).each { |state| scrape_state_page(state) }
 
     puts "\n\nSorry that took so long."
   end
 
+  # Form Data
+  # do I need cookies, etc now?
   def self.scrape_state_page(state)
     puts "Fetching gym data in #{state}..."
-    state_doc = Nokogiri::HTML(open("#{BASE_URL}/Clubs.wp?frm=t&CompanyState=#{state}"))
+    data = {
+      'wp_ClientOrgID' => '',
+      'CompanyParentID' => '',
+      'CompanyName' => '',
+      'CompanyState' => state,
+      'geo_Zip' => '',
+      'geo_Miles' => 25,
+      'submit' => 'Go'
+    }
 
-    # I want it to scrape each state page
-    # I want it to insantiate and save gym objects for each gym on the page
-    # This will just be basic info and I can set other data in the gym class
+    url = URI("#{BASE_URL}/Clubs.wp?frm=t&RF=Zp%2CST")
+    http = Net::HTTP.new(url.host, url.port)
+    http.use_ssl = true
+    request = Net::HTTP::Post.new(url)
+    request["Content-Type"] = 'application/x-www-form-urlencoded'
+    request["cache-control"] = 'no-cache'
+    request.body = "wp_ClientOrgID=&CompanyParentID=&CompanyName=&CompanyState=#{state}&geo_Zip=&geo_Miles=25&submit=Go"
+    response = http.request(request)
+    state_doc = Nokogiri::HTML(response.read_body)
 
-    state_doc.search(".datarow").each do |gym_row|
+    state_doc.search("#wp_Clubs li").each do |gym_row|
+      details = extract_details(gym_row.search("p")[0])
 
-      if gym_row.search(".right+ .left").text.split(" ").size < 5
-        scrape_gym_page(gym_row)
-      else
-        ClosestWeightliftingGem::Gym.new({
-             :name => gym_row.search("a").first.children.text.titleize,
-           :street => gym_row.children[5].children[0].text,
-             :city => gym_row.children[5].children[2].text.split(",").first,
-            :state => state,
-          :zipcode => gym_row.children[5].children[2].text.split(/\W+/).last,
-            :phone => gym_row.children[5].children[4].text,
-         :usaw_url => gym_row.search("a").first.attr("onclick").match(/\/V.+true/)[0]
-        })
-      end
+      ClosestWeightliftingGem::Gym.new({
+           :name => gym_row.search("h3").text.strip,
+         :street => details[:street], #gym_row.search("p").children[0].to_s,
+           :city => details[:city], #gym_row.search("p").children[2].to_s.split(/\W+/)[0],
+          :state => state,
+        :zipcode => details[:zipcode], #gym_row.search("p").children[2].to_s.split(/\W+/)[-1],
+          :phone => details[:phone], #gym_row.search("p").children[4].to_s,
+        :website => details[:website],
+       :director => details[:director]
+      })
     end
   end
 
   def self.scrape_gym_page(gym_row)
     gym_doc = Nokogiri::HTML(open("#{BASE_URL + gym_row.search("a").first.attr("onclick").match(/\/V.+true/)[0]}"))
-    
+
     ClosestWeightliftingGem::Gym.new({
-      :name => gym_doc.search(".fe_vbig_row td").text.titleize,
-      :street => gym_doc.search(".fe_vbig_row+ .fe_big_row td").children.to_s.split("<br>")[0],
-      :city => gym_doc.search(".fe_vbig_row+ .fe_big_row td").children.to_s.split("<br>")[1].split(",")[0],
-      :state => gym_doc.search(".fe_vbig_row+ .fe_big_row td").children.to_s.split(",").last.split(/\W+/)[1],
-      :zipcode => gym_doc.search(".fe_vbig_row+ .fe_big_row td").children.to_s.split(/\W/).last,
-      :phone => gym_doc.search(".fe_big_row:nth-child(4) td").children.last.to_s[1..-1],
+      :name => gym_doc.search("h3").text,
+      :street => gym_doc.search("p").children[0].to_s,
+      :city => gym_row.search("p").children[2].to_s.split(/\W+/)[0],
+      :state => gym_row.search("p").children[2].to_s.split(/\W+/)[1],
+      :zipcode => gym_row.search("p").children[2].to_s.split(/\W+/)[-1],
+      :phone => gym_row.search("p").children[4].to_s,
       :director => gym_doc.search(".fe_big_row:nth-child(2) td+ td").text,
       :coach => gym_doc.search(".fe_big_row+ .fe_big_row td+ td").text,
       :website => gym_doc.text.split("site:")[1].split("\r").first[1..-1],
       :usaw_url => gym_row.search("a").first.attr("onclick").match(/\/V.+true/)[0]
     })
-
   end
 
-  def self.scrape_attributes(gym)
-    gym_doc = Nokogiri::HTML(open("#{BASE_URL + gym.usaw_url}"))
+  def self.extract_details(info_div)
+    details = {
+      "street": "",
+      "city": "",
+      "state": "",
+      "zipcode": "",
+      "phone": "",
+      "website": "",
+      "director": ""
+    }
 
-    gym.add_attributes({
-      :phone => gym_doc.search(".fe_big_row:nth-child(4) td").children.last.to_s[1..-1],
-      :director => gym_doc.search(".fe_big_row:nth-child(2) td+ td").text,
-      :coach => gym_doc.search(".fe_big_row+ .fe_big_row td+ td").text,
-      :website => gym_doc.text.split("site:")[1].split("\r").first[1..-1]
-    })
-  end
+    begin
+      aa = info_div.children.select { |el| !["br", "b"].include?(el.name) }
+      case aa.length
+      when 1
+        details[:phone] = aa[0].text
+      when 3
+        details[:phone] = aa[0].text
+        details[:director] = aa[1].text
+      when 4
+        details[:phone] = aa[0].text
+        details[:website] = aa[1].text
+        details[:director] = aa[2].text
+      else
+        if aa[1].text.include?(", ")
+          city, state_zip = aa[1].text.split(", ")
+        else
+          city, state_zip = aa[2].text.split(", ")
+        end
 
-  def self.get_state_abbreviations(index)
-    index.search("select").children.collect { |child| child.attr("value") }[2..-1]
-  end
+        state, empty, zip = state_zip.split(/[[:space:]]/)
+        details[:street] = aa[0].text
+        details[:city] = city
+        details[:state] = state
+        details[:zipcode] = zip
+        details[:phone] = aa[2].text
+        details[:website] = aa[3]["href"]
+        details[:director] = aa[4].text
+      end
+    rescue StandardError => e
+      puts "[ERROR] extracting details from: #{aa}"
+      puts "[ERROR] #{e}"
+    end
 
-end
+    details.each { |k, v| details[k] = "" if v.nil? }
+    details
+  end
+end

data/lib/closest_weightlifting_gem/version.rb

@@ -1,3 +1,3 @@
 module ClosestWeightliftingGem
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/notes.md

@@ -1,11 +1,18 @@
-ToDo:
+# ToDo:
+1. Website redesign has caused the scraping to fail.  
+  * ~~.scrape_main to scrape the correct address~~
+  * Update gem versions!!!
+  * State pages have changed slightly in HTML structure and I need to identify
+    those new element selectors
 
+
+## Legacy Notes:
 Update Gemspec ToDos
 2. Start with project structure
 3. Start with entry point - file run (bin/closest-weightlifting-gem)
 4. Build to CLI interface
 5. Stub out interface
-6. Start makinig things real
+6. Start making things real
 7. Discover objects
 8. Program
 
@@ -33,5 +40,3 @@ open to web page or google maps
 
 Basic URI for accessing each state's individual page
 https://webpoint.usaweightlifting.org/wp15/Companies/Clubs.wp?frm=t&CompanyValue=#{state}
-
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: closest_weightlifting_gem
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - zacscodingclub
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-26 00:00:00.000000000 Z
+date: 2018-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.3
+        version: 0.10.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.3
+        version: 0.10.4
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -156,7 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: USA Weighlifting Club Gyms