TOSwimScraper 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 74a57bb46c96004254173ffaa32157460ee5f0d6
+  data.tar.gz: df2230bef488a7f8cf0eadbd170b1dfffcd05db2
+SHA512:
+  metadata.gz: 6181c523844d05a449e65f08b6d909537d4e6dd4a0010d40698c2cc742b923d102142a7814cddba775ce6fe8922f1261ff2590d1db21ffbb638e9b8d435b4329
+  data.tar.gz: ecd7f0859e7349378cf2c11c7d6eb4f78948258000d8ce5f597db0e84c3cdb7529334cb0140de14c1ec83360906b5df4e8af724c53e3ac8fdf7910766ba5e6b8

data/bin/scrape

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require_relative '../lib/scraper'
+
+if ARGV.empty?
+    puts "      Usage: scrape UPDATE_TYPE DISPLAY_TYPE \n\n"
+    puts "      Update type:\n"
+    puts "      '-f' Full refresh including regrabbing swim times and pool locations \n"
+    puts "      '-s' Just update swim schedules  \n\n"
+    puts "      Display type: \n"
+    puts "      '-v' Verbose output \n"
+    puts "      '-c' Concise output(default) \n"
+else
+
+  display_mode = ARGV[1] || '-c'
+
+  aliases = {
+    "-c" => "concise",
+    "-v" => "verbose"
+  }
+
+  display_mode = aliases[display_mode] || "concise"
+
+  Scraper.display_mode(display_mode)
+
+  if ARGV.include?('-f')
+    Scraper.gather_pool_info
+    Scraper.gather_pool_swim_times
+    Scraper.gather_pool_program_cost_status
+  elsif ARGV.include?('-s')
+    Scraper.gather_pool_swim_times
+    Scraper.gather_pool_program_cost_status
+  end
+end

data/lib/scraper.rb

@@ -0,0 +1,199 @@
+# Gems
+require 'nokogiri'
+require 'open-uri'
+require 'pry'
+require 'json'
+require 'geocoder'
+
+module Scraper
+  class << self
+
+    def display_mode(display_mode)
+      @display_mode = display_mode
+    end
+
+
+    # faster testing
+    # POOL_LIST_URLS = ["http://www1.toronto.ca/parks/prd/facilities/indoor-pools/index.htm"]
+    # Full list
+    POOL_LIST_URLS = ["http://www1.toronto.ca/parks/prd/facilities/indoor-pools/index.htm",
+            "http://www1.toronto.ca/parks/prd/facilities/indoor-pools/2-indoor_pool.htm",
+            "http://www1.toronto.ca/parks/prd/facilities/outdoor-pools/index.htm",
+            "http://www1.toronto.ca/parks/prd/facilities/outdoor-pools/2-outdoor_pool.htm"]
+
+    Geocoder.configure(:timeout => 10)
+
+    def gather_pool_info
+      @pool_urls, pool_names, pool_addresses, pool_links, pool_coordinates = [],[],[],[],[]
+
+      POOL_LIST_URLS.each do |url|
+        doc = Nokogiri::HTML(open(url))
+        pools = doc.at_css("#pfrBody > div.pfrListing > table > tbody")
+        pool_names += pools.css('a').map { |link| link.children.text }
+        pool_links += pools.css('a').map { |link| link['href'] }
+        pool_addresses += gather_pool_addresses(pools)
+      end
+
+      # Geotag pools
+      puts "\n--- Scraping pool coordinates ---"
+      pool_coordinates = pool_addresses.map { |address| gather_pool_coordinates(address) }
+
+      # Convert Pool Data to Hash
+      pool_names.each_with_index do |pool, index|
+        current_pool = {}
+        current_pool[:name] = pool_names[index]
+        current_pool[:url] = pool_links[index]
+        current_pool[:address] = pool_addresses[index]
+        current_pool[:coordinates] = pool_coordinates[index]
+        @pool_urls << current_pool
+      end
+
+      # Write Hash
+      File.open("pool_urls.json","w") do |f|
+        f.write(@pool_urls.to_json)
+      end
+    end
+
+    def simple_equal(num)
+      2 * num
+    end
+
+    def swim_time_finder(week, lane_swim_row_index)
+      week.at_css("tbody").css('tr')[lane_swim_row_index].children
+      .map do |el|
+        nodes = el.children.find_all(&:text?)
+        if nodes.length == 1
+          nodes = [el.children.text]
+        else
+          nodes.map!(&:text)
+        end
+      end
+    end
+
+    def build_pool_schedule_array_from_html(doc)
+      weeks = {}
+
+      for i in 0..1 #eventually poll more weeks, possibly 4 of available 7
+        week = doc.at_css("#dropin_Swimming_#{i}")
+        !week.nil?? week_dates = week.at_css('tr').children.map(&:text) : next
+
+        !week_dates.nil?? lane_swim_row_index = week.at_css("tbody").css('tr').find_index { |el| el.text=~ /Lane Swim/ } : next
+
+        if !lane_swim_row_index.nil?
+          week_lane_swim_times = swim_time_finder(week, lane_swim_row_index)
+          weeks.merge!(week_dates.zip(week_lane_swim_times).to_h)
+        end
+      end
+
+      # remove days with no swim times
+      weeks.delete_if { |day, time| time == [" "] || time == [] }
+    end
+
+    def gather_pool_addresses(pools)
+      pool_addresses = []
+      address_index_incrementer = pools.css('td').length / pools.css('tr').length
+      pools.css('td').each_with_index do |node, index|
+        # Address is always second column, table width varies for indoor vs. outdoor
+        if index % address_index_incrementer == 1
+          pool_addresses << node.text
+        end
+      end
+      pool_addresses
+    end
+
+    # Method accepting a block that supresses stdout/console logging
+    #  https://gist.github.com/moertel/11091573
+
+    def suppress_output
+      begin
+        original_stderr = $stderr.clone
+        original_stdout = $stdout.clone
+        $stderr.reopen(File.new('/dev/null', 'w'))
+        $stdout.reopen(File.new('/dev/null', 'w'))
+        retval = yield
+      rescue Exception => e
+        $stdout.reopen(original_stdout)
+        $stderr.reopen(original_stderr)
+        raise e
+      ensure
+        $stdout.reopen(original_stdout)
+        $stderr.reopen(original_stderr)
+      end
+      retval
+    end
+
+    def gather_pool_coordinates(address)
+      if @display_mode == "verbose"
+        puts "Geocoding: #{address}"
+      else
+        print "."
+      end
+
+      coordinates_arr = suppress_output{ Geocoder.coordinates("#{address}, Toronto") }
+
+      # To avoid triggering google API limit of 10 queries per second
+      sleep(0.15)
+      return { latitude: coordinates_arr[0], longitude: coordinates_arr[1] }
+    end
+
+    #####Parse Weekly Leisure Swim Data#####
+    def gather_pool_swim_times
+      begin
+        @pool_urls ||= JSON.parse(File.read('pool_urls.json'), symbolize_names: true)
+      rescue => e
+        puts "Couldn't open pool_info, run scrape -f or run in path with pool_urls.json file"
+        exit
+      end
+
+      puts "\n--- Scraping pool swim times ---"
+      @pool_urls.each do |pool|
+
+        if @display_mode == "verbose"
+          puts "Scraping: " + pool[:name]
+        else
+          print "."
+        end
+
+        url = "http://www1.toronto.ca" + pool[:url]
+        doc = Nokogiri::HTML(open(url))
+        pool[:times] = build_pool_schedule_array_from_html(doc)
+      end
+
+      File.open("pools_data.json","w") do |f|
+        f.write(@pool_urls.to_json)
+        puts "\nWriting pools_data.json complete"
+      end
+    end
+
+    def gather_pool_program_cost_status
+      @pools = JSON.parse(File.read('pools_data.json'), symbolize_names: true)
+
+      page = "http://www1.toronto.ca/wps/portal/contentonly?vgnextoid=aaafdada600f0410VgnVCM10000071d60f89RCRD&vgnextchannel=a96adada600f0410VgnVCM10000071d60f89RCRD"
+      doc = Nokogiri::HTML(open(page))
+      free_facility_article = doc.at_css("#maincontent")
+      links = free_facility_article.css('a')
+      all_hrefs = links.map { |link| link.attribute('href').to_s }.uniq.sort.delete_if { |href| href.empty? }
+
+      free_facility_urls_regexed = all_hrefs.keep_if{ |href| href.match("\/parks/prd/facilities/complex\w*") }
+                                            .map{ |url| url.match(/\/parks\/prd\/facilities\/complex\/\d*/).to_s }
+
+      @pools.each do |pool|
+        pool_url_regex = pool[:url].match(/\/parks\/prd\/facilities\/complex\/\d*/).to_s
+        match = free_facility_urls_regexed.find{ |e| pool_url_regex == e }
+        pool[:free_swim] = match ? true : false
+       end
+
+      File.open("pools_data.json","w") do |f|
+        f.write(@pools.to_json)
+        puts "Writing program cost status to pools_data.json complete"
+      end
+    end
+
+  end
+end
+
+# Todo
+# add a test suite
+# remind self how to log name of vars while blown up (smaller method with info passed in probably!)
+#start displaying, filtering?
+#maybe transform date save

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: TOSwimScraper
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Erich Welz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: geocoder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: Scraper to grab City of Toronto lane swim data creating a JSON file with
+  geotagged pools
+email: erichwelz@gmail.com
+executables:
+- scrape
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/scrape
+- lib/scraper.rb
+homepage: https://github.com/erichwelz/TOSwim
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Scraper to grab City of Toronto lane swim data creating a JSON file with
+  geotagged pools
+test_files: []