translink 2.0.0 → 3.0.0

data/test/unit/cli_test.rb

@@ -8,7 +8,7 @@ class CLITtest < MiniTest::Unit::TestCase
 
     end
 
-    def crawl date
+    def crawl date, from_route_url = nil, step = nil
 
     end
   end
@@ -41,20 +41,39 @@ class CLITtest < MiniTest::Unit::TestCase
     assert_includes @out.string, 'Usage'
   end
 
-  def test_execute_scrape_command_with_uri
-    file = File.join TMPDIR, "test_scrape_#{Time.now.to_i}.sqlite3"
+  def test_execute_scape_command_with_date
+    file = File.join TMPDIR, '2011-11-27.sqlite3'
     refute File.exists?(file), 'Expected file not to exist.'
-    @cli.run "scrape 2011-11-27 --uri=sqlite://#{file}"
+    @cli.run 'scrape 2011-11-27'
     assert File.exists?(file), 'Expected file to exist.'
   end
 
-  def test_execute_scrape_command_with_date_writes_sqlite_database
-    file = File.join TMPDIR, '2011-11-27.sqlite3'
+  def test_execute_scape_command_with_date_and_db_path
+    file = File.join TMPDIR, "test_db_path_#{Time.now.to_i}.sqlite3"
     refute File.exists?(file), 'Expected file not to exist.'
-    @cli.run 'scrape 2011-11-27'
+    @cli.run "scrape 2011-11-27 #{file}"
+    assert File.exists?(file), 'Expected file to exist.'
+  end
+
+  def test_execute_scrape_command_with_date_db_path_and_from_route_url
+    file = File.join TMPDIR, "test_db_path_from_route_url_#{Time.now.to_i}.sqlite3"
+    refute File.exists?(file), 'Expected file not to exist.'
+    @cli.run "scrape 2011-11-27 #{file} http://localhost"
     assert File.exists?(file), 'Expected file to exist.'
   end
 
+  def test_execute_scrape_command_with_date_db_path_from_route_url_and_step
+    file = File.join TMPDIR, "test_db_path_from_route_url_step_#{Time.now.to_i}.sqlite3"
+    refute File.exists?(file), 'Expected file not to exist.'
+    @cli.run "scrape 2011-11-27 #{file} http://localhost 0"
+    assert File.exists?(file), 'Expected file to exist.'
+  end
+
+  def test_version_command
+    @cli.run 'version'
+    assert_match Translink::VERSION, @out.string
+  end
+
   def test_execute_scrape_command_without_date_executes_help_command
     @cli.run 'scrape'
     assert_includes @out.string, 'Usage'

data/test/unit/model/route_test.rb

@@ -6,6 +6,7 @@ class Model::RouteTest < MiniTest::Unit::TestCase
       route_page = MiniTest::Mock.new
       route_page.expect :short_name, '130'
       route_page.expect :long_name, 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson'
+      route_page.expect :route_id, '130'
       route_page.expect :route_type, 0
       assert_equal 0, Model::Route.count
       assert_equal Model::Route.find_or_add_route_from_route_page(route_page), Model::Route.find_or_add_route_from_route_page(route_page)
@@ -15,8 +16,8 @@ class Model::RouteTest < MiniTest::Unit::TestCase
 
   def test_add_trip_from_trip_page
     DB.context 'sqlite::memory:', :migrate => true do
-      trip_page   = OpenStruct.new :direction => Direction::REGULAR, :headsign => 'outbound', :service_id => 8550, :trip_id => 1712196
-      route_model = Model::Route.new
+      trip_page   = OpenStruct.new :direction => Direction::REGULAR, :headsign => 'outbound', :trip_id => 1712196
+      route_model = Model::Route.create :id => '130', :short_name => '130', :long_name => 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson', :route_type => 0
       trip_model  = route_model.add_trip_from_trip_page trip_page
       assert trip_model.saved?
     end

data/test/unit/model/trip_test.rb

@@ -2,12 +2,12 @@ require 'helper'
 
 class Model::TripTest < MiniTest::Unit::TestCase
   def test_trip_page_with_bang
-    trip_page  = OpenStruct.new :direction => 'outbound', :service_id => 8550, :trip_id => 1712196
+    trip_page  = OpenStruct.new :direction => Direction::REGULAR, :headsign => 'outbound', :trip_id => 1712196
     trip_model = Model::Trip.new
     trip_model.trip_page! trip_page
     assert_equal trip_model.direction, trip_page.direction
-    assert_equal trip_model.service_id, trip_page.service_id
-    assert_equal trip_model.trip_id, trip_page.trip_id
+    assert_equal trip_model.headsign, trip_page.headsign
+    assert_equal trip_model.id, trip_page.trip_id
   end
 
   def test_add_stop_time_from_stop_time_page

data/test/unit/page/route_test.rb

@@ -4,19 +4,28 @@ class Page::RouteTest < MiniTest::Unit::TestCase
   attr_accessor :route
 
   def setup
-    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-06-21').
+    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-09-24').
       to_return(:status => 200, :body => fixture('verbatim/route.html'), :headers => {'Content-Type' => 'text/html'})
-    @route = Page::Route.new 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-06-21', 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson'
+    @route = Page::Route.new 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-09-24', 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson'
   end
 
   def test_date
-    assert_equal DateTime.parse('21/06/2012 12:00:00 AM'), route.date
+    assert_equal DateTime.parse('24/09/2012 12:00:00 AM'), route.date
+  end
+
+  def test_direction_from_anchor
+    anchor = route.page.search('a.map-link-top').first
+    assert_equal Direction::REGULAR, route.direction_from_anchor(anchor)
   end
 
   def test_headsigns
     assert_equal ['outbound', 'inbound'], route.headsigns
   end
 
+  def test_route_id
+    assert_equal '130', route.route_id
+  end
+
   def test_short_name
     assert_equal '130', route.short_name
   end
@@ -25,10 +34,10 @@ class Page::RouteTest < MiniTest::Unit::TestCase
     trip_pages = route.trip_pages
     trip1 = trip_pages.first
     trip2 = trip_pages.last
-    assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/outbound/8904/1891449/2012-06-20', trip1.url.to_s
+    assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/outbound/9792/2170894/2012-09-24', trip1.url.to_s
     assert_equal route.date.to_date, trip1.date
     assert_equal Direction::REGULAR, trip1.direction
-    assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/inbound/8904/1881543/2012-06-22', trip2.url.to_s
+    assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/inbound/9792/2180194/2012-09-24', trip2.url.to_s
     assert_equal route.date.to_date, trip2.date
     assert_equal Direction::GOOFY, trip2.direction
   end
@@ -60,3 +69,25 @@ class Page::RouteTypeTest < MiniTest::Unit::TestCase
     end
   end
 end
+
+class Page::DateFromAnchorTest < MiniTest::Unit::TestCase
+  attr_accessor :route
+
+  def setup
+    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/120/2012-09-24').
+      to_return(:status => 200, :body => fixture('verbatim/route/date_from_anchor.html'), :headers => {'Content-Type' => 'text/html'})
+    @route = Page::Route.new 'http://jp.translink.com.au/travel-information/network-information/buses/120/2012-09-24', 'City, SthBank, Buranda, Salisbury, Griffith Uni, Garden City'
+  end
+
+  def test_unparsable_date
+    anchor = route.page.search('a.map-link-top')[0]
+    assert_equal DateTime.parse('01/01/1970 12:00:00 AM'), route.date_from_anchor(anchor)
+    anchor = route.page.search('a.map-link-top')[1]
+    assert_equal DateTime.parse('01/01/1970 12:00:00 AM'), route.date_from_anchor(anchor)
+  end
+
+  def test_parsable_date
+    anchor = route.page.search('a.map-link-top')[2]
+    assert_equal DateTime.parse('24/09/2012 12:00:00 AM'), route.date_from_anchor(anchor)
+  end
+end

data/test/unit/page/timetable_test.rb

@@ -17,6 +17,35 @@ class Page::TimetableTest < MiniTest::Unit::TestCase
     assert_equal 'Coomera stn, Dreamworld, Movieworld, Wet\'N Wild, H\'vale stn', route_pages.last.long_name
   end
 
+  def test_route_pages_are_unique
+    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
+      to_return(:status => 200, :body => fixture('verbatim/timetable/duplicate_routes.html'), :headers => {'Content-Type' => 'text/html'})
+    timetable_page = Page::Timetable.new('http://jp.translink.com.au/travel-information/network-information/buses/all-timetables')
+    route_pages    = timetable_page.route_pages
+    assert_equal 440, route_pages.size
+    assert_equal 1, route_pages.select { |route_page| route_page.url.to_s == 'http://jp.translink.com.au/travel-information/network-information/buses/LOOP/2012-09-24' }.size
+  end
+
+  def test_route_pages_omit_routes_before_route_with_url
+    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
+      to_return(:status => 200, :body => fixture('verbatim/timetable.html'), :headers => {'Content-Type' => 'text/html'})
+    url            = URI.parse 'http://jp.translink.com.au/travel-information/network-information/buses/10'
+    timetable_page = Page::Timetable.new 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
+    route_pages    = timetable_page.route_pages url
+    assert_equal 441, route_pages.size
+    assert_equal url, route_pages.first.url
+  end
+
+  def test_route_pages_limited_by_step
+    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
+      to_return(:status => 200, :body => fixture('verbatim/timetable.html'), :headers => {'Content-Type' => 'text/html'})
+    url            = URI.parse 'http://jp.translink.com.au/travel-information/network-information/buses/10'
+    timetable_page = Page::Timetable.new 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
+    route_pages    = timetable_page.route_pages url, 0
+    assert_equal 1, route_pages.size
+    assert_equal url, route_pages.first.url
+  end
+
   def test_timetable_page
     stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
       to_return(:status => 200, :body => fixture('verbatim/timetable.html'), :headers => {'Content-Type' => 'text/html'})

data/test/unit/page/trip_test.rb

@@ -32,10 +32,6 @@ class Page::TripRequestTest < MiniTest::Unit::TestCase
     Page::Trip.new('http://jp.translink.com.au/travel-information/network-information/service-information/outbound/8550/1712196/2012-05-31', Date.today, Direction::REGULAR)
   end
 
-  def test_service_id
-    assert_equal '1712196', stub_trip.service_id
-  end
-
   def test_stop_times
     stop_page1 = Page::Trip::Stop.new.tap { |s| s.stop_name = 'Queen Street station, platform A6'; s.stop_id = '001002'; s.stop_lat = '-27.470677'; s.stop_lon = '153.024747' }
     stop_page2 = Page::Trip::Stop.new.tap { |s| s.stop_name = 'Illaweena St at Waterstone'; s.stop_id = '010764'; s.stop_lat = '-27.639463'; s.stop_lon = '153.052551' }
@@ -52,6 +48,17 @@ class Page::TripRequestTest < MiniTest::Unit::TestCase
   end
 
   def test_trip_id
-    assert_equal '8550', stub_trip.trip_id
+    assert_equal '1712196', stub_trip.trip_id
+  end
+end
+
+class Page::TripDuplicateStopTimesTest < MiniTest::Unit::TestCase
+  def test_stop_times_are_unique
+    stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/service-information/counterclockwise/9792/2171146/2012-09-24').
+      to_return(:status => 200, :body => fixture('verbatim/trip/duplicate_stop_times.html'), :headers => {'Content-Type' => 'text/html'})
+    trip = Page::Trip.new('http://jp.translink.com.au/travel-information/network-information/service-information/counterclockwise/9792/2171146/2012-09-24', Date.parse('2012-09-24'), Direction::REGULAR)
+    stop_times = trip.stop_times
+    assert_equal 10, stop_times.size
+    assert_equal 1, stop_times.select { |stop_time| stop_time.stop_page.stop_id == '00094A' }.size
   end
 end

data/translink.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |s|
   s.authors              = ['Tate Johnson']
   s.email                = ['tate@tatey.com']
   s.homepage             = 'https://github.com/tatey/translink'
-  s.summary              = %q{Scrapes bus routes, trips, stops and times into a relational database.}
-  s.description          = %q{Scrapes bus routes, trips, stops and times into a relational database.}
+  s.summary              = %q{Scrapes public transport data from Translink into GTFS}
+  s.description          = %q{Scrapes public transport data from Translink into GTFS}
 
   s.rubyforge_project = 'translink'
 
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'dm-core', '~> 1.2.0'
   s.add_runtime_dependency 'dm-migrations', '~> 1.2.0'
   s.add_runtime_dependency 'dm-sqlite-adapter', '~> 1.2.0'
-  s.add_runtime_dependency 'mechanize', '~> 2.0.1'
+  s.add_runtime_dependency 'mechanize', '~> 2.5.1'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'rake', '~> 0.9.2'
   s.add_development_dependency 'webmock', '~> 1.7.8'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: translink
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-21 00:00:00.000000000 Z
+date: 2012-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dm-core
-  requirement: &70220170640540 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70220170640540
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: dm-migrations
-  requirement: &70220170639960 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +37,15 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70220170639960
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: dm-sqlite-adapter
-  requirement: &70220170639060 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,21 +53,31 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70220170639060
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70220170638380 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70220170638380
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.1
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &70220170638000 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +85,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70220170638000
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70220170637340 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +101,15 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70220170637340
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: webmock
-  requirement: &70220170636820 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,8 +117,13 @@ dependencies:
         version: 1.7.8
   type: :development
   prerelease: false
-  version_requirements: *70220170636820
-description: Scrapes bus routes, trips, stops and times into a relational database.
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.8
+description: Scrapes public transport data from Translink into GTFS
 email:
 - tate@tatey.com
 executables:
@@ -123,8 +158,11 @@ files:
 - test/fixtures/sample/route.html
 - test/fixtures/sample/timetable.html
 - test/fixtures/verbatim/route.html
+- test/fixtures/verbatim/route/date_from_anchor.html
 - test/fixtures/verbatim/timetable.html
+- test/fixtures/verbatim/timetable/duplicate_routes.html
 - test/fixtures/verbatim/trip.html
+- test/fixtures/verbatim/trip/duplicate_stop_times.html
 - test/helper.rb
 - test/unit/cli_test.rb
 - test/unit/crawler_test.rb
@@ -158,19 +196,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -132775799840568295
+      hash: -1469314078129760019
 requirements: []
 rubyforge_project: translink
-rubygems_version: 1.8.11
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Scrapes bus routes, trips, stops and times into a relational database.
+summary: Scrapes public transport data from Translink into GTFS
 test_files:
 - test/fixtures/sample/route.html
 - test/fixtures/sample/timetable.html
 - test/fixtures/verbatim/route.html
+- test/fixtures/verbatim/route/date_from_anchor.html
 - test/fixtures/verbatim/timetable.html
+- test/fixtures/verbatim/timetable/duplicate_routes.html
 - test/fixtures/verbatim/trip.html
+- test/fixtures/verbatim/trip/duplicate_stop_times.html
 - test/helper.rb
 - test/unit/cli_test.rb
 - test/unit/crawler_test.rb