translink 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,7 +8,7 @@ class CLITtest < MiniTest::Unit::TestCase
8
8
 
9
9
  end
10
10
 
11
- def crawl date
11
+ def crawl date, from_route_url = nil, step = nil
12
12
 
13
13
  end
14
14
  end
@@ -41,20 +41,39 @@ class CLITtest < MiniTest::Unit::TestCase
41
41
  assert_includes @out.string, 'Usage'
42
42
  end
43
43
 
44
- def test_execute_scrape_command_with_uri
45
- file = File.join TMPDIR, "test_scrape_#{Time.now.to_i}.sqlite3"
44
+ def test_execute_scape_command_with_date
45
+ file = File.join TMPDIR, '2011-11-27.sqlite3'
46
46
  refute File.exists?(file), 'Expected file not to exist.'
47
- @cli.run "scrape 2011-11-27 --uri=sqlite://#{file}"
47
+ @cli.run 'scrape 2011-11-27'
48
48
  assert File.exists?(file), 'Expected file to exist.'
49
49
  end
50
50
 
51
- def test_execute_scrape_command_with_date_writes_sqlite_database
52
- file = File.join TMPDIR, '2011-11-27.sqlite3'
51
+ def test_execute_scape_command_with_date_and_db_path
52
+ file = File.join TMPDIR, "test_db_path_#{Time.now.to_i}.sqlite3"
53
53
  refute File.exists?(file), 'Expected file not to exist.'
54
- @cli.run 'scrape 2011-11-27'
54
+ @cli.run "scrape 2011-11-27 #{file}"
55
+ assert File.exists?(file), 'Expected file to exist.'
56
+ end
57
+
58
+ def test_execute_scrape_command_with_date_db_path_and_from_route_url
59
+ file = File.join TMPDIR, "test_db_path_from_route_url_#{Time.now.to_i}.sqlite3"
60
+ refute File.exists?(file), 'Expected file not to exist.'
61
+ @cli.run "scrape 2011-11-27 #{file} http://localhost"
55
62
  assert File.exists?(file), 'Expected file to exist.'
56
63
  end
57
64
 
65
+ def test_execute_scrape_command_with_date_db_path_from_route_url_and_step
66
+ file = File.join TMPDIR, "test_db_path_from_route_url_step_#{Time.now.to_i}.sqlite3"
67
+ refute File.exists?(file), 'Expected file not to exist.'
68
+ @cli.run "scrape 2011-11-27 #{file} http://localhost 0"
69
+ assert File.exists?(file), 'Expected file to exist.'
70
+ end
71
+
72
+ def test_version_command
73
+ @cli.run 'version'
74
+ assert_match Translink::VERSION, @out.string
75
+ end
76
+
58
77
  def test_execute_scrape_command_without_date_executes_help_command
59
78
  @cli.run 'scrape'
60
79
  assert_includes @out.string, 'Usage'
@@ -6,6 +6,7 @@ class Model::RouteTest < MiniTest::Unit::TestCase
6
6
  route_page = MiniTest::Mock.new
7
7
  route_page.expect :short_name, '130'
8
8
  route_page.expect :long_name, 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson'
9
+ route_page.expect :route_id, '130'
9
10
  route_page.expect :route_type, 0
10
11
  assert_equal 0, Model::Route.count
11
12
  assert_equal Model::Route.find_or_add_route_from_route_page(route_page), Model::Route.find_or_add_route_from_route_page(route_page)
@@ -15,8 +16,8 @@ class Model::RouteTest < MiniTest::Unit::TestCase
15
16
 
16
17
  def test_add_trip_from_trip_page
17
18
  DB.context 'sqlite::memory:', :migrate => true do
18
- trip_page = OpenStruct.new :direction => Direction::REGULAR, :headsign => 'outbound', :service_id => 8550, :trip_id => 1712196
19
- route_model = Model::Route.new
19
+ trip_page = OpenStruct.new :direction => Direction::REGULAR, :headsign => 'outbound', :trip_id => 1712196
20
+ route_model = Model::Route.create :id => '130', :short_name => '130', :long_name => 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson', :route_type => 0
20
21
  trip_model = route_model.add_trip_from_trip_page trip_page
21
22
  assert trip_model.saved?
22
23
  end
@@ -2,12 +2,12 @@ require 'helper'
2
2
 
3
3
  class Model::TripTest < MiniTest::Unit::TestCase
4
4
  def test_trip_page_with_bang
5
- trip_page = OpenStruct.new :direction => 'outbound', :service_id => 8550, :trip_id => 1712196
5
+ trip_page = OpenStruct.new :direction => Direction::REGULAR, :headsign => 'outbound', :trip_id => 1712196
6
6
  trip_model = Model::Trip.new
7
7
  trip_model.trip_page! trip_page
8
8
  assert_equal trip_model.direction, trip_page.direction
9
- assert_equal trip_model.service_id, trip_page.service_id
10
- assert_equal trip_model.trip_id, trip_page.trip_id
9
+ assert_equal trip_model.headsign, trip_page.headsign
10
+ assert_equal trip_model.id, trip_page.trip_id
11
11
  end
12
12
 
13
13
  def test_add_stop_time_from_stop_time_page
@@ -4,19 +4,28 @@ class Page::RouteTest < MiniTest::Unit::TestCase
4
4
  attr_accessor :route
5
5
 
6
6
  def setup
7
- stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-06-21').
7
+ stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-09-24').
8
8
  to_return(:status => 200, :body => fixture('verbatim/route.html'), :headers => {'Content-Type' => 'text/html'})
9
- @route = Page::Route.new 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-06-21', 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson'
9
+ @route = Page::Route.new 'http://jp.translink.com.au/travel-information/network-information/buses/130/2012-09-24', 'City, Griffith Uni, Sunnybank Hills, Algester, Parkinson'
10
10
  end
11
11
 
12
12
  def test_date
13
- assert_equal DateTime.parse('21/06/2012 12:00:00 AM'), route.date
13
+ assert_equal DateTime.parse('24/09/2012 12:00:00 AM'), route.date
14
+ end
15
+
16
+ def test_direction_from_anchor
17
+ anchor = route.page.search('a.map-link-top').first
18
+ assert_equal Direction::REGULAR, route.direction_from_anchor(anchor)
14
19
  end
15
20
 
16
21
  def test_headsigns
17
22
  assert_equal ['outbound', 'inbound'], route.headsigns
18
23
  end
19
24
 
25
+ def test_route_id
26
+ assert_equal '130', route.route_id
27
+ end
28
+
20
29
  def test_short_name
21
30
  assert_equal '130', route.short_name
22
31
  end
@@ -25,10 +34,10 @@ class Page::RouteTest < MiniTest::Unit::TestCase
25
34
  trip_pages = route.trip_pages
26
35
  trip1 = trip_pages.first
27
36
  trip2 = trip_pages.last
28
- assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/outbound/8904/1891449/2012-06-20', trip1.url.to_s
37
+ assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/outbound/9792/2170894/2012-09-24', trip1.url.to_s
29
38
  assert_equal route.date.to_date, trip1.date
30
39
  assert_equal Direction::REGULAR, trip1.direction
31
- assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/inbound/8904/1881543/2012-06-22', trip2.url.to_s
40
+ assert_equal 'http://jp.translink.com.au/travel-information/network-information/service-information/inbound/9792/2180194/2012-09-24', trip2.url.to_s
32
41
  assert_equal route.date.to_date, trip2.date
33
42
  assert_equal Direction::GOOFY, trip2.direction
34
43
  end
@@ -60,3 +69,25 @@ class Page::RouteTypeTest < MiniTest::Unit::TestCase
60
69
  end
61
70
  end
62
71
  end
72
+
73
+ class Page::DateFromAnchorTest < MiniTest::Unit::TestCase
74
+ attr_accessor :route
75
+
76
+ def setup
77
+ stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/120/2012-09-24').
78
+ to_return(:status => 200, :body => fixture('verbatim/route/date_from_anchor.html'), :headers => {'Content-Type' => 'text/html'})
79
+ @route = Page::Route.new 'http://jp.translink.com.au/travel-information/network-information/buses/120/2012-09-24', 'City, SthBank, Buranda, Salisbury, Griffith Uni, Garden City'
80
+ end
81
+
82
+ def test_unparsable_date
83
+ anchor = route.page.search('a.map-link-top')[0]
84
+ assert_equal DateTime.parse('01/01/1970 12:00:00 AM'), route.date_from_anchor(anchor)
85
+ anchor = route.page.search('a.map-link-top')[1]
86
+ assert_equal DateTime.parse('01/01/1970 12:00:00 AM'), route.date_from_anchor(anchor)
87
+ end
88
+
89
+ def test_parsable_date
90
+ anchor = route.page.search('a.map-link-top')[2]
91
+ assert_equal DateTime.parse('24/09/2012 12:00:00 AM'), route.date_from_anchor(anchor)
92
+ end
93
+ end
@@ -17,6 +17,35 @@ class Page::TimetableTest < MiniTest::Unit::TestCase
17
17
  assert_equal 'Coomera stn, Dreamworld, Movieworld, Wet\'N Wild, H\'vale stn', route_pages.last.long_name
18
18
  end
19
19
 
20
+ def test_route_pages_are_unique
21
+ stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
22
+ to_return(:status => 200, :body => fixture('verbatim/timetable/duplicate_routes.html'), :headers => {'Content-Type' => 'text/html'})
23
+ timetable_page = Page::Timetable.new('http://jp.translink.com.au/travel-information/network-information/buses/all-timetables')
24
+ route_pages = timetable_page.route_pages
25
+ assert_equal 440, route_pages.size
26
+ assert_equal 1, route_pages.select { |route_page| route_page.url.to_s == 'http://jp.translink.com.au/travel-information/network-information/buses/LOOP/2012-09-24' }.size
27
+ end
28
+
29
+ def test_route_pages_omit_routes_before_route_with_url
30
+ stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
31
+ to_return(:status => 200, :body => fixture('verbatim/timetable.html'), :headers => {'Content-Type' => 'text/html'})
32
+ url = URI.parse 'http://jp.translink.com.au/travel-information/network-information/buses/10'
33
+ timetable_page = Page::Timetable.new 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
34
+ route_pages = timetable_page.route_pages url
35
+ assert_equal 441, route_pages.size
36
+ assert_equal url, route_pages.first.url
37
+ end
38
+
39
+ def test_route_pages_limited_by_step
40
+ stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
41
+ to_return(:status => 200, :body => fixture('verbatim/timetable.html'), :headers => {'Content-Type' => 'text/html'})
42
+ url = URI.parse 'http://jp.translink.com.au/travel-information/network-information/buses/10'
43
+ timetable_page = Page::Timetable.new 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
44
+ route_pages = timetable_page.route_pages url, 0
45
+ assert_equal 1, route_pages.size
46
+ assert_equal url, route_pages.first.url
47
+ end
48
+
20
49
  def test_timetable_page
21
50
  stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables').
22
51
  to_return(:status => 200, :body => fixture('verbatim/timetable.html'), :headers => {'Content-Type' => 'text/html'})
@@ -32,10 +32,6 @@ class Page::TripRequestTest < MiniTest::Unit::TestCase
32
32
  Page::Trip.new('http://jp.translink.com.au/travel-information/network-information/service-information/outbound/8550/1712196/2012-05-31', Date.today, Direction::REGULAR)
33
33
  end
34
34
 
35
- def test_service_id
36
- assert_equal '1712196', stub_trip.service_id
37
- end
38
-
39
35
  def test_stop_times
40
36
  stop_page1 = Page::Trip::Stop.new.tap { |s| s.stop_name = 'Queen Street station, platform A6'; s.stop_id = '001002'; s.stop_lat = '-27.470677'; s.stop_lon = '153.024747' }
41
37
  stop_page2 = Page::Trip::Stop.new.tap { |s| s.stop_name = 'Illaweena St at Waterstone'; s.stop_id = '010764'; s.stop_lat = '-27.639463'; s.stop_lon = '153.052551' }
@@ -52,6 +48,17 @@ class Page::TripRequestTest < MiniTest::Unit::TestCase
52
48
  end
53
49
 
54
50
  def test_trip_id
55
- assert_equal '8550', stub_trip.trip_id
51
+ assert_equal '1712196', stub_trip.trip_id
52
+ end
53
+ end
54
+
55
+ class Page::TripDuplicateStopTimesTest < MiniTest::Unit::TestCase
56
+ def test_stop_times_are_unique
57
+ stub_request(:get, 'http://jp.translink.com.au/travel-information/network-information/service-information/counterclockwise/9792/2171146/2012-09-24').
58
+ to_return(:status => 200, :body => fixture('verbatim/trip/duplicate_stop_times.html'), :headers => {'Content-Type' => 'text/html'})
59
+ trip = Page::Trip.new('http://jp.translink.com.au/travel-information/network-information/service-information/counterclockwise/9792/2171146/2012-09-24', Date.parse('2012-09-24'), Direction::REGULAR)
60
+ stop_times = trip.stop_times
61
+ assert_equal 10, stop_times.size
62
+ assert_equal 1, stop_times.select { |stop_time| stop_time.stop_page.stop_id == '00094A' }.size
56
63
  end
57
64
  end
@@ -9,8 +9,8 @@ Gem::Specification.new do |s|
9
9
  s.authors = ['Tate Johnson']
10
10
  s.email = ['tate@tatey.com']
11
11
  s.homepage = 'https://github.com/tatey/translink'
12
- s.summary = %q{Scrapes bus routes, trips, stops and times into a relational database.}
13
- s.description = %q{Scrapes bus routes, trips, stops and times into a relational database.}
12
+ s.summary = %q{Scrapes public transport data from Translink into GTFS}
13
+ s.description = %q{Scrapes public transport data from Translink into GTFS}
14
14
 
15
15
  s.rubyforge_project = 'translink'
16
16
 
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
19
19
  s.add_runtime_dependency 'dm-core', '~> 1.2.0'
20
20
  s.add_runtime_dependency 'dm-migrations', '~> 1.2.0'
21
21
  s.add_runtime_dependency 'dm-sqlite-adapter', '~> 1.2.0'
22
- s.add_runtime_dependency 'mechanize', '~> 2.0.1'
22
+ s.add_runtime_dependency 'mechanize', '~> 2.5.1'
23
23
  s.add_development_dependency 'pry'
24
24
  s.add_development_dependency 'rake', '~> 0.9.2'
25
25
  s.add_development_dependency 'webmock', '~> 1.7.8'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: translink
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 3.0.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-07-21 00:00:00.000000000 Z
12
+ date: 2012-12-14 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: dm-core
16
- requirement: &70220170640540 !ruby/object:Gem::Requirement
16
+ requirement: !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ~>
@@ -21,10 +21,15 @@ dependencies:
21
21
  version: 1.2.0
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70220170640540
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 1.2.0
25
30
  - !ruby/object:Gem::Dependency
26
31
  name: dm-migrations
27
- requirement: &70220170639960 !ruby/object:Gem::Requirement
32
+ requirement: !ruby/object:Gem::Requirement
28
33
  none: false
29
34
  requirements:
30
35
  - - ~>
@@ -32,10 +37,15 @@ dependencies:
32
37
  version: 1.2.0
33
38
  type: :runtime
34
39
  prerelease: false
35
- version_requirements: *70220170639960
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: 1.2.0
36
46
  - !ruby/object:Gem::Dependency
37
47
  name: dm-sqlite-adapter
38
- requirement: &70220170639060 !ruby/object:Gem::Requirement
48
+ requirement: !ruby/object:Gem::Requirement
39
49
  none: false
40
50
  requirements:
41
51
  - - ~>
@@ -43,21 +53,31 @@ dependencies:
43
53
  version: 1.2.0
44
54
  type: :runtime
45
55
  prerelease: false
46
- version_requirements: *70220170639060
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: 1.2.0
47
62
  - !ruby/object:Gem::Dependency
48
63
  name: mechanize
49
- requirement: &70220170638380 !ruby/object:Gem::Requirement
64
+ requirement: !ruby/object:Gem::Requirement
50
65
  none: false
51
66
  requirements:
52
67
  - - ~>
53
68
  - !ruby/object:Gem::Version
54
- version: 2.0.1
69
+ version: 2.5.1
55
70
  type: :runtime
56
71
  prerelease: false
57
- version_requirements: *70220170638380
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ~>
76
+ - !ruby/object:Gem::Version
77
+ version: 2.5.1
58
78
  - !ruby/object:Gem::Dependency
59
79
  name: pry
60
- requirement: &70220170638000 !ruby/object:Gem::Requirement
80
+ requirement: !ruby/object:Gem::Requirement
61
81
  none: false
62
82
  requirements:
63
83
  - - ! '>='
@@ -65,10 +85,15 @@ dependencies:
65
85
  version: '0'
66
86
  type: :development
67
87
  prerelease: false
68
- version_requirements: *70220170638000
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
69
94
  - !ruby/object:Gem::Dependency
70
95
  name: rake
71
- requirement: &70220170637340 !ruby/object:Gem::Requirement
96
+ requirement: !ruby/object:Gem::Requirement
72
97
  none: false
73
98
  requirements:
74
99
  - - ~>
@@ -76,10 +101,15 @@ dependencies:
76
101
  version: 0.9.2
77
102
  type: :development
78
103
  prerelease: false
79
- version_requirements: *70220170637340
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ~>
108
+ - !ruby/object:Gem::Version
109
+ version: 0.9.2
80
110
  - !ruby/object:Gem::Dependency
81
111
  name: webmock
82
- requirement: &70220170636820 !ruby/object:Gem::Requirement
112
+ requirement: !ruby/object:Gem::Requirement
83
113
  none: false
84
114
  requirements:
85
115
  - - ~>
@@ -87,8 +117,13 @@ dependencies:
87
117
  version: 1.7.8
88
118
  type: :development
89
119
  prerelease: false
90
- version_requirements: *70220170636820
91
- description: Scrapes bus routes, trips, stops and times into a relational database.
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ none: false
122
+ requirements:
123
+ - - ~>
124
+ - !ruby/object:Gem::Version
125
+ version: 1.7.8
126
+ description: Scrapes public transport data from Translink into GTFS
92
127
  email:
93
128
  - tate@tatey.com
94
129
  executables:
@@ -123,8 +158,11 @@ files:
123
158
  - test/fixtures/sample/route.html
124
159
  - test/fixtures/sample/timetable.html
125
160
  - test/fixtures/verbatim/route.html
161
+ - test/fixtures/verbatim/route/date_from_anchor.html
126
162
  - test/fixtures/verbatim/timetable.html
163
+ - test/fixtures/verbatim/timetable/duplicate_routes.html
127
164
  - test/fixtures/verbatim/trip.html
165
+ - test/fixtures/verbatim/trip/duplicate_stop_times.html
128
166
  - test/helper.rb
129
167
  - test/unit/cli_test.rb
130
168
  - test/unit/crawler_test.rb
@@ -158,19 +196,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
158
196
  version: '0'
159
197
  segments:
160
198
  - 0
161
- hash: -132775799840568295
199
+ hash: -1469314078129760019
162
200
  requirements: []
163
201
  rubyforge_project: translink
164
- rubygems_version: 1.8.11
202
+ rubygems_version: 1.8.23
165
203
  signing_key:
166
204
  specification_version: 3
167
- summary: Scrapes bus routes, trips, stops and times into a relational database.
205
+ summary: Scrapes public transport data from Translink into GTFS
168
206
  test_files:
169
207
  - test/fixtures/sample/route.html
170
208
  - test/fixtures/sample/timetable.html
171
209
  - test/fixtures/verbatim/route.html
210
+ - test/fixtures/verbatim/route/date_from_anchor.html
172
211
  - test/fixtures/verbatim/timetable.html
212
+ - test/fixtures/verbatim/timetable/duplicate_routes.html
173
213
  - test/fixtures/verbatim/trip.html
214
+ - test/fixtures/verbatim/trip/duplicate_stop_times.html
174
215
  - test/helper.rb
175
216
  - test/unit/cli_test.rb
176
217
  - test/unit/crawler_test.rb