gtfs-reader 0.2.9 → 0.2.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +42 -3
- data/Rakefile +2 -2
- data/lib/gtfs_reader/config/column.rb +1 -13
- data/lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb +66 -84
- data/lib/gtfs_reader/config/file_definition.rb +1 -18
- data/lib/gtfs_reader/core.rb +2 -1
- data/lib/gtfs_reader/log.rb +1 -0
- data/lib/gtfs_reader/source_updater.rb +88 -51
- data/lib/gtfs_reader/version.rb +3 -3
- metadata +9 -7
- data/lib/gtfs_reader/config/prefixed_column_setter.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6b7400f376dbef697ba8dbce21afce315c971d43
|
4
|
+
data.tar.gz: cd299fb45520ec614be5386a3fc2e66a057cb2e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9fdba13c8c3933340a6770941f42117944f68ac701bc90591cc4fb053d75f6a2efe2361fc8ef71d58ee442a48cb4889ad6131a2334a77418243d3cb4d10ac1e8
|
7
|
+
data.tar.gz: c42a6814a3eda9a3bef8fb364783bf60abca9d4cd0671ebc5cd3da92b29f54ff2fbbab60b296888d537eed8b5b6848c5b1304827b41b938d08345bbcea808271
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# GTFS Reader
|
2
2
|
|
3
3
|
```ruby
|
4
|
-
gem '
|
4
|
+
gem 'gtfs_reader'
|
5
5
|
```
|
6
6
|
|
7
7
|
GTFS Reader is a gem designed to help process the contents of a "[GTFS
|
@@ -26,10 +26,10 @@ require 'gtfs_reader'
|
|
26
26
|
|
27
27
|
GtfsReader.config do
|
28
28
|
return_hashes true
|
29
|
-
|
29
|
+
# verbose true #uncomment for verbose output
|
30
30
|
sources do
|
31
31
|
sample do
|
32
|
-
url 'http://localhost/sample-feed.zip'
|
32
|
+
url 'http://localhost/sample-feed.zip' # you can also use a filepath here
|
33
33
|
before { |etag| puts "Processing source with tag #{etag}..." }
|
34
34
|
handlers do
|
35
35
|
agency {|row| puts "Read Agency: #{row[:agency_name]}" }
|
@@ -55,3 +55,42 @@ Read Route: Stagecoach - Airport Shuttle
|
|
55
55
|
Read Route: City
|
56
56
|
Read Route: Airport - Amargosa Valley
|
57
57
|
```
|
58
|
+
|
59
|
+
## Custom Feed Format
|
60
|
+
|
61
|
+
By default, this gem parses files in the format specified by the [GTFS Feed
|
62
|
+
Spec](https://developers.google.com/transit/gtfs/reference). You can see this
|
63
|
+
`FeedDefinition` in [config/defaults/gtfs_feed_definition.rb](https://github.com/sangster/gtfs_reader/blob/develop/lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb).
|
64
|
+
However, in many cases these feeds are created by people who aren't
|
65
|
+
technically-proficient and may not exactly conform to the spec. In the event
|
66
|
+
that you want to parse a file with a different format, you can do so in the
|
67
|
+
`GtfsReader.config` block:
|
68
|
+
|
69
|
+
```ruby
|
70
|
+
GtfsReader.config do
|
71
|
+
sources do
|
72
|
+
sample do
|
73
|
+
feed_definition do
|
74
|
+
file :drivers, required: true do # for my_file.txt
|
75
|
+
col :licence_number, required: true, unique: true
|
76
|
+
|
77
|
+
# If the sex column contains "1", the symbol :male will be returned,
|
78
|
+
# otherwise :female will be returned
|
79
|
+
col :sex, &output_map( :unspecified, female: ?1, male: ?2 )
|
80
|
+
|
81
|
+
# This will allow you to create a custom parser. Within the given
|
82
|
+
# block you can reference other columns in the current row by name.
|
83
|
+
col :name do |name|
|
84
|
+
case name
|
85
|
+
when :female then "Ms. #{name}"
|
86
|
+
when :male then "Mr. #{name}"
|
87
|
+
else name
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
# ...
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
```
|
data/Rakefile
CHANGED
@@ -17,9 +17,9 @@ rescue Bundler::BundlerError => e
|
|
17
17
|
end
|
18
18
|
|
19
19
|
Jeweler::Tasks.new do |gem|
|
20
|
-
gem.name = '
|
20
|
+
gem.name = 'gtfs_reader'
|
21
21
|
gem.version = GtfsReader::Version.to_s
|
22
|
-
gem.homepage = 'http://github.com/sangster/
|
22
|
+
gem.homepage = 'http://github.com/sangster/gtfs_reader'
|
23
23
|
gem.license = 'GPL 3'
|
24
24
|
gem.summary = 'Read General Transit Feed Specification zip files'
|
25
25
|
gem.description = <<-EOF.strip.gsub /\s+/, ' '
|
@@ -10,20 +10,13 @@ module GtfsReader
|
|
10
10
|
#@param name [String] the name of the column
|
11
11
|
#@option opts [Boolean] :required (false) If this column is required to
|
12
12
|
# appear in the given file
|
13
|
-
#@option opts [String] :alias an alternative name for this column. Many
|
14
|
-
# column names are needlessly prefixed with their filename:
|
15
|
-
# +Stop.stop_name+ could be aliased to +Stop.name+ for example.
|
16
13
|
#@option opts [Boolean] :unique (false) if values in this column need to be
|
17
14
|
# unique among all rows in the file.
|
18
15
|
def initialize(name, opts={}, &parser)
|
19
16
|
@name = name
|
20
17
|
@parser = block_given? ? parser : IDENTITY_PARSER
|
21
18
|
|
22
|
-
@opts = {
|
23
|
-
required: false,
|
24
|
-
unique: false,
|
25
|
-
alias: nil
|
26
|
-
}.merge (opts || {})
|
19
|
+
@opts = { required: false, unique: false }.merge ( opts || {} )
|
27
20
|
end
|
28
21
|
|
29
22
|
def parser(&block)
|
@@ -42,11 +35,6 @@ module GtfsReader
|
|
42
35
|
@opts[:unique]
|
43
36
|
end
|
44
37
|
|
45
|
-
#@return [String,nil] this column's name's alias, if there is one
|
46
|
-
def alias
|
47
|
-
@opts[:alias]
|
48
|
-
end
|
49
|
-
|
50
38
|
#@return [Boolean]
|
51
39
|
def parser?
|
52
40
|
parser != IDENTITY_PARSER
|
@@ -1,34 +1,35 @@
|
|
1
1
|
require_relative '../feed_definition'
|
2
2
|
|
3
|
+
# This default config file creates a [FeedDefinition] that matches the one
|
4
|
+
# specified by Google. You can use this definition in most cases. A custom
|
5
|
+
# definition will only be required if you need to parse a feed that differs in
|
6
|
+
# some critical way (Remember that these feeds are not always created by
|
7
|
+
# technically-proficient people).
|
8
|
+
# See https://developers.google.com/transit/gtfs/reference
|
3
9
|
module GtfsReader
|
4
10
|
module Config
|
5
11
|
module Defaults
|
6
12
|
FEED_DEFINITION = FeedDefinition.new.tap do |feed|
|
7
13
|
feed.instance_exec do
|
8
14
|
file :agency, required: true do
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
col :fare_url
|
17
|
-
end
|
15
|
+
col :agency_name, required: true
|
16
|
+
col :agency_url, required: true
|
17
|
+
col :agency_timezone, required: true
|
18
|
+
col :agency_id, unique: true
|
19
|
+
col :agency_lang
|
20
|
+
col :agency_phone
|
21
|
+
col :agency_fare_url
|
18
22
|
end
|
19
23
|
|
20
24
|
file :stops, required: true do
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
col :timezone
|
30
|
-
end
|
31
|
-
|
25
|
+
col :stop_id, required: true, unique: true
|
26
|
+
col :stop_code
|
27
|
+
col :stop_name, required: true
|
28
|
+
col :stop_desc
|
29
|
+
col :stop_lat, required: true
|
30
|
+
col :stop_lon, required: true
|
31
|
+
col :stop_url
|
32
|
+
col :stop_timezone
|
32
33
|
col :zone_id
|
33
34
|
col :location_type, &output_map( :stop, station: ?1 )
|
34
35
|
col :parent_station
|
@@ -51,59 +52,47 @@ module GtfsReader
|
|
51
52
|
end
|
52
53
|
|
53
54
|
file :routes, required: true do
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
col :type, required: true,
|
55
|
+
col :route_id, required: true, unique: true
|
56
|
+
col :route_short_name, required: true
|
57
|
+
col :route_long_name, required: true
|
58
|
+
col :route_desc
|
59
|
+
col :route_type, required: true,
|
60
60
|
&output_map( :unknown,
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
end
|
73
|
-
|
61
|
+
tram: ?0,
|
62
|
+
subway: ?1,
|
63
|
+
rail: ?2,
|
64
|
+
bus: ?3,
|
65
|
+
ferry: ?4,
|
66
|
+
cable_car: ?5,
|
67
|
+
gondola: ?6,
|
68
|
+
funicular: ?7 )
|
69
|
+
col :route_url
|
70
|
+
col :route_color
|
71
|
+
col :route_text_color
|
74
72
|
col :agency_id
|
75
73
|
end
|
76
74
|
|
77
75
|
file :trips, required: true do
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
col :long_name
|
83
|
-
end
|
84
|
-
|
76
|
+
col :trip_id, required: true, unique: true
|
77
|
+
col :trip_headsign
|
78
|
+
col :trip_short_name
|
79
|
+
col :trip_long_name
|
85
80
|
col :route_id, required: true
|
86
81
|
col :service_id, required: true
|
87
|
-
col :direction_id,
|
88
|
-
&output_map( primary: ?0, opposite: ?1 )
|
82
|
+
col :direction_id, &output_map( primary: ?0, opposite: ?1 )
|
89
83
|
col :block_id
|
90
84
|
col :shape_id
|
91
|
-
col :wheelchair_accessible,
|
92
|
-
|
93
|
-
col :bikes_allowed,
|
94
|
-
&output_map( :unknown, yes: ?1, no: ?2 )
|
85
|
+
col :wheelchair_accessible, &output_map( :unknown, yes: ?1, no: ?2 )
|
86
|
+
col :bikes_allowed, &output_map( :unknown, yes: ?1, no: ?2 )
|
95
87
|
end
|
96
88
|
|
97
89
|
file :stop_times, required: true do
|
98
90
|
col :trip_id, required: true
|
99
91
|
col :arrival_time, required: true
|
100
92
|
col :departure_time, required: true
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
col :sequence, required: true
|
105
|
-
col :headsign
|
106
|
-
end
|
93
|
+
col :stop_id, required: true
|
94
|
+
col :stop_sequence, required: true
|
95
|
+
col :stop_headsign
|
107
96
|
|
108
97
|
col :pickup_type,
|
109
98
|
&output_map( :regular,
|
@@ -122,15 +111,13 @@ module GtfsReader
|
|
122
111
|
|
123
112
|
file :calendar, required: true do
|
124
113
|
col :service_id, required: true, unique: true
|
125
|
-
|
126
|
-
col :
|
127
|
-
col :
|
128
|
-
col :
|
129
|
-
col :
|
130
|
-
col :
|
131
|
-
col :
|
132
|
-
col :sunday, required: true, &output_map( yes: ?1, no: ?0 )
|
133
|
-
|
114
|
+
col :monday, required: true, &output_map( yes: ?1, no: ?0 )
|
115
|
+
col :tuesday, required: true, &output_map( yes: ?1, no: ?0 )
|
116
|
+
col :wednesday, required: true, &output_map( yes: ?1, no: ?0 )
|
117
|
+
col :thursday, required: true, &output_map( yes: ?1, no: ?0 )
|
118
|
+
col :friday, required: true, &output_map( yes: ?1, no: ?0 )
|
119
|
+
col :saturday, required: true, &output_map( yes: ?1, no: ?0 )
|
120
|
+
col :sunday, required: true, &output_map( yes: ?1, no: ?0 )
|
134
121
|
col :start_date
|
135
122
|
col :end_date
|
136
123
|
end
|
@@ -162,13 +149,11 @@ module GtfsReader
|
|
162
149
|
end
|
163
150
|
|
164
151
|
file :shapes do
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
col :dist_traveled
|
171
|
-
end
|
152
|
+
col :shape_id, required: true
|
153
|
+
col :shape_pt_lat, required: true
|
154
|
+
col :shape_pt_lon, required: true
|
155
|
+
col :shape_pt_sequence, required: true
|
156
|
+
col :shape_dist_traveled
|
172
157
|
end
|
173
158
|
|
174
159
|
file :frequencies do
|
@@ -176,8 +161,7 @@ module GtfsReader
|
|
176
161
|
col :start_time, required: true
|
177
162
|
col :end_time, required: true
|
178
163
|
col :headway_secs, required: true
|
179
|
-
col :exact_times,
|
180
|
-
&output_map( :inexact, exact: 1 )
|
164
|
+
col :exact_times, &output_map( :inexact, exact: 1 )
|
181
165
|
end
|
182
166
|
|
183
167
|
file :transfers do
|
@@ -192,14 +176,12 @@ module GtfsReader
|
|
192
176
|
end
|
193
177
|
|
194
178
|
file :feed_info do
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
col :version
|
202
|
-
end
|
179
|
+
col :feed_publisher_name, required: true
|
180
|
+
col :feed_publisher_url, required: true
|
181
|
+
col :feed_lang, required: true
|
182
|
+
col :feed_start_date
|
183
|
+
col :feed_end_date
|
184
|
+
col :feed_version
|
203
185
|
end
|
204
186
|
end
|
205
187
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require_relative 'column'
|
2
|
-
require_relative 'prefixed_column_setter'
|
3
2
|
|
4
3
|
module GtfsReader
|
5
4
|
module Config
|
@@ -13,7 +12,6 @@ module GtfsReader
|
|
13
12
|
def initialize(name, opts={})
|
14
13
|
@name, @columns = name, {}
|
15
14
|
@opts = { required: false }.merge (opts || {})
|
16
|
-
@aliases = {}
|
17
15
|
end
|
18
16
|
|
19
17
|
#@return [Boolean] If this file is required to be in the feed.
|
@@ -62,27 +60,12 @@ module GtfsReader
|
|
62
60
|
#@yieldreturn Any kind of object.
|
63
61
|
#@return [Column] The newly created column.
|
64
62
|
def col(name, *args, &block)
|
65
|
-
name = @aliases[name] if @aliases.key? name
|
66
|
-
|
67
63
|
if @columns.key? name
|
68
64
|
@columns[name].parser &block if block_given?
|
69
65
|
return @columns[name]
|
70
66
|
end
|
71
67
|
|
72
|
-
|
73
|
-
@aliases[col.alias] = name if col.alias
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
# Starts a new block within which any defined columns will have the given
|
78
|
-
# +sym+ prefixed to its name (joined with an underscore). Also, the
|
79
|
-
# defined name given within the block will be aliased to the column.
|
80
|
-
#@param sym the prefix to prefixed to each column within the block
|
81
|
-
#
|
82
|
-
#@example Create a column +route_name+ with the alias +name+
|
83
|
-
# prefix( :route ) { name }
|
84
|
-
def prefix(sym, &blk)
|
85
|
-
PrefixedColumnSetter.new(self, sym.to_s).instance_exec &blk
|
68
|
+
@columns[name] = Column.new name, args.first, &block
|
86
69
|
end
|
87
70
|
|
88
71
|
# Creates an input-output proc to convert column values from one form to
|
data/lib/gtfs_reader/core.rb
CHANGED
@@ -42,12 +42,13 @@ module GtfsReader
|
|
42
42
|
def update_verbosely(name)
|
43
43
|
source = config.sources[name]
|
44
44
|
raise UnknownSourceError, "No source named '#{name}'" if source.nil?
|
45
|
+
|
45
46
|
updater = SourceUpdater.new name, source
|
46
47
|
begin
|
47
48
|
updater.instance_exec do
|
48
49
|
Log.info { "Updating #{name.to_s.green}".underline }
|
49
50
|
before_callbacks
|
50
|
-
|
51
|
+
download_source
|
51
52
|
check_files
|
52
53
|
check_columns
|
53
54
|
process_files
|
data/lib/gtfs_reader/log.rb
CHANGED
@@ -1,12 +1,15 @@
|
|
1
|
+
require 'active_support/core_ext/object/try'
|
2
|
+
require 'csv'
|
1
3
|
require 'net/http'
|
2
4
|
require 'open-uri'
|
3
|
-
require 'zip/filesystem'
|
4
|
-
require 'csv'
|
5
5
|
require 'uri'
|
6
|
+
require 'zip/filesystem'
|
6
7
|
|
7
8
|
require_relative 'file_reader'
|
8
9
|
|
9
10
|
module GtfsReader
|
11
|
+
# Downloads remote Feed files, checks that they are valid, and passes each
|
12
|
+
# file in the feed to the handlers in the given [Source].
|
10
13
|
class SourceUpdater
|
11
14
|
#@param name [String] an arbitrary string describing this source
|
12
15
|
#@param source [Source]
|
@@ -15,54 +18,49 @@ module GtfsReader
|
|
15
18
|
@temp_files = {}
|
16
19
|
end
|
17
20
|
|
21
|
+
# Call the "before" callback set on this source
|
18
22
|
def before_callbacks
|
19
|
-
|
23
|
+
if @source.before
|
24
|
+
@source.before.call fetch_data_set_identifier
|
25
|
+
end
|
20
26
|
end
|
21
27
|
|
22
28
|
# Download the data from the remote server
|
23
|
-
def
|
29
|
+
def download_source
|
24
30
|
Log.debug { " Reading #{@source.url.green}" }
|
25
|
-
|
26
|
-
|
27
|
-
|
31
|
+
zip = Tempfile.new 'gtfs'
|
32
|
+
zip.binmode
|
33
|
+
zip << open(@source.url).read
|
34
|
+
zip.rewind
|
28
35
|
|
29
|
-
|
30
|
-
file = Tempfile.new 'gtfs'
|
31
|
-
file.binmode
|
32
|
-
file << open(@source.url).read
|
33
|
-
file.rewind
|
36
|
+
extract_to_tempfiles zip
|
34
37
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
file.close
|
38
|
+
Log.debug { "Finished reading #{@source.url.green}" }
|
39
|
+
rescue Exception => e
|
40
|
+
Log.error e.message
|
41
|
+
raise e
|
42
|
+
ensure
|
43
|
+
zip.try :close
|
43
44
|
end
|
44
45
|
|
45
46
|
def close
|
46
47
|
@temp_files.values.each &:close
|
47
48
|
end
|
48
49
|
|
50
|
+
# Parse the filenames in the feed and check which required and optional
|
51
|
+
# files are present.
|
52
|
+
#@raise [RequiredFilenamesMissing] if the feed is missing a file which is
|
53
|
+
# marked as "required" in the [FeedDefinition]
|
49
54
|
def check_files
|
50
55
|
@found_files = []
|
51
56
|
check_required_files
|
52
57
|
check_optional_files
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
raise RequiredFilenamesMissing, missing unless missing.empty?
|
60
|
-
end
|
61
|
-
|
62
|
-
def check_optional_files
|
63
|
-
Log.info { 'optional files'.cyan }
|
64
|
-
files = @source.feed_definition.optional_files
|
65
|
-
check_missing_files files, :cyan, :light_yellow
|
58
|
+
# Add feed files of zip to the list of files to be processed
|
59
|
+
@source.feed_definition.files.each do |req|
|
60
|
+
if filenames.include? req.filename
|
61
|
+
@found_files << req
|
62
|
+
end
|
63
|
+
end
|
66
64
|
end
|
67
65
|
|
68
66
|
# Check that every file has its required columns
|
@@ -86,7 +84,16 @@ module GtfsReader
|
|
86
84
|
|
87
85
|
private
|
88
86
|
|
89
|
-
|
87
|
+
def extract_to_tempfiles(zip)
|
88
|
+
Zip::File.open(zip).each do |entry|
|
89
|
+
temp = Tempfile.new "gtfs_file_#{entry.name}"
|
90
|
+
temp << entry.get_input_stream.read
|
91
|
+
temp.close
|
92
|
+
@temp_files[entry.name] = temp
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# Check for the given list of expected filenames in the zip file
|
90
97
|
def check_missing_files(expected, found_color, missing_color)
|
91
98
|
check = '✔'.colorize found_color
|
92
99
|
cross = '✘'.colorize missing_color
|
@@ -95,7 +102,6 @@ module GtfsReader
|
|
95
102
|
filename = req.filename
|
96
103
|
if filenames.include? filename
|
97
104
|
Log.info { "#{filename.rjust filename_width} [#{check}]" }
|
98
|
-
@found_files << req
|
99
105
|
nil
|
100
106
|
else
|
101
107
|
Log.info { "#{filename.rjust filename_width} [#{cross}]" }
|
@@ -104,6 +110,8 @@ module GtfsReader
|
|
104
110
|
end.compact
|
105
111
|
end
|
106
112
|
|
113
|
+
#@return <FixNum> the maximum string-width of the filenames, so they can be
|
114
|
+
# aligned when printed on the console.
|
107
115
|
def filename_width
|
108
116
|
@filename_width ||= @source.feed_definition.files.max do |a, b|
|
109
117
|
a.filename.length <=> b.filename.length
|
@@ -114,31 +122,45 @@ module GtfsReader
|
|
114
122
|
@temp_files.keys
|
115
123
|
end
|
116
124
|
|
117
|
-
#
|
118
|
-
#
|
119
|
-
#
|
125
|
+
# Perform a HEAD request against the source's URL, looking for a unique
|
126
|
+
# identifier for the remote data set. It will choose a header from the
|
127
|
+
# result in the given order of preference:
|
120
128
|
# - ETag
|
121
129
|
# - Last-Modified
|
122
130
|
# - Content-Length (may result in different data sets being considered
|
123
131
|
# the same if they happen to have the same size)
|
124
132
|
# - The current date/time (this will always result in a fresh download)
|
125
133
|
def fetch_data_set_identifier
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
head_request
|
131
|
-
|
132
|
-
Log.warn "No ETag supplied with: #{uri.path}"
|
133
|
-
|
134
|
-
if head_request.key? 'last-modified'
|
135
|
-
head_request['last-modified']
|
136
|
-
elsif head_request.key? 'content-length'
|
137
|
-
head_request['content-length']
|
134
|
+
if @source.url =~ /\A#{URI::regexp}\z/
|
135
|
+
uri = URI @source.url
|
136
|
+
Net::HTTP.start(uri.host) do |http|
|
137
|
+
head_request = http.request_head uri.path
|
138
|
+
if head_request.key? 'etag'
|
139
|
+
head_request['etag']
|
138
140
|
else
|
139
|
-
|
141
|
+
Log.warn "No ETag supplied with: #{uri.path}"
|
142
|
+
fetch_http_fallback_identifier head_request
|
140
143
|
end
|
141
144
|
end
|
145
|
+
else # it's not a url, it may be a file => last modified
|
146
|
+
begin
|
147
|
+
File.mtime @source.url
|
148
|
+
rescue StandardError => e
|
149
|
+
Log.error e
|
150
|
+
raise e
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
# Find a "next best" ID when the HEAD request does not return an "ETag"
|
156
|
+
# header.
|
157
|
+
def fetch_http_fallback_identifier(head_request)
|
158
|
+
if head_request.key? 'last-modified'
|
159
|
+
head_request['last-modified']
|
160
|
+
elsif head_request.key? 'content-length'
|
161
|
+
head_request['content-length']
|
162
|
+
else
|
163
|
+
Time.now.to_s
|
142
164
|
end
|
143
165
|
end
|
144
166
|
|
@@ -153,5 +175,20 @@ module GtfsReader
|
|
153
175
|
@source.handlers.handle_file file.name, reader
|
154
176
|
end
|
155
177
|
end
|
178
|
+
|
179
|
+
#@raise [RequiredFilenamesMissing] if a file is missing a header which is
|
180
|
+
# marked as "required" in the [FeedDefinition]
|
181
|
+
def check_required_files
|
182
|
+
Log.info { 'required files'.magenta }
|
183
|
+
files = @source.feed_definition.required_files
|
184
|
+
missing = check_missing_files files, :green, :red
|
185
|
+
raise RequiredFilenamesMissing, missing unless missing.empty?
|
186
|
+
end
|
187
|
+
|
188
|
+
def check_optional_files
|
189
|
+
Log.info { 'optional files'.cyan }
|
190
|
+
files = @source.feed_definition.optional_files
|
191
|
+
check_missing_files files, :cyan, :light_yellow
|
192
|
+
end
|
156
193
|
end
|
157
194
|
end
|
data/lib/gtfs_reader/version.rb
CHANGED
@@ -3,9 +3,9 @@ module GtfsReader
|
|
3
3
|
# {Bumper} class which will modify this file to increase the version
|
4
4
|
module Version
|
5
5
|
# The following four lines are generated, so don't mess with them.
|
6
|
-
MAJOR =
|
7
|
-
MINOR =
|
8
|
-
PATCH =
|
6
|
+
MAJOR = 1
|
7
|
+
MINOR = 0
|
8
|
+
PATCH = 0
|
9
9
|
BUILD = nil
|
10
10
|
|
11
11
|
#@return [String] the current version in the form of +1.2.3.build+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gtfs-reader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jon Sangster
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-08-
|
11
|
+
date: 2014-08-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: log4r
|
@@ -136,9 +136,7 @@ dependencies:
|
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '4.2'
|
139
|
-
description:
|
140
|
-
can take up quite a bit of memory when inflated, so this gem prefers to read them
|
141
|
-
as a stream of rows. GTFS Spec: https://developers.google.com/transit/gtfs'
|
139
|
+
description: Renamed to gtfs_reader. See https://rubygems.org/gems/gtfs_reader
|
142
140
|
email: jon@ertt.ca
|
143
141
|
executables: []
|
144
142
|
extensions: []
|
@@ -155,7 +153,6 @@ files:
|
|
155
153
|
- lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb
|
156
154
|
- lib/gtfs_reader/config/feed_definition.rb
|
157
155
|
- lib/gtfs_reader/config/file_definition.rb
|
158
|
-
- lib/gtfs_reader/config/prefixed_column_setter.rb
|
159
156
|
- lib/gtfs_reader/config/source.rb
|
160
157
|
- lib/gtfs_reader/config/sources.rb
|
161
158
|
- lib/gtfs_reader/configuration.rb
|
@@ -171,7 +168,12 @@ homepage: http://github.com/sangster/gtfs-reader
|
|
171
168
|
licenses:
|
172
169
|
- GPL 3
|
173
170
|
metadata: {}
|
174
|
-
post_install_message:
|
171
|
+
post_install_message: |
|
172
|
+
! The 'gtfs-reader' gem has been deprecated and has been replaced by
|
173
|
+
! 'gtfs_reader' to follow rubygem.org's naming convention.
|
174
|
+
! See: https://rubygems.org/gems/gtfs_reader
|
175
|
+
! And: https://github.com/sangster/gtfs_reader
|
176
|
+
! And: http://guides.rubygems.org/name-your-gem
|
175
177
|
rdoc_options: []
|
176
178
|
require_paths:
|
177
179
|
- lib
|
@@ -1,26 +0,0 @@
|
|
1
|
-
module GtfsReader
|
2
|
-
module Config
|
3
|
-
class PrefixedColumnSetter
|
4
|
-
def initialize(definition, prefix)
|
5
|
-
@definition, @prefix = definition, prefix.to_sym
|
6
|
-
end
|
7
|
-
|
8
|
-
def col(name_alias, *args, &blk)
|
9
|
-
name = "#{@prefix}_#{name_alias}"
|
10
|
-
opts =
|
11
|
-
case args.first
|
12
|
-
when ::Hash then args.first
|
13
|
-
else {}
|
14
|
-
end
|
15
|
-
opts[:alias] = name_alias
|
16
|
-
args[0] = opts
|
17
|
-
|
18
|
-
@definition.col name.to_sym, *args, &blk
|
19
|
-
end
|
20
|
-
|
21
|
-
def output_map(*args, &block)
|
22
|
-
@definition.output_map *args, &block
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|