gtfs-reader 0.2.9 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +42 -3
- data/Rakefile +2 -2
- data/lib/gtfs_reader/config/column.rb +1 -13
- data/lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb +66 -84
- data/lib/gtfs_reader/config/file_definition.rb +1 -18
- data/lib/gtfs_reader/core.rb +2 -1
- data/lib/gtfs_reader/log.rb +1 -0
- data/lib/gtfs_reader/source_updater.rb +88 -51
- data/lib/gtfs_reader/version.rb +3 -3
- metadata +9 -7
- data/lib/gtfs_reader/config/prefixed_column_setter.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6b7400f376dbef697ba8dbce21afce315c971d43
|
4
|
+
data.tar.gz: cd299fb45520ec614be5386a3fc2e66a057cb2e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9fdba13c8c3933340a6770941f42117944f68ac701bc90591cc4fb053d75f6a2efe2361fc8ef71d58ee442a48cb4889ad6131a2334a77418243d3cb4d10ac1e8
|
7
|
+
data.tar.gz: c42a6814a3eda9a3bef8fb364783bf60abca9d4cd0671ebc5cd3da92b29f54ff2fbbab60b296888d537eed8b5b6848c5b1304827b41b938d08345bbcea808271
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# GTFS Reader
|
2
2
|
|
3
3
|
```ruby
|
4
|
-
gem '
|
4
|
+
gem 'gtfs_reader'
|
5
5
|
```
|
6
6
|
|
7
7
|
GTFS Reader is a gem designed to help process the contents of a "[GTFS
|
@@ -26,10 +26,10 @@ require 'gtfs_reader'
|
|
26
26
|
|
27
27
|
GtfsReader.config do
|
28
28
|
return_hashes true
|
29
|
-
|
29
|
+
# verbose true #uncomment for verbose output
|
30
30
|
sources do
|
31
31
|
sample do
|
32
|
-
url 'http://localhost/sample-feed.zip'
|
32
|
+
url 'http://localhost/sample-feed.zip' # you can also use a filepath here
|
33
33
|
before { |etag| puts "Processing source with tag #{etag}..." }
|
34
34
|
handlers do
|
35
35
|
agency {|row| puts "Read Agency: #{row[:agency_name]}" }
|
@@ -55,3 +55,42 @@ Read Route: Stagecoach - Airport Shuttle
|
|
55
55
|
Read Route: City
|
56
56
|
Read Route: Airport - Amargosa Valley
|
57
57
|
```
|
58
|
+
|
59
|
+
## Custom Feed Format
|
60
|
+
|
61
|
+
By default, this gem parses files in the format specified by the [GTFS Feed
|
62
|
+
Spec](https://developers.google.com/transit/gtfs/reference). You can see this
|
63
|
+
`FeedDefinition` in [config/defaults/gtfs_feed_definition.rb](https://github.com/sangster/gtfs_reader/blob/develop/lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb).
|
64
|
+
However, in many cases these feeds are created by people who aren't
|
65
|
+
technically-proficient and may not exactly conform to the spec. In the event
|
66
|
+
that you want to parse a file with a different format, you can do so in the
|
67
|
+
`GtfsReader.config` block:
|
68
|
+
|
69
|
+
```ruby
|
70
|
+
GtfsReader.config do
|
71
|
+
sources do
|
72
|
+
sample do
|
73
|
+
feed_definition do
|
74
|
+
file :drivers, required: true do # for my_file.txt
|
75
|
+
col :licence_number, required: true, unique: true
|
76
|
+
|
77
|
+
# If the sex column contains "1", the symbol :male will be returned,
|
78
|
+
# otherwise :female will be returned
|
79
|
+
col :sex, &output_map( :unspecified, female: ?1, male: ?2 )
|
80
|
+
|
81
|
+
# This will allow you to create a custom parser. Within the given
|
82
|
+
# block you can reference other columns in the current row by name.
|
83
|
+
col :name do |name|
|
84
|
+
case name
|
85
|
+
when :female then "Ms. #{name}"
|
86
|
+
when :male then "Mr. #{name}"
|
87
|
+
else name
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
# ...
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
```
|
data/Rakefile
CHANGED
@@ -17,9 +17,9 @@ rescue Bundler::BundlerError => e
|
|
17
17
|
end
|
18
18
|
|
19
19
|
Jeweler::Tasks.new do |gem|
|
20
|
-
gem.name = '
|
20
|
+
gem.name = 'gtfs_reader'
|
21
21
|
gem.version = GtfsReader::Version.to_s
|
22
|
-
gem.homepage = 'http://github.com/sangster/
|
22
|
+
gem.homepage = 'http://github.com/sangster/gtfs_reader'
|
23
23
|
gem.license = 'GPL 3'
|
24
24
|
gem.summary = 'Read General Transit Feed Specification zip files'
|
25
25
|
gem.description = <<-EOF.strip.gsub /\s+/, ' '
|
@@ -10,20 +10,13 @@ module GtfsReader
|
|
10
10
|
#@param name [String] the name of the column
|
11
11
|
#@option opts [Boolean] :required (false) If this column is required to
|
12
12
|
# appear in the given file
|
13
|
-
#@option opts [String] :alias an alternative name for this column. Many
|
14
|
-
# column names are needlessly prefixed with their filename:
|
15
|
-
# +Stop.stop_name+ could be aliased to +Stop.name+ for example.
|
16
13
|
#@option opts [Boolean] :unique (false) if values in this column need to be
|
17
14
|
# unique among all rows in the file.
|
18
15
|
def initialize(name, opts={}, &parser)
|
19
16
|
@name = name
|
20
17
|
@parser = block_given? ? parser : IDENTITY_PARSER
|
21
18
|
|
22
|
-
@opts = {
|
23
|
-
required: false,
|
24
|
-
unique: false,
|
25
|
-
alias: nil
|
26
|
-
}.merge (opts || {})
|
19
|
+
@opts = { required: false, unique: false }.merge ( opts || {} )
|
27
20
|
end
|
28
21
|
|
29
22
|
def parser(&block)
|
@@ -42,11 +35,6 @@ module GtfsReader
|
|
42
35
|
@opts[:unique]
|
43
36
|
end
|
44
37
|
|
45
|
-
#@return [String,nil] this column's name's alias, if there is one
|
46
|
-
def alias
|
47
|
-
@opts[:alias]
|
48
|
-
end
|
49
|
-
|
50
38
|
#@return [Boolean]
|
51
39
|
def parser?
|
52
40
|
parser != IDENTITY_PARSER
|
@@ -1,34 +1,35 @@
|
|
1
1
|
require_relative '../feed_definition'
|
2
2
|
|
3
|
+
# This default config file creates a [FeedDefinition] that matches the one
|
4
|
+
# specified by Google. You can use this definition in most cases. A custom
|
5
|
+
# definition will only be required if you need to parse a feed that differs in
|
6
|
+
# some critical way (Remember that these feeds are not always created by
|
7
|
+
# technically-proficient people).
|
8
|
+
# See https://developers.google.com/transit/gtfs/reference
|
3
9
|
module GtfsReader
|
4
10
|
module Config
|
5
11
|
module Defaults
|
6
12
|
FEED_DEFINITION = FeedDefinition.new.tap do |feed|
|
7
13
|
feed.instance_exec do
|
8
14
|
file :agency, required: true do
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
col :fare_url
|
17
|
-
end
|
15
|
+
col :agency_name, required: true
|
16
|
+
col :agency_url, required: true
|
17
|
+
col :agency_timezone, required: true
|
18
|
+
col :agency_id, unique: true
|
19
|
+
col :agency_lang
|
20
|
+
col :agency_phone
|
21
|
+
col :agency_fare_url
|
18
22
|
end
|
19
23
|
|
20
24
|
file :stops, required: true do
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
col :timezone
|
30
|
-
end
|
31
|
-
|
25
|
+
col :stop_id, required: true, unique: true
|
26
|
+
col :stop_code
|
27
|
+
col :stop_name, required: true
|
28
|
+
col :stop_desc
|
29
|
+
col :stop_lat, required: true
|
30
|
+
col :stop_lon, required: true
|
31
|
+
col :stop_url
|
32
|
+
col :stop_timezone
|
32
33
|
col :zone_id
|
33
34
|
col :location_type, &output_map( :stop, station: ?1 )
|
34
35
|
col :parent_station
|
@@ -51,59 +52,47 @@ module GtfsReader
|
|
51
52
|
end
|
52
53
|
|
53
54
|
file :routes, required: true do
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
col :type, required: true,
|
55
|
+
col :route_id, required: true, unique: true
|
56
|
+
col :route_short_name, required: true
|
57
|
+
col :route_long_name, required: true
|
58
|
+
col :route_desc
|
59
|
+
col :route_type, required: true,
|
60
60
|
&output_map( :unknown,
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
end
|
73
|
-
|
61
|
+
tram: ?0,
|
62
|
+
subway: ?1,
|
63
|
+
rail: ?2,
|
64
|
+
bus: ?3,
|
65
|
+
ferry: ?4,
|
66
|
+
cable_car: ?5,
|
67
|
+
gondola: ?6,
|
68
|
+
funicular: ?7 )
|
69
|
+
col :route_url
|
70
|
+
col :route_color
|
71
|
+
col :route_text_color
|
74
72
|
col :agency_id
|
75
73
|
end
|
76
74
|
|
77
75
|
file :trips, required: true do
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
col :long_name
|
83
|
-
end
|
84
|
-
|
76
|
+
col :trip_id, required: true, unique: true
|
77
|
+
col :trip_headsign
|
78
|
+
col :trip_short_name
|
79
|
+
col :trip_long_name
|
85
80
|
col :route_id, required: true
|
86
81
|
col :service_id, required: true
|
87
|
-
col :direction_id,
|
88
|
-
&output_map( primary: ?0, opposite: ?1 )
|
82
|
+
col :direction_id, &output_map( primary: ?0, opposite: ?1 )
|
89
83
|
col :block_id
|
90
84
|
col :shape_id
|
91
|
-
col :wheelchair_accessible,
|
92
|
-
|
93
|
-
col :bikes_allowed,
|
94
|
-
&output_map( :unknown, yes: ?1, no: ?2 )
|
85
|
+
col :wheelchair_accessible, &output_map( :unknown, yes: ?1, no: ?2 )
|
86
|
+
col :bikes_allowed, &output_map( :unknown, yes: ?1, no: ?2 )
|
95
87
|
end
|
96
88
|
|
97
89
|
file :stop_times, required: true do
|
98
90
|
col :trip_id, required: true
|
99
91
|
col :arrival_time, required: true
|
100
92
|
col :departure_time, required: true
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
col :sequence, required: true
|
105
|
-
col :headsign
|
106
|
-
end
|
93
|
+
col :stop_id, required: true
|
94
|
+
col :stop_sequence, required: true
|
95
|
+
col :stop_headsign
|
107
96
|
|
108
97
|
col :pickup_type,
|
109
98
|
&output_map( :regular,
|
@@ -122,15 +111,13 @@ module GtfsReader
|
|
122
111
|
|
123
112
|
file :calendar, required: true do
|
124
113
|
col :service_id, required: true, unique: true
|
125
|
-
|
126
|
-
col :
|
127
|
-
col :
|
128
|
-
col :
|
129
|
-
col :
|
130
|
-
col :
|
131
|
-
col :
|
132
|
-
col :sunday, required: true, &output_map( yes: ?1, no: ?0 )
|
133
|
-
|
114
|
+
col :monday, required: true, &output_map( yes: ?1, no: ?0 )
|
115
|
+
col :tuesday, required: true, &output_map( yes: ?1, no: ?0 )
|
116
|
+
col :wednesday, required: true, &output_map( yes: ?1, no: ?0 )
|
117
|
+
col :thursday, required: true, &output_map( yes: ?1, no: ?0 )
|
118
|
+
col :friday, required: true, &output_map( yes: ?1, no: ?0 )
|
119
|
+
col :saturday, required: true, &output_map( yes: ?1, no: ?0 )
|
120
|
+
col :sunday, required: true, &output_map( yes: ?1, no: ?0 )
|
134
121
|
col :start_date
|
135
122
|
col :end_date
|
136
123
|
end
|
@@ -162,13 +149,11 @@ module GtfsReader
|
|
162
149
|
end
|
163
150
|
|
164
151
|
file :shapes do
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
col :dist_traveled
|
171
|
-
end
|
152
|
+
col :shape_id, required: true
|
153
|
+
col :shape_pt_lat, required: true
|
154
|
+
col :shape_pt_lon, required: true
|
155
|
+
col :shape_pt_sequence, required: true
|
156
|
+
col :shape_dist_traveled
|
172
157
|
end
|
173
158
|
|
174
159
|
file :frequencies do
|
@@ -176,8 +161,7 @@ module GtfsReader
|
|
176
161
|
col :start_time, required: true
|
177
162
|
col :end_time, required: true
|
178
163
|
col :headway_secs, required: true
|
179
|
-
col :exact_times,
|
180
|
-
&output_map( :inexact, exact: 1 )
|
164
|
+
col :exact_times, &output_map( :inexact, exact: 1 )
|
181
165
|
end
|
182
166
|
|
183
167
|
file :transfers do
|
@@ -192,14 +176,12 @@ module GtfsReader
|
|
192
176
|
end
|
193
177
|
|
194
178
|
file :feed_info do
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
col :version
|
202
|
-
end
|
179
|
+
col :feed_publisher_name, required: true
|
180
|
+
col :feed_publisher_url, required: true
|
181
|
+
col :feed_lang, required: true
|
182
|
+
col :feed_start_date
|
183
|
+
col :feed_end_date
|
184
|
+
col :feed_version
|
203
185
|
end
|
204
186
|
end
|
205
187
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require_relative 'column'
|
2
|
-
require_relative 'prefixed_column_setter'
|
3
2
|
|
4
3
|
module GtfsReader
|
5
4
|
module Config
|
@@ -13,7 +12,6 @@ module GtfsReader
|
|
13
12
|
def initialize(name, opts={})
|
14
13
|
@name, @columns = name, {}
|
15
14
|
@opts = { required: false }.merge (opts || {})
|
16
|
-
@aliases = {}
|
17
15
|
end
|
18
16
|
|
19
17
|
#@return [Boolean] If this file is required to be in the feed.
|
@@ -62,27 +60,12 @@ module GtfsReader
|
|
62
60
|
#@yieldreturn Any kind of object.
|
63
61
|
#@return [Column] The newly created column.
|
64
62
|
def col(name, *args, &block)
|
65
|
-
name = @aliases[name] if @aliases.key? name
|
66
|
-
|
67
63
|
if @columns.key? name
|
68
64
|
@columns[name].parser &block if block_given?
|
69
65
|
return @columns[name]
|
70
66
|
end
|
71
67
|
|
72
|
-
|
73
|
-
@aliases[col.alias] = name if col.alias
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
# Starts a new block within which any defined columns will have the given
|
78
|
-
# +sym+ prefixed to its name (joined with an underscore). Also, the
|
79
|
-
# defined name given within the block will be aliased to the column.
|
80
|
-
#@param sym the prefix to prefixed to each column within the block
|
81
|
-
#
|
82
|
-
#@example Create a column +route_name+ with the alias +name+
|
83
|
-
# prefix( :route ) { name }
|
84
|
-
def prefix(sym, &blk)
|
85
|
-
PrefixedColumnSetter.new(self, sym.to_s).instance_exec &blk
|
68
|
+
@columns[name] = Column.new name, args.first, &block
|
86
69
|
end
|
87
70
|
|
88
71
|
# Creates an input-output proc to convert column values from one form to
|
data/lib/gtfs_reader/core.rb
CHANGED
@@ -42,12 +42,13 @@ module GtfsReader
|
|
42
42
|
def update_verbosely(name)
|
43
43
|
source = config.sources[name]
|
44
44
|
raise UnknownSourceError, "No source named '#{name}'" if source.nil?
|
45
|
+
|
45
46
|
updater = SourceUpdater.new name, source
|
46
47
|
begin
|
47
48
|
updater.instance_exec do
|
48
49
|
Log.info { "Updating #{name.to_s.green}".underline }
|
49
50
|
before_callbacks
|
50
|
-
|
51
|
+
download_source
|
51
52
|
check_files
|
52
53
|
check_columns
|
53
54
|
process_files
|
data/lib/gtfs_reader/log.rb
CHANGED
@@ -1,12 +1,15 @@
|
|
1
|
+
require 'active_support/core_ext/object/try'
|
2
|
+
require 'csv'
|
1
3
|
require 'net/http'
|
2
4
|
require 'open-uri'
|
3
|
-
require 'zip/filesystem'
|
4
|
-
require 'csv'
|
5
5
|
require 'uri'
|
6
|
+
require 'zip/filesystem'
|
6
7
|
|
7
8
|
require_relative 'file_reader'
|
8
9
|
|
9
10
|
module GtfsReader
|
11
|
+
# Downloads remote Feed files, checks that they are valid, and passes each
|
12
|
+
# file in the feed to the handlers in the given [Source].
|
10
13
|
class SourceUpdater
|
11
14
|
#@param name [String] an arbitrary string describing this source
|
12
15
|
#@param source [Source]
|
@@ -15,54 +18,49 @@ module GtfsReader
|
|
15
18
|
@temp_files = {}
|
16
19
|
end
|
17
20
|
|
21
|
+
# Call the "before" callback set on this source
|
18
22
|
def before_callbacks
|
19
|
-
|
23
|
+
if @source.before
|
24
|
+
@source.before.call fetch_data_set_identifier
|
25
|
+
end
|
20
26
|
end
|
21
27
|
|
22
28
|
# Download the data from the remote server
|
23
|
-
def
|
29
|
+
def download_source
|
24
30
|
Log.debug { " Reading #{@source.url.green}" }
|
25
|
-
|
26
|
-
|
27
|
-
|
31
|
+
zip = Tempfile.new 'gtfs'
|
32
|
+
zip.binmode
|
33
|
+
zip << open(@source.url).read
|
34
|
+
zip.rewind
|
28
35
|
|
29
|
-
|
30
|
-
file = Tempfile.new 'gtfs'
|
31
|
-
file.binmode
|
32
|
-
file << open(@source.url).read
|
33
|
-
file.rewind
|
36
|
+
extract_to_tempfiles zip
|
34
37
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
file.close
|
38
|
+
Log.debug { "Finished reading #{@source.url.green}" }
|
39
|
+
rescue Exception => e
|
40
|
+
Log.error e.message
|
41
|
+
raise e
|
42
|
+
ensure
|
43
|
+
zip.try :close
|
43
44
|
end
|
44
45
|
|
45
46
|
def close
|
46
47
|
@temp_files.values.each &:close
|
47
48
|
end
|
48
49
|
|
50
|
+
# Parse the filenames in the feed and check which required and optional
|
51
|
+
# files are present.
|
52
|
+
#@raise [RequiredFilenamesMissing] if the feed is missing a file which is
|
53
|
+
# marked as "required" in the [FeedDefinition]
|
49
54
|
def check_files
|
50
55
|
@found_files = []
|
51
56
|
check_required_files
|
52
57
|
check_optional_files
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
raise RequiredFilenamesMissing, missing unless missing.empty?
|
60
|
-
end
|
61
|
-
|
62
|
-
def check_optional_files
|
63
|
-
Log.info { 'optional files'.cyan }
|
64
|
-
files = @source.feed_definition.optional_files
|
65
|
-
check_missing_files files, :cyan, :light_yellow
|
58
|
+
# Add feed files of zip to the list of files to be processed
|
59
|
+
@source.feed_definition.files.each do |req|
|
60
|
+
if filenames.include? req.filename
|
61
|
+
@found_files << req
|
62
|
+
end
|
63
|
+
end
|
66
64
|
end
|
67
65
|
|
68
66
|
# Check that every file has its required columns
|
@@ -86,7 +84,16 @@ module GtfsReader
|
|
86
84
|
|
87
85
|
private
|
88
86
|
|
89
|
-
|
87
|
+
def extract_to_tempfiles(zip)
|
88
|
+
Zip::File.open(zip).each do |entry|
|
89
|
+
temp = Tempfile.new "gtfs_file_#{entry.name}"
|
90
|
+
temp << entry.get_input_stream.read
|
91
|
+
temp.close
|
92
|
+
@temp_files[entry.name] = temp
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# Check for the given list of expected filenames in the zip file
|
90
97
|
def check_missing_files(expected, found_color, missing_color)
|
91
98
|
check = '✔'.colorize found_color
|
92
99
|
cross = '✘'.colorize missing_color
|
@@ -95,7 +102,6 @@ module GtfsReader
|
|
95
102
|
filename = req.filename
|
96
103
|
if filenames.include? filename
|
97
104
|
Log.info { "#{filename.rjust filename_width} [#{check}]" }
|
98
|
-
@found_files << req
|
99
105
|
nil
|
100
106
|
else
|
101
107
|
Log.info { "#{filename.rjust filename_width} [#{cross}]" }
|
@@ -104,6 +110,8 @@ module GtfsReader
|
|
104
110
|
end.compact
|
105
111
|
end
|
106
112
|
|
113
|
+
#@return <FixNum> the maximum string-width of the filenames, so they can be
|
114
|
+
# aligned when printed on the console.
|
107
115
|
def filename_width
|
108
116
|
@filename_width ||= @source.feed_definition.files.max do |a, b|
|
109
117
|
a.filename.length <=> b.filename.length
|
@@ -114,31 +122,45 @@ module GtfsReader
|
|
114
122
|
@temp_files.keys
|
115
123
|
end
|
116
124
|
|
117
|
-
#
|
118
|
-
#
|
119
|
-
#
|
125
|
+
# Perform a HEAD request against the source's URL, looking for a unique
|
126
|
+
# identifier for the remote data set. It will choose a header from the
|
127
|
+
# result in the given order of preference:
|
120
128
|
# - ETag
|
121
129
|
# - Last-Modified
|
122
130
|
# - Content-Length (may result in different data sets being considered
|
123
131
|
# the same if they happen to have the same size)
|
124
132
|
# - The current date/time (this will always result in a fresh download)
|
125
133
|
def fetch_data_set_identifier
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
head_request
|
131
|
-
|
132
|
-
Log.warn "No ETag supplied with: #{uri.path}"
|
133
|
-
|
134
|
-
if head_request.key? 'last-modified'
|
135
|
-
head_request['last-modified']
|
136
|
-
elsif head_request.key? 'content-length'
|
137
|
-
head_request['content-length']
|
134
|
+
if @source.url =~ /\A#{URI::regexp}\z/
|
135
|
+
uri = URI @source.url
|
136
|
+
Net::HTTP.start(uri.host) do |http|
|
137
|
+
head_request = http.request_head uri.path
|
138
|
+
if head_request.key? 'etag'
|
139
|
+
head_request['etag']
|
138
140
|
else
|
139
|
-
|
141
|
+
Log.warn "No ETag supplied with: #{uri.path}"
|
142
|
+
fetch_http_fallback_identifier head_request
|
140
143
|
end
|
141
144
|
end
|
145
|
+
else # it's not a url, it may be a file => last modified
|
146
|
+
begin
|
147
|
+
File.mtime @source.url
|
148
|
+
rescue StandardError => e
|
149
|
+
Log.error e
|
150
|
+
raise e
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
# Find a "next best" ID when the HEAD request does not return an "ETag"
|
156
|
+
# header.
|
157
|
+
def fetch_http_fallback_identifier(head_request)
|
158
|
+
if head_request.key? 'last-modified'
|
159
|
+
head_request['last-modified']
|
160
|
+
elsif head_request.key? 'content-length'
|
161
|
+
head_request['content-length']
|
162
|
+
else
|
163
|
+
Time.now.to_s
|
142
164
|
end
|
143
165
|
end
|
144
166
|
|
@@ -153,5 +175,20 @@ module GtfsReader
|
|
153
175
|
@source.handlers.handle_file file.name, reader
|
154
176
|
end
|
155
177
|
end
|
178
|
+
|
179
|
+
#@raise [RequiredFilenamesMissing] if a file is missing a header which is
|
180
|
+
# marked as "required" in the [FeedDefinition]
|
181
|
+
def check_required_files
|
182
|
+
Log.info { 'required files'.magenta }
|
183
|
+
files = @source.feed_definition.required_files
|
184
|
+
missing = check_missing_files files, :green, :red
|
185
|
+
raise RequiredFilenamesMissing, missing unless missing.empty?
|
186
|
+
end
|
187
|
+
|
188
|
+
def check_optional_files
|
189
|
+
Log.info { 'optional files'.cyan }
|
190
|
+
files = @source.feed_definition.optional_files
|
191
|
+
check_missing_files files, :cyan, :light_yellow
|
192
|
+
end
|
156
193
|
end
|
157
194
|
end
|
data/lib/gtfs_reader/version.rb
CHANGED
@@ -3,9 +3,9 @@ module GtfsReader
|
|
3
3
|
# {Bumper} class which will modify this file to increase the version
|
4
4
|
module Version
|
5
5
|
# The following four lines are generated, so don't mess with them.
|
6
|
-
MAJOR =
|
7
|
-
MINOR =
|
8
|
-
PATCH =
|
6
|
+
MAJOR = 1
|
7
|
+
MINOR = 0
|
8
|
+
PATCH = 0
|
9
9
|
BUILD = nil
|
10
10
|
|
11
11
|
#@return [String] the current version in the form of +1.2.3.build+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gtfs-reader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jon Sangster
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-08-
|
11
|
+
date: 2014-08-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: log4r
|
@@ -136,9 +136,7 @@ dependencies:
|
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '4.2'
|
139
|
-
description:
|
140
|
-
can take up quite a bit of memory when inflated, so this gem prefers to read them
|
141
|
-
as a stream of rows. GTFS Spec: https://developers.google.com/transit/gtfs'
|
139
|
+
description: Renamed to gtfs_reader. See https://rubygems.org/gems/gtfs_reader
|
142
140
|
email: jon@ertt.ca
|
143
141
|
executables: []
|
144
142
|
extensions: []
|
@@ -155,7 +153,6 @@ files:
|
|
155
153
|
- lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb
|
156
154
|
- lib/gtfs_reader/config/feed_definition.rb
|
157
155
|
- lib/gtfs_reader/config/file_definition.rb
|
158
|
-
- lib/gtfs_reader/config/prefixed_column_setter.rb
|
159
156
|
- lib/gtfs_reader/config/source.rb
|
160
157
|
- lib/gtfs_reader/config/sources.rb
|
161
158
|
- lib/gtfs_reader/configuration.rb
|
@@ -171,7 +168,12 @@ homepage: http://github.com/sangster/gtfs-reader
|
|
171
168
|
licenses:
|
172
169
|
- GPL 3
|
173
170
|
metadata: {}
|
174
|
-
post_install_message:
|
171
|
+
post_install_message: |
|
172
|
+
! The 'gtfs-reader' gem has been deprecated and has been replaced by
|
173
|
+
! 'gtfs_reader' to follow rubygem.org's naming convention.
|
174
|
+
! See: https://rubygems.org/gems/gtfs_reader
|
175
|
+
! And: https://github.com/sangster/gtfs_reader
|
176
|
+
! And: http://guides.rubygems.org/name-your-gem
|
175
177
|
rdoc_options: []
|
176
178
|
require_paths:
|
177
179
|
- lib
|
@@ -1,26 +0,0 @@
|
|
1
|
-
module GtfsReader
|
2
|
-
module Config
|
3
|
-
class PrefixedColumnSetter
|
4
|
-
def initialize(definition, prefix)
|
5
|
-
@definition, @prefix = definition, prefix.to_sym
|
6
|
-
end
|
7
|
-
|
8
|
-
def col(name_alias, *args, &blk)
|
9
|
-
name = "#{@prefix}_#{name_alias}"
|
10
|
-
opts =
|
11
|
-
case args.first
|
12
|
-
when ::Hash then args.first
|
13
|
-
else {}
|
14
|
-
end
|
15
|
-
opts[:alias] = name_alias
|
16
|
-
args[0] = opts
|
17
|
-
|
18
|
-
@definition.col name.to_sym, *args, &blk
|
19
|
-
end
|
20
|
-
|
21
|
-
def output_map(*args, &block)
|
22
|
-
@definition.output_map *args, &block
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|