gtfs-reader 0.2.8 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/gtfs_reader/config/column.rb +1 -1
- data/lib/gtfs_reader/core.rb +1 -1
- data/lib/gtfs_reader/source_updater.rb +52 -25
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8334bda2725737d0161d5ec5494a7e32718289f6
|
4
|
+
data.tar.gz: 25172bccca7b0e90f462358a6ce4075208b53142
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 116149373c94c48366bb1c4689be781248a352f5b42ba7657b43c79eeff1efd140bd42bff3a6d3a0416af9e7b39e972326bb1a4b4ee1da972d6ef387b52ae016
|
7
|
+
data.tar.gz: 82c4917f3878cd3c52689c1da7d22017a567091aa29efa61f9f4b1ead162fbbbc20f3d54ebe0ead0d2ba7e03d1d82cd124add70fdc419a69b962dfb748357378
|
data/lib/gtfs_reader/core.rb
CHANGED
@@ -12,28 +12,38 @@ module GtfsReader
|
|
12
12
|
#@param source [Source]
|
13
13
|
def initialize(name, source)
|
14
14
|
@name, @source = name, source
|
15
|
+
@temp_files = {}
|
15
16
|
end
|
16
17
|
|
17
18
|
def before_callbacks
|
18
|
-
@source.before.call
|
19
|
+
@source.before.call fetch_data_set_identifier if @source.before
|
19
20
|
end
|
20
21
|
|
21
22
|
# Download the data from the remote server
|
22
23
|
def read
|
23
24
|
Log.debug { " Reading #{@source.url.green}" }
|
24
|
-
|
25
|
-
@file.binmode
|
26
|
-
@file << open(@source.url).read
|
27
|
-
@file.rewind
|
28
|
-
#binding.pry
|
29
|
-
@zip = Zip::File.open @file
|
25
|
+
extract_zip_to_tempfiles
|
30
26
|
Log.debug { "Finished reading #{@source.url.green}" }
|
31
27
|
end
|
32
28
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
29
|
+
def extract_zip_to_tempfiles
|
30
|
+
file = Tempfile.new 'gtfs'
|
31
|
+
file.binmode
|
32
|
+
file << open(@source.url).read
|
33
|
+
file.rewind
|
34
|
+
|
35
|
+
Zip::File.open(file).each do |entry|
|
36
|
+
temp = Tempfile.new "gtfs_file_#{entry.name}"
|
37
|
+
temp << entry.get_input_stream.read
|
38
|
+
temp.close
|
39
|
+
@temp_files[entry.name] = temp
|
40
|
+
end
|
41
|
+
|
42
|
+
file.close
|
43
|
+
end
|
44
|
+
|
45
|
+
def close
|
46
|
+
@temp_files.values.each &:close
|
37
47
|
end
|
38
48
|
|
39
49
|
def check_files
|
@@ -58,7 +68,7 @@ module GtfsReader
|
|
58
68
|
# Check that every file has its required columns
|
59
69
|
def check_columns
|
60
70
|
@found_files.each do |file|
|
61
|
-
@
|
71
|
+
@temp_files[file.filename].open do |data|
|
62
72
|
FileReader.new data, file, validate: true
|
63
73
|
end
|
64
74
|
end
|
@@ -76,11 +86,12 @@ module GtfsReader
|
|
76
86
|
|
77
87
|
private
|
78
88
|
|
89
|
+
# Checks for the given list of expected filenames in the zip file
|
79
90
|
def check_missing_files(expected, found_color, missing_color)
|
80
91
|
check = '✔'.colorize found_color
|
81
92
|
cross = '✘'.colorize missing_color
|
82
93
|
|
83
|
-
expected.
|
94
|
+
expected.map do |req|
|
84
95
|
filename = req.filename
|
85
96
|
if filenames.include? filename
|
86
97
|
Log.info { "#{filename.rjust filename_width} [#{check}]" }
|
@@ -100,13 +111,34 @@ module GtfsReader
|
|
100
111
|
end
|
101
112
|
|
102
113
|
def filenames
|
103
|
-
@
|
114
|
+
@temp_files.keys
|
104
115
|
end
|
105
116
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
117
|
+
# Performs a HEAD request against the source's URL, in an attempt to
|
118
|
+
# return a unique identifier for the remote data set. It will choose a
|
119
|
+
# header from the result in the given order of preference:
|
120
|
+
# - ETag
|
121
|
+
# - Last-Modified
|
122
|
+
# - Content-Length (may result in different data sets being considered
|
123
|
+
# the same if they happen to have the same size)
|
124
|
+
# - The current date/time (this will always result in a fresh download)
|
125
|
+
def fetch_data_set_identifier
|
126
|
+
uri = URI @source.url
|
127
|
+
Net::HTTP.start(uri.host) do |http|
|
128
|
+
head_request = http.request_head uri.path
|
129
|
+
if head_request.key? 'etag'
|
130
|
+
head_request['etag']
|
131
|
+
else
|
132
|
+
Log.warn "No ETag supplied with: #{uri.path}"
|
133
|
+
|
134
|
+
if head_request.key? 'last-modified'
|
135
|
+
head_request['last-modified']
|
136
|
+
elsif head_request.key? 'content-length'
|
137
|
+
head_request['content-length']
|
138
|
+
else
|
139
|
+
Time.now.to_s
|
140
|
+
end
|
141
|
+
end
|
110
142
|
end
|
111
143
|
end
|
112
144
|
|
@@ -115,15 +147,10 @@ module GtfsReader
|
|
115
147
|
hash = !!GtfsReader.config.return_hashes
|
116
148
|
|
117
149
|
Log.info "Reading file #{file.filename.cyan}..."
|
118
|
-
|
119
|
-
temp = Tempfile.new 'gtfs_file'
|
120
150
|
begin
|
121
|
-
|
122
|
-
|
123
|
-
reader = FileReader.new temp, file, parse: do_parse, hash: hash
|
151
|
+
reader = FileReader.new @temp_files[file.filename], file,
|
152
|
+
parse: do_parse, hash: hash
|
124
153
|
@source.handlers.handle_file file.name, reader
|
125
|
-
ensure
|
126
|
-
temp.close and temp.unlink
|
127
154
|
end
|
128
155
|
end
|
129
156
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gtfs-reader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jon Sangster
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-08-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: log4r
|