gtfs-reader 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7db02f48bbea20078924919318834ec557818099
-  data.tar.gz: 293058b7473f38041d4c9c856da661deb30b14de
+  metadata.gz: 8334bda2725737d0161d5ec5494a7e32718289f6
+  data.tar.gz: 25172bccca7b0e90f462358a6ce4075208b53142
 SHA512:
-  metadata.gz: 2fdb6c72f3d19a0764a9006579afb6dfee61c7274812891be42351f63413843a99d896e22f6988e2f8347c4587d702d33fb65fe5f43bc5091d8ae5728c422418
-  data.tar.gz: b8cc196d3415b28fab85cd557206a831cc25800a2f98c516181d628f51ed5b5e869f6e2b10d9c57dc2ac4c4f8d5b1bade01e322f10bc23f7f5ff5885703a5833
+  metadata.gz: 116149373c94c48366bb1c4689be781248a352f5b42ba7657b43c79eeff1efd140bd42bff3a6d3a0416af9e7b39e972326bb1a4b4ee1da972d6ef387b52ae016
+  data.tar.gz: 82c4917f3878cd3c52689c1da7d22017a567091aa29efa61f9f4b1ead162fbbbc20f3d54ebe0ead0d2ba7e03d1d82cd124add70fdc419a69b962dfb748357378

data/lib/gtfs_reader/config/column.rb

@@ -53,7 +53,7 @@ module GtfsReader
       end
 
       def to_s
-        opts = @opts.collect do |key,value|
+        opts = @opts.map do |key,value|
           case value
             when true then key
             when false,nil then nil

data/lib/gtfs_reader/core.rb

@@ -59,7 +59,7 @@ module GtfsReader
         "#{'Skipping'.red} #{source.name.to_s.yellow}" + msg
       end
     ensure
-      updater.finish
+      updater.close
     end
   end
 

data/lib/gtfs_reader/source_updater.rb

@@ -12,28 +12,38 @@ module GtfsReader
     #@param source [Source]
     def initialize(name, source)
       @name, @source = name, source
+      @temp_files = {}
     end
 
     def before_callbacks
-      @source.before.call fetch_remote_etag if @source.before
+      @source.before.call fetch_data_set_identifier if @source.before
     end
 
     # Download the data from the remote server
     def read
       Log.debug { "         Reading #{@source.url.green}" }
-      @file = Tempfile.new 'gtfs'
-      @file.binmode
-      @file << open(@source.url).read
-      @file.rewind
-      #binding.pry
-      @zip = Zip::File.open @file
+      extract_zip_to_tempfiles
       Log.debug { "Finished reading #{@source.url.green}" }
     end
 
-    # Close any streams still open
-    def finish
-      @zip.close if @zip
-      @file.delete if @file
+    def extract_zip_to_tempfiles
+      file = Tempfile.new 'gtfs'
+      file.binmode
+      file << open(@source.url).read
+      file.rewind
+
+      Zip::File.open(file).each do |entry|
+        temp = Tempfile.new "gtfs_file_#{entry.name}"
+        temp << entry.get_input_stream.read
+        temp.close
+        @temp_files[entry.name] = temp
+      end
+
+      file.close
+    end
+
+    def close
+      @temp_files.values.each &:close
     end
 
     def check_files
@@ -58,7 +68,7 @@ module GtfsReader
     # Check that every file has its required columns
     def check_columns
       @found_files.each do |file|
-        @zip.file.open(file.filename) do |data|
+        @temp_files[file.filename].open do |data|
           FileReader.new data, file, validate: true
         end
       end
@@ -76,11 +86,12 @@ module GtfsReader
 
     private
 
+    # Checks for the given list of expected filenames in the zip file
     def check_missing_files(expected, found_color, missing_color)
       check = '✔'.colorize found_color
       cross = '✘'.colorize missing_color
 
-      expected.collect do |req|
+      expected.map do |req|
         filename = req.filename
         if filenames.include? filename
           Log.info { "#{filename.rjust filename_width} [#{check}]" }
@@ -100,13 +111,34 @@ module GtfsReader
     end
 
     def filenames
-      @filenames ||= @zip.entries.collect &:name
+      @temp_files.keys
     end
 
-    def fetch_remote_etag
-      url = URI @source.url
-      Net::HTTP.start(url.host) do |http|
-        /[^"]+/ === http.request_head(url.path)['etag'] and $&
+    # Performs a HEAD request against the source's URL, in an attempt to
+    # return a unique identifier for the remote data set. It will choose a
+    # header from the result in the given order of preference:
+    # - ETag
+    # - Last-Modified
+    # - Content-Length (may result in different data sets being considered
+    #   the same if they happen to have the same size)
+    # - The current date/time (this will always result in a fresh download)
+    def fetch_data_set_identifier
+      uri = URI @source.url
+      Net::HTTP.start(uri.host) do |http|
+        head_request = http.request_head uri.path
+        if head_request.key? 'etag'
+          head_request['etag']
+        else
+          Log.warn "No ETag supplied with: #{uri.path}"
+
+          if head_request.key? 'last-modified'
+            head_request['last-modified']
+          elsif head_request.key? 'content-length'
+            head_request['content-length']
+          else
+            Time.now.to_s
+          end
+        end
       end
     end
 
@@ -115,15 +147,10 @@ module GtfsReader
       hash = !!GtfsReader.config.return_hashes
 
       Log.info "Reading file #{file.filename.cyan}..."
-
-      temp = Tempfile.new 'gtfs_file'
       begin
-        @zip.file.open(file.filename) { |z| temp.write z.read }
-        temp.rewind
-        reader = FileReader.new temp, file, parse: do_parse, hash: hash
+        reader = FileReader.new @temp_files[file.filename], file,
+                                parse: do_parse, hash: hash
         @source.handlers.handle_file file.name, reader
-      ensure
-        temp.close and temp.unlink
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gtfs-reader
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - Jon Sangster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-29 00:00:00.000000000 Z
+date: 2014-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: log4r