gtfs-reader 0.2.9 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8334bda2725737d0161d5ec5494a7e32718289f6
-  data.tar.gz: 25172bccca7b0e90f462358a6ce4075208b53142
+  metadata.gz: 6b7400f376dbef697ba8dbce21afce315c971d43
+  data.tar.gz: cd299fb45520ec614be5386a3fc2e66a057cb2e2
 SHA512:
-  metadata.gz: 116149373c94c48366bb1c4689be781248a352f5b42ba7657b43c79eeff1efd140bd42bff3a6d3a0416af9e7b39e972326bb1a4b4ee1da972d6ef387b52ae016
-  data.tar.gz: 82c4917f3878cd3c52689c1da7d22017a567091aa29efa61f9f4b1ead162fbbbc20f3d54ebe0ead0d2ba7e03d1d82cd124add70fdc419a69b962dfb748357378
+  metadata.gz: 9fdba13c8c3933340a6770941f42117944f68ac701bc90591cc4fb053d75f6a2efe2361fc8ef71d58ee442a48cb4889ad6131a2334a77418243d3cb4d10ac1e8
+  data.tar.gz: c42a6814a3eda9a3bef8fb364783bf60abca9d4cd0671ebc5cd3da92b29f54ff2fbbab60b296888d537eed8b5b6848c5b1304827b41b938d08345bbcea808271

data/README.md

@@ -1,7 +1,7 @@
 # GTFS Reader
 
 ```ruby
-gem 'gtfs-reader'
+gem 'gtfs_reader'
 ```
 
 GTFS Reader is a gem designed to help process the contents of a "[GTFS
@@ -26,10 +26,10 @@ require 'gtfs_reader'
 
 GtfsReader.config do
   return_hashes true
-
+  # verbose true #uncomment for verbose output
   sources do
     sample do
-      url 'http://localhost/sample-feed.zip'
+      url 'http://localhost/sample-feed.zip' # you can also use a filepath here 
       before { |etag| puts "Processing source with tag #{etag}..." }
       handlers do
         agency {|row| puts "Read Agency: #{row[:agency_name]}" }
@@ -55,3 +55,42 @@ Read Route: Stagecoach - Airport Shuttle
 Read Route: City
 Read Route: Airport - Amargosa Valley
 ```
+
+## Custom Feed Format
+
+By default, this gem parses files in the format specified by the [GTFS Feed
+Spec](https://developers.google.com/transit/gtfs/reference). You can see this
+`FeedDefinition` in [config/defaults/gtfs_feed_definition.rb](https://github.com/sangster/gtfs_reader/blob/develop/lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb).
+However, in many cases these feeds are created by people who aren't
+technically-proficient and may not exactly conform to the spec. In the event
+that you want to parse a file with a different format, you can do so in the
+`GtfsReader.config` block:
+
+```ruby
+GtfsReader.config do
+  sources do
+    sample do
+      feed_definition do
+        file :drivers, required: true do # for my_file.txt
+          col :licence_number, required: true, unique: true
+
+          # If the sex column contains "1", the symbol :male will be returned,
+          # otherwise :female will be returned
+          col :sex, &output_map( :unspecified, female: ?1, male: ?2 )
+
+          # This will allow you to create a custom parser. Within the given
+          # block you can reference other columns in the current row by name.
+          col :name do |name|
+            case name
+              when :female then "Ms. #{name}"
+              when :male   then "Mr. #{name}"
+              else              name
+            end
+          end
+        end
+      end
+      # ...
+    end
+  end
+end
+```

data/Rakefile

@@ -17,9 +17,9 @@ rescue Bundler::BundlerError => e
 end
 
 Jeweler::Tasks.new do |gem|
-  gem.name = 'gtfs-reader'
+  gem.name = 'gtfs_reader'
   gem.version = GtfsReader::Version.to_s
-  gem.homepage = 'http://github.com/sangster/gtfs-reader'
+  gem.homepage = 'http://github.com/sangster/gtfs_reader'
   gem.license = 'GPL 3'
   gem.summary = 'Read General Transit Feed Specification zip files'
   gem.description = <<-EOF.strip.gsub /\s+/, ' '

data/lib/gtfs_reader/config/column.rb

@@ -10,20 +10,13 @@ module GtfsReader
       #@param name [String] the name of the column
       #@option opts [Boolean] :required (false) If this column is required to
       #  appear in the given file
-      #@option opts [String] :alias an alternative name for this column. Many
-      #  column names are needlessly prefixed with their filename:
-      #  +Stop.stop_name+ could be aliased to +Stop.name+ for example.
       #@option opts [Boolean] :unique (false) if values in this column need to be
       #  unique among all rows in the file.
       def initialize(name, opts={}, &parser)
         @name = name
         @parser = block_given? ? parser : IDENTITY_PARSER
 
-        @opts = {
-          required: false,
-          unique: false,
-          alias: nil
-        }.merge (opts || {})
+        @opts = { required: false, unique: false }.merge ( opts || {} )
       end
 
       def parser(&block)
@@ -42,11 +35,6 @@ module GtfsReader
         @opts[:unique]
       end
 
-      #@return [String,nil] this column's name's alias, if there is one
-      def alias
-        @opts[:alias]
-      end
-
       #@return [Boolean]
       def parser?
         parser != IDENTITY_PARSER

data/lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb

@@ -1,34 +1,35 @@
 require_relative '../feed_definition'
 
+# This default config file creates a [FeedDefinition] that matches the one
+# specified by Google. You can use this definition in most cases. A custom
+# definition will only be required if you need to parse a feed that differs in
+# some critical way (Remember that these feeds are not always created by
+# technically-proficient people).
+# See https://developers.google.com/transit/gtfs/reference
 module GtfsReader
   module Config
     module Defaults
       FEED_DEFINITION = FeedDefinition.new.tap do |feed|
         feed.instance_exec do
           file :agency, required: true do
-            prefix :agency do
-              col :name,     required: true
-              col :url,      required: true
-              col :timezone, required: true
-              col :id,                       unique: true
-              col :lang
-              col :phone
-              col :fare_url
-            end
+            col :agency_name,     required: true
+            col :agency_url,      required: true
+            col :agency_timezone, required: true
+            col :agency_id,                       unique: true
+            col :agency_lang
+            col :agency_phone
+            col :agency_fare_url
           end
 
           file :stops, required: true do
-            prefix :stop do
-              col :id,       required: true, unique: true
-              col :code
-              col :name,     required: true
-              col :desc
-              col :lat,      required: true
-              col :lon,      required: true
-              col :url
-              col :timezone
-            end
-
+            col :stop_id,       required: true, unique: true
+            col :stop_code
+            col :stop_name,     required: true
+            col :stop_desc
+            col :stop_lat,      required: true
+            col :stop_lon,      required: true
+            col :stop_url
+            col :stop_timezone
             col :zone_id
             col :location_type, &output_map( :stop, station: ?1 )
             col :parent_station
@@ -51,59 +52,47 @@ module GtfsReader
           end
 
           file :routes, required: true do
-            prefix :route do
-              col :id,         required: true, unique: true
-              col :short_name, required: true
-              col :long_name,  required: true
-              col :desc
-              col :type,       required: true,
+            col :route_id,         required: true, unique: true
+            col :route_short_name, required: true
+            col :route_long_name,  required: true
+            col :route_desc
+            col :route_type,       required: true,
                 &output_map(   :unknown,
-                                   tram: ?0,
-                                 subway: ?1,
-                                   rail: ?2,
-                                    bus: ?3,
-                                  ferry: ?4,
-                              cable_car: ?5,
-                                gondola: ?6,
-                              funicular: ?7 )
-              col :url
-              col :color
-              col :text_color
-            end
-
+                               tram: ?0,
+                               subway: ?1,
+                               rail: ?2,
+                               bus: ?3,
+                               ferry: ?4,
+                               cable_car: ?5,
+                               gondola: ?6,
+                               funicular: ?7 )
+            col :route_url
+            col :route_color
+            col :route_text_color
             col :agency_id
           end
 
           file :trips, required: true do
-            prefix :trip do
-              col :id,         required: true, unique: true
-              col :headsign
-              col :short_name
-              col :long_name
-            end
-
+            col :trip_id,               required: true, unique: true
+            col :trip_headsign
+            col :trip_short_name
+            col :trip_long_name
             col :route_id,              required: true
             col :service_id,            required: true
-            col :direction_id,
-              &output_map( primary: ?0, opposite: ?1 )
+            col :direction_id,          &output_map( primary: ?0, opposite: ?1 )
             col :block_id
             col :shape_id
-            col :wheelchair_accessible,
-              &output_map( :unknown, yes: ?1, no: ?2 )
-            col :bikes_allowed,
-              &output_map( :unknown, yes: ?1, no: ?2 )
+            col :wheelchair_accessible, &output_map( :unknown, yes: ?1, no: ?2 )
+            col :bikes_allowed,         &output_map( :unknown, yes: ?1, no: ?2 )
           end
 
           file :stop_times, required: true do
             col :trip_id,        required: true
             col :arrival_time,   required: true
             col :departure_time, required: true
-
-            prefix :stop do
-              col :id,       required: true
-              col :sequence, required: true
-              col :headsign
-            end
+            col :stop_id,        required: true
+            col :stop_sequence,  required: true
+            col :stop_headsign
 
             col :pickup_type,
               &output_map(    :regular,
@@ -122,15 +111,13 @@ module GtfsReader
 
           file :calendar, required: true do
             col :service_id, required: true, unique: true
-
-            col :monday,    required: true, &output_map( yes: ?1, no: ?0 )
-            col :tuesday,   required: true, &output_map( yes: ?1, no: ?0 )
-            col :wednesday, required: true, &output_map( yes: ?1, no: ?0 )
-            col :thursday,  required: true, &output_map( yes: ?1, no: ?0 )
-            col :friday,    required: true, &output_map( yes: ?1, no: ?0 )
-            col :saturday,  required: true, &output_map( yes: ?1, no: ?0 )
-            col :sunday,    required: true, &output_map( yes: ?1, no: ?0 )
-
+            col :monday,     required: true, &output_map( yes: ?1, no: ?0 )
+            col :tuesday,    required: true, &output_map( yes: ?1, no: ?0 )
+            col :wednesday,  required: true, &output_map( yes: ?1, no: ?0 )
+            col :thursday,   required: true, &output_map( yes: ?1, no: ?0 )
+            col :friday,     required: true, &output_map( yes: ?1, no: ?0 )
+            col :saturday,   required: true, &output_map( yes: ?1, no: ?0 )
+            col :sunday,     required: true, &output_map( yes: ?1, no: ?0 )
             col :start_date
             col :end_date
           end
@@ -162,13 +149,11 @@ module GtfsReader
           end
 
           file :shapes do
-            prefix :shape do
-              col :id,            required: true
-              col :pt_lat,        required: true
-              col :pt_lon,        required: true
-              col :pt_sequence,   required: true
-              col :dist_traveled
-            end
+            col :shape_id,            required: true
+            col :shape_pt_lat,        required: true
+            col :shape_pt_lon,        required: true
+            col :shape_pt_sequence,   required: true
+            col :shape_dist_traveled
           end
 
           file :frequencies do
@@ -176,8 +161,7 @@ module GtfsReader
             col :start_time,   required: true
             col :end_time,     required: true
             col :headway_secs, required: true
-            col :exact_times,
-              &output_map( :inexact, exact: 1 )
+            col :exact_times,  &output_map( :inexact, exact: 1 )
           end
 
           file :transfers do
@@ -192,14 +176,12 @@ module GtfsReader
           end
 
           file :feed_info do
-            prefix :scope do
-              col :publisher_name, required: true
-              col :publisher_url,  required: true
-              col :lang,           required: true
-              col :start_date
-              col :end_date
-              col :version
-            end
+            col :feed_publisher_name, required: true
+            col :feed_publisher_url,  required: true
+            col :feed_lang,           required: true
+            col :feed_start_date
+            col :feed_end_date
+            col :feed_version
           end
         end
       end

data/lib/gtfs_reader/config/file_definition.rb

@@ -1,5 +1,4 @@
 require_relative 'column'
-require_relative 'prefixed_column_setter'
 
 module GtfsReader
   module Config
@@ -13,7 +12,6 @@ module GtfsReader
       def initialize(name, opts={})
         @name, @columns = name, {}
         @opts = { required: false }.merge (opts || {})
-        @aliases = {}
       end
 
       #@return [Boolean] If this file is required to be in the feed.
@@ -62,27 +60,12 @@ module GtfsReader
       #@yieldreturn Any kind of object.
       #@return [Column] The newly created column.
       def col(name, *args, &block)
-        name = @aliases[name] if @aliases.key? name
-
         if @columns.key? name
           @columns[name].parser &block if block_given?
           return @columns[name]
         end
 
-        (@columns[name] = Column.new name, args.first, &block).tap do |col|
-          @aliases[col.alias] = name if col.alias
-        end
-      end
-
-      # Starts a new block within which any defined columns will have the given
-      # +sym+ prefixed to its name (joined with an underscore). Also, the
-      # defined name given within the block will be aliased to the column.
-      #@param sym the prefix to prefixed to each column within the block
-      #
-      #@example Create a column +route_name+ with the alias +name+
-      #  prefix( :route ) { name }
-      def prefix(sym, &blk)
-        PrefixedColumnSetter.new(self, sym.to_s).instance_exec &blk
+        @columns[name] = Column.new name, args.first, &block
       end
 
       # Creates an input-output proc to convert column values from one form to

data/lib/gtfs_reader/core.rb

@@ -42,12 +42,13 @@ module GtfsReader
   def update_verbosely(name)
     source = config.sources[name]
     raise UnknownSourceError, "No source named '#{name}'" if source.nil?
+
     updater = SourceUpdater.new name, source
     begin
       updater.instance_exec do
         Log.info { "Updating #{name.to_s.green}".underline }
         before_callbacks
-        read
+        download_source
         check_files
         check_columns
         process_files

data/lib/gtfs_reader/log.rb

@@ -37,6 +37,7 @@ module GtfsReader
         logger.level
       end
 
+      # Silence the logger for the duration of the given block
       def quiet
         old_logger = @logger
         begin

data/lib/gtfs_reader/source_updater.rb

@@ -1,12 +1,15 @@
+require 'active_support/core_ext/object/try'
+require 'csv'
 require 'net/http'
 require 'open-uri'
-require 'zip/filesystem'
-require 'csv'
 require 'uri'
+require 'zip/filesystem'
 
 require_relative 'file_reader'
 
 module GtfsReader
+  # Downloads remote Feed files, checks that they are valid, and passes each
+  # file in the feed to the handlers in the given [Source].
   class SourceUpdater
     #@param name [String] an arbitrary string describing this source
     #@param source [Source]
@@ -15,54 +18,49 @@ module GtfsReader
       @temp_files = {}
     end
 
+    # Call the "before" callback set on this source
     def before_callbacks
-      @source.before.call fetch_data_set_identifier if @source.before
+      if @source.before
+        @source.before.call fetch_data_set_identifier
+      end
     end
 
     # Download the data from the remote server
-    def read
+    def download_source
       Log.debug { "         Reading #{@source.url.green}" }
-      extract_zip_to_tempfiles
-      Log.debug { "Finished reading #{@source.url.green}" }
-    end
+      zip = Tempfile.new 'gtfs'
+      zip.binmode
+      zip << open(@source.url).read
+      zip.rewind
 
-    def extract_zip_to_tempfiles
-      file = Tempfile.new 'gtfs'
-      file.binmode
-      file << open(@source.url).read
-      file.rewind
+      extract_to_tempfiles zip
 
-      Zip::File.open(file).each do |entry|
-        temp = Tempfile.new "gtfs_file_#{entry.name}"
-        temp << entry.get_input_stream.read
-        temp.close
-        @temp_files[entry.name] = temp
-      end
-
-      file.close
+      Log.debug { "Finished reading #{@source.url.green}" }
+    rescue Exception => e
+      Log.error e.message
+      raise e
+    ensure
+      zip.try :close
     end
 
     def close
       @temp_files.values.each &:close
     end
 
+    # Parse the filenames in the feed and check which required and optional
+    # files are present.
+    #@raise [RequiredFilenamesMissing] if the feed is missing a file which is
+    #  marked as "required" in the [FeedDefinition]
     def check_files
       @found_files = []
       check_required_files
       check_optional_files
-    end
-
-    def check_required_files
-      Log.info { 'required files'.magenta }
-      files = @source.feed_definition.required_files
-      missing = check_missing_files files, :green, :red
-      raise RequiredFilenamesMissing, missing unless missing.empty?
-    end
-
-    def check_optional_files
-      Log.info { 'optional files'.cyan }
-      files = @source.feed_definition.optional_files
-      check_missing_files files, :cyan, :light_yellow
+      # Add feed files of zip to the list of files to be processed
+      @source.feed_definition.files.each do |req|
+        if filenames.include? req.filename
+          @found_files << req
+        end
+      end
     end
 
     # Check that every file has its required columns
@@ -86,7 +84,16 @@ module GtfsReader
 
     private
 
-    # Checks for the given list of expected filenames in the zip file
+    def extract_to_tempfiles(zip)
+      Zip::File.open(zip).each do |entry|
+        temp = Tempfile.new "gtfs_file_#{entry.name}"
+        temp << entry.get_input_stream.read
+        temp.close
+        @temp_files[entry.name] = temp
+      end
+    end
+
+    # Check for the given list of expected filenames in the zip file
     def check_missing_files(expected, found_color, missing_color)
       check = '✔'.colorize found_color
       cross = '✘'.colorize missing_color
@@ -95,7 +102,6 @@ module GtfsReader
         filename = req.filename
         if filenames.include? filename
           Log.info { "#{filename.rjust filename_width} [#{check}]" }
-          @found_files << req
           nil
         else
           Log.info { "#{filename.rjust filename_width} [#{cross}]" }
@@ -104,6 +110,8 @@ module GtfsReader
       end.compact
     end
 
+    #@return <FixNum> the maximum string-width of the filenames, so they can be
+    #  aligned when printed on the console.
     def filename_width
       @filename_width ||= @source.feed_definition.files.max do |a, b|
         a.filename.length <=> b.filename.length
@@ -114,31 +122,45 @@ module GtfsReader
       @temp_files.keys
     end
 
-    # Performs a HEAD request against the source's URL, in an attempt to
-    # return a unique identifier for the remote data set. It will choose a
-    # header from the result in the given order of preference:
+    # Perform a HEAD request against the source's URL, looking for a unique
+    # identifier for the remote data set. It will choose a header from the
+    # result in the given order of preference:
     # - ETag
     # - Last-Modified
     # - Content-Length (may result in different data sets being considered
     #   the same if they happen to have the same size)
     # - The current date/time (this will always result in a fresh download)
     def fetch_data_set_identifier
-      uri = URI @source.url
-      Net::HTTP.start(uri.host) do |http|
-        head_request = http.request_head uri.path
-        if head_request.key? 'etag'
-          head_request['etag']
-        else
-          Log.warn "No ETag supplied with: #{uri.path}"
-
-          if head_request.key? 'last-modified'
-            head_request['last-modified']
-          elsif head_request.key? 'content-length'
-            head_request['content-length']
+      if @source.url =~ /\A#{URI::regexp}\z/
+        uri = URI @source.url
+        Net::HTTP.start(uri.host) do |http|
+          head_request = http.request_head uri.path
+          if head_request.key? 'etag'
+            head_request['etag']
           else
-            Time.now.to_s
+            Log.warn "No ETag supplied with: #{uri.path}"
+            fetch_http_fallback_identifier head_request
           end
         end
+      else # it's not a url, it may be a file => last modified
+        begin
+          File.mtime @source.url
+        rescue StandardError => e
+          Log.error e
+          raise e
+        end
+      end
+    end
+
+    # Find a "next best" ID when the HEAD request does not return an "ETag"
+    # header.
+    def fetch_http_fallback_identifier(head_request)
+      if head_request.key? 'last-modified'
+        head_request['last-modified']
+      elsif head_request.key? 'content-length'
+        head_request['content-length']
+      else
+        Time.now.to_s
       end
     end
 
@@ -153,5 +175,20 @@ module GtfsReader
         @source.handlers.handle_file file.name, reader
       end
     end
+
+    #@raise [RequiredFilenamesMissing] if a file is missing a header which is
+    #  marked as "required" in the [FeedDefinition]
+    def check_required_files
+      Log.info { 'required files'.magenta }
+      files = @source.feed_definition.required_files
+      missing = check_missing_files files, :green, :red
+      raise RequiredFilenamesMissing, missing unless missing.empty?
+    end
+
+    def check_optional_files
+      Log.info { 'optional files'.cyan }
+      files = @source.feed_definition.optional_files
+      check_missing_files files, :cyan, :light_yellow
+    end
   end
 end

data/lib/gtfs_reader/version.rb

@@ -3,9 +3,9 @@ module GtfsReader
   # {Bumper} class which will modify this file to increase the version
   module Version
     # The following four lines are generated, so don't mess with them.
-    MAJOR = 0
-    MINOR = 2
-    PATCH = 8
+    MAJOR = 1
+    MINOR = 0
+    PATCH = 0
     BUILD = nil
 
     #@return [String] the current version in the form of +1.2.3.build+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gtfs-reader
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.2.11
 platform: ruby
 authors:
 - Jon Sangster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-05 00:00:00.000000000 Z
+date: 2014-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: log4r
@@ -136,9 +136,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
-description: 'Reads and parses zip files conforming to Google''s GTFS spec. Such files
-  can take up quite a bit of memory when inflated, so this gem prefers to read them
-  as a stream of rows. GTFS Spec: https://developers.google.com/transit/gtfs'
+description: Renamed to gtfs_reader. See https://rubygems.org/gems/gtfs_reader
 email: jon@ertt.ca
 executables: []
 extensions: []
@@ -155,7 +153,6 @@ files:
 - lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb
 - lib/gtfs_reader/config/feed_definition.rb
 - lib/gtfs_reader/config/file_definition.rb
-- lib/gtfs_reader/config/prefixed_column_setter.rb
 - lib/gtfs_reader/config/source.rb
 - lib/gtfs_reader/config/sources.rb
 - lib/gtfs_reader/configuration.rb
@@ -171,7 +168,12 @@ homepage: http://github.com/sangster/gtfs-reader
 licenses:
 - GPL 3
 metadata: {}
-post_install_message: 
+post_install_message: |
+  !    The 'gtfs-reader' gem has been deprecated and has been replaced by
+  !      'gtfs_reader' to follow rubygem.org's naming convention.
+  !    See: https://rubygems.org/gems/gtfs_reader
+  !    And: https://github.com/sangster/gtfs_reader
+  !    And: http://guides.rubygems.org/name-your-gem
 rdoc_options: []
 require_paths:
 - lib

data/lib/gtfs_reader/config/prefixed_column_setter.rb

@@ -1,26 +0,0 @@
-module GtfsReader
-  module Config
-    class PrefixedColumnSetter
-      def initialize(definition, prefix)
-        @definition, @prefix = definition, prefix.to_sym
-      end
-
-      def col(name_alias, *args, &blk)
-        name = "#{@prefix}_#{name_alias}"
-        opts =
-          case args.first
-          when ::Hash then args.first
-          else {}
-          end
-        opts[:alias] = name_alias
-        args[0] = opts
-
-        @definition.col name.to_sym, *args, &blk
-      end
-
-      def output_map(*args, &block)
-        @definition.output_map *args, &block
-      end
-    end
-  end
-end