gtfs_reader 1.0.0

data/lib/gtfs_reader/config/source.rb

@@ -0,0 +1,68 @@
+require 'active_support/core_ext/hash/reverse_merge'
+
+require_relative 'feed_definition'
+require_relative 'defaults/gtfs_feed_definition'
+require_relative '../feed_handler'
+require_relative '../bulk_feed_handler'
+
+module GtfsReader
+  module Config
+    # A single source of GTFS data
+    class Source
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+        @feed_definition = Config::Defaults::FEED_DEFINITION
+        @feed_handler = FeedHandler.new {}
+      end
+
+      #@param t [String] if given, will be used as the title of this source
+      #@return [String] the title of this source
+      def title(t=nil)
+        @title = t if t.present?
+        @title
+      end
+
+      #@param u [String] if given, will be used as the URL for this source
+      #@return [String] the URL this source's ZIP file
+      def url(u=nil)
+        @url = u if u.present?
+        @url
+      end
+
+      # Define a block to call before the source is read. If this block
+      # returns +false+, cancel processing the source
+      def before(&block)
+        if block_given?
+          @before = block
+        end
+        @before
+      end
+
+      def feed_definition(&block)
+        if block_given?
+          @feed_definition = FeedDefinition.new.tap do |feed|
+            feed.instance_exec feed, &block
+          end
+        end
+
+        @feed_definition
+      end
+
+      def handlers(*args, &block)
+        if block_given?
+          opts = Hash === args.last ? args.pop : {}
+          opts = opts.reverse_merge bulk: nil
+          @feed_handler =
+            if opts[:bulk]
+              BulkFeedHandler.new opts[:bulk], args, &block
+            else
+              FeedHandler.new args, &block
+            end
+        end
+        @feed_handler
+      end
+    end
+  end
+end

data/lib/gtfs_reader/config/sources.rb

@@ -0,0 +1,25 @@
+require_relative 'source'
+
+module GtfsReader
+  module Config
+    class Sources < ::BasicObject
+      def initialize
+        @sources = {}
+      end
+
+      def each(&block)
+        @sources.each &block
+      end
+
+      def [](key)
+        @sources[key]
+      end
+
+      def method_missing(name, *args, &block)
+        (@sources[name] ||= Source.new name).tap do |src|
+          src.instance_exec src, &block if ::Kernel.block_given?
+        end
+      end
+    end
+  end
+end

data/lib/gtfs_reader/configuration.rb

@@ -0,0 +1,27 @@
+module GtfsReader
+  class Configuration
+
+    # Creates simple configuration parameters which may be set by the user
+    #@param names [Array<Symbol>] the names of the parameters to create
+    def parameter(*names)
+      names.each do |name|
+        define_singleton_method name do |*values|
+          if value = values.first
+            instance_variable_set "@#{name}", value
+          else
+            instance_variable_get "@#{name}"
+          end
+        end
+      end
+    end
+
+    def block_parameter(name, obj_class, *init_args)
+      obj = nil
+      define_singleton_method name do |*args, &block|
+        obj ||= obj_class.new *init_args
+        obj.instance_exec( obj, *args, &block ) if block
+        obj
+      end
+    end
+  end
+end

data/lib/gtfs_reader/core.rb

@@ -0,0 +1,78 @@
+require_relative 'configuration'
+require_relative 'config/feed_definition'
+require_relative 'config/sources'
+require_relative 'source_updater'
+
+module GtfsReader
+  extend self
+
+  #@override config(*args, &blk)
+  #  @param args [Array] an array or arguments to pass to the given block
+  #  @param blk [Proc] a block to to call in the context of the configuration
+  #    object. Subsequent calls will use the same configuration for additional
+  #    modification.
+  #  @return [Configuration] the configuration object
+  #
+  #@override config
+  #  @return [Configuration] the configuration object
+  def config(*args, &blk)
+    @cfg ||= create_config
+    if block_given?
+      @cfg.instance_exec *args.unshift(@cfg), &blk
+    elsif args.any?
+      raise ArgumentError, 'arguments given without a block'
+    end
+    @cfg
+  end
+
+  def update_all!
+    config.sources.each {|name, _| update name }
+  end
+
+  def update(name)
+    if config.verbose
+      update_verbosely name
+    else
+      Log.quiet { update_verbosely name }
+    end
+  end
+
+  private
+
+  def update_verbosely(name)
+    source = config.sources[name]
+    raise UnknownSourceError, "No source named '#{name}'" if source.nil?
+
+    updater = SourceUpdater.new name, source
+    begin
+      updater.instance_exec do
+        Log.info { "Updating #{name.to_s.green}".underline }
+        before_callbacks
+        download_source
+        check_files
+        check_columns
+        process_files
+        Log.info { "Finished updating #{name.to_s.green}" }
+      end
+    rescue SkipSourceError => e
+      Log.warn do
+        msg = e.message ? ": #{e.message}" : ''
+        "#{'Skipping'.red} #{source.name.to_s.yellow}" + msg
+      end
+    ensure
+      updater.close
+    end
+  end
+
+  def create_config
+    Configuration.new.tap do |cfg|
+      cfg.instance_exec do
+        parameter :verbose
+        parameter :skip_parsing
+        parameter :return_hashes
+        block_parameter :sources, Config::Sources
+        block_parameter :feed_definition, Config::FeedDefinition
+      end
+    end
+  end
+end

data/lib/gtfs_reader/exceptions.rb

@@ -0,0 +1,29 @@
+module GtfsReader
+
+  class FileReaderError < StandardError; end
+
+  class RequiredColumnsMissing < FileReaderError
+    attr_reader :columns
+    def initialize(columns)
+      @columns = columns
+      super "Required columns missing: #{columns.join ', '}"
+    end
+  end
+
+  class RequiredFilenamesMissing < FileReaderError
+    attr_reader :filenames
+    def initialize(filenames)
+      @filenames = filenames
+      super "Required files missing from zip file: #{filenames.join ', '}"
+    end
+  end
+
+  class UnknownSourceError < StandardError; end
+  class SkipSourceError < StandardError; end
+  class HandlerMissingError < StandardError; end
+
+  module Config
+    class SourceDefinitionError < StandardError; end
+    class FileDefinitionError < StandardError; end
+  end
+end

data/lib/gtfs_reader/feed_handler.rb

@@ -0,0 +1,33 @@
+module GtfsReader
+  # This handler returns each row individually as it is read in from the source.
+  class FeedHandler
+    def initialize(args=[], &block)
+      @read_callbacks = {}
+      FeedHandlerDsl.new(self).instance_exec *args, &block
+    end
+
+    #@param filename [String] the name of the file to handle
+    #@return [Boolean] if this handler can handle the given filename
+    def handler?(filename)
+      @read_callbacks.key? filename
+    end
+
+    def handle_file(filename, enumerator)
+      enumerator.each &@read_callbacks[filename]
+    end
+
+    def create_read_handler(filename, *args, &block)
+      @read_callbacks[filename] = block
+    end
+  end
+
+  class FeedHandlerDsl
+    def initialize(feed_handler)
+      @feed_handler = feed_handler
+    end
+
+    def method_missing(filename, *args, &block)
+      @feed_handler.create_read_handler filename, *args, &block
+    end
+  end
+end

data/lib/gtfs_reader/file_reader.rb

@@ -0,0 +1,120 @@
+require 'active_support/core_ext/object/blank'
+require 'csv'
+require_relative 'file_row'
+
+module GtfsReader
+  CSV_OPTIONS = { headers: :first_row,
+                  return_headers: true,
+                  header_converters: :symbol }
+
+  # Iterates over the rows in a single file using a provided definition.
+  #@see #each
+  class FileReader
+    include Enumerable
+
+    attr_reader :definition, :columns, :col_names
+
+    #@param data [IO,String] CSV data
+    #@param definition [FileDefinition] describes the expected columns in this
+    #  file
+    def initialize(data, definition, opts={})
+      opts = { parse: true, validate: false, hash: true }.merge opts
+
+      @csv = CSV.new data, CSV_OPTIONS
+      @definition, @do_parse, @return_hash =
+        definition, opts[:parse], opts[:hash]
+      @index = 0
+      @csv_headers = @csv.shift.headers
+      @columns = find_columns opts[:validate]
+    end
+
+    def filename
+      @definition.filename
+    end
+
+    #@overload each(&blk)
+    #  @yieldparam hash [Hash] a hash of columns to their values in this row
+    #@overload each
+    #  @return [Enumerator] an {::Enumerator} that iterates of the rows in the
+    #    file
+    #@see FileRow#to_hash
+    def each
+      return to_enum :each unless block_given?
+
+      while row = shift
+        yield(@return_hash ? row.to_hash : row.to_a)
+      end
+    end
+
+    #@return [FileRow,nil] the next row from the file, or +nil+ if the end of
+    #  the file has been reached.
+    def shift
+      if row = @csv.shift
+        file_row(row).tap { @index += 1 }
+      end
+    end
+
+    private
+
+    def file_row(row)
+      FileRow.new @index, @col_names, row, @definition, @do_parse
+    end
+
+    # Check the list of headers in the file against the expected columns in
+    # the definition
+    def find_columns(validate)
+      @found_columns = []
+      prefix = "#{filename.yellow}:"
+
+      required = @definition.required_columns
+      unless required.empty?
+        Log.info { "#{prefix} #{'required columns'.magenta}" } if validate
+
+        missing = check_columns validate, prefix, required, :green, :red
+        raise RequiredColumnsMissing, missing if validate && missing.present?
+      end
+
+      optional = @definition.optional_columns
+      unless optional.empty?
+        Log.info { "#{prefix} #{'optional columns'.cyan}" } if validate
+        check_columns validate, prefix, optional, :cyan, :light_yellow
+      end
+
+      cols = @definition.columns.collect( &:name )
+      headers = @csv_headers.select {|h| cols.include? h }
+
+      @col_names ||= @found_columns.map &:name
+      ::Hash[ *headers.inject([]) {|list,c| list << c << @definition[c] } ]
+    end
+
+    def check_columns(validate, prefix, expected, found_color, missing_color)
+      check = '✔'.colorize found_color
+      cross = '✘'.colorize missing_color
+
+      expected.map do |col|
+        name = col.name
+        missing =
+          if @csv_headers.include? name
+            @found_columns << col
+            nil
+          else
+            name
+          end
+
+        if validate
+          Log.info do
+            mark = missing ? cross : check
+            "#{prefix} #{name.to_s.rjust column_width} [#{mark}]"
+          end
+        end
+        missing
+      end.compact
+    end
+
+    def column_width
+      @column_width ||= @definition.columns.collect( &:name ).max do |a, b|
+        a.length <=> b.length
+      end.length
+    end
+  end
+end

data/lib/gtfs_reader/file_row.rb

@@ -0,0 +1,66 @@
+require 'csv'
+
+module GtfsReader
+  # Contains the contents of a single row read in from the file
+  class FileRow
+    attr_reader :line_number
+
+    #@param line_number [Integer] the line number from the source file
+    #@return [Array<Symbol>]
+    #@param data [CSV::Row] the data for this row
+    #@param definition [FileDefinition] the definition of the columns that the
+    #  data in this row represent
+    def initialize(line_number, headers, data, definition, do_parse)
+      @line_number, @headers, @data, @definition, @do_parse =
+          line_number, headers, data, definition, do_parse
+      @parsed = {}
+    end
+
+    #@return [Array<Symbol>]
+    def headers
+      @headers
+    end
+
+    #@param column [Symbol] the name of the column to fetch
+    #@return the parsed data for the column at this row
+    #@see #raw
+    def [](column)
+      return raw(column) unless @do_parse
+
+      @parsed[column] ||= begin
+        ParserContext.new(column, self).
+            instance_exec raw(column), &@definition[column].parser
+      end
+    end
+
+    #@param (see #[])
+    #@return the data unparsed data from the column at this row
+    def raw(column)
+      @data[column]
+    end
+
+    #@return [Hash] a hash representing this row of data, where each key is the
+    #  column name and each value is the parsed data for this row
+    def to_hash
+      ::Hash[ *headers.inject([]) {|list,h| list << h << self[h] } ]
+    end
+
+    #@return [Array] an array representing this row of data
+    def to_a
+      headers.map {|h| self[h] }
+    end
+  end
+
+  class ParserContext
+    def initialize(column, file_row)
+      @column, @file_row = column, file_row
+    end
+
+    def method_missing(column)
+      if column == @column
+        raise "Parser for '#{column}' cannot refer to itself"
+      end
+      @file_row[column] or super
+    end
+  end
+end