gtfs_reader 1.0.0

data/lib/gtfs_reader/log.rb

@@ -0,0 +1,70 @@
+require 'log4r'
+require 'log4r/formatter/patternformatter'
+require 'colorize'
+
+module GtfsReader
+  module Log
+    class << self
+      def debug(*args, &block); log :debug, *args, &block end
+      def  info(*args, &block); log :info,  *args, &block end
+      def  warn(*args, &block); log :warn,  *args, &block end
+      def error(*args, &block); log :error, *args, &block end
+      def fatal(*args, &block); log :fatal, *args, &block end
+
+      def log(level, *args, &block)
+        logger.send level, *args, &block
+        nil
+      end
+
+      def logger
+        @logger = yield if block_given?
+        @logger ||= create_logger
+      end
+
+      def level=(lev)
+        logger.level =
+            case lev
+            when :debug then logger.levels.index 'DEBUG'
+            when :info  then logger.levels.index 'INFO'
+            when :warn  then logger.levels.index 'WARN'
+            when :error then logger.levels.index 'ERROR'
+            when :fatal then logger.levels.index 'FATAL'
+            else raise "unknown log level '#{lev}'"
+            end
+      end
+
+      def level
+        logger.level
+      end
+
+      # Silence the logger for the duration of the given block
+      def quiet
+        old_logger = @logger
+        begin
+          @logger = NoOpLogger.new
+          yield
+        ensure
+          @logger = old_logger
+        end
+      end
+
+      private
+
+      def create_logger
+        Log4r::Logger.new('GtfsReader').tap do |log|
+          out = Log4r::StdoutOutputter.new('log_stdout')
+          out.formatter = Log4r::PatternFormatter.new pattern: '%d [%l]: %m'
+          log.outputters << out
+          log.level = Log4r::INFO
+          log.debug { 'Starting GtfsReader...'.underline.colorize :yellow }
+        end
+      end
+    end
+
+    class NoOpLogger
+      def method_missing(*args, &block)
+        nil
+      end
+    end
+  end
+end

data/lib/gtfs_reader/source_updater.rb

@@ -0,0 +1,194 @@
+require 'active_support/core_ext/object/try'
+require 'csv'
+require 'net/http'
+require 'open-uri'
+require 'uri'
+require 'zip/filesystem'
+
+require_relative 'file_reader'
+
+module GtfsReader
+  # Downloads remote Feed files, checks that they are valid, and passes each
+  # file in the feed to the handlers in the given [Source].
+  class SourceUpdater
+    #@param name [String] an arbitrary string describing this source
+    #@param source [Source]
+    def initialize(name, source)
+      @name, @source = name, source
+      @temp_files = {}
+    end
+
+    # Call the "before" callback set on this source
+    def before_callbacks
+      if @source.before
+        @source.before.call fetch_data_set_identifier
+      end
+    end
+
+    # Download the data from the remote server
+    def download_source
+      Log.debug { "         Reading #{@source.url.green}" }
+      zip = Tempfile.new 'gtfs'
+      zip.binmode
+      zip << open(@source.url).read
+      zip.rewind
+
+      extract_to_tempfiles zip
+
+      Log.debug { "Finished reading #{@source.url.green}" }
+    rescue Exception => e
+      Log.error e.message
+      raise e
+    ensure
+      zip.try :close
+    end
+
+    def close
+      @temp_files.values.each &:close
+    end
+
+    # Parse the filenames in the feed and check which required and optional
+    # files are present.
+    #@raise [RequiredFilenamesMissing] if the feed is missing a file which is
+    #  marked as "required" in the [FeedDefinition]
+    def check_files
+      @found_files = []
+      check_required_files
+      check_optional_files
+      # Add feed files of zip to the list of files to be processed
+      @source.feed_definition.files.each do |req|
+        if filenames.include? req.filename
+          @found_files << req
+        end
+      end
+    end
+
+    # Check that every file has its required columns
+    def check_columns
+      @found_files.each do |file|
+        @temp_files[file.filename].open do |data|
+          FileReader.new data, file, validate: true
+        end
+      end
+    end
+
+    def process_files
+      @found_files.each do |file|
+        if @source.handlers.handler? file.name
+          process_from_temp_file file
+        else
+          Log.warn { "Skipping #{file.filename.yellow} (no handler)" }
+        end
+      end
+    end
+
+    private
+
+    def extract_to_tempfiles(zip)
+      Zip::File.open(zip).each do |entry|
+        temp = Tempfile.new "gtfs_file_#{entry.name}"
+        temp << entry.get_input_stream.read
+        temp.close
+        @temp_files[entry.name] = temp
+      end
+    end
+
+    # Check for the given list of expected filenames in the zip file
+    def check_missing_files(expected, found_color, missing_color)
+      check = '✔'.colorize found_color
+      cross = '✘'.colorize missing_color
+
+      expected.map do |req|
+        filename = req.filename
+        if filenames.include? filename
+          Log.info { "#{filename.rjust filename_width} [#{check}]" }
+          nil
+        else
+          Log.info { "#{filename.rjust filename_width} [#{cross}]" }
+          filename
+        end
+      end.compact
+    end
+
+    #@return <FixNum> the maximum string-width of the filenames, so they can be
+    #  aligned when printed on the console.
+    def filename_width
+      @filename_width ||= @source.feed_definition.files.max do |a, b|
+        a.filename.length <=> b.filename.length
+      end.filename.length
+    end
+
+    def filenames
+      @temp_files.keys
+    end
+
+    # Perform a HEAD request against the source's URL, looking for a unique
+    # identifier for the remote data set. It will choose a header from the
+    # result in the given order of preference:
+    # - ETag
+    # - Last-Modified
+    # - Content-Length (may result in different data sets being considered
+    #   the same if they happen to have the same size)
+    # - The current date/time (this will always result in a fresh download)
+    def fetch_data_set_identifier
+      if @source.url =~ /\A#{URI::regexp}\z/
+        uri = URI @source.url
+        Net::HTTP.start(uri.host) do |http|
+          head_request = http.request_head uri.path
+          if head_request.key? 'etag'
+            head_request['etag']
+          else
+            Log.warn "No ETag supplied with: #{uri.path}"
+            fetch_http_fallback_identifier head_request
+          end
+        end
+      else # it's not a url, it may be a file => last modified
+        begin
+          File.mtime @source.url
+        rescue StandardError => e
+          Log.error e
+          raise e
+        end
+      end
+    end
+
+    # Find a "next best" ID when the HEAD request does not return an "ETag"
+    # header.
+    def fetch_http_fallback_identifier(head_request)
+      if head_request.key? 'last-modified'
+        head_request['last-modified']
+      elsif head_request.key? 'content-length'
+        head_request['content-length']
+      else
+        Time.now.to_s
+      end
+    end
+
+    def process_from_temp_file(file)
+      do_parse = !GtfsReader.config.skip_parsing
+      hash = !!GtfsReader.config.return_hashes
+
+      Log.info "Reading file #{file.filename.cyan}..."
+      begin
+        reader = FileReader.new @temp_files[file.filename], file,
+                                parse: do_parse, hash: hash
+        @source.handlers.handle_file file.name, reader
+      end
+    end
+
+    #@raise [RequiredFilenamesMissing] if a file is missing a header which is
+    #  marked as "required" in the [FeedDefinition]
+    def check_required_files
+      Log.info { 'required files'.magenta }
+      files = @source.feed_definition.required_files
+      missing = check_missing_files files, :green, :red
+      raise RequiredFilenamesMissing, missing unless missing.empty?
+    end
+
+    def check_optional_files
+      Log.info { 'optional files'.cyan }
+      files = @source.feed_definition.optional_files
+      check_missing_files files, :cyan, :light_yellow
+    end
+  end
+end

data/lib/gtfs_reader/version.rb

@@ -0,0 +1,77 @@
+module GtfsReader
+  # This module both contains the current version of GtfsReader, but also has a
+  # {Bumper} class which will modify this file to increase the version
+  module Version
+    # The following four lines are generated, so don't mess with them.
+    MAJOR = 1
+    MINOR = 0
+    PATCH = 0
+    BUILD = nil
+
+    #@return [String] the current version in the form of +1.2.3.build+
+    def self.to_s
+      [MAJOR, MINOR, PATCH, BUILD].compact.join '.'
+    end
+
+    # A helper class which bumps the version number stored in this file
+    class Bumper
+      PARTS = %i[major minor patch]
+      PATTERN = %r[(\s+)MAJOR = \d+\s+MINOR = \d+\s+PATCH = \d+\s+BUILD = .+]
+
+      #@param filename [String] the file to edit
+      #@param part [String] the part of the version to bump. one of major,
+      #  minor, or patch
+      def initialize(filename=__FILE__, part)
+        raise "#{part} not one of #{PARTS}" unless PARTS.include? part
+        @filename, @part = filename, part
+      end
+
+      # Increase the version number and write it to this file
+      def bump
+        parts = new_version
+        # \1 holds a newline and the indentation from the source
+        text = '\1' + ["MAJOR = #{parts[:major]}",
+                       "MINOR = #{parts[:minor]}",
+                       "PATCH = #{parts[:patch]}",
+                       "BUILD = #{parts[:build] || 'nil'}"].join( '\1' )
+
+        out_data = File.read( @filename ).gsub PATTERN, text
+        #puts out_data
+        File.open( @filename, 'w' ) { |out| out << out_data }
+        puts "Bumped version to #{to_s}"
+      end
+
+      #@return [String] What the new version string will be.
+      def to_s
+        p = new_version
+        [p[:major], p[:minor], p[:patch], p[:build]].compact.join ?.
+      end
+
+      private
+
+      def new_version
+        @vers ||= { major: MAJOR,
+                    minor: MINOR,
+                    patch: PATCH,
+                    build: BUILD }.merge new_parts
+      end
+
+      def new_parts
+        case @part
+        when :major then {
+            major: MAJOR + 1,
+            minor: 0,
+            patch: 0
+        }
+        when :minor then {
+            minor: MINOR + 1,
+            patch: 0
+        }
+        else {
+            patch: PATCH + 1
+        }
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,193 @@
+--- !ruby/object:Gem::Specification
+name: gtfs_reader
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jon Sangster
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: 'Reads and parses zip files conforming to Google''s GTFS spec. Such files
+  can take up quite a bit of memory when inflated, so this gem prefers to read them
+  as a stream of rows. GTFS Spec: https://developers.google.com/transit/gtfs'
+email: jon@ertt.ca
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- LICENSE
+- README.md
+- Rakefile
+- lib/gtfs_reader.rb
+- lib/gtfs_reader/bulk_feed_handler.rb
+- lib/gtfs_reader/config/column.rb
+- lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb
+- lib/gtfs_reader/config/feed_definition.rb
+- lib/gtfs_reader/config/file_definition.rb
+- lib/gtfs_reader/config/source.rb
+- lib/gtfs_reader/config/sources.rb
+- lib/gtfs_reader/configuration.rb
+- lib/gtfs_reader/core.rb
+- lib/gtfs_reader/exceptions.rb
+- lib/gtfs_reader/feed_handler.rb
+- lib/gtfs_reader/file_reader.rb
+- lib/gtfs_reader/file_row.rb
+- lib/gtfs_reader/log.rb
+- lib/gtfs_reader/source_updater.rb
+- lib/gtfs_reader/version.rb
+homepage: http://github.com/sangster/gtfs_reader
+licenses:
+- GPL 3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Read General Transit Feed Specification zip files
+test_files: []