gtfs-reader 0.2.2

data/lib/gtfs_reader/log.rb

@@ -0,0 +1,66 @@
+require 'log4r'
+require 'colorize'
+
+module GtfsReader
+  module Log
+    class << self
+      def debug(*args, &block); log :debug, *args, &block end
+      def  info(*args, &block); log :info,  *args, &block end
+      def  warn(*args, &block); log :warn,  *args, &block end
+      def error(*args, &block); log :error, *args, &block end
+      def fatal(*args, &block); log :fatal, *args, &block end
+
+      def log(level, *args, &block)
+        logger.send level, *args, &block
+        nil
+      end
+
+      def logger
+        @logger = yield if block_given?
+        @logger ||= create_logger
+      end
+
+      def level=(lev)
+        logger.level =
+            case lev
+            when :debug then logger.levels.index 'DEBUG'
+            when :info  then logger.levels.index 'INFO'
+            when :warn  then logger.levels.index 'WARN'
+            when :error then logger.levels.index 'ERROR'
+            when :fatal then logger.levels.index 'FATAL'
+            else raise "unknown log level '#{lev}'"
+            end
+      end
+
+      def level
+        logger.level
+      end
+
+      def quiet
+        old_logger = @logger
+        begin
+          @logger = NoOp.new
+          yield
+        ensure
+          @logger = old_logger
+        end
+      end
+
+      private
+
+      def create_logger
+        Log4r::Logger.new('GtfsReader').tap do |log|
+          log.outputters << Log4r::StdoutOutputter.new('log_stdout')
+          log.level = Log4r::INFO
+          log.debug { 'Starting GtfsReader...'.underline.colorize :yellow }
+        end
+      end
+    end
+
+    class NoOp
+      def method_missing(*args, &block)
+        nil
+      end
+    end
+  end
+end

data/lib/gtfs_reader/source_updater.rb

@@ -0,0 +1,130 @@
+require 'net/http'
+require 'open-uri'
+require 'zip/filesystem'
+require 'csv'
+require 'uri'
+
+require_relative 'file_reader'
+
+module GtfsReader
+  class SourceUpdater
+    #@param name [String] an arbitrary string describing this source
+    #@param source [Source]
+    def initialize(name, source)
+      @name, @source = name, source
+    end
+
+    def before_callbacks
+      @source.before.call fetch_remote_etag if @source.before
+    end
+
+    # Download the data from the remote server
+    def read
+      Log.debug { "         Reading #{@source.url.green}" }
+      @file = Tempfile.new 'gtfs'
+      @file.binmode
+      @file << open(@source.url).read
+      @file.rewind
+      #binding.pry
+      @zip = Zip::File.open @file
+      Log.debug { "Finished reading #{@source.url.green}" }
+    end
+
+    # Close any streams still open
+    def finish
+      @zip.close if @zip
+      @file.delete if @file
+    end
+
+    def check_files
+      @found_files = []
+      check_required_files
+      check_optional_files
+    end
+
+    def check_required_files
+      Log.info { 'required files'.magenta }
+      files = @source.feed_definition.required_files
+      missing = check_missing_files files, :green, :red
+      raise RequiredFilenamesMissing, missing unless missing.empty?
+    end
+
+    def check_optional_files
+      Log.info { 'optional files'.cyan }
+      files = @source.feed_definition.optional_files
+      check_missing_files files, :cyan, :light_yellow
+    end
+
+    # Check that every file has its required columns
+    def check_columns
+      @found_files.each do |file|
+        @zip.file.open(file.filename) do |data|
+          FileReader.new data, file, validate: true
+        end
+      end
+    end
+
+    def process_files
+      @found_files.each do |file|
+        if @source.handlers.handler? file.name
+          process_from_temp_file file
+        else
+          Log.warn { "Skipping #{file.filename.yellow} (no handler)" }
+        end
+      end
+    end
+
+    private
+
+    def check_missing_files(expected, found_color, missing_color)
+      check = '✔'.colorize found_color
+      cross = '✘'.colorize missing_color
+
+      expected.collect do |req|
+        filename = req.filename
+        if filenames.include? filename
+          Log.info { "#{filename.rjust filename_width} [#{check}]" }
+          @found_files << req
+          nil
+        else
+          Log.info { "#{filename.rjust filename_width} [#{cross}]" }
+          filename
+        end
+      end.compact
+    end
+
+    def filename_width
+      @filename_width ||= @source.feed_definition.files.max do |a, b|
+        a.filename.length <=> b.filename.length
+      end.filename.length
+    end
+
+    def filenames
+      @filenames ||= @zip.entries.collect &:name
+    end
+
+    def fetch_remote_etag
+      url = URI @source.url
+      Net::HTTP.start(url.host) do |http|
+        /[^"]+/ === http.request_head(url.path)['etag'] and $&
+      end
+    end
+
+    def process_from_temp_file(file)
+      do_parse = !GtfsReader.config.skip_parsing
+      hash = !!GtfsReader.config.return_hashes
+
+      Log.info "Reading file #{file.filename.cyan}..."
+
+      temp = Tempfile.new 'gtfs_file'
+      begin
+        @zip.file.open(file.filename) { |z| temp.write z.read }
+        temp.rewind
+        reader = FileReader.new temp, file, parse: do_parse, hash: hash
+        @source.handlers.handle_file file.name, reader
+      ensure
+        temp.close and temp.unlink
+      end
+    end
+  end
+end

data/lib/gtfs_reader/version.rb

@@ -0,0 +1,71 @@
+module GtfsReader
+  module Version
+    # The following four lines are generated, so don't mess with them.
+    MAJOR = 0
+    MINOR = 2
+    PATCH = 2
+    BUILD = nil
+
+    def self.to_s
+      [MAJOR, MINOR, PATCH, BUILD].compact.join '.'
+    end
+
+    # A helper class which bumps the version number stored in this file
+    class Bumper
+      PARTS = %i[major minor patch]
+      PATTERN = %r[(\s+)MAJOR = \d+\s+MINOR = \d+\s+PATCH = \d+\s+BUILD = .+]
+
+      def initialize(filename=__FILE__, part)
+        raise "#{part} not one of #{PARTS}" unless PARTS.include? part
+        @filename, @part = filename, part
+      end
+
+      # Increase the version number and write it to this file
+      def bump
+        parts = new_version
+        # \1 holds a newline and the indentation from the source
+        text = '\1' + ["MAJOR = #{parts[:major]}",
+                       "MINOR = #{parts[:minor]}",
+                       "PATCH = #{parts[:patch]}",
+                       "BUILD = #{parts[:build] || 'nil'}"].join( '\1' )
+
+        out_data = File.read( @filename ).gsub PATTERN, text
+        #puts out_data
+        File.open( @filename, 'w' ) { |out| out << out_data }
+        puts "Bumped version to #{to_s}"
+      end
+
+      #@return [String] What the new version string will be.
+      def to_s
+        p = new_version
+        [p[:major], p[:minor], p[:patch], p[:build]].compact.join ?.
+      end
+
+      private
+
+      def new_version
+        @vers ||= { major: MAJOR,
+                    minor: MINOR,
+                    patch: PATCH,
+                    build: BUILD }.merge new_parts
+      end
+
+      def new_parts
+        case @part
+        when :major then {
+            major: MAJOR + 1,
+            minor: 0,
+            patch: 0
+        }
+        when :minor then {
+            minor: MINOR + 1,
+            patch: 0
+        }
+        else {
+            patch: PATCH + 1
+        }
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,195 @@
+--- !ruby/object:Gem::Specification
+name: gtfs-reader
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Jon Sangster
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: 'Reads and parses zip files conforming to Google''s GTFS spec. Such files
+  can take up quite a bit of memory when inflated, so this gem prefers to read them
+  as a stream of rows. GTFS Spec: https://developers.google.com/transit/gtfs'
+email: jon@ertt.ca
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- LICENSE
+- README.md
+- Rakefile
+- lib/gtfs_reader.rb
+- lib/gtfs_reader/bulk_feed_handler.rb
+- lib/gtfs_reader/config/column.rb
+- lib/gtfs_reader/config/defaults/gtfs_feed_definition.rb
+- lib/gtfs_reader/config/feed_definition.rb
+- lib/gtfs_reader/config/file_definition.rb
+- lib/gtfs_reader/config/prefixed_column_setter.rb
+- lib/gtfs_reader/config/source.rb
+- lib/gtfs_reader/config/sources.rb
+- lib/gtfs_reader/configuration.rb
+- lib/gtfs_reader/core.rb
+- lib/gtfs_reader/exceptions.rb
+- lib/gtfs_reader/feed_handler.rb
+- lib/gtfs_reader/file_reader.rb
+- lib/gtfs_reader/file_row.rb
+- lib/gtfs_reader/log.rb
+- lib/gtfs_reader/source_updater.rb
+- lib/gtfs_reader/version.rb
+homepage: http://github.com/sangster/gtfs-reader
+licenses:
+- GPL 3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Read General Transit Feed Specification zip files
+test_files: []
+has_rdoc: