gtfs-engine 1.0.0

data/lib/gtfs_engine/engine.rb

@@ -0,0 +1,28 @@
+# This file is part of the KNOWtime server.
+#
+# The KNOWtime server is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# The KNOWtime server is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the KNOWtime server.  If not, see <http://www.gnu.org/licenses/>.
+module GtfsEngine
+  class Engine < ::Rails::Engine
+    isolate_namespace GtfsEngine
+    engine_name 'gtfs_engine'
+
+    config.generators.instance_exec do
+      template_engine     :jbuilder
+      test_framework      :rspec, fixture: false
+      stylesheets         false
+      javascripts         false
+      fixture_replacement :factory_girl, dir: 'spec/factories'
+    end
+  end
+end

data/lib/gtfs_engine/exceptions.rb

@@ -0,0 +1,35 @@
+# This file is part of the KNOWtime server.
+#
+# The KNOWtime server is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# The KNOWtime server is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the KNOWtime server.  If not, see <http://www.gnu.org/licenses/>.
+module GtfsEngine
+  Error = Class.new StandardError
+
+
+  class UnknownFilters < Error
+    attr_reader :fields
+
+    #@param fields [Array<String>] the names of the unknown fields which the
+    # caller tried to filter with
+    def initialize(fields)
+      @fields = fields
+    end
+
+    def to_hash
+      fields.each_with_object({}) do |field, hash|
+        hash[field] = message
+      end
+    end
+  end
+end
+

data/lib/gtfs_engine/json_responder.rb

@@ -0,0 +1,31 @@
+# This file is part of the KNOWtime server.
+#
+# The KNOWtime server is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# The KNOWtime server is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the KNOWtime server.  If not, see <http://www.gnu.org/licenses/>.
+class GtfsEngine::JsonResponder < ActionController::Responder
+  def to_json
+    if has_errors?
+      display_errors
+    else
+      default_render
+    end
+  end
+
+  protected
+
+  def display_errors
+    status_code = options[:status] || :unprocessable_entity
+    data = { status: 'error', data: resource_errors[:errors] }
+    controller.render format => data, status: status_code
+  end
+end

data/lib/gtfs_engine/sources.rb

@@ -0,0 +1,86 @@
+# This file is part of the KNOWtime server.
+#
+# The KNOWtime server is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# The KNOWtime server is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the KNOWtime server.  If not, see <http://www.gnu.org/licenses/>.
+require 'set'
+require 'activerecord-import'
+
+module GtfsEngine
+  class Sources < ::BasicObject
+    def initialize(config)
+      @config = config
+      @source_ids = ::Set.new
+    end
+
+    def method_missing(name, &block)
+      source = @config.sources.__send__ name
+      add_default_importers source
+      source.instance_exec source, &block if ::Kernel.block_given?
+      source
+    end
+
+    private
+
+    def add_default_importers(source)
+      id = source.__id__
+      return if @source_ids.include? id
+      @source_ids.add id
+
+      get_method = ::Kernel.instance_method(:method).bind self
+      bulk_importer = get_method.call :bulk_importer
+      set_id = ::OpenStruct.new id: -1
+
+      source.before do |etag|
+        if DataSet.where(name: source.name, etag: etag).empty?
+          set = DataSet.create name: source.name,
+                               title: source.title,
+                               etag: etag,
+                               url: source.url
+          set.reload
+          set_id.id = set.id
+        else
+          ::Kernel.raise ::GtfsReader::SkipSourceError,
+                         "Data Pull already exists for #{etag}"
+        end
+      end
+
+      source.handlers bulk_importer, set_id, bulk: 1024 do |importer, set_id|
+        agency          &importer.call( Agency, set_id )
+        calendar        &importer.call( Calendar, set_id )
+        calendar_dates  &importer.call( CalendarDate, set_id )
+        fare_attributes &importer.call( FareAttribute, set_id )
+        fare_rules      &importer.call( FareRule, set_id )
+        feed_info       &importer.call( FeedInfo, set_id )
+        frequencies     &importer.call( Frequency, set_id )
+        stop_times      &importer.call( StopTime, set_id )
+        routes          &importer.call( Route, set_id )
+        shapes          &importer.call( Shape, set_id )
+        stops           &importer.call( Stop, set_id )
+        stop_times      &importer.call( StopTime, set_id )
+        transfers       &importer.call( Transfer, set_id )
+        trips           &importer.call( Trip, set_id )
+      end
+    end
+
+    def bulk_importer(model_class, data_set_id_holder)
+      ::Proc.new do
+        bulk do |values, count, total, columns|
+          id = data_set_id_holder.id
+          values.each {|v| v << id }
+          model_class.import (columns + [:data_set_id]), values
+          ::Rails.logger.info "Inserted #{count} #{model_class} records (total: #{total})"
+        end
+      end
+    end
+  end
+end

data/lib/gtfs_engine/version.rb

@@ -0,0 +1,84 @@
+# This file is part of the KNOWtime server.
+#
+# The KNOWtime server is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# The KNOWtime server is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the KNOWtime server.  If not, see <http://www.gnu.org/licenses/>.
+module GtfsEngine
+  module Version
+    # The following four lines are generated, so don't mess with them.
+    MAJOR = 1
+    MINOR = 0
+    PATCH = 0
+    BUILD = nil
+
+    def self.to_s
+      [MAJOR, MINOR, PATCH, BUILD].compact.join '.'
+    end
+
+    # A helper class which bumps the version number stored in this file
+    class Bumper
+      PARTS = %i[major minor patch]
+      PATTERN = %r[(\s+)MAJOR = \d+\s+MINOR = \d+\s+PATCH = \d+\s+BUILD = .+]
+
+      def initialize(filename=__FILE__, part)
+        raise "#{part} not one of #{PARTS}" unless PARTS.include? part
+        @filename, @part = filename, part
+      end
+
+      # Increase the version number and write it to this file
+      def bump
+        parts = new_version
+        text = '\1' + ["MAJOR = #{parts[:major]}",
+                       "MINOR = #{parts[:minor]}",
+                       "PATCH = #{parts[:patch]}",
+                       "BUILD = #{parts[:build] || 'nil'}"].join( '\1' )
+
+        out_data = File.read( @filename ).gsub PATTERN, text
+        #puts out_data
+        File.open( @filename, 'w' ) { |out| out << out_data }
+        puts "Bumped version to #{to_s}"
+      end
+
+      #@return [String] What the new version string will be.
+      def to_s
+        p = new_version
+        [p[:major], p[:minor], p[:patch], p[:build]].compact.join ?.
+      end
+
+      private
+
+      def new_version
+        @vers ||= { major: MAJOR,
+                    minor: MINOR,
+                    patch: PATCH,
+                    build: BUILD }.merge new_parts
+      end
+
+      def new_parts
+        case @part
+        when :major then {
+            major: MAJOR + 1,
+            minor: 0,
+            patch: 0
+        }
+        when :minor then {
+            minor: MINOR + 1,
+            patch: 0
+        }
+        else {
+            patch: PATCH + 1
+        }
+        end
+      end
+    end
+  end
+end

data/lib/tasks/gtfs_engine_tasks.rake

@@ -0,0 +1,89 @@
+# This file is part of the KNOWtime server.
+#
+# The KNOWtime server is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# The KNOWtime server is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the KNOWtime server.  If not, see <http://www.gnu.org/licenses/>.
+require 'colorize'
+
+namespace :gtfs_engine do
+  desc 'Update sources'
+  task :update, [:name] => :environment do |_, args|
+    GtfsReader.config do
+      skip_parsing true
+      verbose true
+    end
+
+    if args.name
+      GtfsReader.update args.name.to_sym
+    else
+      GtfsReader.update_all!
+    end
+
+    Rails.cache.clear
+  end
+
+  namespace :sets do
+    desc 'List the data sets for each source'
+    task list: :environment do
+      GtfsEngine::DataSet.distinct.pluck(:name).each do |name|
+        list_source_data_sets name
+      end
+    end
+
+    desc 'Delete a data set and all the models from it'
+    task :delete, [:name, :etag] => :environment do |_, args|
+      set = GtfsEngine::DataSet.find_by name: args.name, etag: args.etag
+      name, etag = [args.name.to_s.light_blue, args.etag.yellow]
+
+      if set
+        set.delete
+        puts "Deleted #{name} data set with etag #{etag}"
+      else
+        error = 'Error:'.red
+        puts "#{error} Could not find #{name} data set with etag #{etag}"
+      end
+    end
+
+    desc 'Count the number of models in a data set'
+    task :count, [:name, :etag] => :environment do |_, args|
+      set = GtfsEngine::DataSet.find_by name: args.name, etag: args.etag
+      name, etag = [args.name.to_s.light_blue, args.etag.yellow]
+      puts "Model count for #{name} data set with etag #{etag}"
+
+      width = model_lists.inject(0) {|max, n| [max, n.to_s.length].max }
+      model_lists.each do |list|
+        count = set.send(list).count
+        puts "\t#{list.to_s.rjust(width).yellow}: #{count}"
+      end
+    end
+  end
+end
+
+def list_source_data_sets(name)
+  puts name.underline.green
+
+  sets = GtfsEngine::DataSet.where(name: name).order :created_at
+  if sets.count == 0
+    puts "\tnone".light_black
+  else
+    sets.each do |s|
+      puts "\t#{s.etag.light_blue}: #{s.created_at.to_s.cyan} (#{s.url.yellow})"
+    end
+  end
+end
+
+def model_lists
+  @model_lists ||= begin
+    has_many = GtfsEngine::DataSet.reflect_on_all_associations(:has_many)
+    has_many.collect &:name
+  end
+end

metadata

@@ -0,0 +1,242 @@
+--- !ruby/object:Gem::Specification
+name: gtfs-engine
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jon Sangster
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: gtfs-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pry-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: 'Reads and parses zip files conforming to Google''s GTFS spec. GTFS Spec:
+  https://developers.google.com/transit/gtfs'
+email: jon@ertt.ca
+executables:
+- rails
+extensions: []
+extra_rdoc_files: []
+files:
+- Rakefile
+- app/controllers/gtfs_engine/agencies_controller.rb
+- app/controllers/gtfs_engine/application_controller.rb
+- app/controllers/gtfs_engine/calendar_dates_controller.rb
+- app/controllers/gtfs_engine/calendars_controller.rb
+- app/controllers/gtfs_engine/data_sets_controller.rb
+- app/controllers/gtfs_engine/fare_attributes_controller.rb
+- app/controllers/gtfs_engine/fare_rules_controller.rb
+- app/controllers/gtfs_engine/feed_infos_controller.rb
+- app/controllers/gtfs_engine/frequencies_controller.rb
+- app/controllers/gtfs_engine/routes_controller.rb
+- app/controllers/gtfs_engine/shapes_controller.rb
+- app/controllers/gtfs_engine/stop_times_controller.rb
+- app/controllers/gtfs_engine/stops_controller.rb
+- app/controllers/gtfs_engine/transfers_controller.rb
+- app/controllers/gtfs_engine/trips_controller.rb
+- app/helpers/gtfs_engine/default_views_helper.rb
+- app/helpers/gtfs_engine/fields_helper.rb
+- app/models/gtfs_engine/agency.rb
+- app/models/gtfs_engine/calendar.rb
+- app/models/gtfs_engine/calendar_date.rb
+- app/models/gtfs_engine/data_set.rb
+- app/models/gtfs_engine/fare_attribute.rb
+- app/models/gtfs_engine/fare_rule.rb
+- app/models/gtfs_engine/feed_info.rb
+- app/models/gtfs_engine/frequency.rb
+- app/models/gtfs_engine/route.rb
+- app/models/gtfs_engine/shape.rb
+- app/models/gtfs_engine/stop.rb
+- app/models/gtfs_engine/stop_time.rb
+- app/models/gtfs_engine/transfer.rb
+- app/models/gtfs_engine/trip.rb
+- app/views/gtfs_engine/calendars/dates.json.jbuilder
+- app/views/gtfs_engine/data_sets/index.json.jbuilder
+- app/views/gtfs_engine/data_sets/show.json.jbuilder
+- app/views/gtfs_engine/gtfs/index.json.jbuilder
+- app/views/gtfs_engine/gtfs/show.json.jbuilder
+- app/views/gtfs_engine/transfers/from.json.jbuilder
+- app/views/gtfs_engine/transfers/from_to.json.jbuilder
+- app/views/gtfs_engine/transfers/to.json.jbuilder
+- app/views/gtfs_engine/trips/block.json.jbuilder
+- bin/rails
+- config/initializers/extensions_loader.rb
+- config/routes.rb
+- db/migrate/20140320045108_create_gtfs_engine_data_sets.rb
+- db/migrate/20140320045232_create_gtfs_engine_calendars.rb
+- db/migrate/20140320045751_create_gtfs_engine_calendar_dates.rb
+- db/migrate/20140320050100_create_gtfs_engine_shapes.rb
+- db/migrate/20140320051140_create_gtfs_engine_routes.rb
+- db/migrate/20140320052005_create_gtfs_engine_stops.rb
+- db/migrate/20140320052508_create_gtfs_engine_trips.rb
+- db/migrate/20140320052907_create_gtfs_engine_stop_times.rb
+- db/migrate/20140401032609_create_gtfs_engine_agencies.rb
+- db/migrate/20140405235947_create_gtfs_engine_fare_attributes.rb
+- db/migrate/20140406063500_create_gtfs_engine_fare_rules.rb
+- db/migrate/20140406071922_create_gtfs_engine_frequencies.rb
+- db/migrate/20140406072309_create_gtfs_engine_transfers.rb
+- db/migrate/20140406073548_create_gtfs_engine_feed_infos.rb
+- lib/ext.rb
+- lib/ext/active_record/associations/association.rb
+- lib/gtfs_engine.rb
+- lib/gtfs_engine/concerns.rb
+- lib/gtfs_engine/concerns/controllers.rb
+- lib/gtfs_engine/concerns/controllers/data_access.rb
+- lib/gtfs_engine/concerns/controllers/gtfs.rb
+- lib/gtfs_engine/engine.rb
+- lib/gtfs_engine/exceptions.rb
+- lib/gtfs_engine/json_responder.rb
+- lib/gtfs_engine/sources.rb
+- lib/gtfs_engine/version.rb
+- lib/tasks/gtfs_engine_tasks.rake
+homepage: http://github.com/sangster/gtfs-engine
+licenses:
+- GPL 3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Read General Transit Feed Specification zip files
+test_files: []