southeastern-daily-performance 0.0.3 → 0.0.5

data/README.md

@@ -1,3 +1,9 @@
+## Intro
+
+Southeastern publish [Daily Performance reports](http://www.southeasternrailway.co.uk/your-journey/daily-performance/).  They're not very usable as they are.  This project currently contains the code I've used to parse that data and the parsed data itself.
+
+I've also popped the data into [this Google Fusion table](http://www.google.com/fusiontables/DataSource?dsrcid=359310) and added a bit of a [query tool](http://chrisroos.github.com/southeastern-daily-performance/) and [dashboard](http://chrisroos.github.com/southeastern-daily-performance/dashboard.html) (of sorts).
+
 ## Installation
 
     $ gem install southeastern-daily-performance -r http://gemcutter.org
@@ -13,7 +19,26 @@
     $ curl "http://www.southeasternrailway.co.uk/index.php/cms/pages/view/132" > sedpr.html
     $ sedpr-to-csv sedpr.html
 
-
 ### Implicitly download html and convert
 
-    $ sedpr-to-csv http://www.southeasternrailway.co.uk/index.php/cms/pages/view/132
+    $ sedpr-to-csv http://www.southeasternrailway.co.uk/index.php/cms/pages/view/132
+    
+## Notes
+
+### Combining all csv files into one big file
+
+    $ echo "Date,Problem,Scheduled departure time,Scheduled departure station,Scheduled arrival station,Affect on service" > combined.csv
+    $ cat /path/to/csv/files/*.csv >> combined.csv
+    
+### Combining all csv overview files into one file
+
+    $ echo "Date,Services scheduled,Services run,Services within 5 minutes of schedule" > combined.overview.csv
+    $ cat /path/to/csv/files/*.overview.csv >> combined.overview.csv
+
+## TODO
+
+* 2010-04-21 breaks the parser...
+* Ensure that I'm generating valid CSV (specifically that I'm quoting columns that contain commas)
+* I currently have an empty file for 2010-11-30.csv.  Investigate the cause.
+* Republish the gem
+* I'm getting 0s for the parsed overview from 2010-11-30.  Investigate the cause.

data/Rakefile

@@ -7,19 +7,6 @@ Rake::TestTask.new do |t|
   t.verbose = true
 end
 
-require File.join(File.dirname(__FILE__), 'lib', 'sedpr')
-task 'convert' do
-  if data_dir = ENV['DATA_DIR']
-    Dir[File.join(data_dir, '*.html')].each do |html_file|
-      html = File.read(html_file)
-      puts DailyPerformanceReport.new(html).to_csv
-    end
-  else
-    puts "Usage: DATA_DIR=/path/to/html-reports rake convert"
-    exit 1
-  end
-end
-
 require "rubygems"
 require "rake/gempackagetask"
 require "rake/rdoctask"
@@ -33,7 +20,7 @@ spec = Gem::Specification.new do |s|
 
   # Change these as appropriate
   s.name              = "southeastern-daily-performance"
-  s.version           = "0.0.3"
+  s.version           = "0.0.5"
   s.summary           = "Converts Southeaster Daily Performance reports from HTML to CSV"
   s.author            = "Chris Roos"
   s.email             = "chris@seagul.co.uk"
@@ -53,7 +40,7 @@ spec = Gem::Specification.new do |s|
   s.add_dependency('hpricot')
 
   # If your tests use any gems, include them here
-  # s.add_development_dependency("mocha") # for example
+  s.add_development_dependency("mocha") # for example
 
   # If you want to publish automatically to rubyforge, you'll may need
   # to tweak this, and the publishing task below too.
@@ -82,47 +69,4 @@ end
 desc 'Clear out RDoc and generated packages'
 task :clean => [:clobber_rdoc, :clobber_package] do
   rm "#{spec.name}.gemspec"
-end
-
-# If you want to publish to RubyForge automatically, here's a simple
-# task to help do that. If you don't, just get rid of this.
-# Be sure to set up your Rubyforge account details with the Rubyforge
-# gem; you'll need to run `rubyforge setup` and `rubyforge config` at
-# the very least.
-begin
-  require "rake/contrib/sshpublisher"
-  namespace :rubyforge do
-
-    desc "Release gem and RDoc documentation to RubyForge"
-    task :release => ["rubyforge:release:gem", "rubyforge:release:docs"]
-
-    namespace :release do
-      desc "Release a new version of this gem"
-      task :gem => [:package] do
-        require 'rubyforge'
-        rubyforge = RubyForge.new
-        rubyforge.configure
-        rubyforge.login
-        rubyforge.userconfig['release_notes'] = spec.summary
-        path_to_gem = File.join(File.dirname(__FILE__), "pkg", "#{spec.name}-#{spec.version}.gem")
-        puts "Publishing #{spec.name}-#{spec.version.to_s} to Rubyforge..."
-        rubyforge.add_release(spec.rubyforge_project, spec.name, spec.version.to_s, path_to_gem)
-      end
-
-      desc "Publish RDoc to RubyForge."
-      task :docs => [:rdoc] do
-        config = YAML.load(
-            File.read(File.expand_path('~/.rubyforge/user-config.yml'))
-        )
-
-        host = "#{config['username']}@rubyforge.org"
-        remote_dir = "/var/www/gforge-projects/southeastern-daily-performance/" # Should be the same as the rubyforge project name
-        local_dir = 'rdoc'
-
-        Rake::SshDirPublisher.new(host, remote_dir, local_dir).upload
-      end
-    end
-  end
-rescue LoadError
-  puts "Rake SshDirPublisher is unavailable or your rubyforge environment is not configured."
-end
+end

data/bin/convert-all-html-data

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+FORMATS = ['full', 'overview']
+
+def print_usage_instruction_and_exit
+  puts "Usage: #{File.basename(__FILE__)} [#{FORMATS.join('|')}] html-input-directory csv-output-directory"
+  exit 1
+end
+
+format, html_input_dir, csv_output_dir = ARGV
+print_usage_instruction_and_exit unless format         && FORMATS.include?(format)
+print_usage_instruction_and_exit unless html_input_dir && File.directory?(html_input_dir)
+print_usage_instruction_and_exit unless csv_output_dir && File.directory?(csv_output_dir)
+  
+# ROOT          = File.expand_path('../../', __FILE__)
+# LIB_DIR       = File.join(ROOT, 'lib')
+# CONVERTOR     = File.join(ROOT, 'bin', 'sedpr-to-csv')
+OPTIONAL_ARGS = (format == 'overview') ? 'overview' : ''
+
+Dir[File.join(html_input_dir, '*.html')].each do |file|
+  p file
+  
+  html_filename = File.basename(file)
+  csv_filename  = html_filename.sub(/\.html/, '.csv')
+  csv_filename  = csv_filename.sub(/\.csv/, '.overview.csv') if format == 'overview'
+  
+  # cmd = "ruby -I#{LIB_DIR} #{CONVERTOR} #{file} #{OPTIONAL_ARGS} > #{File.join(csv_output_dir, csv_filename)}"
+  cmd = "sedpr-to-csv #{file} #{OPTIONAL_ARGS} > #{File.join(csv_output_dir, csv_filename)}"
+  `#{cmd}`
+end

data/bin/sedpr-to-csv

@@ -1,12 +1,18 @@
 #! /usr/bin/env ruby
 
 require 'open-uri'
-require File.join(File.dirname(__FILE__), '..', 'lib', 'sedpr')
+require 'southeastern_daily_performance'
 
 unless html_location = ARGV[0]
-  puts "Usage: sedpr-to-csv location-of-html # file or uri"
+  msg = []
+  msg << "Usage: sedpr-to-csv location-of-html [overview]"
+  msg << " * location-of-html - Can be a file or url."
+  msg << " * overview         - Optional. Generate the overview data."
+  puts msg.join("\n")
   exit 1
 end
 
+format = ARGV[1]
+
 io = open(html_location)
-puts SoutheasternDailyPerformance::DailyPerformanceReport.new(io.read).to_csv
+puts SoutheasternDailyPerformance::DailyPerformanceReport.new(io.read).to_csv(format)

data/lib/southeastern_daily_performance.rb

@@ -0,0 +1,3 @@
+require 'southeastern_daily_performance/affected_service'
+require 'southeastern_daily_performance/affected_services_report'
+require 'southeastern_daily_performance/daily_performance_report'

data/lib/{sedpr → southeastern_daily_performance}/affected_service.rb

@@ -7,12 +7,13 @@ module SoutheasternDailyPerformance
   
     def initialize(reason_for_disruption, incident_text)
       @reason_for_disruption = reason_for_disruption
+      incident_text.gsub!(/–/, '-') # Replace ndash with normal dash
       if incident_text =~ /(\d\d:\d\d) (.*?) ?(?:-|to|\?) (.*)/
         @scheduled_start_time, @scheduled_start_station = $1, $2
         destination_and_effect_on_service = $3
         @scheduled_start_station.gsub!(/[^a-zA-Z ]/, '')
         reasons = [
-          'cancelled', 'started', 'delayed by', 'did not call', 'terminated at', 'diverted'
+          'cancelled', 'started', 'delayed by', 'did not call', 'terminated( at)?', 'diverted', 'ran fast', 'called'
         ]
         matches = reasons.collect do |reason|
           destination_and_effect_on_service =~ /#{reason}/i

data/lib/{sedpr → southeastern_daily_performance}/daily_performance_report.rb

@@ -22,7 +22,7 @@ module SoutheasternDailyPerformance
     end
   
     def scheduled_services
-      report[/(\d+) train services were scheduled/, 1].to_i
+      report[/(\d+) (train )?services were scheduled/, 1].to_i
     end
   
     def actual_services
@@ -30,17 +30,21 @@ module SoutheasternDailyPerformance
     end
   
     def services_within_five_minutes_of_schedule
-      report[/(\d+)% of services ran within 5 minutes of schedule/, 1].to_i
+      report[/(\d+\.?\d+?)% of services ran within 5 minutes of schedule/, 1].to_f
     end
   
     def affected_services
       AffectedServicesReport.new(report_container.inner_html).affected_services
     end
   
-    def to_csv
-      affected_services.collect do |service|
-        CSV.generate_line [date, service.reason_for_disruption, service.scheduled_start_time, service.scheduled_start_station, service.scheduled_destination_station, service.effect_on_service]
-      end.join("\n")
+    def to_csv(type=nil)
+      if "#{type}" == 'overview'
+        CSV.generate_line [date, scheduled_services, actual_services, services_within_five_minutes_of_schedule]
+      else
+        affected_services.collect do |service|
+          CSV.generate_line [date, service.reason_for_disruption, service.scheduled_start_time, service.scheduled_start_station, service.scheduled_destination_station, service.effect_on_service]
+        end.join("\n")
+      end
     end
   
     private
@@ -50,7 +54,11 @@ module SoutheasternDailyPerformance
       end
     
       def report_container
-        (@doc/'h1').first.parent.next_sibling
+        if (@doc/'#mainblock').any?
+          (@doc/'h1').first.next_sibling
+        else
+          (@doc/'h1').first.parent.next_sibling
+        end
       end
     
   end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: southeastern-daily-performance
 version: !ruby/object:Gem::Version 
+  hash: 21
   prerelease: false
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Chris Roos
@@ -14,24 +15,41 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-09 00:00:00 +01:00
+date: 2010-12-22 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: hpricot
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
 description: 
 email: chris@seagul.co.uk
 executables: 
+- convert-all-html-data
 - sedpr-to-csv
 extensions: []
 
@@ -40,10 +58,12 @@ extra_rdoc_files:
 files: 
 - README.md
 - Rakefile
-- lib/sedpr/affected_service.rb
-- lib/sedpr/affected_services_report.rb
-- lib/sedpr/daily_performance_report.rb
-- lib/sedpr.rb
+- lib/southeastern_daily_performance/affected_service.rb
+- lib/southeastern_daily_performance/affected_services_report.rb
+- lib/southeastern_daily_performance/daily_performance_report.rb
+- lib/southeastern_daily_performance.rb
+- bin/convert-all-html-data
+- bin/sedpr-to-csv
 has_rdoc: true
 homepage: http://github.com/chrisroos/southeastern-daily-performance
 licenses: []
@@ -55,23 +75,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: southeastern-daily-performance
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Converts Southeaster Daily Performance reports from HTML to CSV

data/lib/sedpr.rb

@@ -1,3 +0,0 @@
-require File.expand_path '../sedpr/affected_service',         __FILE__
-require File.expand_path '../sedpr/affected_services_report', __FILE__
-require File.expand_path '../sedpr/daily_performance_report', __FILE__