translink 2.0.0 → 3.0.0

data/lib/translink/cli.rb

@@ -1,6 +1,7 @@
 module Translink
   class CLI
-    RUNNABLE = ['help', 'scrape']
+    RUNNABLE = ['help', 'scrape', 'version']
+    URL      = 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
 
     attr_accessor :out, :pwd, :__crawler__
 
@@ -23,23 +24,48 @@ module Translink
 
     def help input
       tomorrow = Date.today + 1
-      log 'Usage: translink scrape <DATE> [URI]'
+      log 'Usage: translink scrape <DATE> [DB_PATH] [FROM_ROUTE_URL] [STEP]'
+      log '       translink version'
       log ''
       log 'Examples:'
       log "    translink scrape #{tomorrow}"
-      log "    translink scrape #{tomorrow} sqlite://~/Desktop/#{tomorrow}.sqlite3"
+      log "    translink scrape #{tomorrow} ~/Desktop/#{tomorrow}.sqlite3"
+      log "    translink scrape #{tomorrow} ~/Desktop/#{tomorrow}.sqlite3 http://jp.translink.com.au/travel-information/network-information/buses/435"
+      log "    translink scrape #{tomorrow} ~/Desktop/#{tomorrow}.sqlite3 http://jp.translink.com.au/travel-information/network-information/buses/435/#{tomorrow} 0"
     end
 
     def scrape input
-      return help nil unless input =~ /^(\d{4}-\d{2}-\d{2})(\s+--uri="?(.+)"?)?$/
-      date = Date.parse $1
-      uri  = $3 || 'sqlite://' + File.join(pwd, "#{date}.sqlite3")
-      DB.context uri, :migrate => true do
-        crawler = __crawler__.new 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
-        crawler.crawl date
+      args = (input || '').split /\s/
+      case args.size
+      when 1
+        date    = Date.parse args[0]
+        db_path = File.join(pwd, "#{date}.sqlite3")
+      when 2
+        date    = Date.parse args[0]
+        db_path = File.expand_path args[1]
+      when 3
+        date           = Date.parse args[0]
+        db_path        = File.expand_path args[1]
+        from_route_url = URI.parse args[2]
+      when 4
+        date           = Date.parse args[0]
+        db_path        = File.expand_path args[1]
+        from_route_url = URI.parse args[2]
+        step           = args[3].to_i
+      else
+        help nil
+        return
+      end
+      DB.context "sqlite://#{db_path}", :migrate => !File.exists?(db_path) do
+        crawler = __crawler__.new URL
+        crawler.crawl date, from_route_url, step
       end
     end
 
+    def version input
+      log VERSION
+    end
+
     def log message
       out.puts message
     end

data/lib/translink/crawler.rb

@@ -1,20 +1,53 @@
 module Translink
   class Crawler
-    attr_reader :url
+    MAX_RETRY_COUNT = 2 # Maximum number of times to attempt a HTTP request.
+    SLEEP_DURATION  = 5 # Base amount of time to sleep in seconds before retrying.
+
+    attr_accessor :out
+    attr_reader   :url
 
     def initialize url
       @url = URI.parse url
+      @out = $stdout
     end
 
-    def crawl date
+    def crawl date, from_route_url = nil, step = nil
       timetable_page = Page::Timetable.new(url.to_s).timetable_page date
-      timetable_page.route_pages.each do |route_page|
-        route_model = Model::Route.find_or_add_route_from_route_page route_page
-        route_page.trip_pages.each do |trip_page|
-          trip_model = route_model.add_trip_from_trip_page trip_page
-          trip_model.add_stop_times_from_stop_time_pages trip_page.stop_times
-        end
+      timetable_page.route_pages(from_route_url, step).each do |route_page|
+        crawl_route_page route_page
+      end
+    end
+
+    def crawl_route_page route_page, retry_count = 0
+      route_model = Model::Route.find_or_add_route_from_route_page route_page
+      route_page.trip_pages.each do |trip_page|
+        crawl_trip_page route_model, trip_page
+      end
+    rescue Mechanize::ResponseCodeError, Page::UnexpectedParserError => exception
+      if retry_count <= MAX_RETRY_COUNT
+        sleep SLEEP_DURATION * retry_count
+        crawl_route_page route_page, retry_count + 1
+      else
+        out.puts "Skipping route page (#{route_page.url}) because of #{exception}"
+      end
+    rescue => exception
+      out.puts "Skipping route page (#{route_page.url}) because of #{exception}"
+      out.puts exception.backtrace
+    end
+
+    def crawl_trip_page route_model, trip_page, retry_count = 0
+      trip_model = route_model.add_trip_from_trip_page trip_page
+      trip_model.add_stop_times_from_stop_time_pages trip_page.stop_times
+    rescue Mechanize::ResponseCodeError, Page::UnexpectedParserError => exception
+      if retry_count <= MAX_RETRY_COUNT
+        sleep SLEEP_DURATION * retry_count
+        crawl_trip_page route_model, trip_page, retry_count + 1
+      else
+        out.puts "Skipping trip page (#{trip_page.url}) because of #{exception}"
       end
+    rescue => exception
+      out.puts "Skipping trip page (#{trip_page.url}) because of #{exception}"
+      out.puts exception.backtrace
     end
   end
 end

data/lib/translink/db.rb

@@ -2,11 +2,55 @@ module Translink
   module DB
     def self.context uri, options = {}
       DataMapper.setup :default, uri
-      DataMapper.repository :default do
+      DataMapper.repository :default do |repository|
         DataMapper.finalize
-        DataMapper.auto_migrate! if options[:migrate]
+        repository.adapter.execute <<-SQL
+          PRAGMA foreign_keys=ON;
+        SQL
+        if options[:migrate]
+          repository.adapter.execute <<-SQL
+            DROP TABLE IF EXISTS "routes";
+            CREATE TABLE "routes" (
+              "route_id" TEXT NOT NULL PRIMARY KEY UNIQUE,
+              "short_name" TEXT NOT NULL,
+              "long_name" TEXT NOT NULL,
+              "route_type" INTEGER NOT NULL
+            );
+
+            DROP TABLE IF EXISTS "trips";
+            CREATE TABLE "trips" (
+              "trip_id" INTEGER NOT NULL PRIMARY KEY UNIQUE,
+              "direction" INTEGER NOT NULL,
+              "headsign" TEXT NOT NULL,
+              "route_id" TEXT NOT NULL,
+              FOREIGN KEY ("route_id") REFERENCES "routes" ("route_id") ON DELETE CASCADE
+            );
+
+            CREATE INDEX "index_trips_on_route_id" ON "trips" ("route_id");
+
+            DROP TABLE IF EXISTS "stop_times";
+            CREATE TABLE "stop_times" (
+              "arrival_time" STRING NOT NULL,
+              "stop_sequence" INTEGER NOT NULL,
+              "stop_id" TEXT NOT NULL,
+              "trip_id" INTEGER NOT NULL,
+              PRIMARY KEY ("arrival_time", "stop_id", "trip_id"),
+              FOREIGN KEY ("stop_id") REFERENCES "stops" ("stop_id") ON DELETE RESTRICT,
+              FOREIGN KEY ("trip_id") REFERENCES "trips" ("trip_id") ON DELETE CASCADE
+            );
+
+            DROP TABLE IF EXISTS "stops";
+            CREATE TABLE "stops" (
+              "stop_id" TEXT NOT NULL PRIMARY KEY UNIQUE,
+              "stop_name" TEXT NOT NULL,
+              "stop_lat" REAL NOT NULL,
+              "stop_lon" REAL NOT NULL
+            );
+          SQL
+        end
         yield if block_given?
       end
     end
   end
 end
+

data/lib/translink/model/route.rb

@@ -5,12 +5,19 @@ module Translink
 
       storage_names[:default] = 'routes'
 
-      property :id,         Serial
-      property :short_name, String  # Route code. Eg "130".
-      property :long_name,  String  # Suburbs serviced or destination. Eg "City, Sunnybank, Algester". 
-      property :route_type, Integer # Type of transporation. Eg "Bus".
+      # Primary key. Same as +short_name+ because that's the only unique ID we've got.
+      property :id, String, :field => 'route_id', :key => true, :unique => true, :unique_index => true
 
-      has n, :trips
+      # Route code. Eg "130".
+      property :short_name, String
+
+      # Suburbs serviced or destination. Eg "City, Sunnybank, Algester".
+      property :long_name, String
+
+      # Type of transporation. Eg "Bus".
+      property :route_type, Integer
+
+      has n, :trips, :child_key => [:route_id]
 
       # Route model for the given +route_page+. Will create the route if it
       # doesn't exist.
@@ -18,7 +25,8 @@ module Translink
       # @param route_pate [Page::Route] HTML page that represents the route.
       # @return [Model::Route] DataMapper record.
       def self.find_or_add_route_from_route_page route_page
-        first_or_create :short_name => route_page.short_name,
+        first_or_create :id         => route_page.route_id,
+                        :short_name => route_page.short_name,
                         :long_name  => route_page.long_name,
                         :route_type => route_page.route_type
       end

data/lib/translink/model/stop.rb

@@ -5,8 +5,7 @@ module Translink
 
       storage_names[:default] = 'stops'
 
-      property :id,        Serial
-      property :stop_id,   String
+      property :id,        String, :field => 'stop_id', :key => true, :unique => true, :unique_index => true
       property :stop_name, String
       property :stop_lat,  Float
       property :stop_lon,  Float
@@ -19,7 +18,7 @@ module Translink
       # @param route_pate [Page::Route] HTML page representing the stop.
       # @return [Model::Stop] DataMapper record.
       def self.find_or_add_from_stop_page stop_page
-        first_or_create :stop_id   => stop_page.stop_id,
+        first_or_create :id        => stop_page.stop_id,
                         :stop_name => stop_page.stop_name,
                         :stop_lat  => stop_page.stop_lat,
                         :stop_lon  => stop_page.stop_lon

data/lib/translink/model/stop_time.rb

@@ -5,9 +5,10 @@ module Translink
 
       storage_names[:default] = 'stop_times'
 
-      property :id,            Serial
-      property :arrival_time,  String
+      property :arrival_time,  String,  :key => true
       property :stop_sequence, Integer
+      property :stop_id,       String,  :key => true
+      property :trip_id,       Integer, :key => true
 
       belongs_to :stop
       belongs_to :trip

data/lib/translink/model/trip.rb

@@ -5,11 +5,14 @@ module Translink
 
       storage_names[:default] = 'trips'
 
-      property :id,         Serial
-      property :direction,  Integer # Travel in one direction (Regular) or the opposite (Goofy) direction.
-      property :headsign,   String  # Name of the direction. Eg "Inbound".
-      property :service_id, Integer # Service belongs to a trip. Assigned by Translink.
-      property :trip_id,    Integer # Unique ID assigned by Translink.
+      # Primary key. Unique ID assigned by Translink.
+      property :id, Serial, :field => 'trip_id'
+
+      # Travel in one direction (Regular) or the opposite (Goofy) direction.
+      property :direction, Integer
+
+      # Name of the direction. Eg "Inbound".
+      property :headsign, String
 
       belongs_to :route
 
@@ -40,10 +43,9 @@ module Translink
       # @param trip_page [Trip::Page] HTML page that represents the trip.
       # @return [void]
       def trip_page! trip_page
-       self.direction  = trip_page.direction
-       self.headsign   = trip_page.headsign
-       self.service_id = trip_page.service_id
-       self.trip_id    = trip_page.trip_id
+        self.id         = trip_page.trip_id
+        self.direction  = trip_page.direction
+        self.headsign   = trip_page.headsign
       end
     end
   end

data/lib/translink/page.rb

@@ -1,20 +1,30 @@
 module Translink
   class Page
+    class UnexpectedParserError < StandardError
+    end
+
     USER_AGENT = "Mozilla/5.0 (Translink/#{VERSION} Ruby/#{RUBY_VERSION} (https://github.com/tatey/translink))"
-    
+
     attr_accessor :agent, :page, :url
-    
+
     def initialize url
       @agent = Mechanize.new.tap { |mechanize| mechanize.user_agent = USER_AGENT }
       @url   = URI.parse url
     end
-        
+
     def page
-      @page ||= agent.get url.to_s
+      @page ||= begin
+        page = agent.get url.to_s
+        if page.instance_of? Mechanize::Page
+          page
+        else
+          raise UnexpectedParserError, "Expected instance of Mechanize::Page. Got #{page.class}"
+        end
+      end
     end
-    
+
   protected
-  
+
     def url_from_href href
       url.scheme + '://' + url.host + href
     end

data/lib/translink/page/route.rb

@@ -16,11 +16,28 @@ module Translink
       @long_name = long_name
     end
 
+    # Get the route's unique ID assigned by Translink. This is the same
+    # as the +short_name+.
+    #
+    # @return [String]
+    def route_id
+      @route_id ||= page.search('div#headingBar h1').first.text.sub('Route ', '')
+    end
+
     # Gets the route's code.
     #
     # @return [String]
     def short_name
-      page.search('div#headingBar h1').first.text.sub('Route ', '')
+      case route_id
+      when 'CGLD'
+        'CityGlider'
+      when 'LOOP'
+        'City Loop'
+      when 'SHLP'
+        'Spring Hill City Loop'
+      else
+        route_id
+      end
     end
 
     # Get the date this route is running. Trip pages are bound by this
@@ -47,11 +64,33 @@ module Translink
       headsigns.index anchor.ancestors('div.route-timetable').search('h3').first.text.downcase
     end
 
+    # Get the date of the trip. If the trip does not have a date, the UNIX
+    # epoc is returned.
+    #
+    # Examples:
+    #
+    #   "/travel-information/network-information/service-information/outbound/9792/2173523/2012-09-24"
+    #   ... becomes
+    #   DateTime.new('2012-09-24')
+    #
+    #   "/travel-information/network-information/service-information/outbound/9792/2173523"
+    #   ... becomes
+    #   DateTime.new('1970-01-01')
+    #
+    # @return [DateTime]
+    def date_from_anchor anchor
+      match = anchor[:href].match /\d{4}-\d{2}-\d{2}$/
+      date  = match ? match[0] : '1970-01-01'
+      DateTime.parse date
+    end
+
     # Builds an array of trip pages.
     #
     # @return [Array<Page::Trip>]
     def trip_pages
-      page.search('a.map-link-top').map do |anchor|
+      page.search('a.map-link-top').select do |anchor|
+        date_from_anchor(anchor) == date
+      end.map do |anchor|
         Trip.new url_from_href(anchor[:href]), date, direction_from_anchor(anchor)
       end
     end

data/lib/translink/page/timetable.rb

@@ -1,13 +1,20 @@
 module Translink
   class Page::Timetable < Page
-    # Builds an array of route pages.
+    # Builds an unique array of route pages.
     #
+    # @param url [URI] Omit routes before the route with +url+.
     # @return [Array<Page::Route>]
-    def route_pages
-      page.search('table tr td:last-child a').reduce Array.new do |pages, anchor|
-        route = Route.new url_from_href(anchor['href']), anchor.text
-        pages << route
-        pages
+    def route_pages url = nil, step = nil
+      routes = page.search('table tr td:last-child a').reduce(Array.new) do |routes, anchor|
+        route     = Route.new url_from_href(anchor['href']), anchor.text
+        duplicate = routes.find { |duplicate| duplicate.url == route.url }
+        routes << route unless duplicate
+        routes
+      end
+      if url
+        routes.drop_while { |route| route.url != url }.slice 0..(step || routes.size)
+      else
+        routes
       end
     end
 

data/lib/translink/page/trip.rb

@@ -35,13 +35,6 @@ module Translink
       attr_accessor :stop_page     # [Page::Trip::Stop] Stop associated with the +arrival_time+.
       attr_accessor :stop_sequence # [Integer] Order in which this stop is visited in the trip.
 
-      # Creates a new stop time.
-      #
-      # @param stop_sequence [Integer] Order in which this stop is visited.
-      def initialize stop_sequence
-        @stop_sequence = stop_sequence
-      end
-      
       # Time vehicle starts from the +stop+. Translink doesn't provide an
       # explicit +departure_time+ so we use the +arrival_time+.
       #
@@ -66,7 +59,7 @@ module Translink
                              # or the opposite (Goofy) direction.
 
     # Creates a new trip.
-    # 
+    #
     # @param url [String] URL to fetch the page from.
     # @param date [Date] Date the trip runs on.
     def initialize url, date, direction
@@ -87,24 +80,25 @@ module Translink
     #
     # @return [String]
     def trip_id
-      url.to_s =~ /information\/[a-z]+\/([^\/]+)/
-      $1
-    end
-
-    # Get the trip's service ID.
-    #
-    # @return [String]
-    def service_id
       url.to_s =~ /information\/[a-z]+\/[^\/]+\/([^\/]+)/
       $1
     end
 
-    # Builds an array of stop times.
+    # Builds an unique array of stop times.
     #
     # @return [Array<Page::Trip::StopTime>]
     def stop_times
-      page.search('table#trip-details tbody tr').each_with_index.map do |table_row, index|
-        StopTime.new(index).html! table_row
+      page.search('table#trip-details tbody tr').reduce(Array.new) do |stop_times, table_row|
+        stop_time = StopTime.new.html! table_row
+        duplicate = stop_times.find do |duplicate|
+          duplicate.stop_page.stop_id == stop_time.stop_page.stop_id &&
+          duplicate.arrival_time == stop_time.arrival_time
+        end
+        stop_times << stop_time unless duplicate
+        stop_times
+      end.each_with_index.map do |stop_time, index|
+        stop_time.stop_sequence = index
+        stop_time
       end
     end
   end