translink 0.0.1 → 1.0.0

data/lib/translink/cli.rb

@@ -1,12 +1,11 @@
 module Translink
   class CLI
-    RUNNABLE = ['extract', 'help', 'scrape']
+    RUNNABLE = ['help', 'scrape']
 
-    attr_accessor :out, :pwd, :__crawler__, :__stop__
+    attr_accessor :out, :pwd, :__crawler__
 
     def initialize pwd
       self.__crawler__ = Translink::Crawler
-      self.__stop__    = Model::Stop
       self.out         = $stdout
       self.pwd         = pwd
     end
@@ -22,26 +21,21 @@ module Translink
 
   protected
 
-    def extract input
-      return help nil unless input =~ /[A-Za-z]:\/\/.+/
-      DB.new input do
-        __stop__.all.each do |stop|
-          stop.extract!
-          stop.save!
-        end
-      end
-    end
-
     def help input
-      log 'help'
+      tomorrow = Date.today + 1
+      log 'Usage: translink scrape <DATE> [URI]'
+      log ''
+      log 'Examples:'
+      log "    translink scrape #{tomorrow}"
+      log "    translink scrape #{tomorrow} sqlite://~/Desktop/#{tomorrow}.sqlite3"
     end
 
     def scrape input
       return help nil unless input =~ /^(\d{4}-\d{2}-\d{2})(\s+--uri="?(.+)"?)?$/
       date = Date.parse $1
       uri  = $3 || 'sqlite://' + File.join(pwd, "#{date}.sqlite3")
-      DB.new uri do
-        crawler = __crawler__.new 'http://jp.translink.com.au/travel-information/services-and-timetables/buses/all-bus-timetables'
+      DB.context uri, :migrate => true do
+        crawler = __crawler__.new 'http://jp.translink.com.au/travel-information/network-information/buses/all-timetables'
         crawler.crawl date
       end
     end

data/lib/translink/crawler.rb

@@ -1,19 +1,18 @@
 module Translink
   class Crawler
-    attr_accessor :__model__
-    attr_reader   :url
+    attr_reader :url
 
     def initialize url
-      @__model__ = Model::Route
-      @url       = URI.parse url
+      @url = URI.parse url
     end
 
     def crawl date
       timetable_page = Page::Timetable.new(url.to_s).timetable_page date
       timetable_page.route_pages.each do |route_page|
-        model = __model__.find_or_add_from_route_page route_page
+        route_model = Model::Route.find_or_add_route_from_route_page route_page
         route_page.trip_pages.each do |trip_page|
-          model.add_service_from_trip_page trip_page
+          trip_model = route_model.add_trip_from_trip_page trip_page
+          trip_model.add_stop_times_from_stop_time_pages trip_page.stop_times
         end
       end
     end

data/lib/translink/db.rb

@@ -1,20 +1,12 @@
 module Translink
-  class DB
-    attr_reader :name, :uri
-    
-    def initialize uri, &block
-      @uri  = uri
-      @name = :default
-      DataMapper.setup name, uri
-      DataMapper.repository name do
+  module DB
+    def self.context uri, options = {}
+      DataMapper.setup :default, uri
+      DataMapper.repository :default do
         DataMapper.finalize
-        DataMapper.auto_migrate!
+        DataMapper.auto_migrate! if options[:migrate]
+        yield if block_given?
       end
-      use &block if block
-    end
-    
-    def use &block
-      DataMapper.repository(name) { block.call }
     end
   end
 end

data/lib/translink/model/route.rb

@@ -2,26 +2,37 @@ module Translink
   module Model
     class Route
       include DataMapper::Resource
-      
-      property :id,           Serial
-      property :code,         String
-      property :name,         String
-      property :translink_id, Integer
-      
-      has n, :services
-      has n, :stops, :through => :services
-      
-      def add_service_from_trip_page trip_page
-        trip_page.trips.each do |trip|
-          services << Service.build_from_trip(trip)
-          services.last.save
-        end
+
+      storage_names[:default] = 'routes'
+
+      property :id,         Serial
+      property :short_name, String  # Route code. Eg "130".
+      property :long_name,  String  # Suburbs serviced or destination. Eg "City, Sunnybank, Algester". 
+      property :route_type, Integer # Type of transporation. Eg "Bus".
+
+      has n, :trips
+
+      # Route model for the given +route_page+. Will create the route if it
+      # doesn't exist.
+      #
+      # @param route_pate [Page::Route] HTML page that represents the route.
+      # @return [Model::Route] DataMapper record.
+      def self.find_or_add_route_from_route_page route_page
+        first_or_create :short_name => route_page.short_name,
+                        :long_name  => route_page.long_name,
+                        :route_type => route_page.route_type
       end
 
-      def self.find_or_add_from_route_page route_page
-        first_or_create :code         => route_page.code,
-                        :name         => route_page.name,
-                        :translink_id => route_page.translink_id
+      # Create a trip.
+      #
+      # @param trip_page [Page::Trip] HTML page that represents the trip.
+      # @return [Model::Trip] DataMapper record.
+      def add_trip_from_trip_page trip_page
+        Trip.new.tap do |trip|
+          trip.route = self
+          trip.trip_page! trip_page
+          trip.save
+        end
       end
     end
   end

data/lib/translink/model/stop.rb

@@ -3,28 +3,26 @@ module Translink
     class Stop
       include DataMapper::Resource
 
-      property :id,       Serial
-      property :name,     String
-      property :summary,  String
-      property :street1,  String
-      property :street2,  String
-      property :locality, String
+      storage_names[:default] = 'stops'
 
-      has n, :services
-      has n, :routes, :through => :services
+      property :id,        Serial
+      property :stop_id,   String
+      property :stop_name, String
+      property :stop_lat,  Float
+      property :stop_lon,  Float
 
-      attr_accessor :__extractor__
+      has n, :stop_times
 
-      def self.find_or_add_from_stop stop
-        Stop.first_or_create :name => stop.name, :summary => stop.summary
-      end
-
-      def extract!
-        __extractor__.new(self).extract!
-      end
-
-      def __extractor__
-        @__extractor__ ||= Extractor
+      # Stop model for the given +stop_page+. Will create the route if it
+      # doesn't exist.
+      #
+      # @param route_pate [Page::Route] HTML page representing the stop.
+      # @return [Model::Stop] DataMapper record.
+      def self.find_or_add_from_stop_page stop_page
+        first_or_create :stop_id   => stop_page.stop_id,
+                        :stop_name => stop_page.stop_name,
+                        :stop_lat  => stop_page.stop_lat,
+                        :stop_lon  => stop_page.stop_lon
       end
     end
   end

data/lib/translink/model/stop_time.rb

@@ -0,0 +1,26 @@
+module Translink
+  module Model
+    class StopTime
+      include DataMapper::Resource
+
+      storage_names[:default] = 'stop_times'
+
+      property :id,            Serial
+      property :arrival_time,  String
+      property :stop_sequence, Integer
+
+      belongs_to :stop
+      belongs_to :trip
+
+      # Sets attributes from the +stop_time_page+.
+      #
+      # @param stop_time_page [Page::Stop::StopTime]
+      # @return [void]
+      def stop_time_page! stop_time_page
+        self.arrival_time  = stop_time_page.arrival_time
+        self.stop          = Stop.find_or_add_from_stop_page stop_time_page.stop_page
+        self.stop_sequence = stop_time_page.stop_sequence
+      end
+    end
+  end
+end

data/lib/translink/model/trip.rb

@@ -0,0 +1,48 @@
+module Translink
+  module Model
+    class Trip
+      include DataMapper::Resource
+
+      storage_names[:default] = 'trips'
+
+      property :id,         Serial
+      property :direction,  String
+      property :service_id, Integer # Service belongs to a trip. Assigned by Translink.
+      property :trip_id,    Integer # Unique ID assigned by Translink.
+
+      belongs_to :route
+
+      has n, :stop_times
+
+      # Creates a +Model::StopTime+ record and associates it with this
+      # trip.
+      #
+      # @param stop_time_page [Page::Stop::StopTime] HTML page representing the
+      #   stop-time.
+      # @return [void]
+      def add_stop_time_from_stop_time_page stop_time_page
+        StopTime.new.tap do |stop_time|
+          stop_time.trip = self
+          stop_time.stop_time_page! stop_time_page
+          stop_time.save
+        end
+      end
+
+      def add_stop_times_from_stop_time_pages stop_time_pages
+        stop_time_pages.map do |stop_time_page|
+          add_stop_time_from_stop_time_page stop_time_page
+        end
+      end
+
+      # Sets properties from the given +trip_page+. 
+      #
+      # @param trip_page [Trip::Page] HTML page that represents the trip.
+      # @return [void]
+      def trip_page! trip_page
+       self.direction  = trip_page.direction
+       self.service_id = trip_page.service_id
+       self.trip_id    = trip_page.trip_id
+      end
+    end
+  end
+end

data/lib/translink/page/route.rb

@@ -1,29 +1,53 @@
 module Translink
   class Page::Route < Page
-    def code
-      page.search('div#contentleftCol table th:nth-child(2)').first.text.strip
+    class UnknownRouteTypeError < StandardError
     end
-    
-    def date
-      Date.parse page.search('div#contentleftCol div.content p span').text
-    end
-          
-    def direction
-      page.search('div#contentleftCol div.content p').text.match(/(\S+)$/).captures.first
+
+    ROUTE_TYPES = {'buses' => 3, 'ferries' => 4, 'trains' => 0} # Maps to Google Transit route type.
+
+    attr_reader :long_name # [String] Usually a list of suburbs.
+
+    # Creates a new route.
+    #
+    # @param url [String] URL to fetch the page from.
+    # @param name [String] Route's long name.
+    def initialize url, long_name
+      super url
+      @long_name = long_name
     end
-    
-    def name
-      page.search('div#headingBar h1').text
+
+    # Gets the route's code.
+    #
+    # @return [String]
+    def short_name
+      page.search('div#headingBar h1').first.text.sub('Route ', '')
     end
-    
-    def translink_id
-      url.path.match(/([^\/]+)$/).captures.last
+
+    # Get the date this route is running. Trip pages are bound by this
+    # date.
+    #
+    # @return [DateTime]
+    def date
+      DateTime.parse page.search('select#TimetableDate option[selected]').first['value']
     end
-    
+
+    # Builds an array of trip pages.
+    #
+    # @return [Array<Page::Trip>]
     def trip_pages
-      page.search('table:not(:last-child) tfoot a').map do |anchor|
-        Trip.new url_from_href(anchor[:href])
+      page.search('a.map-link-top').map do |anchor|
+        Trip.new url_from_href(anchor[:href]), date
       end
     end
+
+    # Get the type of transportation used on the route.
+    #
+    # @return [Integer]
+    # @raise [UnknownRouteTypeError] if the route type is not defined in 
+    #   +ROUTE_TYPES+.
+    def route_type
+      url.to_s =~ /(buses|ferries|trains)/
+      ROUTE_TYPES.fetch($1) { raise UnknownRouteTypeError }
+    end
   end
 end

data/lib/translink/page/timetable.rb

@@ -1,31 +1,28 @@
 module Translink
   class Page::Timetable < Page
-    attr_accessor :code_class
-
-    def code_class
-      @code_class ||= Code
-    end
-
+    # Builds an array of route pages.
+    #
+    # @return [Array<Page::Route>]
     def route_pages
-      page.search('table tr td:first-child a').reduce Array.new do |pages, anchor|
-        route = Route.new url_from_href anchor['href']
-        pages << route if code_class.brisbane? extract_code_from_anchor(anchor)
+      page.search('table tr td:last-child a').reduce Array.new do |pages, anchor|
+        route = Route.new url_from_href(anchor['href']), anchor.text
+        pages << route
         pages
       end
     end
 
-    def timetable_page date
-      form = page.forms[1]
-      form.field_with(:name => 'TimetableDate').value = date.to_s
+    # Returns a timetable page with routes running on the given date.
+    #
+    # @param timestamp [DateTime] Filter by this date.
+    #   TZ should be Australia/Brisbane and the time should be at midnight.
+    # @return [Timetable]
+    def timetable_page timestamp
+      form  = page.forms[1]
+      value = timestamp.strftime('%-d/%m/%y %I:%M:%S %p') # Eg: "4/06/2012 12:00:00 AM"
+      form.field_with(:name => 'Date').value = value
       self.class.new(url_from_href(form.action)).tap do |page|
         page.page = form.submit
       end
     end
-
-  protected
-
-    def extract_code_from_anchor anchor
-      anchor.text.gsub(/\-\s(Inbound|Outbound)/, '').strip
-    end
   end
 end

data/lib/translink/page/trip.rb

@@ -1,35 +1,108 @@
 module Translink
   class Page::Trip < Page
-    Trip = Struct.new :stop, :time
-    Stop = Struct.new :name, :summary
+    class Stop
+      attr_accessor :stop_id   # [String] Unique ID.
+      attr_accessor :stop_name # [String] Eg: "Queen Street station, platform A6".
+      attr_accessor :stop_lat  # [String] Eg: "27.470677".
+      attr_accessor :stop_lon  # [String] Eg: "153.024747".
 
-    def date
-      Date.parse page.search('div#contentleftCol p span').text
+      # Tests equality with +other+. Considered equal if +stop_id+ and 
+      # +stop_name+ are equal.
+      #
+      # @param other [Page::Trip::Stop] Stop to compare.
+      # @return [TrueClass, FalseClass] 
+      def == other
+        stop_id == other.stop_id &&
+        stop_name == other.stop_name
+      end
+
+      # Sets attributes by extracting attributes from the HTML fragment.
+      #
+      # @param node_set [Nokogiri::XML::NodeSet] The HTML fragment to search.
+      # @return [Page::Trip::Stop] 
+      def html! node_set
+        anchor = node_set.search('td a').first
+        anchor['href'] =~ /([^\/]+)$/
+        @stop_id = $1
+        @stop_name = anchor.text
+        @stop_lat, @stop_lon = node_set['data-position'].split(',')
+        self
+      end
     end
 
-    def stops
-      table_rows.search('td:first-child').map do |td|
-        attributes = td.text.strip.split "\n"
-        Stop.new *attributes
+    class StopTime
+      attr_accessor :arrival_time  # [String] The time the vehicle arrives at the +stop+.
+      attr_accessor :stop_page     # [Page::Trip::Stop] Stop associated with the +arrival_time+.
+      attr_accessor :stop_sequence # [Integer] Order in which this stop is visited in the trip.
+
+      # Creates a new stop time.
+      #
+      # @param stop_sequence [Integer] Order in which this stop is visited.
+      def initialize stop_sequence
+        @stop_sequence = stop_sequence
+      end
+      
+      # Time vehicle starts from the +stop+. Translink doesn't provide an
+      # explicit +departure_time+ so we use the +arrival_time+.
+      #
+      # @return [String] Eg: "10:00 A.M."
+      def departure_time
+        arrival_time
+      end
+
+      # Sets attributes by extracting attributes from the HTML fragment.
+      #
+      # @param node_set [Nokogiri::XML::NodeSet] The HTML fragment to search.
+      # @return [Page::Trip::Stop] 
+      def html! node_set
+        @stop_page    = Stop.new.html! node_set
+        @arrival_time = node_set.search('td').first.text.sub('.', ':').sub(/(a|p)(m)$/, ' \1.M.').upcase # "12:25pm" -> "12:25 P.M"
+        self
       end
     end
 
-    def times
-      table_rows.search('td:last-child').map { |td| date_time td.text.strip }
+    attr_accessor :date # [Date] Date the trip runs on.
+
+    # Creates a new trip.
+    # 
+    # @param url [String] URL to fetch the page from.
+    # @param date [Date] Date the trip runs on.
+    def initialize url, date
+      super url
+      @date = date.to_date
     end
 
-    def trips
-      stops.zip(times).map { |attributes| Trip.new *attributes }
+    # Get the trip's direction of travel.
+    #
+    # @return [String] "inbound" or "outbound".
+    def direction
+      url.to_s =~ /information\/([a-z]+)\//
+      $1
     end
 
-  protected
+    # Get the trip's unique ID.
+    #
+    # @return [String]
+    def trip_id
+      url.to_s =~ /information\/[a-z]+\/([^\/]+)/
+      $1
+    end
 
-    def date_time time_string
-      DateTime.parse "#{date} #{time_string.sub('.', ':')} +1000"
+    # Get the trip's service ID.
+    #
+    # @return [String]
+    def service_id
+      url.to_s =~ /information\/[a-z]+\/[^\/]+\/([^\/]+)/
+      $1
     end
 
-    def table_rows
-      @table_rows ||= page.search 'tbody tr'
+    # Builds an array of stop times.
+    #
+    # @return [Array<Page::Trip::StopTime>]
+    def stop_times
+      page.search('table#trip-details tbody tr').each_with_index.map do |table_row, index|
+        StopTime.new(index).html! table_row
+      end
     end
   end
 end

data/lib/translink/page.rb

@@ -1,6 +1,6 @@
 module Translink
   class Page
-    USER_AGENT = "Translink/#{VERSION} Ruby/#{RUBY_VERSION} (https://github.com/tatey/translink)"
+    USER_AGENT = "Mozilla/5.0 (Translink/#{VERSION} Ruby/#{RUBY_VERSION} (https://github.com/tatey/translink))"
     
     attr_accessor :agent, :page, :url
     

data/lib/translink/version.rb

@@ -1,3 +1,3 @@
 module Translink
-  VERSION = '0.0.1'
+  VERSION = '1.0.0'
 end

data/lib/translink.rb

@@ -5,13 +5,12 @@ require 'uri'
 
 require 'translink/version'
 require 'translink/cli'
-require 'translink/code'
 require 'translink/crawler'
 require 'translink/db'
 require 'translink/model/route'
-require 'translink/model/service'
 require 'translink/model/stop'
-require 'translink/model/stop/extractor'
+require 'translink/model/stop_time'
+require 'translink/model/trip'
 require 'translink/page'
 require 'translink/page/route'
 require 'translink/page/timetable'