rail_feeds 0.0.1

data/lib/rail_feeds/network_rail/schedule/fetcher.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'net/http'
+
+module RailFeeds
+  module NetworkRail
+    module Schedule
+      # A class for fetching the schedule data files.
+      class Fetcher
+        include Logging
+
+        # Initialize a new schedule
+        # @param [RailFeeds::NetworkRail::Credentials] credentials
+        #   The credentials for connecting to the feed.
+        # @param [Logger, nil] logger
+        #   The logger for outputting events, if nil the global logger will be used.
+        def initialize(credentials: Credentials, logger: nil)
+          @credentials = credentials
+          self.logger = logger unless logger.nil?
+        end
+
+        # Download the full schedule.
+        # @param [:json, :cif] format
+        #   The format to download the schedule in.
+        # @param [String] file
+        #   The path to the file to save the .json.gz / .cif.gz download in.
+        def download_all_full(format, file)
+          download 'ALL', 'full', format, file
+        end
+
+        # Download the daily update to the full schedule.
+        # @param [String, #to_s] day
+        #   The day to get the update schedule for ("mon", "tue", "wed", ...).
+        #   Defaults to the current day.
+        # @param [:json, :cif] format
+        #   The format to fetch the schedule in.
+        # @param [String] file
+        #   The path to the file to save the .json.gz / .cif.gz download in.
+        def download_all_update(day, format, file)
+          download 'ALL', day, format, file
+        end
+
+        # Fetch the freight schedule.
+        # @param [String] file
+        #   The path to the file to save the .json.gz download in.
+        def download_freight_full(file)
+          download 'FREIGHT', 'full', :json, file
+        end
+
+        # Fetch the daily update to the freight schedule.
+        # @param [String, #to_s] day
+        #   The day to get the update schedule for ("mon", "tue", "wed", ...).
+        #   Defaults to the current day.
+        # @param [String] file
+        #   The path to the file to save the .json.gz download in.
+        def download_freight_update(day, file)
+          download 'FREIGHT', day, :json, file
+        end
+
+        # Fetch the schedule for a TOC.
+        # @param [String, #to_s, nil] toc
+        #   The TOC to get the schedule for.
+        # @param [String] file
+        #   The path to the file to save the .json.gz download in.
+        def download_toc_full(toc, file)
+          download toc, 'full', :json, file
+        end
+
+        # Fetch the daily update for a TOC.
+        # @param [String, #to_s, nil] toc
+        #   The TOC to get the schedule for.
+        # @param [String, #to_s] day
+        #   The day to get the update schedule for ("mon", "tue", "wed", ...).
+        #   Defaults to the current day.
+        # @param [String] file
+        #   The path to the file to save the .json.gz download in.
+        def download_toc_update(toc, day, file)
+          download toc, day, :json, file
+        end
+
+        # Fetch the full schedule.
+        # @param [:json, :cif] format
+        #   The format to fetch the schedule in.
+        # @yield [file] Once the block has run the temp file will be deleted.
+        #   @yieldparam [Zlib::GzipReader] file The unzippable content of the file.
+        def fetch_all_full(format, &block)
+          fetch 'ALL', 'full', format, &block
+        end
+
+        # Fetch the daily update to the full schedule.
+        # @param [String, #to_s] day
+        #   The day to get the update schedule for ("mon", "tue", "wed", ...).
+        #   Defaults to the current day.
+        # @param [:json, :cif] format
+        #   The format to fetch the schedule in.
+        # @yield [file] Once the block has run the temp file will be deleted.
+        #   @yieldparam [Zlib::GzipReader] file The unzippable content of the file.
+        def fetch_all_update(day, format, &block)
+          fetch 'ALL', day, format, &block
+        end
+
+        # Fetch the freight schedule.
+        # @yield [file] Once the block has run the temp file will be deleted.
+        #   @yieldparam [Zlib::GzipReader] file The unzippable content of the file.
+        def fetch_freight_full(&block)
+          fetch 'FREIGHT', 'full', :json, &block
+        end
+
+        # Fetch the daily update to the freight schedule.
+        # @param [String, #to_s] day
+        #   The day to get the update schedule for ("mon", "tue", "wed", ...).
+        #   Defaults to the current day.
+        # @yield [file] Once the block has run the temp file will be deleted.
+        #   @yieldparam [TempFile] file The unzippable content of the file.
+        def fetch_freight_update(day, &block)
+          fetch 'FREIGHT', day, :json, &block
+        end
+
+        # Fetch the schedule for a TOC.
+        # @param [String, #to_s, nil] toc
+        #   The TOC to get the schedule for.
+        # @yield [file] Once the block has run the temp file will be deleted.
+        #   @yieldparam [Zlib::GzipReader] file The unzippable content of the file.
+        def fetch_toc_full(toc, &block)
+          fetch toc, 'full', :json, &block
+        end
+
+        # Fetch the daily update for a TOC.
+        # @param [String, #to_s, nil] toc
+        #   The TOC to get the schedule for.
+        # @param [String, #to_s] day
+        #   The day to get the update schedule for ("mon", "tue", "wed", ...).
+        #   Defaults to the current day.
+        # @yield [file] Once the block has run the temp file will be deleted.
+        #   @yieldparam [Zlib::GzipReader] file The unzippable content of the file.
+        def fetch_toc_update(toc, day, &block)
+          fetch toc, day, :json, &block
+        end
+
+        private
+
+        # Fetch a schedule.
+        def fetch(toc, day, format, &block)
+          path = path_for toc, day, format
+          client = HTTPClient.new(credentials: @credentials, logger: logger)
+          client.fetch_unzipped(path, &block)
+        end
+
+        # Download a schedule.
+        def download(toc, day, format, file)
+          path = path_for toc, day, format
+          client = HTTPClient.new(credentials: @credentials, logger: logger)
+          client.download(path, file)
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/PerceivedComplexity
+        # Get the path for a schedule
+        def path_for(toc, day, format)
+          toc = "#{toc}_TOC" unless %w[ALL FREIGHT].include?(toc)
+
+          if format.eql?(:cif)
+            unless toc.eql?('ALL')
+              fail ArgumentError, 'CIF format is only available for the all schedule'
+            end
+          else
+            unless format.eql?(:json)
+              fail ArgumentError, 'format must be either :json or :cif'
+            end
+          end
+
+          if day.eql?('full')
+            day = 'toc-full'
+            type = "CIF_#{toc}_FULL_DAILY"
+          else
+            unless %w[mon tue wed thu fri sat sun].include?(day)
+              fail ArgumentError, 'day is invalid'
+            end
+            day = "toc-update-#{day}"
+            type = "CIF_#{toc}_UPDATE_DAILY"
+          end
+
+          path = "ntrod/CifFileAuthenticate?type=#{type}&day=#{day}"
+          format.eql?(:cif) ? "#{path}.CIF.gz" : path
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Metrics/MethodLength
+        # rubocop:enable Metrics/PerceivedComplexity
+      end
+    end
+  end
+end

data/lib/rail_feeds/network_rail/schedule/header/cif.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module RailFeeds
+  module NetworkRail
+    module Schedule
+      module Header
+        # A class to hole the information from the header row of a cif file
+        class CIF
+          # @!attribute [rw] file_identity
+          # @!attribute [rw] extracted_at
+          #   @return [Time] When the BTD extract happened.
+          # @!attribute [rw] current_file_reference
+          #   @return [String] Unique reference for the current file.
+          # @!attribute [rw] previous_file_reference
+          #   @return [String, nil] Unique reference for the previous file
+          #   (the one to apply the update to).
+          # @!attribute [rw] update_indicator
+          #   @return [String] 'F' for a full extract, 'U' for an update extract.
+          # @!attribute [rw] version
+          #   @return [String] The version of the software that generated the CIF file.
+          # @!attribute [rw] start_date
+          #   @return [Date]
+          # @!attribute [rw] end_date
+          #   @return [Date]
+
+          attr_accessor :file_identity, :extracted_at,
+                        :current_file_reference, :previous_file_reference,
+                        :update_indicator, :version, :start_date, :end_date
+
+          def initialize(**attributes)
+            attributes.each do |attribute, value|
+              send "#{attribute}=", value
+            end
+          end
+
+          # rubocop:disable Metrics/AbcSize
+          # Initialize a new header from a CIF file line
+          def self.from_cif(line)
+            fail ArgumentError, "Invalid line:\n#{line}" unless line[0..1].eql?('HD')
+
+            new(
+              file_identity: line[2..21].strip,
+              extracted_at: Time.strptime(line[22..31] + 'UTC', '%d%m%y%H%M%Z'),
+              current_file_reference: line[32..38].strip,
+              previous_file_reference: line[39..45].strip,
+              update_indicator: line[46].strip,
+              version: line[47].strip,
+              start_date: Date.strptime(line[48..53], '%d%m%y'),
+              end_date: Date.strptime(line[54..59], '%d%m%y')
+            )
+          end
+          # rubocop:enable Metrics/AbcSize
+
+          # Test if this is a header for an update file
+          def update?
+            update_indicator.eql?('U')
+          end
+
+          # Test if this is a header for a full file
+          def full?
+            update_indicator.eql?('F')
+          end
+
+          def ==(other)
+            hash == other&.hash
+          end
+
+          def hash
+            current_file_reference&.dup
+          end
+
+          # rubocop:disable Metrics/AbcSize
+          def to_cif
+            format('%-80.80s', [
+              'HD',
+              format('%-20.20s', file_identity),
+              # rubocop:disable Style/FormatStringToken
+              format('%-10.10s', extracted_at&.strftime('%d%m%y%H%M')),
+              # rubocop:enable Style/FormatStringToken
+              format('%-7.7s', current_file_reference),
+              format('%-7.7s', previous_file_reference),
+              format('%-1.1s', update_indicator),
+              format('%-1.1s', version),
+              # rubocop:disable Style/FormatStringToken
+              format('%-6.6s', start_date&.strftime('%d%m%y')),
+              format('%-6.6s', end_date&.strftime('%d%m%y'))
+              # rubocop:enable Style/FormatStringToken
+            ].join) + "\n"
+          end
+          # rubocop:enable Metrics/AbcSize
+
+          def to_s
+            "File #{file_identity.inspect} (version #{version}) " \
+              "at #{extracted_at.strftime('%Y-%m-%d %H:%M')}. " \
+              "#{full? ? 'A full' : 'An update'} extract " \
+              "for #{start_date} to #{end_date}."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rail_feeds/network_rail/schedule/header/json.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module RailFeeds
+  module NetworkRail
+    module Schedule
+      module Header
+        # A class to hole the information from the header row of a json file
+        class JSON
+          # @!attribute [rw] extracted_at
+          #   @return [Time] When the BTD extract happened.
+          # @!attribute [rw] sequence
+          #   @return [Integer] Where this file appears in the sequence of extracts.
+          # (Appears to be days since 2012-06-13)
+          # @!attribute [r] start_date
+          #   @return [Date] Infered from sequence
+
+          attr_accessor :extracted_at, :sequence
+
+          START_DATE = Date.new 2012, 6, 13
+          private_constant :START_DATE
+
+          def initialize(**attributes)
+            attributes.each do |attribute, value|
+              send "#{attribute}=", value
+            end
+          end
+
+          # Initialize a new header from a JSON file line
+          def self.from_json(line)
+            data = ::JSON.parse(line)['JsonTimetableV1']
+            metadata = data['Metadata']
+
+            new(
+              extracted_at: Time.strptime(data['timestamp'].to_s, '%s').utc,
+              sequence: metadata['sequence']
+            )
+          end
+
+          def start_date
+            START_DATE + sequence.to_i
+          end
+
+          def <=>(other)
+            hash <=> other&.hash
+          end
+
+          def hash
+            sequence&.dup
+          end
+
+          # rubocop:disable Metrics/MethodLength
+          def to_json
+            {
+              'JsonTimetableV1' => {
+                'classification' => 'public',
+                'timestamp' => extracted_at.to_i,
+                'owner' => 'Network Rail',
+                'Sender' => {
+                  'organisation': '',
+                  'application' => 'NTROD',
+                  'component' => 'SCHEDULE'
+                },
+                'Metadata' => {
+                  'type' => 'full',
+                  'sequence' => sequence
+                }
+              }
+            }.to_json
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          def to_s
+            "Sequence #{sequence}, proabbly from #{start_date}."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rail_feeds/network_rail/schedule/header.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative 'header/cif'
+require_relative 'header/json'
+
+module RailFeeds
+  module NetworkRail
+    module Schedule
+      module Header # :nodoc:
+        # Initialize a new header from a CIF file line
+        def self.from_cif(line)
+          CIF.from_cif line
+        end
+
+        # Initialize a new header from a JSON file line
+        def self.from_json(line)
+          JSON.from_json line
+        end
+      end
+    end
+  end
+end

data/lib/rail_feeds/network_rail/schedule/parser/cif.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module RailFeeds
+  module NetworkRail
+    module Schedule
+      class Parser
+        # A class for parsing schedule data read from CIF schedule file(s).
+        class CIF < Parser
+          UNDERSTOOD_ROWS = %w[
+            HD TI TA TD AAN AAD AAR BSN BSD BSR BX LO LI LT CR ZZ
+          ].freeze
+
+          # Parse the data on a single CIF line
+          # @param [String] line
+          def parse_line(line)
+            catch :line_parsed do
+              UNDERSTOOD_ROWS.each do |record_type|
+                if line.start_with?(record_type)
+                  send "parse_#{record_type.downcase}_line", line.chomp
+                  throw :line_parsed
+                end
+              end
+
+              if line[0].eql?('/')
+                parse_comment_line line.chomp
+                throw :line_parsed
+              end
+
+              logger.error "Can't understand line: #{line.chomp.inspect}"
+            end
+          end
+
+          private
+
+          # Header record
+          def parse_hd_line(line)
+            header = Header.from_cif(line)
+            logger.info "Starting Parse. #{header}"
+            @on_header&.call self, header
+          end
+
+          # TIPLOC Insert record
+          def parse_ti_line(line)
+            @on_tiploc_create&.call self, Tiploc.from_cif(line)
+          end
+
+          # TIPLOC Amend record
+          def parse_ta_line(line)
+            tiploc = Tiploc.from_cif(line)
+            old_id = tiploc.tiploc
+            tiploc.tiploc = line[2..8].strip
+            @on_tiploc_update&.call self, old_id, tiploc
+          end
+
+          # TIPLOC Delete record
+          def parse_td_line(line)
+            @on_tiploc_delete&.call self, Tiploc.from_cif(line).tiploc
+          end
+
+          # Association New record
+          def parse_aan_line(line)
+            @on_association_create&.call self, Association.from_cif(line)
+          end
+
+          # Association Revise record
+          def parse_aar_line(line)
+            @on_association_update&.call self, Association.from_cif(line)
+          end
+
+          # Association Delete record
+          def parse_aad_line(line)
+            @on_association_delete&.call self, Association.from_cif(line)
+          end
+
+          # Train schedule record - basic schedule - new
+          def parse_bsn_line(line)
+            finish_current_train
+            @current_train = TrainSchedule.new
+            @current_train.update_from_cif line
+            @current_train_action = :create
+          end
+
+          # Train schedule record - basic schedule - delete
+          def parse_bsd_line(line)
+            finish_current_train
+            train = TrainSchedule.new
+            train.update_from_cif line
+            @on_train_schedule_delete&.call self, train
+          end
+
+          # Train schedule record - basic schedule - revise
+          def parse_bsr_line(line)
+            finish_current_train
+            @current_train = TrainSchedule.new
+            @current_train.update_from_cif line
+            @current_train_action = :update
+          end
+
+          # Train schedule record - basic schedule extra details
+          def parse_bx_line(line)
+            @current_train.update_from_cif line
+          end
+
+          # Train schedule record - origin location
+          alias parse_lo_line parse_bx_line
+          # Train schedule record - intermediate location
+          alias parse_li_line parse_bx_line
+          # Train schedule record - change en route
+          alias parse_cr_line parse_bx_line
+          # Train schedule record - terminating location
+          alias parse_lt_line parse_bx_line
+
+          def finish_current_train
+            return if @current_train.nil?
+
+            case @current_train_action
+            when :create
+              @on_train_schedule_create&.call self, @current_train
+            when :update
+              @on_train_schedule_update&.call self, @current_train
+            end
+
+            @current_train = nil
+          end
+
+          # Trailer record
+          def parse_zz_line(_line)
+            finish_current_train
+            @file_ended = true
+            @on_trailer&.call self
+          end
+
+          # Comment
+          def parse_comment_line(line)
+            @on_comment&.call self, line[1..-1]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rail_feeds/network_rail/schedule/parser/json.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RailFeeds
+  module NetworkRail
+    module Schedule
+      class Parser
+        # A class for parsing schedule data read from JSON schedule file(s).
+        class JSON < Parser
+          def parse_line(line)
+            if line.start_with? '{"TiplocV1":'
+              parse_tiploc_line line
+            elsif line.start_with? '{"JsonAssociationV1":'
+              parse_association_line line
+            elsif line.start_with? '{"JsonScheduleV1":'
+              parse_schedule_line line
+            elsif line.start_with? '{"JsonTimetableV1":'
+              parse_header_line line
+            elsif line.start_with? '{"EOF":'
+              parse_trailer_line line
+            else
+              logger.error "Can't understand line: #{line.chomp}"
+            end
+          end
+
+          private
+
+          def parse_header_line(line)
+            header = Header.from_json(line)
+            logger.info "Starting Parse. #{header}"
+            @on_header&.call self, header
+          end
+
+          def parse_trailer_line(_line)
+            @file_ended = true
+            @on_trailer&.call self
+          end
+
+          def parse_tiploc_line(line)
+            hash = ::JSON.parse(line)['TiplocV1']
+
+            case hash['transaction_type'].downcase
+            when 'create'
+              @on_tiploc_create&.call self, Tiploc.from_json(line)
+            when 'delete'
+              @on_tiploc_delete&.call self, hash['tiploc_code']
+            else
+              logger.error 'Don\'t know how to ' \
+                           "#{hash['transaction_type'].inspect} a Tiploc: #{line.chomp}"
+            end
+          end
+
+          def parse_association_line(line)
+            hash = ::JSON.parse(line)['JsonAssociationV1']
+
+            case hash['transaction_type'].downcase
+            when 'create'
+              @on_association_create&.call self, Association.from_json(line)
+            when 'delete'
+              @on_association_delete&.call self, Association.from_json(line)
+            else
+              logger.error 'Don\'t know how to ' \
+                           "#{hash['transaction_type'].inspect} an Association: " \
+                           "#{line.chomp}"
+            end
+          end
+
+          def parse_schedule_line(line)
+            hash = ::JSON.parse(line)['JsonScheduleV1']
+
+            case hash['transaction_type'].downcase
+            when 'create'
+              @on_train_schedule_create&.call self, TrainSchedule.from_json(line)
+            when 'delete'
+              @on_train_schedule_delete&.call self, TrainSchedule.from_json(line)
+            else
+              logger.error 'Don\'t know how to ' \
+                           "#{hash['transaction_type'].inspect} a Train Schedule: " \
+                           "#{line.chomp}"
+            end
+          end
+        end
+      end
+    end
+  end
+end