adams-visiona 1.0.0

data/lib/cosmos/dart/lib/dart_decommutator.rb

@@ -0,0 +1,304 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+require 'dart_logging'
+require 'packet_log_entry'
+require 'cosmos/io/json_drb_object'
+
+class DartDecommutatorStatus
+  attr_accessor :count
+  attr_accessor :error_count
+  attr_accessor :message
+  attr_accessor :message_time
+
+  def initialize
+    @count = 0
+    @error_count = 0
+    @message = ''
+    @message_time = Time.now
+    @cached_meta_ple = nil
+    @cached_system_meta = nil
+    @cached_system_meta_id = nil
+    @cached_system_config = nil
+  end
+end
+
+class DartDecommutator
+  include DartCommon
+
+  # Wait 60s before giving up on the PacketConfig becoming ready
+  PACKET_CONFIG_READY_TIMEOUT = 60
+
+  # Delay between updating the DART status packet.   Simply throttles this rarely viewed status
+  STATUS_UPDATE_PERIOD_SECONDS = 60.seconds
+
+  def initialize(worker_id = 0, num_workers = 1)
+    sync_targets_and_packets()
+    @worker_id = worker_id
+    @num_workers = num_workers
+    @status = DartDecommutatorStatus.new
+    @master = Cosmos::JsonDRbObject.new(Cosmos::System.connect_hosts['DART_MASTER'], Cosmos::System.ports['DART_MASTER'], 1.0, Cosmos::System.x_csrf_token)
+  end
+
+  def timeit(message, &block)
+    time_start = Time.now
+    yield
+    Cosmos::Logger.info("#{Time.now - time_start}s #{message}")
+  end
+
+  # Run forever looking for data to decommutate
+  def run
+    status_time = Time.now + STATUS_UPDATE_PERIOD_SECONDS
+    while true
+      ple_id = nil
+      start_time = nil
+      end_time = nil
+      begin
+        ple_ids = @master.get_decom_ple_ids()
+      rescue DRb::DRbConnError
+        sleep(1)
+        next
+      end
+      if ple_ids and ple_ids.length > 0
+        ple_ids.each do |ple_id|
+          begin
+            ple = PacketLogEntry.find(ple_id)
+            meta_ple = get_meta_ple(ple)
+            next unless meta_ple
+            system_meta = get_system_meta(ple, meta_ple)
+            next unless system_meta
+            system_config = get_system_config(ple, system_meta)
+            next unless system_config
+            packet = get_packet(ple)
+            next unless packet
+            packet_config = get_packet_config(ple, packet, system_config)
+            next unless packet_config
+            # If we timeout this code will simply exit the application
+            wait_for_ready_packet_config(packet_config)
+            decom_packet(ple, packet, packet_config)
+
+            # Update status
+            if Time.now > status_time
+              status_time = Time.now + 60.seconds
+              status = Status.first
+              if (Time.now - @status.message_time) <= 60.0
+                status.decom_message = @status.message
+                status.decom_message_time = @status.message_time
+                status.save!
+              end
+              if @status.count > 0 or @status.error_count > 0
+                Status.update_counters(status.id, :decom_count => @status.count, :decom_error_count => @status.error_count)
+                @status.count = 0
+                @status.error_count = 0
+              end
+            end
+          rescue => err
+            handle_error("PLE:#{ple.id}:ERROR\n#{err.formatted}")
+          end
+        end
+      else
+        sleep(1)
+      end
+    end
+  rescue Interrupt
+    Cosmos::Logger.info("Dart Worker Closing From Signal...")
+    @master.shutdown
+  end
+
+  protected
+
+  def get_meta_ple(ple)
+    return @cached_meta_ple if @cached_meta_ple and @cached_meta_ple.id == ple.meta_id
+
+    if ple.meta_id != ple.id
+      meta_ple = PacketLogEntry.find(ple.meta_id)
+    else
+      meta_ple = ple
+    end
+    @cached_meta_ple = meta_ple
+    return meta_ple
+  rescue => err
+    ple.decom_state = PacketLogEntry::NO_META_PLE
+    ple.save!
+    handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
+    nil
+  end
+
+  def get_system_meta(ple, meta_ple)
+    return @cached_system_meta if @cached_system_meta and @cached_system_meta_id == meta_ple.id
+
+    system_meta = read_packet_from_ple(meta_ple)
+    if system_meta
+      @cached_system_meta_id = meta_ple.id
+      @cached_system_meta = system_meta
+      return system_meta
+    end
+
+    ple.decom_state = PacketLogEntry::NO_META_PACKET
+    ple.save!
+    handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
+    nil
+  end
+
+  def get_system_config(ple, system_meta)
+    system_config_name = system_meta.read("CONFIG")
+    if @cached_system_config and @cached_system_config.name == system_config_name
+      system_config = @cached_system_config
+    else
+      system_config = SystemConfig.where("name = ?", system_config_name).first
+      unless system_config
+        begin
+          # Try to create a new SystemConfig since it didn't exist
+          system_config = SystemConfig.create(:name => system_config_name)
+        rescue
+          # Another thread probably already created it - Try to get it one more time
+          system_config = SystemConfig.where("name = ?", system_config_name).first
+        end
+      end
+    end
+    unless system_config
+      ple.decom_state = PacketLogEntry::NO_SYSTEM_CONFIG
+      ple.save!
+      handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
+      return nil
+    end
+    @cached_system_config = system_config
+
+    # Switch to this system_config
+    begin
+      switch_and_get_system_config(system_config_name)
+    rescue
+      Cosmos::Logger.error("Could not load system_config: #{system_config_name}")
+      ple.decom_state = PacketLogEntry::NO_CONFIG
+      ple.save!
+      handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
+      return nil
+    end
+    system_config
+  end
+
+  def get_packet(ple)
+    packet = read_packet_from_ple(ple)
+    return packet if packet
+
+    ple.decom_state = PacketLogEntry::NO_PACKET
+    ple.save!
+    handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
+    nil
+  end
+
+  def get_packet_config(ple, packet, system_config)
+    packet_id = ple.packet_id
+    packet_config = PacketConfig.where("packet_id = ? and name = ?", packet_id, packet.config_name).first
+    return packet_config if packet_config
+
+    # The PacketConfig didnt't exist so create one
+    begin
+      Cosmos::Logger.info("Creating PacketConfig: #{packet.config_name}...")
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => system_config.id)
+      setup_packet_config(packet, packet_id, packet_config)
+      Cosmos::Logger.info("Successfully Created PacketConfig: #{packet.config_name}")
+    rescue => err
+      handle_error(err.formatted)
+      # Another thread probably already created it - Try to get it one more time
+      packet_config = PacketConfig.where("packet_id = ? and name = ?", packet_id, packet.config_name).first
+    end
+    unless packet_config
+      ple.decom_state = PacketLogEntry::NO_PACKET_CONFIG
+      ple.save!
+      handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
+      return nil
+    end
+    packet_config
+  end
+
+  def wait_for_ready_packet_config(packet_config)
+    first = true
+    while !packet_config.ready
+      if first
+        ready_wait_start = Time.now
+        Cosmos::Logger.info("Waiting for PacketConfig:#{packet_config.id}...")
+        first = false
+      end
+      sleep(1)
+      # Reload the attributes from the database
+      packet_config.reload
+
+      if (Time.now - ready_wait_start) > PACKET_CONFIG_READY_TIMEOUT
+        handle_error("Timeout waiting for ready on PacketConfig:#{packet_config.id}")
+        exit(1)
+      end
+    end
+  end
+
+  def get_values(packet)
+    values = []
+    packet.sorted_items.each do |item|
+      next unless decommutate_item?(item)
+      if separate_raw_con?(item)
+        values << packet.read_item(item, :RAW)
+        values << packet.read_item(item, :CONVERTED)
+      else
+        values << packet.read_item(item, :RAW)
+      end
+    end
+    values
+  end
+
+  def decom_packet(ple, packet, packet_config)
+    ActiveRecord::Base.transaction do
+      # Update packet config times
+      if !packet_config.start_time or (packet.packet_time < packet_config.start_time)
+        packet_config.start_time = packet.packet_time
+        packet_config.save!
+      end
+      if !packet_config.end_time or (packet.packet_time > packet_config.end_time)
+        packet_config.end_time = packet.packet_time
+        packet_config.save!
+      end
+
+      values = get_values(packet)
+
+      table_index = 0
+      rows = []
+      # Create rows in the decommutation table model
+      values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
+        model = get_decom_table_model(packet_config.id, table_index)
+        row = model.new
+        row.time = ple.time
+        row.ple_id = ple.id
+        row.packet_log_id = ple.packet_log_id
+        row.meta_id = ple.meta_id
+        row.reduced_state = READY_TO_REDUCE
+        table_values.each_with_index do |value, index|
+          item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
+          row.write_attribute("i#{item_index}", value)
+        end
+        row.save!
+        rows << row
+        table_index += 1
+      end
+
+      # The log entry has been decommutated, mark COMPLETE
+      ple.decom_state = PacketLogEntry::COMPLETE
+      ple.save!
+      @status.count += 1
+      Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
+    end
+  end
+
+  def handle_error(message)
+    Cosmos::Logger.error(message)
+    @status.error_count += 1
+    @status.message = message
+    @status.message_time = Time.now
+  end
+end

data/lib/cosmos/dart/lib/dart_importer.rb

@@ -0,0 +1,184 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+
+# Import COSMOS binary files into the DART database.
+# This code must be run on the database server.
+# The file to be imported should be placed in its final storage location.
+# Note that files are imported in place with algorithms that attempt to prevent
+# duplicate creation of Database entries.
+class DartImporter
+  include DartCommon
+
+  # @param filename [String] Fully qualified filename to import
+  # @param force [Boolean] Whether to reverify all packets in a log file
+  #   are in the database or just abort if the first and last are found.
+  def import(filename, force)
+    Cosmos::Logger.level = Cosmos::Logger::INFO
+
+    # Ensure all defined target and packets are in the database
+    sync_targets_and_packets()
+
+    dart_data_dir = File.expand_path(Cosmos::System.paths['DART_DATA'])
+    directory = File.dirname(File.expand_path(filename))
+    if directory != dart_data_dir
+      Cosmos::Logger.fatal("Imported files must be in \"#{dart_data_dir}\"")
+      Cosmos::Logger.fatal("  File is in: \"#{directory}\"")
+      return 1
+    end
+
+    # Make sure this file isn't already imported
+    packet_log = PacketLog.where("filename = ?", filename).first
+    if packet_log
+      Cosmos::Logger.warn("PacketLog already exists in database: #{filename}")
+    end
+
+    # Determine if this is a command or telemetry packet log
+    begin
+      plr = Cosmos::PacketLogReader.new
+      plr.open(filename)
+      if plr.log_type == :TLM
+        is_tlm = true
+      else
+        is_tlm = false
+      end
+    rescue
+      Cosmos::Logger.fatal("Unable to open #{filename}. Exiting...")
+      return 1
+    end
+
+    # Check if first and last packet in the log are already in the database
+    last_packet = plr.last
+    first_packet = plr.first
+    plr.close
+    unless first_packet and last_packet
+      Cosmos::Logger.fatal("No packets found in file. Exiting...")
+      return 1
+    end
+
+    first_ple = find_packet_log_entry(first_packet, is_tlm)
+    last_ple = find_packet_log_entry(last_packet, is_tlm)
+
+    fast = false
+    if first_ple and last_ple
+      Cosmos::Logger.warn("First and Last Packet in File Already in Database.")
+      if force
+        Cosmos::Logger.warn("Reverifying all packets in database due to force...")
+      else
+        Cosmos::Logger.info("Complete")
+        return 1
+      end
+    elsif !first_ple and !last_ple
+      Cosmos::Logger.info("First and Last Packet in File not in database")
+
+      # Check if time range of packets is not present in database
+      ple = PacketLogEntry.where("time >= ? and time <= ?", first_packet.packet_time, last_packet.packet_time).first
+      if !ple # Can go fast if not present at all
+        Cosmos::Logger.info("  Fast Import Enabled...")
+        fast = true
+      else
+        Cosmos::Logger.warn("Time range already in database. Will verify each packet before adding")
+      end
+    else
+      Cosmos::Logger.warn("File partially in database. Will verify each packet before adding")
+    end
+
+    unless packet_log
+      Cosmos::Logger.info("Creating PacketLog entry for file: #{filename}")
+      packet_log = PacketLog.create(:filename => filename, :is_tlm => is_tlm)
+    end
+
+    # Read File and Create PacketLogEntries
+    count = 0
+    meta_id = nil
+    ple_data = ""
+    ple_data_count = 0
+    plr.open(filename)
+    data_offset = plr.bytes_read
+    plr.each(filename) do |packet|
+      target_name = packet.target_name
+      target_name = 'UNKNOWN' unless target_name
+      packet_name = packet.packet_name
+      packet_name = 'UNKNOWN' unless packet_name
+
+      target_id, packet_id = lookup_target_and_packet_id(target_name, packet_name, is_tlm)
+
+      # If packets aren't found in the database we don't have to bother looking
+      # for PacketLogEntrys in the database and can simply create new entries
+      if fast
+        ple = nil
+      else # File is partially in the DB so see if the packet already exists
+        ple = find_packet_log_entry(packet, is_tlm)
+      end
+
+      # No PacketLogEntry was found so create one from scratch
+      unless ple
+        # SYSTEM META packets are special in that their meta_id is their own
+        # PacketLogEntry ID from the database. All other packets have meta_id
+        # values which point back to the last SYSTEM META PacketLogEntry ID.
+        if target_name == 'SYSTEM'.freeze and packet_name == 'META'.freeze
+          if ple_data.length > 0
+            ActiveRecord::Base.connection.execute("INSERT INTO packet_log_entries (target_id, packet_id, time, packet_log_id, data_offset, meta_id, is_tlm, ready) VALUES #{ple_data}")
+            ple_data.clear
+            ple_data_count = 0
+          end
+
+          ple = PacketLogEntry.new
+          ple.target_id = target_id
+          ple.packet_id = packet_id
+          ple.time = packet.packet_time
+          ple.packet_log_id = packet_log.id
+          ple.data_offset = data_offset
+          ple.meta_id = meta_id
+          ple.is_tlm = is_tlm
+          ple.ready = false
+          ple.save!(validate: false)
+
+          # Need to update meta_id for this and all subsequent packets
+          meta_id = ple.id
+          ple.meta_id = meta_id
+          ple.ready = true
+          ple.save!(validate: false)
+        else
+          ple_data << "," if ple_data.length > 0
+          ple_data << "(#{target_id},#{packet_id},'#{packet.packet_time.dup.utc.iso8601(6)}',#{packet_log.id},#{data_offset},#{meta_id},#{is_tlm},true)"
+          ple_data_count += 1
+        end
+
+        count += 1
+      else # A PacketLogEntry was found so this packet is skipped
+        # If the packet is a SYSTEM META packet we keep track of the meta_id
+        # for use in subsequent packets that aren't already in the database.
+        if target_name == 'SYSTEM'.freeze and packet_name == 'META'.freeze
+          # Need to update meta_id for subsequent packets
+          meta_id = ple.id
+        end
+      end
+
+      if ple_data_count >= 1000
+        ActiveRecord::Base.connection.execute("INSERT INTO packet_log_entries (target_id, packet_id, time, packet_log_id, data_offset, meta_id, is_tlm, ready) VALUES #{ple_data}")
+        ple_data.clear
+        ple_data_count = 0
+      end
+
+      data_offset = plr.bytes_read
+    end
+
+    if ple_data.length > 0
+      ActiveRecord::Base.connection.execute("INSERT INTO packet_log_entries (target_id, packet_id, time, packet_log_id, data_offset, meta_id, is_tlm, ready) VALUES #{ple_data}")
+      ple_data.clear
+      ple_data_count = 0
+    end
+
+    Cosmos::Logger.info("Added #{count} packet log entries to database")
+    return 0 # Success code
+  end
+end

data/lib/cosmos/dart/lib/dart_logging.rb

@@ -0,0 +1,50 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'cosmos'
+require 'stringio'
+
+# Creates a MessageLog in the DART_LOGS System path for DART logging
+class DartLogging
+  def initialize(message_log_name)
+    @output_sleeper = Cosmos::Sleeper.new
+    @string_output = StringIO.new("", "r+")
+    $stdout = @string_output
+    @message_log = Cosmos::MessageLog.new(message_log_name, Cosmos::System.paths['DART_LOGS'])
+
+    @output_thread = Thread.new do
+      while true
+        handle_string_output()
+        break if @output_sleeper.sleep(1)
+      end
+    end
+  end
+
+  def handle_string_output
+    if @string_output.string[-1..-1] == "\n"
+      string = @string_output.string.clone
+      @string_output.string = @string_output.string[string.length..-1]
+      @message_log.write(string, true)
+      STDOUT.print string if STDIN.isatty # Have a console
+    end
+  end
+
+  def graceful_kill
+    # Do Nothing
+  end
+
+  def stop
+    handle_string_output()
+    @output_sleeper.cancel
+    Cosmos.kill_thread(self, @output_thread)
+    handle_string_output()
+    @message_log.stop
+  end
+end

data/lib/cosmos/dart/lib/dart_master_query.rb

@@ -0,0 +1,71 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+require 'dart_logging'
+require 'thread'
+
+# Rails Json screws up COSMOS handling of Nan, etc.
+require "active_support/core_ext/object/json"
+module ActiveSupport
+  module ToJsonWithActiveSupportEncoder # :nodoc:
+    def to_json(options = nil)
+      super(options)
+    end
+  end
+end
+
+# JsonDRb server which responds to queries for decommutated and reduced data
+# from the database.
+class DartMasterQuery
+  include DartCommon
+
+  def initialize(ples_per_request = 5)
+    # Keep a thread to make sure we have the current list of items to decom
+    @ples_per_request = ples_per_request
+    @mutex = Mutex.new
+    @decom_list = []
+    @thread = Thread.new do
+      loop do
+        # Get all entries that are ready and decommutation hasn't started
+        if @decom_list.length <= 0
+          @mutex.synchronize do
+            begin
+              @decom_list.replace(PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").order("id ASC").limit(1000).pluck(:id))
+            rescue Exception => error
+              Cosmos::Logger.error("Error getting packets to decom\n#{error.formatted}")
+            end
+          end
+        else
+          sleep(1)
+        end
+      end
+    end
+  end
+
+  # Returns the id of a ple that needs to be decommed next
+  #
+  def get_decom_ple_ids()
+    begin
+      @mutex.synchronize do
+        result = []
+        @ples_per_request.times do
+          ple_id = @decom_list.shift
+          result << ple_id if ple_id
+        end
+        return result
+      end
+    rescue Exception => error
+      msg = "Master Error: #{error.message}"
+      raise $!, msg, $!.backtrace
+    end
+  end
+
+end