cosmos 4.5.2 → 5.0.2.pre.beta2

data/lib/cosmos/dart/lib/dart_decommutator.rb

@@ -1,304 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-require 'dart_common'
-require 'dart_logging'
-require 'packet_log_entry'
-require 'cosmos/io/json_drb_object'
-
-class DartDecommutatorStatus
-  attr_accessor :count
-  attr_accessor :error_count
-  attr_accessor :message
-  attr_accessor :message_time
-
-  def initialize
-    @count = 0
-    @error_count = 0
-    @message = ''
-    @message_time = Time.now
-    @cached_meta_ple = nil
-    @cached_system_meta = nil
-    @cached_system_meta_id = nil
-    @cached_system_config = nil
-  end
-end
-
-class DartDecommutator
-  include DartCommon
-
-  # Wait 60s before giving up on the PacketConfig becoming ready
-  PACKET_CONFIG_READY_TIMEOUT = 60
-
-  # Delay between updating the DART status packet.   Simply throttles this rarely viewed status
-  STATUS_UPDATE_PERIOD_SECONDS = 60.seconds
-
-  def initialize(worker_id = 0, num_workers = 1)
-    sync_targets_and_packets()
-    @worker_id = worker_id
-    @num_workers = num_workers
-    @status = DartDecommutatorStatus.new
-    @master = Cosmos::JsonDRbObject.new(Cosmos::System.connect_hosts['DART_MASTER'], Cosmos::System.ports['DART_MASTER'], 1.0, Cosmos::System.x_csrf_token)
-  end
-
-  def timeit(message, &block)
-    time_start = Time.now
-    yield
-    Cosmos::Logger.info("#{Time.now - time_start}s #{message}")
-  end
-
-  # Run forever looking for data to decommutate
-  def run
-    status_time = Time.now + STATUS_UPDATE_PERIOD_SECONDS
-    while true
-      ple_id = nil
-      start_time = nil
-      end_time = nil
-      begin
-        ple_ids = @master.get_decom_ple_ids()
-      rescue DRb::DRbConnError
-        sleep(1)
-        next
-      end
-      if ple_ids and ple_ids.length > 0
-        ple_ids.each do |ple_id|
-          begin
-            ple = PacketLogEntry.find(ple_id)
-            meta_ple = get_meta_ple(ple)
-            next unless meta_ple
-            system_meta = get_system_meta(ple, meta_ple)
-            next unless system_meta
-            system_config = get_system_config(ple, system_meta)
-            next unless system_config
-            packet = get_packet(ple)
-            next unless packet
-            packet_config = get_packet_config(ple, packet, system_config)
-            next unless packet_config
-            # If we timeout this code will simply exit the application
-            wait_for_ready_packet_config(packet_config)
-            decom_packet(ple, packet, packet_config)
-
-            # Update status
-            if Time.now > status_time
-              status_time = Time.now + 60.seconds
-              status = Status.first
-              if (Time.now - @status.message_time) <= 60.0
-                status.decom_message = @status.message
-                status.decom_message_time = @status.message_time
-                status.save!
-              end
-              if @status.count > 0 or @status.error_count > 0
-                Status.update_counters(status.id, :decom_count => @status.count, :decom_error_count => @status.error_count)
-                @status.count = 0
-                @status.error_count = 0
-              end
-            end
-          rescue => err
-            handle_error("PLE:#{ple.id}:ERROR\n#{err.formatted}")
-          end
-        end
-      else
-        sleep(1)
-      end
-    end
-  rescue Interrupt
-    Cosmos::Logger.info("Dart Worker Closing From Signal...")
-    @master.shutdown
-  end
-
-  protected
-
-  def get_meta_ple(ple)
-    return @cached_meta_ple if @cached_meta_ple and @cached_meta_ple.id == ple.meta_id
-
-    if ple.meta_id != ple.id
-      meta_ple = PacketLogEntry.find(ple.meta_id)
-    else
-      meta_ple = ple
-    end
-    @cached_meta_ple = meta_ple
-    return meta_ple
-  rescue => err
-    ple.decom_state = PacketLogEntry::NO_META_PLE
-    ple.save!
-    handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
-    nil
-  end
-
-  def get_system_meta(ple, meta_ple)
-    return @cached_system_meta if @cached_system_meta and @cached_system_meta_id == meta_ple.id
-
-    system_meta = read_packet_from_ple(meta_ple)
-    if system_meta
-      @cached_system_meta_id = meta_ple.id
-      @cached_system_meta = system_meta
-      return system_meta
-    end
-
-    ple.decom_state = PacketLogEntry::NO_META_PACKET
-    ple.save!
-    handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
-    nil
-  end
-
-  def get_system_config(ple, system_meta)
-    system_config_name = system_meta.read("CONFIG")
-    if @cached_system_config and @cached_system_config.name == system_config_name
-      system_config = @cached_system_config
-    else
-      system_config = SystemConfig.where("name = ?", system_config_name).first
-      unless system_config
-        begin
-          # Try to create a new SystemConfig since it didn't exist
-          system_config = SystemConfig.create(:name => system_config_name)
-        rescue
-          # Another thread probably already created it - Try to get it one more time
-          system_config = SystemConfig.where("name = ?", system_config_name).first
-        end
-      end
-    end
-    unless system_config
-      ple.decom_state = PacketLogEntry::NO_SYSTEM_CONFIG
-      ple.save!
-      handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
-      return nil
-    end
-    @cached_system_config = system_config
-
-    # Switch to this system_config
-    begin
-      switch_and_get_system_config(system_config_name)
-    rescue
-      Cosmos::Logger.error("Could not load system_config: #{system_config_name}")
-      ple.decom_state = PacketLogEntry::NO_CONFIG
-      ple.save!
-      handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
-      return nil
-    end
-    system_config
-  end
-
-  def get_packet(ple)
-    packet = read_packet_from_ple(ple)
-    return packet if packet
-
-    ple.decom_state = PacketLogEntry::NO_PACKET
-    ple.save!
-    handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
-    nil
-  end
-
-  def get_packet_config(ple, packet, system_config)
-    packet_id = ple.packet_id
-    packet_config = PacketConfig.where("packet_id = ? and name = ?", packet_id, packet.config_name).first
-    return packet_config if packet_config
-
-    # The PacketConfig didnt't exist so create one
-    begin
-      Cosmos::Logger.info("Creating PacketConfig: #{packet.config_name}...")
-      packet_config = PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => system_config.id)
-      setup_packet_config(packet, packet_id, packet_config)
-      Cosmos::Logger.info("Successfully Created PacketConfig: #{packet.config_name}")
-    rescue => err
-      handle_error(err.formatted)
-      # Another thread probably already created it - Try to get it one more time
-      packet_config = PacketConfig.where("packet_id = ? and name = ?", packet_id, packet.config_name).first
-    end
-    unless packet_config
-      ple.decom_state = PacketLogEntry::NO_PACKET_CONFIG
-      ple.save!
-      handle_error("PLE:#{ple.id}:#{ple.decom_state_string}")
-      return nil
-    end
-    packet_config
-  end
-
-  def wait_for_ready_packet_config(packet_config)
-    first = true
-    while !packet_config.ready
-      if first
-        ready_wait_start = Time.now
-        Cosmos::Logger.info("Waiting for PacketConfig:#{packet_config.id}...")
-        first = false
-      end
-      sleep(1)
-      # Reload the attributes from the database
-      packet_config.reload
-
-      if (Time.now - ready_wait_start) > PACKET_CONFIG_READY_TIMEOUT
-        handle_error("Timeout waiting for ready on PacketConfig:#{packet_config.id}")
-        exit(1)
-      end
-    end
-  end
-
-  def get_values(packet)
-    values = []
-    packet.sorted_items.each do |item|
-      next unless decommutate_item?(item)
-      if separate_raw_con?(item)
-        values << packet.read_item(item, :RAW)
-        values << packet.read_item(item, :CONVERTED)
-      else
-        values << packet.read_item(item, :RAW)
-      end
-    end
-    values
-  end
-
-  def decom_packet(ple, packet, packet_config)
-    ActiveRecord::Base.transaction do
-      # Update packet config times
-      if !packet_config.start_time or (packet.packet_time < packet_config.start_time)
-        packet_config.start_time = packet.packet_time
-        packet_config.save!
-      end
-      if !packet_config.end_time or (packet.packet_time > packet_config.end_time)
-        packet_config.end_time = packet.packet_time
-        packet_config.save!
-      end
-
-      values = get_values(packet)
-
-      table_index = 0
-      rows = []
-      # Create rows in the decommutation table model
-      values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
-        model = get_decom_table_model(packet_config.id, table_index)
-        row = model.new
-        row.time = ple.time
-        row.ple_id = ple.id
-        row.packet_log_id = ple.packet_log_id
-        row.meta_id = ple.meta_id
-        row.reduced_state = READY_TO_REDUCE
-        table_values.each_with_index do |value, index|
-          item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
-          row.write_attribute("i#{item_index}", value)
-        end
-        row.save!
-        rows << row
-        table_index += 1
-      end
-
-      # The log entry has been decommutated, mark COMPLETE
-      ple.decom_state = PacketLogEntry::COMPLETE
-      ple.save!
-      @status.count += 1
-      Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
-    end
-  end
-
-  def handle_error(message)
-    Cosmos::Logger.error(message)
-    @status.error_count += 1
-    @status.message = message
-    @status.message_time = Time.now
-  end
-end

data/lib/cosmos/dart/lib/dart_importer.rb

@@ -1,184 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-require 'dart_common'
-
-# Import COSMOS binary files into the DART database.
-# This code must be run on the database server.
-# The file to be imported should be placed in its final storage location.
-# Note that files are imported in place with algorithms that attempt to prevent
-# duplicate creation of Database entries.
-class DartImporter
-  include DartCommon
-
-  # @param filename [String] Fully qualified filename to import
-  # @param force [Boolean] Whether to reverify all packets in a log file
-  #   are in the database or just abort if the first and last are found.
-  def import(filename, force)
-    Cosmos::Logger.level = Cosmos::Logger::INFO
-
-    # Ensure all defined target and packets are in the database
-    sync_targets_and_packets()
-
-    dart_data_dir = File.expand_path(Cosmos::System.paths['DART_DATA'])
-    directory = File.dirname(File.expand_path(filename))
-    if directory != dart_data_dir
-      Cosmos::Logger.fatal("Imported files must be in \"#{dart_data_dir}\"")
-      Cosmos::Logger.fatal("  File is in: \"#{directory}\"")
-      return 1
-    end
-
-    # Make sure this file isn't already imported
-    packet_log = PacketLog.where("filename = ?", filename).first
-    if packet_log
-      Cosmos::Logger.warn("PacketLog already exists in database: #{filename}")
-    end
-
-    # Determine if this is a command or telemetry packet log
-    begin
-      plr = Cosmos::PacketLogReader.new
-      plr.open(filename)
-      if plr.log_type == :TLM
-        is_tlm = true
-      else
-        is_tlm = false
-      end
-    rescue
-      Cosmos::Logger.fatal("Unable to open #{filename}. Exiting...")
-      return 1
-    end
-
-    # Check if first and last packet in the log are already in the database
-    last_packet = plr.last
-    first_packet = plr.first
-    plr.close
-    unless first_packet and last_packet
-      Cosmos::Logger.fatal("No packets found in file. Exiting...")
-      return 1
-    end
-
-    first_ple = find_packet_log_entry(first_packet, is_tlm)
-    last_ple = find_packet_log_entry(last_packet, is_tlm)
-
-    fast = false
-    if first_ple and last_ple
-      Cosmos::Logger.warn("First and Last Packet in File Already in Database.")
-      if force
-        Cosmos::Logger.warn("Reverifying all packets in database due to force...")
-      else
-        Cosmos::Logger.info("Complete")
-        return 1
-      end
-    elsif !first_ple and !last_ple
-      Cosmos::Logger.info("First and Last Packet in File not in database")
-
-      # Check if time range of packets is not present in database
-      ple = PacketLogEntry.where("time >= ? and time <= ?", first_packet.packet_time, last_packet.packet_time).first
-      if !ple # Can go fast if not present at all
-        Cosmos::Logger.info("  Fast Import Enabled...")
-        fast = true
-      else
-        Cosmos::Logger.warn("Time range already in database. Will verify each packet before adding")
-      end
-    else
-      Cosmos::Logger.warn("File partially in database. Will verify each packet before adding")
-    end
-
-    unless packet_log
-      Cosmos::Logger.info("Creating PacketLog entry for file: #{filename}")
-      packet_log = PacketLog.create(:filename => filename, :is_tlm => is_tlm)
-    end
-
-    # Read File and Create PacketLogEntries
-    count = 0
-    meta_id = nil
-    ple_data = ""
-    ple_data_count = 0
-    plr.open(filename)
-    data_offset = plr.bytes_read
-    plr.each(filename) do |packet|
-      target_name = packet.target_name
-      target_name = 'UNKNOWN' unless target_name
-      packet_name = packet.packet_name
-      packet_name = 'UNKNOWN' unless packet_name
-
-      target_id, packet_id = lookup_target_and_packet_id(target_name, packet_name, is_tlm)
-
-      # If packets aren't found in the database we don't have to bother looking
-      # for PacketLogEntrys in the database and can simply create new entries
-      if fast
-        ple = nil
-      else # File is partially in the DB so see if the packet already exists
-        ple = find_packet_log_entry(packet, is_tlm)
-      end
-
-      # No PacketLogEntry was found so create one from scratch
-      unless ple
-        # SYSTEM META packets are special in that their meta_id is their own
-        # PacketLogEntry ID from the database. All other packets have meta_id
-        # values which point back to the last SYSTEM META PacketLogEntry ID.
-        if target_name == 'SYSTEM'.freeze and packet_name == 'META'.freeze
-          if ple_data.length > 0
-            ActiveRecord::Base.connection.execute("INSERT INTO packet_log_entries (target_id, packet_id, time, packet_log_id, data_offset, meta_id, is_tlm, ready) VALUES #{ple_data}")
-            ple_data.clear
-            ple_data_count = 0
-          end
-
-          ple = PacketLogEntry.new
-          ple.target_id = target_id
-          ple.packet_id = packet_id
-          ple.time = packet.packet_time
-          ple.packet_log_id = packet_log.id
-          ple.data_offset = data_offset
-          ple.meta_id = meta_id
-          ple.is_tlm = is_tlm
-          ple.ready = false
-          ple.save!(validate: false)
-
-          # Need to update meta_id for this and all subsequent packets
-          meta_id = ple.id
-          ple.meta_id = meta_id
-          ple.ready = true
-          ple.save!(validate: false)
-        else
-          ple_data << "," if ple_data.length > 0
-          ple_data << "(#{target_id},#{packet_id},'#{packet.packet_time.dup.utc.iso8601(6)}',#{packet_log.id},#{data_offset},#{meta_id},#{is_tlm},true)"
-          ple_data_count += 1
-        end
-
-        count += 1
-      else # A PacketLogEntry was found so this packet is skipped
-        # If the packet is a SYSTEM META packet we keep track of the meta_id
-        # for use in subsequent packets that aren't already in the database.
-        if target_name == 'SYSTEM'.freeze and packet_name == 'META'.freeze
-          # Need to update meta_id for subsequent packets
-          meta_id = ple.id
-        end
-      end
-
-      if ple_data_count >= 1000
-        ActiveRecord::Base.connection.execute("INSERT INTO packet_log_entries (target_id, packet_id, time, packet_log_id, data_offset, meta_id, is_tlm, ready) VALUES #{ple_data}")
-        ple_data.clear
-        ple_data_count = 0
-      end
-
-      data_offset = plr.bytes_read
-    end
-
-    if ple_data.length > 0
-      ActiveRecord::Base.connection.execute("INSERT INTO packet_log_entries (target_id, packet_id, time, packet_log_id, data_offset, meta_id, is_tlm, ready) VALUES #{ple_data}")
-      ple_data.clear
-      ple_data_count = 0
-    end
-
-    Cosmos::Logger.info("Added #{count} packet log entries to database")
-    return 0 # Success code
-  end
-end

data/lib/cosmos/dart/lib/dart_logging.rb

@@ -1,50 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-require 'cosmos'
-require 'stringio'
-
-# Creates a MessageLog in the DART_LOGS System path for DART logging
-class DartLogging
-  def initialize(message_log_name)
-    @output_sleeper = Cosmos::Sleeper.new
-    @string_output = StringIO.new("", "r+")
-    $stdout = @string_output
-    @message_log = Cosmos::MessageLog.new(message_log_name, Cosmos::System.paths['DART_LOGS'])
-
-    @output_thread = Thread.new do
-      while true
-        handle_string_output()
-        break if @output_sleeper.sleep(1)
-      end
-    end
-  end
-
-  def handle_string_output
-    if @string_output.string[-1..-1] == "\n"
-      string = @string_output.string.clone
-      @string_output.string = @string_output.string[string.length..-1]
-      @message_log.write(string, true)
-      STDOUT.print string if STDIN.isatty # Have a console
-    end
-  end
-
-  def graceful_kill
-    # Do Nothing
-  end
-
-  def stop
-    handle_string_output()
-    @output_sleeper.cancel
-    Cosmos.kill_thread(self, @output_thread)
-    handle_string_output()
-    @message_log.stop
-  end
-end

data/lib/cosmos/dart/lib/dart_master_query.rb

@@ -1,71 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-require 'dart_common'
-require 'dart_logging'
-require 'thread'
-
-# Rails Json screws up COSMOS handling of Nan, etc.
-require "active_support/core_ext/object/json"
-module ActiveSupport
-  module ToJsonWithActiveSupportEncoder # :nodoc:
-    def to_json(options = nil)
-      super(options)
-    end
-  end
-end
-
-# JsonDRb server which responds to queries for decommutated and reduced data
-# from the database.
-class DartMasterQuery
-  include DartCommon
-
-  def initialize(ples_per_request = 5)
-    # Keep a thread to make sure we have the current list of items to decom
-    @ples_per_request = ples_per_request
-    @mutex = Mutex.new
-    @decom_list = []
-    @thread = Thread.new do
-      loop do
-        # Get all entries that are ready and decommutation hasn't started
-        if @decom_list.length <= 0
-          @mutex.synchronize do
-            begin
-              @decom_list.replace(PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").order("id ASC").limit(1000).pluck(:id))
-            rescue Exception => error
-              Cosmos::Logger.error("Error getting packets to decom\n#{error.formatted}")
-            end
-          end
-        else
-          sleep(1)
-        end
-      end
-    end
-  end
-
-  # Returns the id of a ple that needs to be decommed next
-  #
-  def get_decom_ple_ids()
-    begin
-      @mutex.synchronize do
-        result = []
-        @ples_per_request.times do
-          ple_id = @decom_list.shift
-          result << ple_id if ple_id
-        end
-        return result
-      end
-    rescue Exception => error
-      msg = "Master Error: #{error.message}"
-      raise $!, msg, $!.backtrace
-    end
-  end
-
-end