cosmos 4.1.1 → 4.2.0

data/lib/cosmos/dart/lib/dart_database_cleaner.rb

@@ -0,0 +1,172 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+
+class DartDatabaseCleaner
+  include DartCommon
+
+  # Clean the DART database of various issues. Note that this code can exit(1)
+  # if the DART packet logs were moved and force is false (default). This is
+  # deliberate because force causes all the lost (or moved) files to be deleted
+  # which forces them to be re-imported at their new location.
+  def self.clean(force)
+    Cosmos::Logger::info("Starting database cleanup...")
+    cleaner = DartDatabaseCleaner.new
+    cleaner.clean_system_configs()
+    cleaner.clean_packet_logs(force)
+    cleaner.clean_packet_configs()
+    cleaner.clean_packet_log_entries()
+    cleaner.clean_decommutation_tables()
+    cleaner.clean_reductions()
+    Cosmos::Logger::info("Database cleanup complete!")
+  end
+
+  # Ensure we have all the System Configs locally on the DART machine
+  def clean_system_configs
+    Cosmos::Logger::info("Cleaning up SystemConfig...")
+    system_config = SystemConfig.all.each do |sc|
+       begin
+        # This attempts to load the system config and if it can't be found
+        # it is copied from the server to the local DART machine
+        switch_and_get_system_config(sc.name)
+      rescue => err
+        Cosmos::Logger.error("Could not load system_config: #{sc.name}: #{err.message}")
+        next
+      end
+    end
+    Cosmos::System.load_configuration
+  end
+
+  # Ensure all packet log files still exist
+  def clean_packet_logs(force)
+    Cosmos::Logger::info("Cleaning up PacketLog...")
+    PacketLog.find_each do |pl|
+      unless File.exist?(pl.filename)
+        # Try to see if it is in the current DART_DATA folder
+        moved_filename = File.join(Cosmos::System.paths['DART_DATA'], File.basename(pl.filename))
+        if File.exist?(moved_filename)
+          pl.filename = moved_filename
+          pl.save!
+        else
+          if force
+            Cosmos::Logger.error("Packet Log File Missing: #{pl.filename}")
+            pl.destroy # This also destroys linked PacketLogEntry(s)
+          else
+            Cosmos::Logger.fatal("Packet Log File Missing (Cleanup with --force-cleanup): #{pl.filename}")
+            exit(1)
+          end
+        end
+      end
+    end
+  end
+
+  # Check for bad packet configs and cleanup. This typically doesn't happen because
+  # PacketConfig entries are created quickly but if something happens during the setup
+  # we attempt to fix it here.
+  def clean_packet_configs
+    Cosmos::Logger::info("Cleaning up PacketConfig...")
+    packet_configs = PacketConfig.where("ready != true")
+    return unless packet_configs.length > 0
+    Cosmos::Logger::info("Num PacketConfigs requiring cleanup: #{packet_configs.length}")
+    packet_configs.each do |packet_config|
+      begin
+        system_config = SystemConfig.find(packet_config.first_system_config_id)
+        packet_model = Packet.find(packet_config.packet_id)
+        target_model = Target.find(packet_model.target_id)
+        current_config, error = Cosmos::System.load_configuration(system_config.name)
+        if current_config == system_config.name
+          if packet_model.is_tlm
+            packet = Cosmos::System.telemetry.packet(target_model.name, packet_model.name)
+          else
+            packet = Cosmos::System.commands.packet(target_model.name, packet_model.name)
+          end
+          setup_packet_config(packet, packet_model.id, packet_config)
+          Cosmos::Logger::info("Successfully cleaned up packet_config: #{packet_config.id}")
+        else
+          Cosmos::Logger::error("Could not switch to system config: #{system_config.name}: #{error}")
+        end
+      rescue => err
+        Cosmos::Logger::error("Error cleaning up packet config: #{packet_config.id}: #{err.formatted}")
+        raise "Cleanup failure - Database requires manual correction"
+      end
+    end
+  end
+
+  # Remove not ready packet log entries as well as partially decommutated data.
+  def clean_packet_log_entries
+    Cosmos::Logger::info("Cleaning up PacketLogEntry...")
+    # Remove not ready packet log entries. Packet log entries remain not ready until the
+    # log file containing the packet has been flushed to disk. Thus there are always
+    # outstanding entries which are not ready while packets are being received.
+    # Note the normal shutdown process attempts to flush the log file and mark
+    # all outstanding entries as ready so this would only happen during a crash.
+    ples = PacketLogEntry.where("ready != true")
+    return unless ples.length > 0
+    Cosmos::Logger::info("Removing unready packet log entries: #{ples.length}")
+    ples.destroy_all
+  end
+
+  def clean_decommutation_tables
+    Cosmos::Logger::info("Cleaning up Decommutation tables (tX_Y)...")
+    # Check for partially decom data and remove. The DartWorker periodically checks the
+    # database for a PacketLogEntry which is ready to be decommutated and starts the
+    # process of writing into the decommutation table. If this process is interrupted
+    # the state could be IN_PROGRESS instead of COMPLETE. Thus delete all the decommutation
+    # table rows which were created and allow this process to start from scratch.
+    ples = PacketLogEntry.where("decom_state = #{PacketLogEntry::IN_PROGRESS}")
+    return unless ples.length > 0
+    Cosmos::Logger::info("Num PacketLogEntries requiring cleanup: #{ples.length}")
+    ples.each do |ple|
+      begin
+        packet = read_packet_from_ple(ple)
+        packet_config = PacketConfig.where("packet_id = ? and name = ?", ple.packet_id, packet.config_name).first
+        # Need to delete any rows for these ples in the table for this packet_config
+        packet_config.max_table_index.times do |table_index|
+          model = get_decom_table_model(packet_config.id, table_index)
+          model.where("ple_id = ?", ple.id).destroy_all
+        end
+        ple.decom_state = PacketLogEntry::NOT_STARTED
+        ple.save!
+      rescue => err
+        Cosmos::Logger::error("Error cleaning up packet log entry: #{ple.id}: #{err.formatted}")
+      end
+    end
+  end
+
+  def clean_reductions
+    Cosmos::Logger::info("Cleaning up Reductions...")
+    # TBR: This cleanup may be too slow to be worth it for a large data set...
+    each_decom_and_reduced_table() do |packet_config_id, table_index, decom_model, minute_model, hour_model, day_model|
+      decom_model.where("reduced_state = #{REDUCED} and reduced_id IS NULL").update_all(:reduced_state => READY_TO_REDUCE)
+      minute_model.where("reduced_state = #{REDUCED} and reduced_id IS NULL").update_all(:reduced_state => READY_TO_REDUCE)
+      hour_model.where("reduced_state = #{REDUCED} and reduced_id IS NULL").update_all(:reduced_state => READY_TO_REDUCE)
+      # Note: These should be destroyed when cleaning up decom tables
+      rows = decom_model.where("reduced_state = #{INITIALIZING}")
+      rows.destroy_all
+      # This cleanup is only here
+      rows = minute_model.where("reduced_state = #{INITIALIZING}")
+      rows.each do |row|
+        decom_model.where("reduced_id = ?", row.id).update_all(:reduced_state => READY_TO_REDUCE, :reduced_id => nil)
+      end
+      rows.destroy_all
+      rows = hour_model.where("reduced_state = #{INITIALIZING}")
+      rows.each do |row|
+        minute_model.where("reduced_id = ?", row.id).update_all(:reduced_state => READY_TO_REDUCE, :reduced_id => nil)
+      end
+      rows.destroy_all
+      rows = day_model.where("reduced_state = #{INITIALIZING}")
+      rows.each do |row|
+        hour_model.where("reduced_id = ?", row.id).update_all(:reduced_state => READY_TO_REDUCE, :reduced_id => nil)
+      end
+      rows.destroy_all
+    end
+  end
+end

data/lib/cosmos/dart/lib/dart_decom_query.rb

@@ -0,0 +1,184 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+require 'dart_logging'
+
+# Rails Json screws up COSMOS handling of Nan, etc.
+require "active_support/core_ext/object/json"
+module ActiveSupport
+  module ToJsonWithActiveSupportEncoder # :nodoc:
+    def to_json(options = nil)
+      super(options)
+    end
+  end
+end
+
+# JsonDRb server which responds to queries for decommutated and reduced data
+# from the database.
+class DartDecomQuery
+  include DartCommon
+
+  # Returns data from the decommutated database tables including the reduced data tables.
+  #
+  # @param request [Hash] Request for data.
+  #   The hash must contain the following items:
+  #     start_time_sec => Start time in UTC seconds
+  #     start_time_usec => Microseconds to add to start time
+  #     end_time_sec => End time in UTC seconds
+  #     end_time_usec => Microseconds to add to end time
+  #     item => [target name, packet name, item name] Names are all strings
+  #     reduction => "NONE", "MINUTE", "HOUR", "DAY" for how to reduce the data
+  #     value_type => "RAW", "RAW_MAX", "RAW_MIN", "RAW_AVG",
+  #       "CONVERTED", "CONVERTED_MAX", "CONVERTED_MIN", "CONVERTED_AVG"
+  #
+  #   The request can also contain the following optional items:
+  #     meta_ids => Optional IDs related to the meta data you want to filter by. This requires
+  #       making a separate request for the particular meta data in question and recording
+  #       the returned meta_ids for use in a subsequent request.
+  #     limit => Maximum number of data items to return, must be less than 10000
+  #     offset => Offset into the data stream. Since the maximum number of values allowed
+  #       is 10000, you can set the offset to 10000, then 20000, etc to get additional values.
+  #       By default the offset is 0.
+  #     cmd_tlm => Whether the item is a command or telemetry. Default is telemetry.
+  # @return [Array<Array<String, Integer, Integer, Integer, Integer>>] Array of arrays containing
+  #   the item name, item seconds, item microseconds, samples (always 1 for NONE reduction, varies
+  #   for other reduction values), and meta_id.
+  def query(request)
+    request_start_time = Time.now
+    Cosmos::Logger.info("#{request_start_time.formatted}: QUERY: #{request}")
+
+    begin
+      start_time_sec = request['start_time_sec']
+      start_time_usec = request['start_time_usec']
+      end_time_sec = request['end_time_sec']
+      end_time_usec = request['end_time_usec']
+
+      start_time = nil
+      end_time = nil
+      begin
+        start_time = Time.at(start_time_sec, start_time_usec) if start_time_sec and start_time_usec
+      rescue
+        raise "Invalid start time: #{start_time_sec}, #{start_time_usec}"
+      end
+      begin
+        end_time = Time.at(end_time_sec, end_time_usec) if end_time_sec and end_time_usec
+      rescue
+        raise "Invalid end time: #{end_time_sec}, #{end_time_usec}"
+      end
+
+      item = request['item']
+      raise "Item \"#{item}\" invalid" if !item || item.length != 3
+
+      reduction = request['reduction'].to_s.upcase
+      case reduction
+      when "", "NONE"
+        reduction = :NONE
+        reduction_modifier = ""
+      when "MINUTE"
+        reduction = :MINUTE
+        reduction_modifier = "_m"
+      when "HOUR"
+        reduction = :HOUR
+        reduction_modifier = "_h"
+      when "DAY"
+        reduction = :DAY
+        reduction_modifier = "_d"
+      else
+        raise "Unknown reduction: #{reduction}"
+      end
+
+      requested_value_type = request['value_type'].to_s.upcase
+      case requested_value_type
+      when 'RAW'
+        value_type = ItemToDecomTableMapping::CONVERTED
+        item_name_modifier = ""
+        raise "RAW value_type is only valid with NONE reduction" if reduction != :NONE
+      when 'RAW_MAX', 'RAW_MIN', 'RAW_AVG', 'RAW_STDDEV'
+        value_type = ItemToDecomTableMapping::CONVERTED
+        item_name_modifier = requested_value_type.split('_')[1].downcase
+        raise "#{requested_value_type} value_type is not valid with NONE reduction" if reduction == :NONE
+      when 'CONVERTED'
+        value_type = ItemToDecomTableMapping::RAW
+        item_name_modifier = ""
+        raise "CONVERTED value_type is only valid with NONE reduction" if reduction != :NONE
+      when 'CONVERTED_MAX', 'CONVERTED_MIN', 'CONVERTED_AVG', 'CONVERTED_STDDEV'
+        value_type = ItemToDecomTableMapping::RAW
+        item_name_modifier = requested_value_type.split('_')[1].downcase
+        raise "#{requested_value_type} value_type is not valid with NONE reduction" if reduction == :NONE
+      else
+        raise "Unknown value_type: #{requested_value_type}"
+      end
+
+      cmd_tlm = request['cmd_tlm']
+      if cmd_tlm
+        if cmd_tlm.to_s.upcase == 'CMD'
+          is_tlm = false
+        elsif cmd_tlm.to_s.upcase == 'TLM'
+          is_tlm = true
+        else
+          raise "Unknown cmd_tlm: #{cmd_tlm}"
+        end
+      else
+        is_tlm = true
+      end
+
+      meta_ids = request['meta_ids']
+      meta_ids ||= []
+
+      unless meta_ids.length > 0
+        meta_filters = request['meta_filters']
+        meta_filters ||= []
+        
+        if meta_filters.length > 0
+          meta_ids = process_meta_filters(meta_filters, is_tlm, end_time)
+        end
+      end
+
+      limit = request['limit'].to_i
+      limit = 10000 if limit <= 0 or limit > 10000
+
+      offset = request['offset'].to_i
+      offset = 0 if offset < 0
+
+      return query_decom_reduced(
+        item[0], item[1], item[2], 
+        value_type, is_tlm, 
+        start_time, end_time, 
+        reduction, reduction_modifier, 
+        item_name_modifier, limit, offset, meta_ids)
+
+    rescue Exception => error
+      msg = "Query Error: #{error.message}"
+      raise $!, msg, $!.backtrace
+    end
+  end
+
+  # Gets the list of item names for a given packet
+  #
+  # @param target_name Target name
+  # @param packet_name Packet name
+  # @param is_tlm true or false
+  # @return [Array<String>] Array of item names
+  def item_names(target_name, packet_name, is_tlm = true)
+    target = Target.where("name = ?", target_name).first
+    raise "Target #{target_name} not found" unless target
+
+    packet = Packet.where("target_id = ? and name = ? and is_tlm = ?", target.id, packet_name, is_tlm).first
+    raise "Packet #{target_name} #{packet_name} not found" unless packet
+
+    items = Item.where("packet_id = ?", packet.id).select("name")
+    item_names = []
+    items.each { |item| item_names << item.name }
+
+    return item_names
+  end
+
+end

data/lib/cosmos/dart/lib/dart_decommutator.rb

@@ -0,0 +1,235 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'dart_common'
+require 'dart_logging'
+require 'packet_log_entry'
+
+class DartDecommutator
+  include DartCommon
+
+  # Wait 60s before giving up on the PacketConfig becoming ready
+  PACKET_CONFIG_READY_TIMEOUT = 60
+
+  def initialize(worker_id = 0, num_workers = 1)
+    sync_targets_and_packets()
+    @worker_id = worker_id
+    @num_workers = num_workers
+  end
+
+  # Run forever looking for data to decommutate
+  def run
+    while true
+      time_start = Time.now # Remember start time so we can throttle
+      # Get all entries that are ready and decommutation hasn't started
+      PacketLogEntry.where("decom_state = #{PacketLogEntry::NOT_STARTED} and ready = true").
+                     # Mod the ID to allow distribution of effort, in_batches processes 1000 at a time
+                     where("id % #{@num_workers} = #{@worker_id}").in_batches do |group|
+        group.each do |ple|
+          begin
+            # TODO - Optimize and cache meta and system config and packet config lookup
+            meta_ple = get_meta_ple(ple)
+            next unless meta_ple
+            system_meta = get_system_meta(ple, meta_ple)
+            next unless system_meta
+            system_config = get_system_config(ple, system_meta)
+            next unless system_config
+            packet = get_packet(ple)
+            next unless packet
+            packet_config = get_packet_config(ple, packet, system_config)
+            next unless packet_config
+            # If we timeout this code will simply exit the application
+            wait_for_ready_packet_config(packet_config)
+            decom_packet(ple, packet, packet_config)
+          rescue => err
+            Cosmos::Logger.error("PLE:#{ple.id}:ERROR")
+            Cosmos::Logger.error(err.formatted)
+          end
+        end # each ple
+      end # batches
+      # Throttle to no faster than 1 Hz
+      delta = Time.now - time_start
+      sleep(1 - delta) if delta < 1 && delta > 0
+    end
+  rescue Interrupt
+    Cosmos::Logger.info("Dart Worker Closing From Signal...")
+  end
+
+  protected
+
+  def get_meta_ple(ple)
+    if ple.meta_id != ple.id
+      PacketLogEntry.find(ple.meta_id)
+    else
+      ple
+    end
+  rescue => err
+    ple.decom_state = PacketLogEntry::NO_META_PLE
+    ple.save!
+    Cosmos::Logger.error("PLE:#{ple.id}:#{ple.decom_state_string}")
+    nil
+  end
+
+  def get_system_meta(ple, meta_ple)
+    system_meta = read_packet_from_ple(meta_ple)
+    return system_meta if system_meta
+
+    ple.decom_state = PacketLogEntry::NO_META_PACKET
+    ple.save!
+    Cosmos::Logger.error("PLE:#{ple.id}:#{ple.decom_state_string}")
+    nil
+  end
+
+  def get_system_config(ple, system_meta)
+    system_config_name = system_meta.read("CONFIG")
+    system_config = SystemConfig.where("name = ?", system_config_name).first
+    unless system_config
+      begin
+        # Try to create a new SystemConfig since it didn't exist
+        system_config = SystemConfig.create(:name => system_config_name)
+      rescue
+        # Another thread probably already created it - Try to get it one more time
+        system_config = SystemConfig.where("name = ?", system_config_name).first
+      end
+    end
+    unless system_config
+      ple.decom_state = PacketLogEntry::NO_SYSTEM_CONFIG
+      ple.save!
+      Cosmos::Logger.error("PLE:#{ple.id}:#{ple.decom_state_string}")
+      return nil
+    end
+
+    # Switch to this system_config
+    begin
+      switch_and_get_system_config(system_config_name)
+    rescue
+      Cosmos::Logger.error("Could not load system_config: #{system_config_name}")
+      ple.decom_state = PacketLogEntry::NO_CONFIG
+      ple.save!
+      Cosmos::Logger.error("PLE:#{ple.id}:#{ple.decom_state_string}")
+      return nil
+    end
+    system_config
+  end
+
+  def get_packet(ple)
+    packet = read_packet_from_ple(ple)
+    return packet if packet
+
+    ple.decom_state = PacketLogEntry::NO_PACKET
+    ple.save!
+    Cosmos::Logger.error("PLE:#{ple.id}:#{ple.decom_state_string}")
+    nil
+  end
+
+  def get_packet_config(ple, packet, system_config)
+    packet_id = ple.packet_id
+    packet_config = PacketConfig.where("packet_id = ? and name = ?", packet_id, packet.config_name).first
+    return packet_config if packet_config
+
+    # The PacketConfig didnt't exist so create one
+    begin
+      Cosmos::Logger.info("Creating PacketConfig: #{packet.config_name}...")
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => system_config.id)
+      setup_packet_config(packet, packet_id, packet_config)
+      Cosmos::Logger.info("Successfully Created PacketConfig: #{packet.config_name}")
+    rescue => err
+      Cosmos::Logger.error(err.formatted)
+      # Another thread probably already created it - Try to get it one more time
+      packet_config = PacketConfig.where("packet_id = ? and name = ?", packet_id, packet.config_name).first
+    end
+    unless packet_config
+      ple.decom_state = PacketLogEntry::NO_PACKET_CONFIG
+      ple.save!
+      Cosmos::Logger.error("PLE:#{ple.id}:#{ple.decom_state_string}")
+      return nil
+    end
+    packet_config
+  end
+
+  def wait_for_ready_packet_config(packet_config)
+    first = true
+    while !packet_config.ready
+      if first
+        ready_wait_start = Time.now
+        Cosmos::Logger.info("Waiting for PacketConfig:#{packet_config.id}...")
+        first = false
+      end
+      sleep(1)
+      # Reload the attributes from the database
+      packet_config.reload
+
+      if (Time.now - ready_wait_start) > PACKET_CONFIG_READY_TIMEOUT
+        Cosmos::Logger.fatal("Timeout waiting for ready on PacketConfig:#{packet_config.id}")
+        exit(1)
+      end
+    end
+  end
+
+  def get_values(packet)
+    values = []
+    packet.sorted_items.each do |item|
+      next unless decommutate_item?(item)
+      if separate_raw_con?(item)
+        values << packet.read_item(item, :RAW)
+        values << packet.read_item(item, :CONVERTED)
+      else
+        values << packet.read_item(item, :RAW)
+      end
+    end
+    values
+  end
+
+  def decom_packet(ple, packet, packet_config)
+    # Update packet config times
+    if !packet_config.start_time or (packet.received_time < packet_config.start_time)
+      packet_config.start_time = packet.received_time
+      packet_config.save!
+    end
+    if !packet_config.end_time or (packet.received_time > packet_config.end_time)
+      packet_config.end_time = packet.received_time
+      packet_config.save!
+    end
+
+    # Mark the log entry IN_PROGRESS as we decommutate the data
+    ple.decom_state = PacketLogEntry::IN_PROGRESS
+    ple.save!
+    values = get_values(packet)
+
+    table_index = 0
+    rows = []
+    # Create rows in the decommutation table model
+    values.each_slice(MAX_COLUMNS_PER_TABLE) do |table_values|
+      model = get_decom_table_model(packet_config.id, table_index)
+      row = model.new
+      row.time = ple.time
+      row.ple_id = ple.id
+      row.meta_id = ple.meta_id
+      row.reduced_state = INITIALIZING
+      table_values.each_with_index do |value, index|
+        item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
+        row.write_attribute("i#{item_index}", value)
+      end
+      row.save!
+      rows << row
+      table_index += 1
+    end
+    # Mark ready to reduce
+    rows.each do |row|
+      row.reduced_state = READY_TO_REDUCE
+      row.save!
+    end
+
+    # The log entry has been decommutated, mark COMPLETE
+    ple.decom_state = PacketLogEntry::COMPLETE
+    ple.save!
+    Cosmos::Logger.debug("PLE:#{ple.id}:#{ple.decom_state_string}")
+  end
+end