cosmos 4.5.2 → 5.0.2.pre.beta2

data/lib/cosmos/dart/lib/dart_constants.rb

@@ -1,15 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-# Implement constants common to DART (Data Archival Retrieval and Trending).
-module DartConstants
-  # @return [Integer] maximum number of results returned from a decommutation query
-  MAX_DECOM_RESULTS = 10000
-end

data/lib/cosmos/dart/lib/dart_database_cleaner.rb

@@ -1,223 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-require 'dart_common'
-require 'cosmos/packet_logs/packet_log_reader'
-
-class DartDatabaseCleaner
-  include DartCommon
-
-  # Clean the DART database of various issues. Note that this code can exit(1)
-  # if the DART packet logs were moved and force is false (default). This is
-  # deliberate because force causes all the lost (or moved) files to be deleted
-  # which forces them to be re-imported at their new location.
-  def self.clean(force, full = false)
-    Cosmos::Logger::info("Starting database cleanup...")
-    cleaner = DartDatabaseCleaner.new
-    cleaner.clean_system_configs()
-    cleaner.clean_packet_logs(force)
-    cleaner.clean_packet_configs()
-    cleaner.clean_packet_log_entries()
-    if full
-      cleaner.clean_decommutation_tables()
-      cleaner.clean_reductions()
-    end
-    Cosmos::Logger::info("Database cleanup complete!")
-  end
-
-  def remove_packet_log(filename)
-    filename = filename.gsub("\\", "/") # Fix slashes
-    filename = File.expand_path(filename, Cosmos::System.paths['DART_DATA']) # Make absolute path
-    if File.exists?(filename)
-      packet_log = PacketLog.where("filename = ?", filename).first
-      if packet_log
-        size = File.size(filename)
-        reader = Cosmos::PacketLogReader.new
-        reader.open(filename)
-        first_packet = nil
-        last_packet = nil
-        start_time = nil
-        end_time = nil
-        begin
-          first_packet = reader.first
-          last_packet = reader.last
-          start_time = first_packet.packet_time
-          end_time = last_packet.packet_time
-        rescue
-          if size == Cosmos::PacketLogReader::COSMOS2_HEADER_LENGTH or size == 0
-            Cosmos::Logger::error("File contains no packets: #{filename}")
-          else
-            Cosmos::Logger::error("Error analyzing file: #{filename}")
-          end
-          exit(1)
-        ensure
-          reader.close
-        end
-        num_deleted = PacketLogEntry.where("time >= ? and time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.minute, packet_log.id).delete_all
-        Cosmos::Logger::info("Removed #{num_deleted} PacketLogEntries")
-        each_decom_and_reduced_table() do |packet_config_id, table_index, decom_model, minute_model, hour_model, day_model|
-          num_deleted = 0
-          num_deleted += decom_model.where("time >= ? and time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.minute, packet_log.id).delete_all
-          num_deleted += minute_model.where("start_time >= ? and start_time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 2.minutes, packet_log.id).delete_all
-          num_deleted += hour_model.where("start_time >= ? and start_time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.hour + 1.minutes, packet_log.id).delete_all
-          num_deleted += day_model.where("start_time >= ? and start_time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.hour + 1.day, packet_log.id).delete_all
-          Cosmos::Logger::info("Deleted #{num_deleted} rows from #{decom_model.table_name} and reductions")
-        end
-        packet_log.delete
-        Cosmos::Logger::info("Packet Log Id #{packet_log.id} deleted")
-      else
-        Cosmos::Logger::error("File does not exist in database: #{filename}")
-      end
-    else
-      Cosmos::Logger::error("File does not exist: #{filename}")
-    end
-  end
-
-  # Ensure we have all the System Configs locally on the DART machine
-  def clean_system_configs
-    Cosmos::Logger::info("Cleaning up SystemConfig...")
-    system_config = SystemConfig.all.each do |sc|
-       begin
-        # This attempts to load the system config and if it can't be found
-        # it is copied from the server to the local DART machine
-        switch_and_get_system_config(sc.name)
-      rescue => err
-        Cosmos::Logger.error("Could not load system_config: #{sc.name}: #{err.message}")
-        next
-      end
-    end
-    Cosmos::System.load_configuration
-  end
-
-  # Ensure all packet log files still exist
-  def clean_packet_logs(force)
-    Cosmos::Logger::info("Cleaning up PacketLog...")
-    PacketLog.find_each do |pl|
-      unless File.exist?(pl.filename)
-        # Try to see if it is in the current DART_DATA folder
-        moved_filename = File.join(Cosmos::System.paths['DART_DATA'], File.basename(pl.filename))
-        if File.exist?(moved_filename)
-          pl.filename = moved_filename
-          pl.save!
-        else
-          if force
-            Cosmos::Logger.error("Packet Log File Missing: #{pl.filename}")
-            pl.destroy # This also destroys linked PacketLogEntry(s)
-          else
-            Cosmos::Logger.fatal("Packet Log File Missing (Cleanup with --force-cleanup): #{pl.filename}")
-            exit(1)
-          end
-        end
-      end
-    end
-  end
-
-  # Check for bad packet configs and cleanup. This typically doesn't happen because
-  # PacketConfig entries are created quickly but if something happens during the setup
-  # we attempt to fix it here.
-  def clean_packet_configs
-    Cosmos::Logger::info("Cleaning up PacketConfig...")
-    packet_configs = PacketConfig.where("ready != true")
-    return unless packet_configs.length > 0
-    Cosmos::Logger::info("Num PacketConfigs requiring cleanup: #{packet_configs.length}")
-    packet_configs.each do |packet_config|
-      begin
-        system_config = SystemConfig.find(packet_config.first_system_config_id)
-        packet_model = Packet.find(packet_config.packet_id)
-        target_model = Target.find(packet_model.target_id)
-        current_config, error = Cosmos::System.load_configuration(system_config.name)
-        if current_config == system_config.name
-          if packet_model.is_tlm
-            packet = Cosmos::System.telemetry.packet(target_model.name, packet_model.name)
-          else
-            packet = Cosmos::System.commands.packet(target_model.name, packet_model.name)
-          end
-          setup_packet_config(packet, packet_model.id, packet_config)
-          Cosmos::Logger::info("Successfully cleaned up packet_config: #{packet_config.id}")
-        else
-          Cosmos::Logger::error("Could not switch to system config: #{system_config.name}: #{error}")
-        end
-      rescue => err
-        Cosmos::Logger::error("Error cleaning up packet config: #{packet_config.id}: #{err.formatted}")
-        raise $!, "Cleanup failure - Database requires manual correction: #{err.message}", $!.backtrace
-      end
-    end
-  end
-
-  # Remove not ready packet log entries as well as partially decommutated data.
-  def clean_packet_log_entries
-    Cosmos::Logger::info("Cleaning up PacketLogEntry...")
-    # Remove not ready packet log entries. Packet log entries remain not ready until the
-    # log file containing the packet has been flushed to disk. Thus there are always
-    # outstanding entries which are not ready while packets are being received.
-    # Note the normal shutdown process attempts to flush the log file and mark
-    # all outstanding entries as ready so this would only happen during a crash.
-    ples = PacketLogEntry.where("decom_state = 0 and ready != true")
-    return unless ples.length > 0
-    Cosmos::Logger::info("Removing unready packet log entries: #{ples.length}")
-    ples.destroy_all
-  end
-
-  def clean_decommutation_tables
-    Cosmos::Logger::info("Cleaning up Decommutation tables (tX_Y)...")
-    # Check for partially decom data and remove. The DartWorker periodically checks the
-    # database for a PacketLogEntry which is ready to be decommutated and starts the
-    # process of writing into the decommutation table. If this process is interrupted
-    # the state could be IN_PROGRESS instead of COMPLETE. Thus delete all the decommutation
-    # table rows which were created and allow this process to start from scratch.
-    ples = PacketLogEntry.where("decom_state = #{PacketLogEntry::IN_PROGRESS}")
-    return unless ples.length > 0
-    Cosmos::Logger::info("Num PacketLogEntries requiring cleanup: #{ples.length}")
-    ples.each do |ple|
-      begin
-        packet = read_packet_from_ple(ple)
-        packet_config = PacketConfig.where("packet_id = ? and name = ?", ple.packet_id, packet.config_name).first
-        # Need to delete any rows for these ples in the table for this packet_config
-        packet_config.max_table_index.times do |table_index|
-          model = get_decom_table_model(packet_config.id, table_index)
-          model.where("ple_id = ?", ple.id).destroy_all
-        end
-        ple.decom_state = PacketLogEntry::NOT_STARTED
-        ple.save!
-      rescue => err
-        Cosmos::Logger::error("Error cleaning up packet log entry: #{ple.id}: #{err.formatted}")
-      end
-    end
-  end
-
-  def clean_reductions
-    Cosmos::Logger::info("Cleaning up Reductions...")
-    # TBR: This cleanup may be too slow to be worth it for a large data set...
-    each_decom_and_reduced_table() do |packet_config_id, table_index, decom_model, minute_model, hour_model, day_model|
-      decom_model.where("reduced_state = #{REDUCED} and reduced_id IS NULL").update_all(:reduced_state => READY_TO_REDUCE)
-      minute_model.where("reduced_state = #{REDUCED} and reduced_id IS NULL").update_all(:reduced_state => READY_TO_REDUCE)
-      hour_model.where("reduced_state = #{REDUCED} and reduced_id IS NULL").update_all(:reduced_state => READY_TO_REDUCE)
-      # Note: These should be destroyed when cleaning up decom tables
-      rows = decom_model.where("reduced_state = #{INITIALIZING}")
-      rows.destroy_all
-      # This cleanup is only here
-      rows = minute_model.where("reduced_state = #{INITIALIZING}")
-      rows.each do |row|
-        decom_model.where("reduced_id = ?", row.id).update_all(:reduced_state => READY_TO_REDUCE, :reduced_id => nil)
-      end
-      rows.destroy_all
-      rows = hour_model.where("reduced_state = #{INITIALIZING}")
-      rows.each do |row|
-        minute_model.where("reduced_id = ?", row.id).update_all(:reduced_state => READY_TO_REDUCE, :reduced_id => nil)
-      end
-      rows.destroy_all
-      rows = day_model.where("reduced_state = #{INITIALIZING}")
-      rows.each do |row|
-        hour_model.where("reduced_id = ?", row.id).update_all(:reduced_state => READY_TO_REDUCE, :reduced_id => nil)
-      end
-      rows.destroy_all
-    end
-  end
-end

data/lib/cosmos/dart/lib/dart_decom_query.rb

@@ -1,285 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2018 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-
-require 'dart_common'
-require 'dart_logging'
-
-# Rails Json screws up COSMOS handling of Nan, etc.
-require "active_support/core_ext/object/json"
-module ActiveSupport
-  module ToJsonWithActiveSupportEncoder # :nodoc:
-    def to_json(options = nil)
-      super(options)
-    end
-  end
-end
-
-# JsonDRb server which responds to queries for decommutated and reduced data
-# from the database.
-class DartDecomQuery
-  include DartCommon
-
-  # Returns data from the decommutated database tables including the reduced data tables.
-  #
-  # @param request [Hash] Request for data.
-  #   The hash must contain the following items:
-  #     start_time_sec => Start time in UTC seconds
-  #     start_time_usec => Microseconds to add to start time
-  #     end_time_sec => End time in UTC seconds
-  #     end_time_usec => Microseconds to add to end time
-  #     item => [target name, packet name, item name] Names are all strings
-  #     reduction => "NONE", "MINUTE", "HOUR", "DAY" for how to reduce the data
-  #     value_type => "RAW", "RAW_MAX", "RAW_MIN", "RAW_AVG",
-  #       "CONVERTED", "CONVERTED_MAX", "CONVERTED_MIN", "CONVERTED_AVG"
-  #
-  #   The request can also contain the following optional items:
-  #     meta_ids => Optional IDs related to the meta data you want to filter by. This requires
-  #       making a separate request for the particular meta data in question and recording
-  #       the returned meta_ids for use in a subsequent request.
-  #     limit => Maximum number of data items to return, must be less than DartCommon::MAX_DECOM_RESULTS
-  #     offset => Offset into the data stream. Use this to get more than the DartCommon::MAX_DECOM_RESULTS
-  #       by making multipe requests with multiples of the DartCommon::MAX_DECOM_RESULTS value.
-  #     cmd_tlm => Whether the item is a command or telemetry. Default is telemetry.
-  # @return [Array<Array<String, Integer, Integer, Integer, Integer>>] Array of arrays containing
-  #   the item name, item seconds, item microseconds, samples (always 1 for NONE reduction, varies
-  #   for other reduction values), and meta_id.
-  def query(request)
-    request_start_time = Time.now
-    Cosmos::Logger.info("#{request_start_time.formatted}: query: #{request}")
-
-    begin
-      start_time_sec = request['start_time_sec']
-      start_time_usec = request['start_time_usec']
-      end_time_sec = request['end_time_sec']
-      end_time_usec = request['end_time_usec']
-
-      start_time = nil
-      end_time = nil
-      begin
-        start_time = Time.at(start_time_sec, start_time_usec) if start_time_sec and start_time_usec
-      rescue
-        raise "Invalid start time: #{start_time_sec}, #{start_time_usec}"
-      end
-      begin
-        end_time = Time.at(end_time_sec, end_time_usec) if end_time_sec and end_time_usec
-      rescue
-        raise "Invalid end time: #{end_time_sec}, #{end_time_usec}"
-      end
-
-      item = request['item']
-      raise "Item \"#{item}\" invalid" if !item || item.length != 3
-
-      reduction = request['reduction'].to_s.upcase
-      case reduction
-      when "", "NONE"
-        reduction = :NONE
-        reduction_modifier = ""
-      when "MINUTE"
-        reduction = :MINUTE
-        reduction_modifier = "_m"
-      when "HOUR"
-        reduction = :HOUR
-        reduction_modifier = "_h"
-      when "DAY"
-        reduction = :DAY
-        reduction_modifier = "_d"
-      else
-        raise "Unknown reduction: #{reduction}"
-      end
-
-      requested_value_type = request['value_type'].to_s.upcase
-      case requested_value_type
-      when 'RAW'
-        value_type = ItemToDecomTableMapping::CONVERTED
-        item_name_modifier = ""
-        raise "RAW value_type is only valid with NONE reduction" if reduction != :NONE
-      when 'RAW_MAX', 'RAW_MIN', 'RAW_AVG', 'RAW_STDDEV'
-        value_type = ItemToDecomTableMapping::CONVERTED
-        item_name_modifier = requested_value_type.split('_')[1].downcase
-        raise "#{requested_value_type} value_type is not valid with NONE reduction" if reduction == :NONE
-      when 'CONVERTED'
-        value_type = ItemToDecomTableMapping::RAW
-        item_name_modifier = ""
-        raise "CONVERTED value_type is only valid with NONE reduction" if reduction != :NONE
-      when 'CONVERTED_MAX', 'CONVERTED_MIN', 'CONVERTED_AVG', 'CONVERTED_STDDEV'
-        value_type = ItemToDecomTableMapping::RAW
-        item_name_modifier = requested_value_type.split('_')[1].downcase
-        raise "#{requested_value_type} value_type is not valid with NONE reduction" if reduction == :NONE
-      else
-        raise "Unknown value_type: #{requested_value_type}"
-      end
-
-      cmd_tlm = request['cmd_tlm']
-      if cmd_tlm
-        if cmd_tlm.to_s.upcase == 'CMD'
-          is_tlm = false
-        elsif cmd_tlm.to_s.upcase == 'TLM'
-          is_tlm = true
-        else
-          raise "Unknown cmd_tlm: #{cmd_tlm}"
-        end
-      else
-        is_tlm = true
-      end
-
-      meta_ids = request['meta_ids']
-      meta_ids ||= []
-
-      unless meta_ids.length > 0
-        meta_filters = request['meta_filters']
-        meta_filters ||= []
-
-        if meta_filters.length > 0
-          meta_ids = process_meta_filters(meta_filters, is_tlm, end_time)
-        end
-      end
-
-      limit = request['limit'].to_i
-      limit = MAX_DECOM_RESULTS if limit <= 0 or limit > MAX_DECOM_RESULTS
-
-      offset = request['offset'].to_i
-      offset = 0 if offset < 0
-
-      return query_decom_reduced(
-        item[0], item[1], item[2],
-        value_type, is_tlm,
-        start_time, end_time,
-        reduction, reduction_modifier,
-        item_name_modifier, limit, offset, meta_ids)
-
-    rescue Exception => error
-      msg = "Query Error: #{error.message}"
-      raise $!, msg, $!.backtrace
-    end
-  end
-
-  # Gets the list of item names for a given packet
-  #
-  # @param target_name Target name
-  # @param packet_name Packet name
-  # @param is_tlm true or false
-  # @return [Array<String>] Array of item names
-  def item_names(target_name, packet_name, is_tlm = true)
-    Cosmos::Logger.info("#{Time.now.formatted}: item_names")
-
-    target = Target.where("name = ?", target_name).first
-    raise "Target #{target_name} not found" unless target
-
-    packet = Packet.where("target_id = ? and name = ? and is_tlm = ?", target.id, packet_name, is_tlm).first
-    raise "Packet #{target_name} #{packet_name} not found" unless packet
-
-    items = Item.where("packet_id = ?", packet.id).select("name")
-    item_names = []
-    items.each { |item| item_names << item.name }
-
-    return item_names
-  end
-
-  # Returns status on the DART Database
-  def dart_status
-    start_time = Time.now
-    Cosmos::Logger.info("#{start_time.formatted}: dart_status")
-    result = {}
-    status = Status.first
-
-    # Ingest Status
-    # ---------------
-    # Last PacketLogEntry Id
-    last_ple = PacketLogEntry.select("id").last
-    if last_ple
-      result[:LAST_PLE_ID] = last_ple.id
-    else
-      result[:LAST_PLE_ID] = -1
-    end
-    # Num PacketLogEntries Needing Decom state = 0
-    result[:PLE_STATE_NEED_DECOM] = PacketLogEntry.where("decom_state = 0").count
-    # Num PacketLogEntries Errored - state >= 3
-    result[:PLE_STATE_ERROR] = PacketLogEntry.where("decom_state >= 3").count
-    # First Time in Database
-    sort_first_ple = PacketLogEntry.order("time ASC").select("time").first
-    if sort_first_ple
-      result[:PLE_FIRST_TIME_S] = sort_first_ple.time.tv_sec
-      result[:PLE_FIRST_TIME_US] = sort_first_ple.time.tv_usec
-    else
-      result[:PLE_FIRST_TIME_S] = 0
-      result[:PLE_FIRST_TIME_US] = 0
-    end
-    # Last Time in Database
-    sort_last_ple = PacketLogEntry.order("time DESC").select("time").first
-    if sort_last_ple
-      result[:PLE_LAST_TIME_S] = sort_last_ple.time.tv_sec
-      result[:PLE_LAST_TIME_US] = sort_last_ple.time.tv_usec
-    else
-      result[:PLE_LAST_TIME_S] = 0
-      result[:PLE_LAST_TIME_US] = 0
-    end
-
-    # Decom Status
-    # ---------------
-    # Decom Count
-    result[:DECOM_COUNT] = status.decom_count
-    # Decom Errors
-    result[:DECOM_ERROR_COUNT] = status.decom_error_count
-    # Decom Message
-    result[:DECOM_MESSAGE] = status.decom_message
-    # Decom Message Time
-    result[:DECOM_MESSAGE_TIME_S] = status.decom_message_time.tv_sec
-    result[:DECOM_MESSAGE_TIME_US] = status.decom_message_time.tv_usec
-
-    # Reduction Status
-    # ---------------
-    # Reduction Count
-    result[:REDUCTION_COUNT] = status.reduction_count
-    # Reduction Errors
-    result[:REDUCTION_ERROR_COUNT] = status.reduction_error_count
-    # Reduction Message
-    result[:REDUCTION_MESSAGE] = status.reduction_message
-    # Reduction Time
-    result[:REDUCTION_MESSAGE_TIME_S] = status.reduction_message_time.tv_sec
-    result[:REDUCTION_MESSAGE_TIME_US] = status.reduction_message_time.tv_usec
-
-    # Storage
-    # ---------------
-    Cosmos.set_working_dir do
-      # Size of outputs/dart/data folder
-      result[:DART_DATA_BYTES] = Dir.glob(File.join(Cosmos::System.paths['DART_DATA'], '**', '*')).map{ |f| File.size(f) }.inject(:+)
-      # Size of outputs/dart/logs folder
-      result[:DART_LOGS_BYTES] = Dir.glob(File.join(Cosmos::System.paths['DART_LOGS'], '**', '*')).map{ |f| File.size(f) }.inject(:+)
-    end
-    # Size of SQL Database
-    begin
-      result[:DART_DATABASE_BYTES] = ActiveRecord::Base.connection.execute("select pg_database_size('#{ActiveRecord::Base.connection_config[:database]}');")[0]['pg_database_size']
-    rescue
-      result[:DART_DATABASE_BYTES] = -1
-    end
-
-    end_time = Time.now
-    delta = end_time - start_time
-    result[:DART_STATUS_SECONDS] = delta
-
-    return result
-  end
-
-  def clear_errors
-    time = Time.now
-    Cosmos::Logger.info("#{time.formatted}: clear_errors")
-    status = Status.first
-    status.decom_error_count = 0
-    status.decom_message = ''
-    status.decom_message_time = time
-    status.reduction_error_count = 0
-    status.reduction_message = ''
-    status.reduction_message_time = time
-    status.save!
-
-    return nil
-  end
-
-end