cosmos 4.2.3-java → 4.2.4-java

data/demo/config/targets/DART/cmd_tlm_server.txt

@@ -0,0 +1,6 @@
+# This is a segment of the main cmd_tlm_server.txt that will be used with
+# AUTO_INTERFACE_TARGETS or INTERFACE_TARGET
+
+INTERFACE DART_INT dart_status_interface.rb
+  TARGET DART
+  DONT_CONNECT

data/demo/config/targets/DART/screens/status.txt

@@ -0,0 +1,54 @@
+SCREEN AUTO AUTO 1.0 FIXED
+
+VERTICAL
+
+  TITLE "DART Status"
+
+  HORIZONTAL
+    VERTICALBOX
+      SECTIONHEADER "Ingest Status"
+
+      LABELVALUE DART STATUS LAST_PLE_ID
+      LABELVALUE DART STATUS PLE_STATE_ERROR
+      LABELVALUE DART STATUS PLE_FIRST_TIME WITH_UNITS 22
+      LABELVALUE DART STATUS PLE_LAST_TIME WITH_UNITS 22
+    END
+
+    VERTICALBOX
+      SECTIONHEADER "Decom Status"
+
+      LABELVALUE DART STATUS PLE_STATE_NEED_DECOM
+      LABELVALUE DART STATUS DECOM_COUNT
+      LABELVALUE DART STATUS DECOM_ERROR_COUNT
+      LABELVALUE DART STATUS DECOM_MESSAGE_TIME WITH_UNITS 22
+      LABEL DECOM_MESSAGE:
+      VALUE DART STATUS DECOM_MESSAGE CONVERTED 50
+    END
+  END
+  HORIZONTAL
+    VERTICALBOX
+      SECTIONHEADER "Reduction Status"
+
+      LABELVALUE DART STATUS REDUCTION_COUNT
+      LABELVALUE DART STATUS REDUCTION_ERROR_COUNT
+      LABELVALUE DART STATUS REDUCTION_MESSAGE_TIME WITH_UNITS 22
+      LABEL REDUCTION_MESSAGE:
+      VALUE DART STATUS REDUCTION_MESSAGE CONVERTED 50
+    END
+
+    VERTICALBOX
+      SECTIONHEADER "Dart Database Status"
+
+      LABELVALUE DART STATUS DART_DATA_BYTES
+      LABELVALUE DART STATUS DART_LOGS_BYTES
+      LABELVALUE DART STATUS DART_DATABASE_BYTES
+      LABELVALUE DART STATUS DART_STATUS_SECONDS
+    END
+
+  END
+
+  VERTICALBOX
+    LABELVALUE DART STATUS RECEIVED_TIMEFORMATTED WITH_UNITS 22
+  END
+END
+

data/demo/config/targets/DART/target.txt

@@ -0,0 +1,7 @@
+# Ignored Parameters
+# IGNORE_PARAMETER parameter_name
+IGNORE_PARAMETER PACKET_ID
+
+# Ignored Items
+# IGNORE_ITEM item_name
+IGNORE_ITEM PACKET_ID

data/ext/cosmos/ext/structure/structure.c

@@ -46,6 +46,7 @@ static ID id_method_slice = 0;
 static ID id_method_reverse = 0;
 static ID id_method_Integer = 0;
 static ID id_method_Float = 0;
+static ID id_method_kind_of = 0;
 
 static ID id_ivar_buffer = 0;
 static ID id_ivar_bit_offset = 0;
@@ -1186,15 +1187,25 @@ static VALUE read_item(int argc, VALUE* argv, VALUE self)
  * offset.
  */
 static VALUE structure_item_spaceship(VALUE self, VALUE other_item) {
-  int bit_offset = FIX2INT(rb_ivar_get(self, id_ivar_bit_offset));
-  int other_bit_offset = FIX2INT(rb_ivar_get(other_item, id_ivar_bit_offset));
+  int bit_offset = 0;
+  int other_bit_offset = 0;
   int bit_size = 0;
   int other_bit_size = 0;
   int create_index = 0;
   int other_create_index = 0;
   int have_create_index = 0;
-  volatile VALUE v_create_index = rb_ivar_get(self, id_ivar_create_index);
-  volatile VALUE v_other_create_index = rb_ivar_get(other_item, id_ivar_create_index);
+  volatile VALUE v_create_index = Qnil;
+  volatile VALUE v_other_create_index = Qnil;
+
+  if (!RTEST(rb_funcall(other_item, id_method_kind_of, 1, cStructureItem))) {
+    return Qnil;
+  }
+
+  bit_offset = FIX2INT(rb_ivar_get(self, id_ivar_bit_offset));
+  other_bit_offset = FIX2INT(rb_ivar_get(other_item, id_ivar_bit_offset));
+
+  v_create_index = rb_ivar_get(self, id_ivar_create_index);
+  v_other_create_index = rb_ivar_get(other_item, id_ivar_create_index);
   if (RTEST(v_create_index) && RTEST(v_other_create_index)) {
     create_index = FIX2INT(v_create_index);
     other_create_index = FIX2INT(v_other_create_index);
@@ -1373,6 +1384,7 @@ void Init_structure (void)
   id_method_reverse = rb_intern("reverse");
   id_method_Integer = rb_intern("Integer");
   id_method_Float = rb_intern("Float");
+  id_method_kind_of = rb_intern("kind_of?");
 
   MIN_INT8 = INT2NUM(-128);
   MAX_INT8 = INT2NUM(127);

data/lib/cosmos/dart/app/models/status.rb

@@ -0,0 +1,2 @@
+class Status < ApplicationRecord
+end

data/lib/cosmos/dart/db/migrate/20180423205644_create_statuses.rb

@@ -0,0 +1,22 @@
+class CreateStatuses < ActiveRecord::Migration[5.1]
+  def change
+    create_table :statuses do |t|
+      t.bigint :decom_count, :default => 0
+      t.bigint :decom_error_count, :default => 0
+      t.text :decom_message, :default => ""
+      t.datetime :decom_message_time
+
+      t.bigint :reduction_count, :default => 0
+      t.bigint :reduction_error_count, :default => 0
+      t.text :reduction_message, :default => ""
+      t.datetime :reduction_message_time
+
+      t.timestamps
+    end
+    status = Status.new
+    time = Time.utc(1970, 1, 1)
+    status.decom_message_time = time
+    status.reduction_message_time = time
+    status.save!
+  end
+end

data/lib/cosmos/dart/db/migrate/20180425211340_add_decom_state_index.rb

@@ -0,0 +1,5 @@
+class AddDecomStateIndex < ActiveRecord::Migration[5.1]
+  def change
+    add_index :packet_log_entries, [:decom_state]
+  end
+end

data/lib/cosmos/dart/db/migrate/20180509204705_remove_indexes.rb

@@ -0,0 +1,29 @@
+class RemoveIndexes < ActiveRecord::Migration[5.1]
+
+  def change
+    remove_index :packet_log_entries, :is_tlm
+    remove_index :packet_log_entries, :target_id
+    remove_index :packet_log_entries, :meta_id
+    remove_index :packet_log_entries, :ready
+    remove_index :packet_log_entries, :packet_id
+    remove_index :packet_log_entries, :packet_log_id
+
+    ActiveRecord::Base.connection.tables.each do |table|
+      # Since the decommutation tables are created dynamically we search
+      # through all the tables looking for tables named something like
+      # tXXX_YYY where XXX is the PacketConfig ID and YYY is the table index
+      if table.to_s =~ /^t(\d+)_(\d+)$/ # ASCII art? No! Regex!
+        packet_config_id = $1.to_i
+        table_index = $2.to_i
+
+        ["", "_m", "_h", "_d"].each do |modifier|
+          table_name = table + modifier
+          remove_index table_name, :meta_id
+          remove_index table_name, :reduced_id
+          remove_index table_name, :reduced_state
+          add_index table_name, :reduced_state, :where => "reduced_state < 2"
+        end
+      end
+    end
+  end
+end

data/lib/cosmos/dart/db/migrate/20180510160002_remove_timestamps_from_ples.rb

@@ -0,0 +1,6 @@
+class RemoveTimestampsFromPles < ActiveRecord::Migration[5.1]
+  def change
+    remove_column :packet_log_entries, :created_at
+    remove_column :packet_log_entries, :updated_at
+  end
+end

data/lib/cosmos/dart/db/migrate/20180511194944_add_packet_log_id_to_tables.rb

@@ -0,0 +1,18 @@
+class AddPacketLogIdToTables < ActiveRecord::Migration[5.1]
+  def change
+    ActiveRecord::Base.connection.tables.each do |table|
+      # Since the decommutation tables are created dynamically we search
+      # through all the tables looking for tables named something like
+      # tXXX_YYY where XXX is the PacketConfig ID and YYY is the table index
+      if table.to_s =~ /^t(\d+)_(\d+)$/ # ASCII art? No! Regex!
+        packet_config_id = $1.to_i
+        table_index = $2.to_i
+
+        ["", "_m", "_h", "_d"].each do |modifier|
+          table_name = table + modifier
+          add_column table_name, :packet_log_id, :integer
+        end
+      end
+    end
+  end
+end

data/lib/cosmos/dart/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180116214338) do
+ActiveRecord::Schema.define(version: 20180511194944) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -54,18 +54,11 @@ ActiveRecord::Schema.define(version: 20180116214338) do
     t.datetime "time", null: false
     t.integer "packet_log_id", null: false
     t.bigint "data_offset", null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.bigint "meta_id"
     t.boolean "is_tlm", null: false
     t.integer "decom_state", default: 0
     t.boolean "ready", default: false
-    t.index ["is_tlm"], name: "index_packet_log_entries_on_is_tlm"
-    t.index ["meta_id"], name: "index_packet_log_entries_on_meta_id"
-    t.index ["packet_id"], name: "index_packet_log_entries_on_packet_id"
-    t.index ["packet_log_id"], name: "index_packet_log_entries_on_packet_log_id"
-    t.index ["ready"], name: "index_packet_log_entries_on_ready"
-    t.index ["target_id"], name: "index_packet_log_entries_on_target_id"
+    t.index ["decom_state"], name: "index_packet_log_entries_on_decom_state"
     t.index ["time"], name: "index_packet_log_entries_on_time"
   end
 
@@ -86,6 +79,19 @@ ActiveRecord::Schema.define(version: 20180116214338) do
     t.index ["target_id", "name", "is_tlm"], name: "index_packets_on_target_id_and_name_and_is_tlm", unique: true
   end
 
+  create_table "statuses", force: :cascade do |t|
+    t.bigint "decom_count", default: 0
+    t.bigint "decom_error_count", default: 0
+    t.text "decom_message", default: ""
+    t.datetime "decom_message_time"
+    t.bigint "reduction_count", default: 0
+    t.bigint "reduction_error_count", default: 0
+    t.text "reduction_message", default: ""
+    t.datetime "reduction_message_time"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "system_configs", force: :cascade do |t|
     t.string "name", null: false
     t.datetime "created_at", null: false

data/lib/cosmos/dart/db/seeds.rb

@@ -5,3 +5,11 @@
 #
 #   movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
 #   Character.create(name: 'Luke', movie: movies.first)
+status = Status.first
+unless status
+  status = Status.new
+  time = Time.utc(1970, 1, 1)
+  status.decom_message_time = time
+  status.reduction_message_time = time
+  status.save!
+end

data/lib/cosmos/dart/lib/dart_common.rb

@@ -165,6 +165,7 @@ module DartCommon
         t.bigint :ple_id
         t.bigint :meta_id
         t.bigint :reduced_id
+        t.integer :packet_log_id
         t.integer :reduced_state, :default => 0
         table_data_types.each_with_index do |data_type, index|
           item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
@@ -183,9 +184,7 @@ module DartCommon
           end
         end
         t.index :time
-        t.index :meta_id
-        t.index :reduced_state
-        t.index :reduced_id
+        t.index :reduced_state, :where => "reduced_state < 2"
       end
       create_reduction_table("t#{packet_config.id}_#{table_index}_h", table_data_types, table_index) # hour
       create_reduction_table("t#{packet_config.id}_#{table_index}_m", table_data_types, table_index) # month
@@ -524,6 +523,7 @@ module DartCommon
   # @param array_size [Integer, nil] Size of the array or nil if no array
   # @return [Symbol] Database type such as :integer, :bigint, :string, etc.
   def cosmos_data_type_to_db_type(data_type, bit_size, array_size)
+    return nil if data_type.nil? or bit_size.nil?
     db_type = nil
     case data_type
     when :INT
@@ -674,6 +674,7 @@ module DartCommon
       t.integer :num_samples
       t.bigint :meta_id
       t.bigint :reduced_id
+      t.integer :packet_log_id
       t.integer :reduced_state, :default => 0
       table_data_types.each_with_index do |data_type, index|
         item_index = (table_index * MAX_COLUMNS_PER_TABLE) + index
@@ -686,9 +687,7 @@ module DartCommon
         end
       end
       t.index :start_time
-      t.index :meta_id
-      t.index :reduced_state
-      t.index :reduced_id
+      t.index :reduced_state, :where => "reduced_state < 2"
     end
   end
 

data/lib/cosmos/dart/lib/dart_database_cleaner.rb

@@ -9,6 +9,7 @@
 # attribution addendums as found in the LICENSE.txt
 
 require 'dart_common'
+require 'cosmos/packet_logs/packet_log_reader'
 
 class DartDatabaseCleaner
   include DartCommon
@@ -17,18 +18,68 @@ class DartDatabaseCleaner
   # if the DART packet logs were moved and force is false (default). This is
   # deliberate because force causes all the lost (or moved) files to be deleted
   # which forces them to be re-imported at their new location.
-  def self.clean(force)
+  def self.clean(force, full = false)
     Cosmos::Logger::info("Starting database cleanup...")
     cleaner = DartDatabaseCleaner.new
     cleaner.clean_system_configs()
     cleaner.clean_packet_logs(force)
     cleaner.clean_packet_configs()
     cleaner.clean_packet_log_entries()
-    cleaner.clean_decommutation_tables()
-    cleaner.clean_reductions()
+    if full
+      cleaner.clean_decommutation_tables()
+      cleaner.clean_reductions()
+    end
     Cosmos::Logger::info("Database cleanup complete!")
   end
 
+  def remove_packet_log(filename)
+    filename = filename.gsub("\\", "/") # Fix slashes
+    filename = File.expand_path(filename, Cosmos::System.paths['DART_DATA']) # Make absolute path
+    if File.exists?(filename)
+      packet_log = PacketLog.where("filename = ?", filename).first
+      if packet_log
+        size = File.size(filename)
+        reader = Cosmos::PacketLogReader.new
+        reader.open(filename)
+        first_packet = nil
+        last_packet = nil
+        start_time = nil
+        end_time = nil
+        begin
+          first_packet = reader.first
+          last_packet = reader.last
+          start_time = first_packet.received_time
+          end_time = last_packet.received_time
+        rescue
+          if size == Cosmos::PacketLogReader::COSMOS2_HEADER_LENGTH or size == 0
+            Cosmos::Logger::error("File contains no packets: #{filename}")
+          else
+            Cosmos::Logger::error("Error analyzing file: #{filename}")
+          end
+          exit(1)
+        ensure
+          reader.close
+        end
+        num_deleted = PacketLogEntry.where("time >= ? and time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.minute, packet_log.id).delete_all
+        Cosmos::Logger::info("Removed #{num_deleted} PacketLogEntries")
+        each_decom_and_reduced_table() do |packet_config_id, table_index, decom_model, minute_model, hour_model, day_model|
+          num_deleted = 0
+          num_deleted += decom_model.where("time >= ? and time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.minute, packet_log.id).delete_all
+          num_deleted += minute_model.where("start_time >= ? and start_time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 2.minutes, packet_log.id).delete_all
+          num_deleted += hour_model.where("start_time >= ? and start_time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.hour + 1.minutes, packet_log.id).delete_all
+          num_deleted += day_model.where("start_time >= ? and start_time <= ? and packet_log_id = ?", start_time - 1.minute, end_time + 1.hour + 1.day, packet_log.id).delete_all
+          Cosmos::Logger::info("Deleted #{num_deleted} rows from #{decom_model.table_name} and reductions")
+        end
+        packet_log.delete
+        Cosmos::Logger::info("Packet Log Id #{packet_log.id} deleted")
+      else
+        Cosmos::Logger::error("File does not exist in database: #{filename}")
+      end
+    else
+      Cosmos::Logger::error("File does not exist: #{filename}")
+    end
+  end
+
   # Ensure we have all the System Configs locally on the DART machine
   def clean_system_configs
     Cosmos::Logger::info("Cleaning up SystemConfig...")
@@ -95,7 +146,7 @@ class DartDatabaseCleaner
         end
       rescue => err
         Cosmos::Logger::error("Error cleaning up packet config: #{packet_config.id}: #{err.formatted}")
-        raise "Cleanup failure - Database requires manual correction"
+        raise $!, "Cleanup failure - Database requires manual correction: #{err.message}", $!.backtrace
       end
     end
   end
@@ -108,7 +159,7 @@ class DartDatabaseCleaner
     # outstanding entries which are not ready while packets are being received.
     # Note the normal shutdown process attempts to flush the log file and mark
     # all outstanding entries as ready so this would only happen during a crash.
-    ples = PacketLogEntry.where("ready != true")
+    ples = PacketLogEntry.where("decom_state = 0 and ready != true")
     return unless ples.length > 0
     Cosmos::Logger::info("Removing unready packet log entries: #{ples.length}")
     ples.destroy_all

data/lib/cosmos/dart/lib/dart_decom_query.rb

@@ -53,7 +53,7 @@ class DartDecomQuery
   #   for other reduction values), and meta_id.
   def query(request)
     request_start_time = Time.now
-    Cosmos::Logger.info("#{request_start_time.formatted}: QUERY: #{request}")
+    Cosmos::Logger.info("#{request_start_time.formatted}: query: #{request}")
 
     begin
       start_time_sec = request['start_time_sec']
@@ -136,7 +136,7 @@ class DartDecomQuery
       unless meta_ids.length > 0
         meta_filters = request['meta_filters']
         meta_filters ||= []
-        
+
         if meta_filters.length > 0
           meta_ids = process_meta_filters(meta_filters, is_tlm, end_time)
         end
@@ -149,10 +149,10 @@ class DartDecomQuery
       offset = 0 if offset < 0
 
       return query_decom_reduced(
-        item[0], item[1], item[2], 
-        value_type, is_tlm, 
-        start_time, end_time, 
-        reduction, reduction_modifier, 
+        item[0], item[1], item[2],
+        value_type, is_tlm,
+        start_time, end_time,
+        reduction, reduction_modifier,
         item_name_modifier, limit, offset, meta_ids)
 
     rescue Exception => error
@@ -168,6 +168,8 @@ class DartDecomQuery
   # @param is_tlm true or false
   # @return [Array<String>] Array of item names
   def item_names(target_name, packet_name, is_tlm = true)
+    Cosmos::Logger.info("#{time.formatted}: item_names")
+
     target = Target.where("name = ?", target_name).first
     raise "Target #{target_name} not found" unless target
 
@@ -181,4 +183,104 @@ class DartDecomQuery
     return item_names
   end
 
+  # Returns status on the DART Database
+  def dart_status
+    start_time = Time.now
+    Cosmos::Logger.info("#{start_time.formatted}: dart_status")
+    result = {}
+    status = Status.first
+
+    # Ingest Status
+    # ---------------
+    # Last PacketLogEntry Id
+    last_ple = PacketLogEntry.select("id").last
+    if last_ple
+      result[:LAST_PLE_ID] = last_ple.id
+    else
+      result[:LAST_PLE_ID] = -1
+    end
+    # Num PacketLogEntries Needing Decom state = 0
+    result[:PLE_STATE_NEED_DECOM] = PacketLogEntry.where("decom_state = 0").count
+    # Num PacketLogEntries Errored - state >= 3
+    result[:PLE_STATE_ERROR] = PacketLogEntry.where("decom_state >= 3").count
+    # First Time in Database
+    sort_first_ple = PacketLogEntry.order("time ASC").select("time").first
+    if sort_first_ple
+      result[:PLE_FIRST_TIME_S] = sort_first_ple.time.tv_sec
+      result[:PLE_FIRST_TIME_US] = sort_first_ple.time.tv_usec
+    else
+      result[:PLE_FIRST_TIME_S] = 0
+      result[:PLE_FIRST_TIME_US] = 0
+    end
+    # Last Time in Database
+    sort_last_ple = PacketLogEntry.order("time DESC").select("time").first
+    if sort_last_ple
+      result[:PLE_LAST_TIME_S] = sort_last_ple.time.tv_sec
+      result[:PLE_LAST_TIME_US] = sort_last_ple.time.tv_usec
+    else
+      result[:PLE_LAST_TIME_S] = 0
+      result[:PLE_LAST_TIME_US] = 0
+    end
+
+    # Decom Status
+    # ---------------
+    # Decom Count
+    result[:DECOM_COUNT] = status.decom_count
+    # Decom Errors
+    result[:DECOM_ERROR_COUNT] = status.decom_error_count
+    # Decom Message
+    result[:DECOM_MESSAGE] = status.decom_message
+    # Decom Message Time
+    result[:DECOM_MESSAGE_TIME_S] = status.decom_message_time.tv_sec
+    result[:DECOM_MESSAGE_TIME_US] = status.decom_message_time.tv_usec
+
+    # Reduction Status
+    # ---------------
+    # Reduction Count
+    result[:REDUCTION_COUNT] = status.reduction_count
+    # Reduction Errors
+    result[:REDUCTION_ERROR_COUNT] = status.reduction_error_count
+    # Reduction Message
+    result[:REDUCTION_MESSAGE] = status.reduction_message
+    # Reduction Time
+    result[:REDUCTION_MESSAGE_TIME_S] = status.reduction_message_time.tv_sec
+    result[:REDUCTION_MESSAGE_TIME_US] = status.reduction_message_time.tv_usec
+
+    # Storage
+    # ---------------
+    Cosmos.set_working_dir do
+      # Size of outputs/dart/data folder
+      result[:DART_DATA_BYTES] = Dir.glob(File.join(Cosmos::System.paths['DART_DATA'], '**', '*')).map{ |f| File.size(f) }.inject(:+)
+      # Size of outputs/dart/logs folder
+      result[:DART_LOGS_BYTES] = Dir.glob(File.join(Cosmos::System.paths['DART_LOGS'], '**', '*')).map{ |f| File.size(f) }.inject(:+)
+    end
+    # Size of SQL Database
+    begin
+      result[:DART_DATABASE_BYTES] = ActiveRecord::Base.connection.execute("select pg_database_size('#{ActiveRecord::Base.connection_config[:database]}');")[0]['pg_database_size']
+    rescue
+      result[:DART_DATABASE_BYTES] = -1
+    end
+
+    end_time = Time.now
+    delta = end_time - start_time
+    result[:DART_STATUS_SECONDS] = delta
+
+    return result
+  end
+
+  def clear_errors
+    time = Time.now
+    Cosmos::Logger.info("#{time.formatted}: clear_errors")
+    status = Status.first
+    status.decom_error_count = 0
+    status.decom_message = ''
+    status.decom_message_time = time
+    status.reduction_error_count = 0
+    status.reduction_message = ''
+    status.reduction_message_time = time
+    status.save!
+
+    return nil
+  end
+
 end