cosmos 4.1.1 → 4.2.0

data/lib/cosmos/dart/processes/dart_reducer.rb

@@ -0,0 +1,27 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_reducer_manager'
+require 'dart_logging'
+
+# Start the DartReducer
+Cosmos.catch_fatal_exception do
+  DartCommon.handle_argv
+
+  Cosmos::Logger.level = Cosmos::Logger::INFO
+  dart_logging = DartLogging.new('dart_reducer')
+  num_threads = ENV['DART_NUM_REDUCERS']
+  num_threads ||= 5
+  num_threads = num_threads.to_i
+  drm = DartReducerManager.new(num_threads)
+  drm.run
+  dart_logging.stop
+end

data/lib/cosmos/dart/processes/dart_stream_server.rb

@@ -0,0 +1,31 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_tcpip_server_interface'
+require 'dart_logging'
+
+Cosmos.catch_fatal_exception do
+  DartCommon.handle_argv
+
+  Cosmos::Logger.level = Cosmos::Logger::INFO
+  dart_logging = DartLogging.new('dart_stream_server')
+
+  dts = DartTcpipServerInterface.new
+  dts.connect
+  Cosmos::Logger.info("Dart Stream Server Started...")
+  begin
+    sleep(1) while true
+  rescue Interrupt
+    Cosmos::Logger.info("Dart Stream Server Closing...")
+    dart_logging.stop
+    exit(0)
+  end
+end

data/lib/cosmos/dart/processes/dart_worker.rb

@@ -0,0 +1,37 @@
+# encoding: ascii-8bit
+
+# Copyright 2018 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require File.expand_path('../../config/environment', __FILE__)
+require 'dart_common'
+require 'dart_logging'
+require 'dart_decommutator'
+
+Cosmos.catch_fatal_exception do
+  DartCommon.handle_argv
+
+  # 0-based worker ID
+  worker_id = ARGV[0]
+  worker_id ||= 0
+  worker_id = worker_id.to_i
+  # Total number of workers
+  num_workers = ARGV[1]
+  num_workers ||= 1
+  num_workers = num_workers.to_i
+
+  Cosmos::Logger.level = Cosmos::Logger::INFO
+  dart_logging = DartLogging.new("dart_worker_#{worker_id}")
+  Cosmos::Logger.info("Dart Worker Starting...")
+  raise "Worker count #{num_workers} invalid" if num_workers < 1
+  raise "Worker id #{worker_id} too high for worker count of #{num_workers}" if worker_id >= num_workers
+  decom = DartDecommutator.new(worker_id, num_workers)
+  decom.run # Blocks forever
+  shutdown_cmd_tlm()
+  dart_logging.stop
+end

data/lib/cosmos/dart/spec/dart/dart_common_spec.rb

@@ -0,0 +1,333 @@
+# encoding: ascii-8bit
+
+# Copyright 2014 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'rails_helper'
+require 'dart_common'
+require 'database_cleaner'
+require 'cosmos/tools/cmd_tlm_server/cmd_tlm_server'
+require 'cosmos/tools/cmd_tlm_server/api'
+
+describe DartCommon do
+  let(:common) { Object.new.extend(DartCommon) }
+
+  describe "sync_targets_and_packets" do
+    it "configures the database" do
+      targets = Cosmos::System.telemetry.all.keys.sort
+      expect(targets).to eql Cosmos::System.commands.all.keys.sort
+
+      # Put all the known targets and packets into the DB
+      common.sync_targets_and_packets
+
+      # Verify the targets
+      Target.all.order(:name).each_with_index do |target, i|
+        expect(target.name).to eq targets[i]
+      end
+      # Verify the telemetry packets
+      Cosmos::System.telemetry.all.each do |target_name, packets|
+        target = Target.find_by_name(target_name)
+        expect(target.name).to eq target_name
+        packets.each do |name, packet|
+          pkt = Packet.where({target: target, name: name, is_tlm: true}).first
+          expect(pkt.name).to eq name
+          expect(pkt.is_tlm).to eq true
+          expect(pkt.target.name).to eq target_name
+        end
+      end
+      # Verify the command packets
+      Cosmos::System.commands.all.each do |target_name, packets|
+        target = Target.find_by_name(target_name)
+        expect(target.name).to eq target_name
+        packets.each do |name, packet|
+          pkt = Packet.where({target: target, name: name, is_tlm: false}).first
+          expect(pkt.name).to eq name
+          expect(pkt.is_tlm).to eq false
+          expect(pkt.target.name).to eq target_name
+        end
+      end
+
+      num_tgts = Target.all.length
+      tgt_created_at = Target.first.created_at
+      num_pkts = Packet.all.length
+      pkt_created_at = Packet.first.created_at
+      #num_items = Item.all.length
+      #item_created_at = Item.first.created_at
+
+      # Try to add the known targets and packets into the DB again
+      common.sync_targets_and_packets
+      # Verify nothing was added
+      expect(Target.all.length).to eq num_tgts
+      expect(Packet.all.length).to eq num_pkts
+      #expect(Item.all.length).to eq num_items
+      expect(Target.first.created_at).to eq tgt_created_at
+      expect(Packet.first.created_at).to eq pkt_created_at
+      #expect(Item.first.created_at).to eq item_created_at
+    end
+  end
+
+  describe "setup_packet_config" do
+    it "builds the packet configuration in the DB" do
+      common.sync_targets_and_packets
+      target_id, packet_id = common.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet_log = PacketLog.create(:filename => "filename", :is_tlm => true)
+      ple = PacketLogEntry.new
+      ple.target_id = target_id
+      ple.packet_id = packet_id
+      ple.time = Time.now
+      ple.packet_log_id = packet_log.id
+      ple.data_offset = 0
+      ple.meta_id = 0
+      ple.is_tlm = true
+      ple.ready = true
+      ple.save!
+
+      packet = Cosmos::System.telemetry.packet("INST", "HEALTH_STATUS")
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => 0)
+      common.setup_packet_config(packet, packet_id, packet_config)
+
+      model = common.get_decom_table_model(packet_config.id, 0)
+      expect(model.column_names).to include("i0")
+      model = common.get_decom_table_model(packet_config.id, 0, '_m')
+      expect(model.column_names).to include("i0min")
+      expect(model.column_names).to include("i0max")
+      expect(model.column_names).to include("i0avg")
+      expect(model.column_names).to include("i0stddev")
+      model = common.get_decom_table_model(packet_config.id, 0, '_h')
+      expect(model.column_names).to include("i0min")
+      expect(model.column_names).to include("i0max")
+      expect(model.column_names).to include("i0avg")
+      expect(model.column_names).to include("i0stddev")
+      model = common.get_decom_table_model(packet_config.id, 0, '_d')
+      expect(model.column_names).to include("i0min")
+      expect(model.column_names).to include("i0max")
+      expect(model.column_names).to include("i0avg")
+      expect(model.column_names).to include("i0stddev")
+
+      # Useful debugging to pring out all the mapping items
+      # ItemToDecomTableMapping.all.each do |map|
+      #   item = Item.find(map.item_id)
+      #   puts "item name:#{item.name} id:#{item.id} val:#{map.value_type} reduced:#{map.reduced} config:#{map.packet_config_id} table_i:#{map.table_index} item_i:#{map.item_index}"
+      # end
+
+      ###################################
+      # Spot check some interesting items
+      ###################################
+
+      item = Item.find_by_name("RECEIVED_TIMESECONDS")
+      mapping = ItemToDecomTableMapping.where("item_id = ?", item.id)
+      # RECEIVED_TIMESECONDS is derived so it has a single mapping
+      expect(mapping.length).to eq 1
+      map = mapping.first
+      expect(map.value_type).to eq ItemToDecomTableMapping::RAW_CON
+      expect(map.reduced).to eq true # Seconds is an integer so it can be reduced
+      expect(map.packet_config_id).to eq packet_config.id
+      expect(map.table_index).to eq 0 # Not enough values to span multiple tables
+
+      item = Item.find_by_name("RECEIVED_TIMEFORMATTED")
+      mapping = ItemToDecomTableMapping.where("item_id = ?", item.id)
+      # RECEIVED_TIMEFORMATTED is derived so it has a single mapping
+      expect(mapping.length).to eq 1
+      map = mapping.first
+      expect(map.value_type).to eq ItemToDecomTableMapping::RAW_CON
+      expect(map.reduced).to eq false # Formatted is a string so it can't be reduced
+      expect(map.packet_config_id).to eq packet_config.id
+      expect(map.table_index).to eq 0 # Not enough values to span multiple tables
+
+      item = Item.find_by_name("TEMP1")
+      mapping = ItemToDecomTableMapping.where("item_id = ?", item.id)
+      # TEMP1 creates a separate RAW and CONVERTED table since it has a conversion
+      expect(mapping.length).to eq 2
+      map = mapping.where("value_type = ?", ItemToDecomTableMapping::RAW).first
+      expect(map.value_type).to eq ItemToDecomTableMapping::RAW
+      expect(map.reduced).to eq true # UINT32 can be reduced
+      expect(map.packet_config_id).to eq packet_config.id
+      expect(map.table_index).to eq 0 # Not enough values to span multiple tables
+      map = mapping.where("value_type = ?", ItemToDecomTableMapping::CONVERTED).first
+      expect(map.value_type).to eq ItemToDecomTableMapping::CONVERTED
+      expect(map.reduced).to eq true # UINT32 can be reduced
+      expect(map.packet_config_id).to eq packet_config.id
+      expect(map.table_index).to eq 0 # Not enough values to span multiple tables
+
+      item = Item.find_by_name("GROUND1STATUS")
+      mapping = ItemToDecomTableMapping.where("item_id = ?", item.id)
+      # GROUND1STATUS creates a separate RAW and CONVERTED table since it has states
+      expect(mapping.length).to eq 2
+      map = mapping.where("value_type = ?", ItemToDecomTableMapping::RAW).first
+      expect(map.value_type).to eq ItemToDecomTableMapping::RAW
+      expect(map.reduced).to eq true # Raw value can be reduced
+      expect(map.packet_config_id).to eq packet_config.id
+      expect(map.table_index).to eq 0 # Not enough values to span multiple tables
+      map = mapping.where("value_type = ?", ItemToDecomTableMapping::CONVERTED).first
+      expect(map.value_type).to eq ItemToDecomTableMapping::CONVERTED
+      expect(map.reduced).to eq false # Converted is a string and can't be reduced
+      expect(map.packet_config_id).to eq packet_config.id
+      expect(map.table_index).to eq 0 # Not enough values to span multiple tables
+    end
+
+    it "spans multiple DB tables with big packets" do
+      # Create a new packet with a bunch of items
+      packet = Cosmos::Packet.new('INST', 'BIGGIE', :BIG_ENDIAN)
+      num_items = DartCommon::MAX_COLUMNS_PER_TABLE + 1
+      num_items.times do |index|
+        packet.append_item("ITEM#{index}", 8, :INT)
+      end
+      # Push the new packet into the PacketConfig telemetry hash
+      Cosmos::System.telemetry.config.telemetry['INST']['BIGGIE'] = packet
+      common.sync_targets_and_packets
+      target_id, packet_id = common.lookup_target_and_packet_id("INST", "BIGGIE", true)
+      packet_log = PacketLog.create(:filename => "filename", :is_tlm => true)
+      ple = PacketLogEntry.new
+      ple.target_id = target_id
+      ple.packet_id = packet_id
+      ple.time = Time.now
+      ple.packet_log_id = packet_log.id
+      ple.data_offset = 0
+      ple.meta_id = 0
+      ple.is_tlm = true
+      ple.ready = true
+      ple.save!
+
+      packet = Cosmos::System.telemetry.packet("INST", "BIGGIE")
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => 0)
+      common.setup_packet_config(packet, packet_id, packet_config)
+
+      # Verify the decommutation and reduction tables were created
+      (0..1).each do |table_index|
+        model = common.get_decom_table_model(packet_config.id, table_index)
+        if table_index == 0
+          expect(model.column_names).to include("i0")
+          expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE - 1}")
+          # The base decommutation table doesn't average
+          expect(model.column_names).to_not include("i0avg")
+          expect(model.column_names).to_not include("i0stddev")
+        else
+          expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE}")
+          expect(model.column_names).to_not include("i#{DartCommon::MAX_COLUMNS_PER_TABLE + 1}")
+        end
+
+        %w(m h d).each do |suffix|
+          model = common.get_decom_table_model(packet_config.id, table_index, "_#{suffix}")
+          if table_index == 0
+            # The decommutation tables have min max avg only
+            expect(model.column_names).to_not include("i0")
+            expect(model.column_names).to include("i0min")
+            expect(model.column_names).to include("i0max")
+            expect(model.column_names).to include("i0avg")
+            expect(model.column_names).to include("i0stddev")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE - 1}min")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE - 1}max")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE - 1}avg")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE - 1}stddev")
+          else
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE}min")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE}max")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE}avg")
+            expect(model.column_names).to include("i#{DartCommon::MAX_COLUMNS_PER_TABLE}stddev")
+          end
+        end
+      end
+
+      num_items.times do |index|
+        item = Item.where("name = ? AND packet_id = ?", "ITEM#{index}", packet_id).first
+        map = ItemToDecomTableMapping.where("item_id = ?", item.id).first
+        if index < DartCommon::MAX_COLUMNS_PER_TABLE
+          expect(map.table_index).to eq 0
+        else
+          expect(map.table_index).to eq 1 # Mapping spans another table
+        end
+      end
+    end
+  end
+
+  describe "switch_and_get_system_config" do
+    before(:each) do
+      allow_any_instance_of(Cosmos::Interface).to receive(:connected?)
+      allow_any_instance_of(Cosmos::Interface).to receive(:connect)
+      allow_any_instance_of(Cosmos::Interface).to receive(:disconnect)
+      allow_any_instance_of(Cosmos::Interface).to receive(:write_raw)
+      allow_any_instance_of(Cosmos::Interface).to receive(:read)
+      allow_any_instance_of(Cosmos::Interface).to receive(:write)
+      @tlm_file = File.join(Cosmos::USERPATH,'config','targets','SYSTEM','cmd_tlm','test1_tlm.txt')
+      FileUtils.rm @tlm_file if File.exist? @tlm_file
+    end
+
+    it "raises an error if the configuration can't be found" do
+      api = Cosmos::CmdTlmServer.new
+      expect { common.switch_and_get_system_config("abcd") }.to raise_error(/No saved config/)
+      api.stop
+    end
+
+    it "does nothing if loading the current config" do
+      api = Cosmos::CmdTlmServer.new
+      current = Cosmos::System.configuration_name
+      common.switch_and_get_system_config(current)
+      expect(Cosmos::System.configuration_name).to eq current
+      api.stop
+    end
+
+    it "loads new configuration" do
+      api = Cosmos::CmdTlmServer.new
+      initial_config = Cosmos::System.configuration_name
+
+      # Create a new configuration by writing another telemetry file
+      File.open(@tlm_file, 'w') do |file|
+        file.puts "TELEMETRY SYSTEM TEST1 BIG_ENDIAN"
+        file.puts "  APPEND_ITEM DATA 240 STRING"
+      end
+      # Reset the instance variable so it will create the new configuration
+      Cosmos::System.class_eval('@@instance = nil')
+      Cosmos::System.telemetry # Create the new config
+      new_config = Cosmos::System.configuration_name
+      expect(new_config).to_not eq initial_config
+
+      # Stub find_configuration to first return nil (not found) and then work
+      # This allows the switch_and_get_system_config to act like the local copy
+      # could not be found and then properly finds it
+      allow(Cosmos::System.instance).to receive(:find_configuration).and_return(nil, Cosmos::System.instance.find_configuration(initial_config))
+      messages = []
+      allow(Cosmos::Logger).to receive(:info) { |msg| messages << msg }
+
+      common.switch_and_get_system_config(initial_config)
+      expect(Cosmos::System.configuration_name).to eq initial_config
+      messages.each do |msg|
+        if msg =~ /Configuration retrieved/
+          expect(msg).to match(/#{initial_config}/)
+        end
+      end
+
+      api.stop
+      FileUtils.rm @tlm_file
+    end
+  end
+
+  describe "read_packet_from_ple" do
+    # NOTE: most of read_packet_from_ple is tested in the dart_packet_log_writer_spec
+    it "raises an error if the packet can't be found" do
+      common.sync_targets_and_packets
+      target_id, packet_id = common.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet_log = PacketLog.create(:filename => "filename", :is_tlm => true)
+      ple = PacketLogEntry.new
+      ple.target_id = target_id
+      ple.packet_id = packet_id
+      ple.time = Time.now
+      ple.packet_log_id = packet_log.id
+      ple.data_offset = 999
+      ple.meta_id = 999
+      ple.is_tlm = true
+      ple.ready = true
+      ple.save!
+
+      expect(Cosmos::Logger).to receive(:error) do |msg|
+        expect(msg).to match(/Error Reading Packet/)
+      end
+      packet = common.read_packet_from_ple(ple)
+      expect(packet).to be_nil
+    end
+  end
+end

data/lib/cosmos/dart/spec/dart/dart_database_cleaner_spec.rb

@@ -0,0 +1,455 @@
+# encoding: ascii-8bit
+
+# Copyright 2014 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+
+require 'rails_helper'
+require 'dart_database_cleaner'
+require 'dart_packet_log_writer'
+require 'dart_decommutator'
+
+describe DartDatabaseCleaner do
+  before(:each) do
+    DatabaseCleaner.strategy = :truncation
+    DatabaseCleaner.clean
+    @cleaner = DartDatabaseCleaner.new
+  end
+
+  describe "clean" do
+    it "calls the cleaning methods" do
+      messages = []
+      allow(Cosmos::Logger).to receive(:info) { |msg| messages << msg }
+      DartDatabaseCleaner.clean(false)
+      expect(messages.select{|m| m =~ /Cleaning up SystemConfig/}.length).to eq 1
+      expect(messages.select{|m| m =~ /Cleaning up PacketLog\./}.length).to eq 1
+      expect(messages.select{|m| m =~ /Cleaning up PacketConfig/}.length).to eq 1
+      expect(messages.select{|m| m =~ /Cleaning up PacketLogEntry/}.length).to eq 1
+      expect(messages.select{|m| m =~ /Cleaning up Decommutation/}.length).to eq 1
+      expect(messages.select{|m| m =~ /Database cleanup complete/}.length).to eq 1
+    end
+  end
+
+  describe "clean_system_configs" do
+    it "ensures all system configs are local" do
+      # Create a bogus SystemConfig to cause an error
+      SystemConfig.create(:name => "test")
+
+      expect(Cosmos::Logger).to receive(:error) do |msg|
+        expect(msg).to match(/Could not load system_config: test/)
+      end
+      config, error = @cleaner.clean_system_configs
+      # Ensure the configuration is loaded
+      expect(config).to eq Cosmos::System.configuration_name
+    end
+  end
+
+  describe "clean_packet_logs" do
+    before(:each) do
+      @target = Target.create(:name => "test")
+      @packet = Packet.create(:target_id => @target.id, :name => "test", :is_tlm => true)
+    end
+
+    it "ensures all packet logs exist" do
+      # Create a good file which exists where we expect
+      filename = File.join(Cosmos::System.paths['DART_DATA'], 'test_file.bin')
+      File.open(filename, 'w') { |file| file.puts "This is test data" }
+      packet_log = PacketLog.create(:filename => filename, :is_tlm => true)
+
+      ple = PacketLogEntry.new
+      ple.target_id = @target.id
+      ple.packet_id = @packet.id
+      ple.time = Time.now
+      ple.packet_log_id = packet_log.id
+      ple.data_offset = 0
+      ple.meta_id = 0
+      ple.is_tlm = true
+      ple.ready = true
+      ple.save!
+
+      @cleaner.clean_packet_logs(false)
+      # Ensure nothing changed
+      log = PacketLog.find(1)
+      expect(log.filename).to eq filename
+      entry = PacketLogEntry.find(1)
+      expect(entry.packet_log.filename).to eq filename
+    end
+
+    it "checks for files moved to DART_DATA and updates the path" do
+      # Create a good file which exists where we expect
+      filename = File.join(Cosmos::System.paths['DART_DATA'], 'test_file.bin')
+      File.open(filename, 'w') { |file| file.puts "This is test data" }
+      # Create a PacketLog with a bad path
+      bad_path = File.join("C:/somewhere/that/does/not/exist", File.basename(filename))
+      packet_log = PacketLog.create(:filename => bad_path, :is_tlm => true)
+
+      ple = PacketLogEntry.new
+      ple.target_id = @target.id
+      ple.packet_id = @packet.id
+      ple.time = Time.now
+      ple.packet_log_id = packet_log.id
+      ple.data_offset = 0
+      ple.meta_id = 0
+      ple.is_tlm = true
+      ple.ready = true
+      ple.save!
+
+      @cleaner.clean_packet_logs(false)
+      # Ensure the path is updated
+      log = PacketLog.find(1)
+      expect(log.filename).to eq filename
+      entry = PacketLogEntry.find(1)
+      expect(entry.packet_log.filename).to eq filename
+    end
+
+    it "exits if a file is missing and force == false" do
+      # Create a PacketLog with a non-existant file
+      packet_log = PacketLog.create(:filename => "C:/no/file/here/test.bin", :is_tlm => true)
+      expect { @cleaner.clean_packet_logs(false) }.to raise_error(SystemExit)
+    end
+
+    it "deletes if a file is missing and force == true" do
+      # Create a PacketLog with a non-existant file
+      packet_log = PacketLog.create(:filename => "C:/no/file/here/test.bin", :is_tlm => true)
+      10.times do
+        ple = PacketLogEntry.new
+        ple.target_id = @target.id
+        ple.packet_id = @packet.id
+        ple.time = Time.now
+        ple.packet_log_id = packet_log.id
+        ple.data_offset = 0
+        ple.meta_id = 0
+        ple.is_tlm = true
+        ple.ready = true
+        ple.save!
+      end
+
+      @cleaner.clean_packet_logs(true)
+      expect { PacketLog.find(1) }.to raise_error(ActiveRecord::RecordNotFound)
+      # Also any PacketLogEntries linked to that PacketLog are deleted
+      expect(PacketLogEntry.all.count).to eq 0
+    end
+  end
+
+  describe "clean_packet_configs" do
+    it "raises if the SystemConfig can't be found" do
+      target = Target.create(:name => "test")
+      packet = Packet.create(:target_id => target.id, :name => "test", :is_tlm => true)
+      PacketConfig.create(:packet_id => packet.id, :name => "test", :first_system_config_id => 1)
+      expect { @cleaner.clean_packet_configs }.to raise_error(/Cleanup failure/)
+    end
+
+    it "complains about bad SystemConfigs" do
+      @cleaner.sync_targets_and_packets
+      target_id, packet_id = @cleaner.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet = Cosmos::System.telemetry.packet("INST", "HEALTH_STATUS")
+      sys_config = SystemConfig.create(:name => "test")
+      PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => sys_config.id)
+
+      expect(Cosmos::Logger).to receive(:error) do |msg|
+        expect(msg).to match(/Could not switch to system config: test/)
+      end
+      @cleaner.clean_packet_configs
+    end
+
+    it "recreates the PacketConfig if it is not 'ready'" do
+      @cleaner.sync_targets_and_packets
+      target_id, packet_id = @cleaner.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet = Cosmos::System.telemetry.packet("INST", "HEALTH_STATUS")
+      meta_packet = Cosmos::System.telemetry.packet("SYSTEM", "META")
+      sys_config = SystemConfig.create(:name => meta_packet.read("CONFIG"))
+      PacketConfig.create(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => sys_config.id)
+
+      # By default new PacketConfig items should not be marked ready
+      packet_config = PacketConfig.where(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => sys_config.id).first
+      expect(packet_config.ready).to be false
+
+      item = Item.create(:packet_id => packet_id, :name => "TEST_ITEM")
+      # Create an item to decom table mapping to ensure it gets cleaned up
+      ItemToDecomTableMapping.create(
+        :item_id => item.id,
+        :value_type => ItemToDecomTableMapping::RAW,
+        :reduced => true,
+        :packet_config_id => packet_config.id,
+        :table_index => 0,
+        :item_index => 999999) # Bogus value we can check
+      expect(ItemToDecomTableMapping.find_by_item_index(999999)).to_not be_nil
+      # Create a decommutation table to ensure it gets cleaned up
+      table_name = "t#{packet_config.id}_0"
+      table = ActiveRecord::Base.connection.create_table(table_name) do |table|
+        table.integer :delete_me # Bogus column
+      end
+      # Need to create model
+      model = Class.new(ActiveRecord::Base) do
+        self.table_name = table_name.dup
+      end
+      model.reset_column_information
+      model_name = table_name.upcase
+      Cosmos.send(:remove_const, model_name) if Cosmos.const_defined?(model_name)
+      Cosmos.const_set(model_name, model)
+
+      expect(model.column_names).to include("delete_me")
+      # Create reduction tables to ensure they are cleaned up
+      %w(_h _m _d).each do |id|
+        table_name = "t#{packet_config.id}_0#{id}"
+        table = ActiveRecord::Base.connection.create_table(table_name) do |table|
+          table.integer :delete_me # Bogus column
+        end
+        # Need to create model
+        model = Class.new(ActiveRecord::Base) do
+          self.table_name = table_name.dup
+        end
+        model.reset_column_information
+        model_name = table_name.upcase
+        Cosmos.send(:remove_const, model_name) if Cosmos.const_defined?(model_name)
+        Cosmos.const_set(model_name, model)
+
+        expect(model.column_names).to include("delete_me")
+      end
+      messages = [] # Store all Logger.info messages
+      allow(Cosmos::Logger).to receive(:info) { |msg| messages << msg }
+
+      @cleaner.clean_packet_configs # <--- PERFORM THE TEST
+
+      # Check messages for success
+      expect(messages.select {|msg| msg =~ /Successfully cleaned/ }[0]).to match(/packet_config: 1/)
+      packet_config = PacketConfig.where(:packet_id => packet_id, :name => packet.config_name, :first_system_config_id => sys_config.id).first
+      expect(packet_config.ready).to be true # ready is not true
+      # The item should still be there
+      expect(Item.find(1).name).to eq "TEST_ITEM"
+      # The old ItemToDecomTableMapping should be removed
+      expect(ItemToDecomTableMapping.find_by_item_index(999999)).to be_nil
+      # All new items should have the PacketConfig ID set
+      ItemToDecomTableMapping.all.each do |item|
+        expect(item.packet_config_id).to eq packet_config.id
+      end
+      # The decom table should have been recreated
+      model = @cleaner.get_decom_table_model(packet_config.id, 0)
+      expect(model.column_names).to_not include("delete_me")
+      expect(model.column_names).to include("i0") # as well as i1, i2, etc
+      # The reduction table should have been recreated
+      %w(_h _m _d).each do |id|
+      model = @cleaner.get_decom_table_model(packet_config.id, 0, id)
+        expect(model.column_names).to_not include("delete_me")
+        expect(model.column_names).to include("i0min")
+        expect(model.column_names).to include("i0max")
+        expect(model.column_names).to include("i0avg")
+        expect(model.column_names).to include("i0stddev")
+      end
+    end
+  end
+
+  describe "clean_packet_log_entries" do
+    it "removes PacketLogEntry rows where ready == false" do
+      target = Target.create(:name => "test")
+      packet = Packet.create(:target_id => target.id, :name => "test", :is_tlm => true)
+      packet_log = PacketLog.create(:filename => "filename", :is_tlm => true)
+      ple = PacketLogEntry.new
+      ple.target_id = target.id
+      ple.packet_id = packet.id
+      ple.time = Time.now
+      ple.packet_log_id = packet_log.id
+      ple.data_offset = 1
+      ple.meta_id = 1
+      ple.is_tlm = true
+      ple.ready = false
+      ple.save!
+      expect(PacketLogEntry.all.length).to eq 1
+      @cleaner.clean_packet_log_entries
+      expect(PacketLogEntry.all.length).to eq 0
+    end
+  end
+
+  describe "clean_decommutation_tables" do
+    it "removes decommutation rows which are in progress" do
+      writer = DartPacketLogWriter.new(
+        :TLM,    # Log telemetry
+        'clean_decom_', # File name suffix
+        true,    # Enable logging
+        nil,     # Don't cycle on time
+        2_000_000_000, # Cycle the log at 2GB
+        Cosmos::System.paths['DART_DATA']) # Log into the DART_DATA dir
+
+      hs_packet = Cosmos::System.telemetry.packet("INST", "HEALTH_STATUS")
+      # Write three packets. The first packet is always SYSTEM META.
+      3.times do
+        hs_packet.received_time = Time.now
+        writer.write(hs_packet)
+        sleep 0.01
+      end
+      writer.shutdown
+      sleep 0.1
+
+      # Create a valid SystemConfig in order to create a valid PacketConfig
+      meta = Cosmos::System.telemetry.packet("SYSTEM", "META")
+      system_config = SystemConfig.create(:name => meta.read("CONFIG"))
+      target_id, packet_id = writer.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => hs_packet.config_name, :first_system_config_id => system_config.id)
+      writer.setup_packet_config(hs_packet, packet_id, packet_config)
+
+      decom = writer.get_decom_table_model(packet_config.id, 0)
+      PacketLogEntry.all.each do |ple|
+        # By default all PacketLogEntries should be marked NOT_STARTED
+        expect(ple.decom_state).to eq PacketLogEntry::NOT_STARTED
+        case ple.target.name
+        when 'SYSTEM'
+          ple.decom_state = PacketLogEntry::COMPLETE
+        when 'INST'
+          ple.decom_state = PacketLogEntry::IN_PROGRESS
+          row = decom.new
+          row.time = Time.now
+          row.reduced_state = DartDecommutator::INITIALIZING
+          row.ple_id = ple.id
+          row.save!
+        end
+        ple.save!
+      end
+      expect(decom.all.count).to eq 3
+
+      @cleaner.clean_decommutation_tables # <--- PERFORM THE TEST
+
+      PacketLogEntry.all.each do |ple|
+        case ple.target.name
+        when 'SYSTEM'
+          expect(ple.decom_state).to eq PacketLogEntry::COMPLETE
+        when 'INST'
+          # All INST should be now marked NOT_STARTED
+          expect(ple.decom_state).to eq PacketLogEntry::NOT_STARTED
+        end
+      end
+      # The decommutation table has been cleaned
+      decom = writer.get_decom_table_model(packet_config.id, 0)
+      expect(decom.all.count).to eq 0
+    end
+  end
+
+  describe "clean_decommutation_tables" do
+    it "removes decommutation rows which are in progress" do
+      writer = DartPacketLogWriter.new(
+        :TLM,    # Log telemetry
+        'clean_decom_', # File name suffix
+        true,    # Enable logging
+        nil,     # Don't cycle on time
+        2_000_000_000, # Cycle the log at 2GB
+        Cosmos::System.paths['DART_DATA']) # Log into the DART_DATA dir
+
+      hs_packet = Cosmos::System.telemetry.packet("INST", "HEALTH_STATUS")
+      # Write three packets. The first packet is always SYSTEM META.
+      3.times do
+        hs_packet.received_time = Time.now
+        writer.write(hs_packet)
+        sleep 0.01
+      end
+      writer.shutdown
+      sleep 0.1
+
+      # Create a valid SystemConfig in order to create a valid PacketConfig
+      meta = Cosmos::System.telemetry.packet("SYSTEM", "META")
+      system_config = SystemConfig.create(:name => meta.read("CONFIG"))
+      target_id, packet_id = writer.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => hs_packet.config_name, :first_system_config_id => system_config.id)
+      writer.setup_packet_config(hs_packet, packet_id, packet_config)
+
+      decom = writer.get_decom_table_model(packet_config.id, 0)
+      PacketLogEntry.all.each do |ple|
+        # By default all PacketLogEntries should be marked NOT_STARTED
+        expect(ple.decom_state).to eq PacketLogEntry::NOT_STARTED
+        case ple.target.name
+        when 'SYSTEM'
+          ple.decom_state = PacketLogEntry::COMPLETE
+        when 'INST'
+          ple.decom_state = PacketLogEntry::IN_PROGRESS
+          row = decom.new
+          row.time = Time.now
+          row.reduced_state = DartDecommutator::INITIALIZING
+          row.ple_id = ple.id
+          row.save!
+        end
+        ple.save!
+      end
+      expect(decom.all.count).to eq 3
+
+      @cleaner.clean_decommutation_tables # <--- PERFORM THE TEST
+
+      PacketLogEntry.all.each do |ple|
+        case ple.target.name
+        when 'SYSTEM'
+          expect(ple.decom_state).to eq PacketLogEntry::COMPLETE
+        when 'INST'
+          # All INST should be now marked NOT_STARTED
+          expect(ple.decom_state).to eq PacketLogEntry::NOT_STARTED
+        end
+      end
+      # The decommutation table has been cleaned
+      decom = writer.get_decom_table_model(packet_config.id, 0)
+      expect(decom.all.count).to eq 0
+    end
+  end
+
+  describe "clean_reductions" do
+    it "removes decommutation rows which are in progress" do
+      writer = DartPacketLogWriter.new(
+        :TLM,    # Log telemetry
+        'clean_reduction_', # File name suffix
+        true,    # Enable logging
+        nil,     # Don't cycle on time
+        2_000_000_000, # Cycle the log at 2GB
+        Cosmos::System.paths['DART_DATA']) # Log into the DART_DATA dir
+
+      hs_packet = Cosmos::System.telemetry.packet("INST", "HEALTH_STATUS")
+      # Write three packets. The first packet is always SYSTEM META.
+      3.times do
+        hs_packet.received_time = Time.now
+        writer.write(hs_packet)
+        sleep 0.01
+      end
+      writer.shutdown
+      sleep 0.1
+
+      # Create a valid SystemConfig in order to create a valid PacketConfig
+      meta = Cosmos::System.telemetry.packet("SYSTEM", "META")
+      system_config = SystemConfig.create(:name => meta.read("CONFIG"))
+      target_id, packet_id = writer.lookup_target_and_packet_id("INST", "HEALTH_STATUS", true)
+      packet_config = PacketConfig.create(:packet_id => packet_id, :name => hs_packet.config_name, :first_system_config_id => system_config.id)
+      writer.setup_packet_config(hs_packet, packet_id, packet_config)
+
+      decom = writer.get_decom_table_model(packet_config.id, 0)
+      PacketLogEntry.all.each do |ple|
+        # By default all PacketLogEntries should be marked NOT_STARTED
+        expect(ple.decom_state).to eq PacketLogEntry::NOT_STARTED
+        case ple.target.name
+        when 'SYSTEM'
+          ple.decom_state = PacketLogEntry::COMPLETE
+        when 'INST'
+          ple.decom_state = PacketLogEntry::IN_PROGRESS
+          row = decom.new
+          row.time = Time.now
+          row.reduced_state = DartDecommutator::INITIALIZING
+          row.ple_id = ple.id
+          row.save!
+        end
+        ple.save!
+      end
+      expect(decom.all.count).to eq 3
+
+      @cleaner.clean_decommutation_tables # <--- PERFORM THE TEST
+
+      PacketLogEntry.all.each do |ple|
+        case ple.target.name
+        when 'SYSTEM'
+          expect(ple.decom_state).to eq PacketLogEntry::COMPLETE
+        when 'INST'
+          # All INST should be now marked NOT_STARTED
+          expect(ple.decom_state).to eq PacketLogEntry::NOT_STARTED
+        end
+      end
+      # The decommutation table has been cleaned
+      decom = writer.get_decom_table_model(packet_config.id, 0)
+      expect(decom.all.count).to eq 0
+    end
+  end
+end