cosmos 4.5.0 → 5.0.2.pre.beta2

data/lib/cosmos/operators/operator.rb

@@ -0,0 +1,278 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
+
+require 'childprocess'
+require 'cosmos'
+require 'fileutils'
+require 'tempfile'
+
+module Cosmos
+  class OperatorProcess
+    attr_accessor :process_definition
+    attr_accessor :work_dir
+    attr_accessor :env
+    attr_accessor :new_temp_dir
+    attr_reader :temp_dir
+    attr_reader :scope
+
+    def self.setup
+      # Perform any setup steps necessary
+    end
+
+    def initialize(process_definition, work_dir: '/cosmos/lib/cosmos/microservices', temp_dir: nil, env: {}, scope:, container: nil) # container is not used, it's just here for Enterprise
+      @process = nil
+      @process_definition = process_definition
+      @work_dir = work_dir
+      @temp_dir = temp_dir
+      @new_temp_dir = temp_dir
+      @env = env
+      @scope = scope
+    end
+
+    def start
+      @temp_dir = @new_temp_dir
+      @new_temp_dir = nil
+      Logger.info("Starting: #{@process_definition.join(' ')}", scope: @scope)
+      @process = ChildProcess.build(*@process_definition)
+      # This lets the ChildProcess use the parent IO ... but it breaks unit tests
+      # @process.io.inherit!
+      @process.cwd = @work_dir
+      # Spawned process should not be controlled by same Bundler constraints as spawning process
+      ENV.each do |key, value|
+        if key =~ /^BUNDLER/
+          @process.environment[key] = nil
+        end
+      end
+      @env['RUBYOPT'] = nil # Removes loading bundler setup
+      @env.each do |key, value|
+        @process.environment[key] = value
+      end
+      @process.environment['COSMOS_SCOPE'] = @scope
+      @process.io.stdout = Tempfile.new("child-output")
+      @process.io.stderr = Tempfile.new("child-output")
+      @process.start
+    end
+
+    def alive?
+      if @process
+        @process.alive?
+      else
+        false
+      end
+    end
+
+    def exit_code
+      if @process
+        @process.exit_code
+      else
+        nil
+      end
+    end
+
+    def soft_stop
+      Thread.new do
+        Logger.info("Soft shutting down process: #{@process_definition.join(' ')}", scope: @scope)
+        Process.kill("SIGINT", @process.pid) if @process # Signal the process to stop
+      end
+    end
+
+    def hard_stop
+      if @process and !@process.exited?
+        Logger.info("Hard shutting down process: #{@process_definition.join(' ')}", scope: @scope)
+        @process.stop
+      end
+      FileUtils.remove_entry(@temp_dir) if @temp_dir and File.exist?(@temp_dir)
+      @process = nil
+    end
+
+    def stdout
+      @process.io.stdout
+    end
+
+    def stderr
+      @process.io.stderr
+    end
+
+    def extract_output(max_length_stdout = 65536, max_length_stderr = 65536)
+      if @process
+        @process.io.stdout.rewind
+        output = @process.io.stdout.read
+        @process.io.stdout.close
+        @process.io.stdout.unlink
+        @process.io.stderr.rewind
+        err_output = @process.io.stderr.read
+        @process.io.stderr.close
+        @process.io.stderr.unlink
+        return "Stdout:\n#{output[-max_length_stdout..-1] || output}\n\nStderr:\n#{err_output[-max_length_stderr..-1] || err_output}\n"
+      else
+        return ""
+      end
+    end
+  end
+
+  class Operator
+    attr_reader :processes, :cycle_time
+
+    @@instance = nil
+
+    CYCLE_TIME = 15.0 # cycle time to check for new microservices
+
+    def initialize
+      Logger.level = Logger::INFO
+      # TODO: This is pretty generic. Can we pass in more information to help identify the operator?
+      Logger.microservice_name = 'MicroserviceOperator'
+      Logger.tag = "operator.log"
+
+      OperatorProcess.setup()
+      @cycle_time = (ENV['OPERATOR_CYCLE_TIME'] and ENV['OPERATOR_CYCLE_TIME'].to_f) || CYCLE_TIME # time in seconds
+
+      @ruby_process_name = ENV['COSMOS_RUBY']
+      if RUBY_ENGINE != 'ruby'
+        @ruby_process_name ||= 'jruby'
+      else
+        @ruby_process_name ||= 'ruby'
+      end
+
+      @processes = {}
+      @new_processes = {}
+      @changed_processes = {}
+      @removed_processes = {}
+      @mutex = Mutex.new
+      @shutdown = false
+      @shutdown_complete = false
+    end
+
+    def update
+      raise "Implement in subclass"
+    end
+
+    def start_new
+      @mutex.synchronize do
+        if @new_processes.length > 0
+          # Start all the processes
+          Logger.info("#{self.class} starting each new process...")
+          @new_processes.each { |name, p| p.start }
+          @new_processes = {}
+        end
+      end
+    end
+
+    def respawn_changed
+      @mutex.synchronize do
+        if @changed_processes.length > 0
+          Logger.info("Cycling #{@changed_processes.length} changed microservices...")
+          shutdown_processes(@changed_processes)
+          break if @shutdown
+
+          @changed_processes.each { |name, p| p.start }
+          @changed_processes = {}
+        end
+      end
+    end
+
+    def remove_old
+      @mutex.synchronize do
+        if @removed_processes.length > 0
+          Logger.info("Shutting down #{@removed_processes.length} removed microservices...")
+          shutdown_processes(@removed_processes)
+          @removed_processes = {}
+        end
+      end
+    end
+
+    def respawn_dead
+      @mutex.synchronize do
+        @processes.each do |name, p|
+          break if @shutdown
+
+          unless p.alive?
+            # Respawn process
+            p.stdout.rewind
+            output = p.stdout.read
+            p.stdout.close
+            p.stdout.unlink
+            p.stderr.rewind
+            err_output = p.stderr.read
+            p.stderr.close
+            p.stderr.unlink
+            Logger.error("Unexpected process died... respawning! #{p.process_definition.join(' ')}\nStdout:\n#{output}\nStderr:\n#{err_output}\n", scope: p.scope)
+            p.start
+          end
+        end
+      end
+    end
+
+    def shutdown_processes(processes)
+      processes.each { |name, p| p.soft_stop }
+      sleep(2) # TODO: This is an arbitrary sleep of 2s ...
+      processes.each { |name, p| p.hard_stop }
+    end
+
+    def run
+      # Use at_exit to shutdown cleanly
+      at_exit do
+        @shutdown = true
+        @mutex.synchronize do
+          Logger.info("Shutting down processes...")
+          shutdown_processes(@processes)
+          @shutdown_complete = true
+        end
+      end
+
+      # Monitor processes and respawn if died
+      Logger.info("#{self.class} Monitoring processes every #{@cycle_time} sec...")
+      loop do
+        update()
+        remove_old()
+        respawn_changed()
+        start_new()
+        respawn_dead()
+        break if @shutdown
+
+        sleep(@cycle_time)
+        break if @shutdown
+      end
+
+      loop do
+        break if @shutdown_complete
+
+        sleep(1)
+      end
+    ensure
+      Logger.info("#{self.class} shutdown complete")
+    end
+
+    def stop
+      @shutdown = true
+    end
+
+    def self.run
+      @@instance = self.new
+      @@instance.run
+    end
+
+    def self.processes
+      @@instance.processes
+    end
+
+    def self.instance
+      @@instance
+    end
+  end
+end

data/lib/cosmos/packets/binary_accessor.rb

@@ -1,12 +1,21 @@
 # encoding: ascii-8bit
 
-# Copyright 2014 Ball Aerospace & Technologies Corp.
+# Copyright 2022 Ball Aerospace & Technologies Corp.
 # All Rights Reserved.
 #
 # This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Lesser General Public License
+# under the terms of the GNU Affero General Public License
 # as published by the Free Software Foundation; version 3 with
 # attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
 
 # This file contains the implementation of the BinaryAccessor class.
 # This class allows for easy reading and writing of binary data in Ruby
@@ -14,10 +23,8 @@
 require 'cosmos/ext/packet' if RUBY_ENGINE == 'ruby' and !ENV['COSMOS_NO_EXT']
 
 module Cosmos
-
   # Provides methods for binary reading and writing
   class BinaryAccessor
-
     # Constants for ruby packing directives
     PACK_8_BIT_INT = 'c'
     PACK_NATIVE_16_BIT_INT = 's'
@@ -50,19 +57,19 @@ module Cosmos
     PACK_BIG_ENDIAN_32_BIT_FLOAT_ARRAY = 'g*'
     PACK_BIG_ENDIAN_64_BIT_FLOAT_ARRAY = 'G*'
 
-    if RUBY_ENGINE != 'ruby' or ENV['COSMOS_NO_ENV']
+    if RUBY_ENGINE != 'ruby' or ENV['COSMOS_NO_EXT']
       MIN_INT8 = -128
       MAX_INT8 = 127
       MAX_UINT8 = 255
       MIN_INT16 = -32768
       MAX_INT16 = 32767
       MAX_UINT16 = 65535
-      MIN_INT32  = -(2 ** 31)
-      MAX_INT32  = (2 ** 31) - 1
-      MAX_UINT32 = (2 ** 32) - 1
-      MIN_INT64  = -(2 ** 63)
-      MAX_INT64  = (2 ** 63) - 1
-      MAX_UINT64 = (2 ** 64) - 1
+      MIN_INT32  = -(2**31)
+      MAX_INT32  = (2**31) - 1
+      MAX_UINT32 = (2**32) - 1
+      MIN_INT64  = -(2**63)
+      MAX_INT64  = (2**63) - 1
+      MAX_UINT64 = (2**64) - 1
     end
 
     # Additional Constants
@@ -221,9 +228,9 @@ module Cosmos
             # Handle :INT and :UINT Bitfields
             ##########################
 
-            #Extract Data for Bitfield
+            # Extract Data for Bitfield
             if endianness == :LITTLE_ENDIAN
-              #Bitoffset always refers to the most significant bit of a bitfield
+              # Bitoffset always refers to the most significant bit of a bitfield
               num_bytes = (((bit_offset % 8) + bit_size - 1) / 8) + 1
               upper_bound = bit_offset / 8
               lower_bound = upper_bound - num_bytes + 1
@@ -237,7 +244,7 @@ module Cosmos
               temp_data = buffer[lower_bound..upper_bound]
             end
 
-            #Determine temp upper bound
+            # Determine temp upper bound
             temp_upper = upper_bound - lower_bound
 
             # Handle Bitfield
@@ -246,20 +253,20 @@ module Cosmos
             total_bits = (temp_upper + 1) * 8
             right_shift = total_bits - start_bits - bit_size
 
-            #Mask off unwanted bits at beginning
+            # Mask off unwanted bits at beginning
             temp = temp_data.getbyte(0) & start_mask
 
             if upper_bound > lower_bound
-              #Combine bytes into a FixNum
-              temp_data[1..temp_upper].each_byte {|temp_value| temp = temp << 8; temp = temp + temp_value }
+              # Combine bytes into a FixNum
+              temp_data[1..temp_upper].each_byte { |temp_value| temp = temp << 8; temp = temp + temp_value }
             end
 
             # Shift off unwanted bits at end
             temp = temp >> right_shift
 
             if data_type == :INT
-              #Convert to negative if necessary
-              if ((bit_size > 1) && (temp[bit_size - 1] == 1))
+              # Convert to negative if necessary
+              if (bit_size > 1) && (temp[bit_size - 1] == 1)
                 temp = -((1 << bit_size) - temp)
               end
             end
@@ -356,7 +363,6 @@ module Cosmos
                 # String was completely empty
                 if end_bytes > 0
                   # Preserve bytes at end of buffer
-                  buffer_length = buffer.length
                   buffer << "\000" * value.length
                   buffer[lower_bound + value.length, end_bytes] = buffer[lower_bound, end_bytes]
                 end
@@ -368,7 +374,6 @@ module Cosmos
                 buffer[upper_bound + 1, old_upper_bound - upper_bound] = ''
               elsif (upper_bound > old_upper_bound) && (end_bytes > 0)
                 # Preserve bytes at end of buffer
-                buffer_length = buffer.length
                 diff = upper_bound - old_upper_bound
                 buffer << "\000" * diff
                 buffer[upper_bound + 1, end_bytes] = buffer[old_upper_bound + 1, end_bytes]
@@ -477,6 +482,7 @@ module Cosmos
               if lower_bound < 0
                 raise(ArgumentError, "LITTLE_ENDIAN bitfield with bit_offset #{given_bit_offset} and bit_size #{given_bit_size} is invalid")
               end
+
               temp_data = buffer[lower_bound..upper_bound].reverse
             else
               temp_data = buffer[lower_bound..upper_bound]
@@ -496,7 +502,7 @@ module Cosmos
             temp = temp_data.getbyte(0) & start_mask
 
             # Adjust value to correct number of bits
-            temp_mask = (2 ** bit_size) - 1
+            temp_mask = (2**bit_size) - 1
             temp_value = value & temp_mask
 
             # Add in New Data
@@ -593,12 +599,12 @@ module Cosmos
         # Sanity check buffer size
         if upper_bound >= buffer_length
           # If it's not the special case of little endian bit field then we fail and return false
-          if !( (endianness == :LITTLE_ENDIAN) &&
+          if !((endianness == :LITTLE_ENDIAN) &&
                  ((data_type == :INT) || (data_type == :UINT)) &&
                  # Not byte aligned with an even bit size
-                 (!( (byte_aligned(bit_offset)) && (even_bit_size(bit_size)) )) &&
+                 (!((byte_aligned(bit_offset)) && (even_bit_size(bit_size)))) &&
                  (lower_bound < buffer_length)
-             )
+              )
             result = false
           end
         end
@@ -645,20 +651,20 @@ module Cosmos
           if data_type == :INT
             # Note signed integers must allow up to the maximum unsigned value to support values given in hex
             if bit_size > 1
-              max_value = 2 ** (bit_size - 1)
+              max_value = 2**(bit_size - 1)
               # min_value = -(2 ** bit_size - 1)
               min_value = -max_value
               # max_value = (2 ** bit_size - 1) - 1
               max_value -= 1
               # hex_max_value = (2 ** bit_size) - 1
-              hex_max_value = (2 ** bit_size) - 1
+              hex_max_value = (2**bit_size) - 1
             else # 1-bit signed
               min_value = -1
               max_value = 1
               hex_max_value = 1
             end
           else
-            max_value = (2 ** bit_size) - 1
+            max_value = (2**bit_size) - 1
             hex_max_value = max_value
           end
         end
@@ -675,6 +681,7 @@ module Cosmos
       end
 
       public
+
     end
 
     # Reads an array of binary data of any data type from a buffer
@@ -721,6 +728,7 @@ module Cosmos
       # Calculate number of items in the array
       # If there is a remainder then we have a problem
       raise ArgumentError, "array_size #{given_array_size} not a multiple of bit_size #{given_bit_size}" if array_size % bit_size != 0
+
       num_items = array_size / bit_size
 
       # Define bounds of string to access this item
@@ -1195,7 +1203,5 @@ module Cosmos
       end
       values
     end
-
   end # class BinaryAccessor
-
 end # module Cosmos

data/lib/cosmos/packets/commands.rb

@@ -1,17 +1,25 @@
 # encoding: ascii-8bit
 
-# Copyright 2014 Ball Aerospace & Technologies Corp.
+# Copyright 2022 Ball Aerospace & Technologies Corp.
 # All Rights Reserved.
 #
 # This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU General Public License
+# under the terms of the GNU Affero General Public License
 # as published by the Free Software Foundation; version 3 with
 # attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
 
 require 'cosmos/packets/packet_config'
 
 module Cosmos
-
   # Commands uses PacketConfig to parse the command and telemetry
   # configuration files. It contains all the knowledge of which command packets
   # exist in the system and how to access them. This class is the API layer
@@ -51,6 +59,7 @@ module Cosmos
     def packets(target_name)
       target_packets = @config.commands[target_name.to_s.upcase]
       raise "Command target '#{target_name.to_s.upcase}' does not exist" unless target_packets
+
       target_packets
     end
 
@@ -62,6 +71,7 @@ module Cosmos
       target_packets = packets(target_name)
       packet = target_packets[packet_name.to_s.upcase]
       raise "Command packet '#{target_name.to_s.upcase} #{packet_name.to_s.upcase}' does not exist" unless packet
+
       packet
     end
 
@@ -91,7 +101,6 @@ module Cosmos
 
       target_names.each do |target_name|
         target_name = target_name.to_s.upcase
-        
         target_packets = nil
         begin
           target_packets = packets(target_name)
@@ -117,15 +126,15 @@ module Cosmos
             hash = @config.cmd_id_value_hash[target_name]
             identified_packet = hash[key]
             identified_packet = hash['CATCHALL'.freeze] unless identified_packet
-          end          
+          end
         end
 
         if identified_packet
+          identified_packet.received_count += 1
           identified_packet = identified_packet.clone
           identified_packet.received_time = nil
           identified_packet.stored = false
           identified_packet.extra = nil
-          identified_packet.received_count = 0
           identified_packet.buffer = packet_data
           break
         end
@@ -151,7 +160,9 @@ module Cosmos
       packet_upcase = packet_name.to_s.upcase
 
       # Lookup the command and create a light weight copy
-      command = packet(target_upcase, packet_upcase).clone
+      pkt = packet(target_upcase, packet_upcase)
+      pkt.received_count += 1
+      command = pkt.clone
 
       # Restore the command's buffer to a zeroed string of defined length
       # This will undo any side effects from earlier commands that may have altered the size
@@ -181,7 +192,7 @@ module Cosmos
         items = packet.read_all(:FORMATTED)
         raw = false
       end
-      items.delete_if {|item_name, item_value| ignored_parameters.include?(item_name)}
+      items.delete_if { |item_name, item_value| ignored_parameters.include?(item_name) }
       return build_cmd_output_string(packet.target_name, packet.packet_name, items, raw)
     end
 
@@ -202,6 +213,7 @@ module Cosmos
 
         params = []
         cmd_params.each do |key, value|
+          next if Packet::RESERVED_ITEM_NAMES.include?(key)
 
           begin
             item_type = command_items[key].data_type
@@ -223,7 +235,7 @@ module Cosmos
             if value.length > 256
               value = value[0..255] + "...'"
             end
-            value.tr!('"',"'")
+            value.tr!('"', "'")
           elsif value.is_a?(Array)
             value = "[#{value.join(", ")}]"
           end
@@ -293,11 +305,13 @@ module Cosmos
           splash.progress = index / total
         end
 
-        ignored_items = System.targets[target_name].ignored_items
+        # TODO: This wasn't being used ... should it be
+        # ignored_items = System.targets[target_name].ignored_items
 
         packets(target_name).each do |command_name, packet|
           # We don't audit against hidden or disabled packets/commands
           next if !include_hidden and (packet.hidden || packet.disabled)
+
           strings << "#{target_name} #{command_name}"
         end
       end
@@ -355,7 +369,5 @@ module Cosmos
         end
       end
     end
-
   end # class Commands
-
 end # module Cosmos