openc3 5.0.11 → 5.1.0

data/lib/openc3/operators/operator.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'childprocess'
 require 'openc3'
@@ -23,6 +26,56 @@ require 'fileutils'
 require 'tempfile'
 
 module OpenC3
+  # Class to prevent an infinitely growing log file
+  class OperatorProcessIO < Tempfile
+    def initialize(label, max_start_lines: 100, max_end_lines: 100)
+      super(label)
+      @max_start_lines = max_start_lines
+      @max_end_lines = max_end_lines
+      @start_lines = []
+      @end_lines = []
+    end
+
+    def extract
+      rewind()
+      data = read()
+      truncate(0)
+      rewind()
+
+      # Save a set number of lines for unexpected death messages
+      lines = data.split("\n")
+      lines.each do |line|
+        if @start_lines.length < @max_start_lines
+          @start_lines << line
+        else
+          @end_lines << line
+        end
+      end
+      if @end_lines.length > @max_end_lines
+        @end_lines = @end_lines[(@end_lines.length - @max_end_lines)..-1]
+      end
+
+      return data
+    end
+
+    def finalize
+      extract()
+      close()
+      unlink()
+
+      output = ''
+      output << @start_lines.join("\n")
+      if @end_lines.length >= @max_end_lines
+        output << "\n...\n"
+        output << @end_lines.join("\n")
+      elsif @end_lines.length > 0
+        output << @end_lines.join("\n")
+      end
+      output << "\n"
+      output
+    end
+  end
+
   class OperatorProcess
     attr_accessor :process_definition
     attr_accessor :work_dir
@@ -48,21 +101,25 @@ module OpenC3
       @config = config
     end
 
-    def start
-      @temp_dir = @new_temp_dir
-      @new_temp_dir = nil
-
+    def cmd_line
       # In ProcessManager processes, the process_definition is the actual thing run
       # e.g. OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "load", ...])
       # However, if the MicroserviceOperator is spawning the proceses it sets
       # process_definition = ["ruby", "plugin_microservice.rb"]
       # which then calls exec(*@config["cmd"]) to actually run
       # So check if the @config['cmd'] is defined to give the user more info in the log
-      cmd = @process_definition.join(' ')
+      cmd_line_text = @process_definition.join(' ')
       if @config && @config['cmd']
-        cmd = @config['cmd'].join(' ')
+        cmd_line_text = @config['cmd'].join(' ')
       end
-      Logger.info("Starting: #{cmd}", scope: @scope)
+      return cmd_line_text
+    end
+
+    def start
+      @temp_dir = @new_temp_dir
+      @new_temp_dir = nil
+
+      Logger.info("Starting: #{cmd_line()}", scope: @scope)
 
       @process = ChildProcess.build(*@process_definition)
       # This lets the ChildProcess use the parent IO ... but it breaks unit tests
@@ -79,8 +136,8 @@ module OpenC3
         @process.environment[key] = value
       end
       @process.environment['OPENC3_SCOPE'] = @scope
-      @process.io.stdout = Tempfile.new("child-output")
-      @process.io.stderr = Tempfile.new("child-output")
+      @process.io.stdout = OperatorProcessIO.new('microservice-stdout')
+      @process.io.stderr = OperatorProcessIO.new('microservice-stderr')
       @process.start
     end
 
@@ -102,14 +159,18 @@ module OpenC3
 
     def soft_stop
       Thread.new do
-        Logger.info("Soft shutting down process: #{@process_definition.join(' ')}", scope: @scope)
+        Logger.info("Soft shutting down process: #{cmd_line()}", scope: @scope)
         Process.kill("SIGINT", @process.pid) if @process # Signal the process to stop
       end
     end
 
     def hard_stop
       if @process and !@process.exited?
-        Logger.info("Hard shutting down process: #{@process_definition.join(' ')}", scope: @scope)
+        # Redis may be down at this point so just catch any Logger errors
+        begin
+          Logger.info("Hard shutting down process: #{cmd_line()}", scope: @scope)
+        rescue Exception
+        end
         @process.stop
       end
       FileUtils.remove_entry(@temp_dir) if @temp_dir and File.exist?(@temp_dir)
@@ -124,20 +185,37 @@ module OpenC3
       @process.io.stderr
     end
 
-    def extract_output(max_length_stdout = 65536, max_length_stderr = 65536)
+    def output_increment
       if @process
-        @process.io.stdout.rewind
-        output = @process.io.stdout.read
-        @process.io.stdout.close
-        @process.io.stdout.unlink
-        @process.io.stderr.rewind
-        err_output = @process.io.stderr.read
-        @process.io.stderr.close
-        @process.io.stderr.unlink
-        return "Stdout:\n#{output[-max_length_stdout..-1] || output}\n\nStderr:\n#{err_output[-max_length_stderr..-1] || err_output}\n"
-      else
-        return ""
+        stdout = @process.io.stdout.extract
+        if stdout.length > 0
+          STDOUT.puts "STDOUT #{stdout.length} bytes from #{cmd_line()}:"
+          STDOUT.puts stdout
+        end
+        stderr = @process.io.stderr.extract
+        if stderr.length > 0
+          STDERR.puts "STDERR #{stderr.length} bytes from #{cmd_line()}:"
+          STDERR.puts stderr
+        end
+      end
+    end
+
+    # This is method is used in here and in ProcessManager
+    def extract_output
+      output = ''
+      if @process
+        stdout = @process.io.stdout.finalize
+        stderr = @process.io.stderr.finalize
+
+        # Always include the Stdout header for consistency and to show the option
+        output << "Stdout:\n"
+        output << stdout
+
+        # Always include the nStderr header for consistency and to show the option
+        output << "\nStderr:\n"
+        output << stderr
       end
+      output
     end
   end
 
@@ -215,18 +293,11 @@ module OpenC3
       @mutex.synchronize do
         @processes.each do |name, p|
           break if @shutdown
-
+          p.output_increment
           unless p.alive?
             # Respawn process
-            p.stdout.rewind
-            output = p.stdout.read
-            p.stdout.close
-            p.stdout.unlink
-            p.stderr.rewind
-            err_output = p.stderr.read
-            p.stderr.close
-            p.stderr.unlink
-            Logger.error("Unexpected process died... respawning! #{p.process_definition.join(' ')}\nStdout:\n#{output}\nStderr:\n#{err_output}\n", scope: p.scope)
+            output = p.extract_output
+            Logger.error("Unexpected process died... respawning! #{p.cmd_line}\n#{output}\n", scope: p.scope)
             p.start
           end
         end
@@ -235,7 +306,7 @@ module OpenC3
 
     def shutdown_processes(processes)
       processes.each { |name, p| p.soft_stop }
-      sleep(2) # TODO: This is an arbitrary sleep of 2s ...
+      sleep(4) # TODO: This is an arbitrary sleep of 4s ...
       processes.each { |name, p| p.hard_stop }
     end
 

data/lib/openc3/packets/commands.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet_config'
 

data/lib/openc3/packets/json_packet.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3'
 require 'json'
@@ -30,20 +33,63 @@ module OpenC3
     attr_accessor :stored
     attr_accessor :json_hash
 
-    def initialize(cmd_or_tlm, target_name, packet_name, time_nsec_from_epoch, stored, json_data)
+    def initialize(cmd_or_tlm, target_name, packet_name, time_nsec_from_epoch, stored, json_data, key_map = nil)
       @cmd_or_tlm = cmd_or_tlm.intern
       @target_name = target_name
       @packet_name = packet_name
       @packet_time = ::Time.from_nsec_from_epoch(time_nsec_from_epoch)
       @stored = ConfigParser.handle_true_false(stored)
-      @json_hash = JSON.parse(json_data, :allow_nan => true, :create_additions => true)
+      @json_hash = json_data
+      @json_hash = JSON.parse(json_data, :allow_nan => true, :create_additions => true) if String === json_data
+      if key_map
+        uncompressed = {}
+        @json_hash.each do |key, value|
+          uncompressed_key = key_map[key]
+          uncompressed_key = key unless uncompressed_key
+          uncompressed[uncompressed_key] = value
+        end
+        @json_hash = uncompressed
+      end
     end
 
     # Read an item in the packet by name
     #
     # @param name [String] Name of the item to read - Should already by upcase
     # @param value_type (see #read_item)
-    def read(name, value_type = :CONVERTED)
+    def read(name, value_type = :CONVERTED, reduced_type = nil)
+      if reduced_type
+        raise "Reduced types only support RAW or CONVERTED value types: #{value_type} unsupported" if value_type == :WITH_UNITS or value_type == :FORMATTED
+        if value_type == :CONVERTED
+          case reduced_type
+          when :AVG
+            value = @json_hash["#{name}__CA"]
+            return value if value
+          when :STDDEV
+            value = @json_hash["#{name}__CS"]
+            return value if value
+          when :MIN
+            value = @json_hash["#{name}__CN"]
+            return value if value
+          when :MAX
+            value = @json_hash["#{name}__CX"]
+            return value if value
+          end
+        end
+        case reduced_type
+        when :AVG
+          value = @json_hash["#{name}__A"]
+          return value if value
+        when :STDDEV
+          value = @json_hash["#{name}__S"]
+          return value if value
+        when :MIN
+          value = @json_hash["#{name}__N"]
+          return value if value
+        when :MAX
+          value = @json_hash["#{name}__X"]
+          return value if value
+        end
+      end
       if value_type == :WITH_UNITS
         value = @json_hash["#{name}__U"]
         return value if value
@@ -68,8 +114,8 @@ module OpenC3
       return value if value
     end
 
-    def read_with_limits_state(name, value_type = :CONVERTED)
-      value = read(name, value_type)
+    def read_with_limits_state(name, value_type = :CONVERTED, reduced_type = nil)
+      value = read(name, value_type, reduced_type)
       limits_state = @json_hash["#{name}__L"]
       limits_state.intern if limits_state
       return [value, limits_state]
@@ -79,11 +125,11 @@ module OpenC3
     #   [[item name, item value], ...]
     #
     # @param value_type (see #read_item)
-    def read_all(value_type = :CONVERTED, names = nil)
+    def read_all(value_type = :CONVERTED, reduced_type = nil, names = nil)
       result = {}
       names = read_all_names() unless names
       names.each do |name|
-        result[name] = read(name, value_type)
+        result[name] = read(name, value_type, reduced_type)
       end
       return result
     end
@@ -92,21 +138,48 @@ module OpenC3
     #   [[item name, item value], [item limits state], ...]
     #
     # @param value_type (see #read_all)
-    def read_all_with_limits_states(value_type = :CONVERTED, names = nil)
+    def read_all_with_limits_states(value_type = :CONVERTED, reduced_type = nil, names = nil)
       result = {}
       names = read_all_names() unless names
       names.each do |name|
-        result[name] = read_with_limits_state(name, value_type)
+        result[name] = read_with_limits_state(name, value_type, reduced_type)
       end
       return result
     end
 
     # Read all the names of items in the packet
     # Note: This is not very efficient, ideally only call once for discovery purposes
-    def read_all_names
+    def read_all_names(value_type = nil, reduced_type = nil)
       result = {}
-      @json_hash.each do |key, value|
-        result[key.split("__")[0]] = true
+      if value_type
+        case value_type
+        when :RAW
+          postfix = ''
+        when :CONVERTED
+          postfix = 'C'
+        when :FORMATTED
+          postfix = 'F'
+        when :WITH_UNITS
+          postfix = 'U'
+        end
+        case reduced_type
+        when :MIN
+          postfix << 'N'
+        when :MAX
+          postfix << 'X'
+        when :AVG
+          postfix << 'A'
+        when :STDDEV
+          postfix << 'S'
+        else
+          postfix = nil if value_type == :RAW
+        end
+        @json_hash.each do |key, value|
+          key_split = key.split("__")
+          result[key_split[0]] = true if key_split[1] == postfix
+        end
+      else
+        @json_hash.each { |key, value| result[key.split("__")[0]] = true }
       end
       return result.keys
     end
@@ -115,12 +188,12 @@ module OpenC3
     #
     # @param value_type (see #read_item)
     # @param indent (see Structure#formatted)
-    def formatted(value_type = :CONVERTED, names = nil, indent = 0)
+    def formatted(value_type = :CONVERTED, reduced_type = nil, names = nil, indent = 0)
       names = read_all_names() unless names
       indent_string = ' ' * indent
       string = ''
       names.each do |name|
-        value = read(name, value_type)
+        value = read(name, value_type, reduced_type)
         if String === value and value =~ File::NON_ASCII_PRINTABLE
           string << "#{indent_string}#{name}:\n"
           string << value.formatted(1, 16, ' ', indent + 2)

data/lib/openc3/packets/limits.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet_config'
 
@@ -234,7 +237,7 @@ module OpenC3
         limits_for_set[4] = green_low.to_f
         limits_for_set[5] = green_high.to_f
       end
-      limits.enabled = enabled
+      limits.enabled = enabled if not enabled.nil?
       limits.persistence_setting = Integer(persistence) if persistence
       packet.update_limits_items_cache(item)
       @config.limits_sets << limits_set

data/lib/openc3/packets/limits_response.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 # This file implements a class to handle responses to limits state changes.
 

data/lib/openc3/packets/packet.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'digest'
 require 'openc3/packets/structure'
@@ -558,7 +561,8 @@ module OpenC3
     #   can be any type.
     def read_item(item, value_type = :CONVERTED, buffer = @buffer, given_raw = nil)
       if given_raw
-        value = given_raw
+        # Must clone this since value is returned
+        value = given_raw.clone
       else
         value = super(item, :RAW, buffer)
       end

data/lib/openc3/packets/packet_config.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/config/config_parser'
 require 'openc3/packets/packet'

data/lib/openc3/packets/packet_item.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/structure_item'
 require 'openc3/packets/packet_item_limits'
@@ -464,6 +467,8 @@ module OpenC3
       config['write_conversion'] = self.write_conversion.as_json(*a) if self.write_conversion
 
       if self.limits
+        config['limits'] ||= {}
+        config['limits']['enabled'] = true if self.limits.enabled
         if self.limits.values
           config['limits'] ||= {}
           config['limits']['persistence_setting'] = self.limits.persistence_setting
@@ -527,17 +532,18 @@ module OpenC3
         end
       end
 
+      item.limits = PacketItemLimits.new
       if hash['limits']
-        item.limits = PacketItemLimits.new
         # Delete these keys so the only ones left are limits sets
-        item.limits.persistence_setting = hash['limits'].delete('persistence_setting')
+        persistence_setting = hash['limits'].delete('persistence_setting')
+        item.limits.persistence_setting = persistence_setting if persistence_setting
         item.limits.enabled = true if hash['limits'].delete('enabled')
         values = {}
         hash['limits'].each do |set, items|
           values[set.to_sym] = [items['red_low'], items['yellow_low'], items['yellow_high'], items['red_high']]
           values[set.to_sym].concat([items['green_low'], items['green_high']]) if items['green_low'] && items['green_high']
         end
-        item.limits.values = values
+        item.limits.values = values if values.length > 0
       end
       item.meta = hash['meta']
       item

data/lib/openc3/packets/packet_item_limits.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/limits_response'
 

data/lib/openc3/packets/parsers/format_string_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   class FormatStringParser

data/lib/openc3/packets/parsers/limits_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   class LimitsParser

data/lib/openc3/packets/parsers/limits_response_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   class LimitsResponseParser

data/lib/openc3/packets/parsers/packet_item_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet_item'
 

data/lib/openc3/packets/parsers/packet_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet'
 

data/lib/openc3/packets/parsers/processor_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/processors'
 

data/lib/openc3/packets/parsers/state_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet_item'
 

data/lib/openc3/packets/parsers/xtce_converter.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'nokogiri'
 require 'openc3/packets/parsers/xtce_parser'

data/lib/openc3/packets/parsers/xtce_parser.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'nokogiri'
 require 'ostruct'

data/lib/openc3/packets/structure.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/binary_accessor'
 require 'openc3/packets/structure_item'

data/lib/openc3/packets/structure_item.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/ext/packet' if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']
 

data/lib/openc3/packets/telemetry.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet_config'
 require 'openc3/ext/telemetry' if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']

data/lib/openc3/processors/processor.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   class Processor

data/lib/openc3/processors/statistics_processor.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/processors/processor'
 

data/lib/openc3/processors/watermark_processor.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/processors/processor'