openc3 5.0.6

data/lib/openc3/utilities/logger.rb

@@ -0,0 +1,187 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'openc3/core_ext/class'
+require 'openc3/core_ext/time'
+require 'openc3/topics/topic'
+require 'socket'
+require 'logger'
+require 'time'
+require 'json'
+
+module OpenC3
+  # Supports different levels of logging and only writes if the level
+  # is exceeded.
+  class Logger
+    # @return [Boolean] Whether to output the message to stdout
+    instance_attr_accessor :stdout
+
+    # @return [Integer] The logging level
+    instance_attr_accessor :level
+
+    # @return [String] Additional detail to add to messages
+    instance_attr_accessor :detail_string
+
+    # @return [String] Fluent tag
+    instance_attr_accessor :tag
+
+    # @return [String] Microservice name
+    instance_attr_accessor :microservice_name
+
+    # @return [String] Scope
+    instance_attr_accessor :scope
+
+    @@mutex = Mutex.new
+    @@instance = nil
+
+    # DEBUG only prints DEBUG messages
+    DEBUG = ::Logger::DEBUG
+    # INFO prints INFO, DEBUG messages
+    INFO  = ::Logger::INFO
+    # WARN prints WARN, INFO, DEBUG messages
+    WARN  = ::Logger::WARN
+    # ERROR prints ERROR, WARN, INFO, DEBUG messages
+    ERROR = ::Logger::ERROR
+    # FATAL prints FATAL, ERROR, WARN, INFO, DEBUG messages
+    FATAL = ::Logger::FATAL
+
+    DEBUG_SEVERITY_STRING = 'DEBUG'
+    INFO_SEVERITY_STRING = 'INFO'
+    WARN_SEVERITY_STRING = 'WARN'
+    ERROR_SEVERITY_STRING = 'ERROR'
+    FATAL_SEVERITY_STRING = 'FATAL'
+
+    # @param level [Integer] The initial logging level
+    def initialize(level = Logger::INFO)
+      @stdout = true
+      @level = level
+      @scope = nil
+      @detail_string = nil
+      @container_name = Socket.gethostname
+      @microservice_name = nil
+      @tag = @container_name + ".log"
+      @mutex = Mutex.new
+      @no_store = ENV['OPENC3_NO_STORE']
+    end
+
+    # @param message [String] The message to print if the log level is at or
+    #   below the method name log level.
+    # @param block [Proc] Block to call which should return a string to append
+    #   to the log message
+    def debug(message = nil, scope: @scope, user: nil, &block)
+      log_message(DEBUG_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= DEBUG
+    end
+
+    # (see #debug)
+    def info(message = nil, scope: @scope, user: nil, &block)
+      log_message(INFO_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= INFO
+    end
+
+    # (see #debug)
+    def warn(message = nil, scope: @scope, user: nil, &block)
+      log_message(WARN_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= WARN
+    end
+
+    # (see #debug)
+    def error(message = nil, scope: @scope, user: nil, &block)
+      log_message(ERROR_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= ERROR
+    end
+
+    # (see #debug)
+    def fatal(message = nil, scope: @scope, user: nil, &block)
+      log_message(FATAL_SEVERITY_STRING, message, scope: scope, user: user, &block) if @level <= FATAL
+    end
+
+    # (see #debug)
+    def self.debug(message = nil, scope: nil, user: nil, &block)
+      args = {}
+      args[:scope] = scope if scope
+      args[:user] = user if user
+      self.instance.debug(message, **args, &block)
+    end
+
+    # (see #debug)
+    def self.info(message = nil, scope: nil, user: nil, &block)
+      args = {}
+      args[:scope] = scope if scope
+      args[:user] = user if user
+      self.instance.info(message, **args, &block)
+    end
+
+    # (see #debug)
+    def self.warn(message = nil, scope: nil, user: nil, &block)
+      args = {}
+      args[:scope] = scope if scope
+      args[:user] = user if user
+      self.instance.warn(message, **args, &block)
+    end
+
+    # (see #debug)
+    def self.error(message = nil, scope: nil, user: nil, &block)
+      args = {}
+      args[:scope] = scope if scope
+      args[:user] = user if user
+      self.instance.error(message, **args, &block)
+    end
+
+    # (see #debug)
+    def self.fatal(message = nil, scope: nil, user: nil, &block)
+      args = {}
+      args[:scope] = scope if scope
+      args[:user] = user if user
+      self.instance.fatal(message, **args, &block)
+    end
+
+    # @return [Logger] The logger instance
+    def self.instance
+      return @@instance if @@instance
+
+      @@mutex.synchronize do
+        @@instance ||= self.new
+      end
+      @@instance
+    end
+
+    protected
+
+    def log_message(severity_string, message, scope:, user:)
+      @mutex.synchronize do
+        data = { time: Time.now.to_nsec_from_epoch, '@timestamp' => Time.now.xmlschema(3), severity: severity_string }
+        data[:microservice_name] = @microservice_name if @microservice_name
+        data[:detail] = @detail_string if @detail_string
+        data[:user] = user['name'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
+        if block_given?
+          message = yield
+        end
+        data[:container_name] = @container_name
+        data[:log] = message
+        puts data.as_json(:allow_nan => true).to_json(:allow_nan => true) if @stdout
+        unless @no_store
+          if scope
+            Topic.write_topic("#{scope}__openc3_log_messages", data)
+          else
+            # The base openc3_log_messages doesn't have an associated logger
+            # so it must be limited to prevent unbounded stream growth
+            Topic.write_topic("openc3_log_messages", data, '*', 1000)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/openc3/utilities/message_log.rb

@@ -0,0 +1,91 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'openc3/config/config_parser'
+require 'openc3/system/system'
+require 'fileutils'
+require 'openc3/utilities/s3'
+
+module OpenC3
+  # Handles writing message logs to a file
+  class MessageLog
+    # @return [String] The name of the message log file. Empty string until the
+    #   write or start methods are called at which point it is set to the
+    #   filename. Retains the last filename even after stop is called.
+    attr_reader :filename
+
+    # @param tool_name [String] The name of the tool creating the message log.
+    #   This will be inserted into the message log filename to help identify it.
+    # @param log_dir [String] The filesystem path to store the message log file.
+    def initialize(tool_name, log_dir, scope:)
+      @remote_log_directory = "#{scope}/tool_logs/#{tool_name}/"
+      @tool_name = tool_name
+      @log_dir = log_dir
+      @filename = ''
+      @file = nil
+      @start_day = nil
+      @mutex = Mutex.new
+    end
+
+    # Ensures the log file is opened and ready to write. It then writes the
+    # message to the log and flushes it to force the write.
+    #
+    # @param message [String] Message to write to the log
+    def write(message, flush = false)
+      @mutex.synchronize do
+        if @file.nil? or @file.closed? or (not File.exist?(@filename))
+          start(false)
+        end
+
+        @file.write(message)
+        @file.flush if flush
+      end
+    end
+
+    # Closes the message log and marks it read only
+    def stop(take_mutex = true, s3_object_metadata: {})
+      @mutex.lock if take_mutex
+      if @file and not @file.closed?
+        @file.close
+        File.chmod(0444, @filename)
+        s3_key = File.join(@remote_log_directory, @start_day, File.basename(@filename))
+        begin
+          thread = S3Utilities.move_log_file_to_s3(@filename, s3_key, metadata: s3_object_metadata)
+          thread.join
+        rescue StandardError => e
+          Logger.error e.formatted
+        end
+      end
+      @mutex.unlock if take_mutex
+    end
+
+    # Creates a new message log and sets the filename
+    def start(take_mutex = true)
+      @mutex.lock if take_mutex
+      # Prevent starting files too fast
+      sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename([@tool_name, 'messages'])))
+      stop(false)
+      timed_filename = File.build_timestamped_filename([@tool_name, 'messages'])
+      @start_day = timed_filename[0..9].gsub("_", "") # YYYYMMDD
+      @filename = File.join(@log_dir, timed_filename)
+      @file = File.open(@filename, 'a')
+      @mutex.unlock if take_mutex
+    end
+  end
+end

data/lib/openc3/utilities/metric.rb

@@ -0,0 +1,141 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'openc3/models/metric_model'
+require 'thread'
+
+module OpenC3
+  class Metric
+    # This class is designed to output metrics to the openc3-cmd-tlm-api
+    # InternalMetricsController. Output format can be read about here
+    # https://prometheus.io/docs/concepts/data_model/
+    #
+    # Warning contains some sorcery.
+    #
+    # examples:
+    #    TYPE foobar histogram
+    #    HELP foobar internal metric generated from openc3/utilities/metric.rb
+    #    foobar{code="200",method="get",path="/metrics"} 5.0
+    #
+    # items = {"name|labels" => [value_array], ...}
+
+    attr_reader :items
+    attr_accessor :size
+    attr_reader :scope
+    attr_reader :microservice
+
+    def initialize(microservice:, scope:)
+      if microservice.include? '|' or scope.include? '|'
+        raise ArgumentError.new('invalid input must not contain: |')
+      end
+
+      @items = {}
+      @scope = scope
+      @microservice = microservice
+      @size = 5000
+      @mutex = Mutex.new
+    end
+
+    def add_sample(name:, value:, labels:)
+      # add a value to the metric to report out later or a seperate thread
+      # name is a string often function_name_duration_seconds
+      #    name: debug_duration_seconds
+      # value is a numerical value to add to a round robin array.
+      #    value: 0.1211
+      # labels is a hash of values that could have an effect on the value.
+      #    labels: {"code"=>200,"method"=>"get","path"=>"/metrics"}
+      # internal:
+      # the internal items hash is used as a lookup table to store unique
+      # varients of a similar metric. these varients are values that could
+      # cause a difference in run time to add context to the metric. the
+      # microservice and scope are added to the labels the labels are
+      # converted to a string and joined with the name to create a unique
+      # metric item. this is looked up in the items hash and if not found
+      # the key is created and an array of size @size is allocated. then
+      # the value is added to @items and the count of the value is increased
+      # if the count of the values exceed the size of the array it sets the
+      # count back to zero and the array will over write older data.
+      @mutex.synchronize do
+        key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
+        if not @items.has_key?(key)
+          Logger.debug("new data for #{@scope}, #{key}")
+          @items[key] = { 'values' => Array.new(@size), 'count' => 0 }
+        end
+        count = @items[key]['count']
+        # Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
+        @items[key]['values'][count] = value
+        @items[key]['count'] = count + 1 >= @size ? 0 : count + 1
+      end
+    end
+
+    def percentile(sorted_values, percentile)
+      # get the percentile out of an ordered array
+      len = sorted_values.length
+      return sorted_values.first if len == 1
+
+      k = ((percentile / 100.0) * (len - 1) + 1).floor - 1
+      f = ((percentile / 100.0) * (len - 1) + 1).modulo(1)
+      return sorted_values[k] + (f * (sorted_values[k + 1] - sorted_values[k]))
+    end
+
+    def output
+      # Output percentile based metrics to Redis under the key of the
+      # #{@scope}__openc3__metric we will use hset with a subkey.
+      # internal:
+      # loop over the key value pairs within the @items hash, remove nil
+      # and sort the values added via the add_sample method. calculate the
+      # different percentiles. the labels are still only contained in the
+      # key of the @items hash to extract these you split the key on | the
+      # name|labels then to make the labels back into a hash we split the ,
+      # into an array ["foo=bar", ...] and split again this time on the =
+      # into [["foo","bar"], ...] to map the internal array into a hash
+      # [{"foo"=>"bar"}, ...] finally reducing the array into a single hash
+      # to add the percentile and percentile value. this hash is added to an
+      # array. to store the array as the value with the metric name again joined
+      # with the @microservice and @scope.
+      Logger.debug("#{@microservice} #{@scope} sending metrics to redis, #{@items.length}") if @items.length > 0
+      @mutex.synchronize do
+        @items.each do |key, values|
+          label_list = []
+          name, labels = key.split('|')
+          metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
+          sorted_values = values['values'].compact.sort
+          for percentile_value in [10, 50, 90, 95, 99]
+            percentile_result = percentile(sorted_values, percentile_value)
+            labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
+            labels['percentile'] = percentile_value
+            labels['metric__value'] = percentile_result
+            label_list.append(labels)
+          end
+          begin
+            Logger.debug("sending metrics summary to redis key: #{@microservice}")
+            metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
+            metric.create(force: true)
+          rescue RuntimeError
+            Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
+          end
+        end
+      end
+    end
+
+    def destroy
+      MetricModel.destroy(scope: @scope, name: @microservice)
+    end
+  end
+end

data/lib/openc3/utilities/process_manager.rb

@@ -0,0 +1,139 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'openc3/operators/operator'
+require 'openc3/models/process_status_model'
+require 'openc3/models/scope_model'
+require 'openc3/utilities/logger'
+require 'socket'
+
+module OpenC3
+  class ProcessManagerProcess < OperatorProcess
+    attr_accessor :process_type
+    attr_accessor :detail
+    attr_accessor :expires_at
+    attr_accessor :status
+
+    def initialize(cmd_array, process_type, detail, expires_at, **kw_args)
+      super(cmd_array, **kw_args)
+      @process_type = process_type
+      @detail = detail
+      @expires_at = expires_at
+      @status = nil
+    end
+
+    def start
+      super()
+      if @process
+        @status = ProcessStatusModel.new(name: "#{Socket.gethostname}__#{@process.pid}", process_type: @process_type, detail: @detail, state: "Running", scope: @scope)
+        @status.create
+      end
+    end
+  end
+
+  # Spawns short lived processes and ensures they complete
+  class ProcessManager
+    MONITOR_CYCLE_SECONDS = 10
+    CLEANUP_CYCLE_SECONDS = 600
+
+    @@instance = nil
+
+    def self.instance
+      @@instance = ProcessManager.new unless @@instance
+      return @@instance
+    end
+
+    def initialize
+      @processes = []
+      @monitor_thread = Thread.new do
+        begin
+          monitor()
+        rescue => err
+          raise "ProcessManager unexpectedly died\n#{err.formatted}"
+        end
+      end
+    end
+
+    def spawn(cmd_array, process_type, detail, expires_at, **kw_args)
+      process = ProcessManagerProcess.new(cmd_array, process_type, detail, expires_at, **kw_args)
+      process.start
+      @processes << process
+    end
+
+    def monitor
+      processes_to_delete = []
+      cleanup_time = Time.now
+      while true
+        current_time = Time.now
+
+        # Monitor Active Processes
+        @processes.each do |process|
+          # Check if the process is still alive
+          if !process.alive?
+            if process.exit_code != 0
+              process.status.state = "Crashed"
+            else
+              process.status.state = "Complete"
+            end
+            output = process.extract_output
+            process.status.output = output
+            process.hard_stop
+            processes_to_delete << process
+          elsif process.expires_at < current_time
+            process.status.state = "Expired"
+            output = process.extract_output
+            process.status.output = output
+            process.hard_stop
+            processes_to_delete << process
+          end
+
+          # Update Process Status
+          process.status.update
+        end
+        processes_to_delete.each do |process|
+          if process.status.state == "Complete"
+            Logger.info "Process #{process.status.name}:#{process.process_type}:#{process.detail} completed with state #{process.status.state}"
+          else
+            Logger.error "Process #{process.status.name}:#{process.process_type}:#{process.detail} completed with state #{process.status.state}"
+            Logger.error "Process Output:\n#{process.status.output}"
+          end
+
+          @processes.delete(process)
+        end
+        processes_to_delete.clear
+
+        # Cleanup Old Process Status
+        if (current_time - cleanup_time) > CLEANUP_CYCLE_SECONDS
+          scopes = ScopeModel.names
+          scopes.each do |scope|
+            statuses = ProcessStatusModel.get_all_models(scope: scope)
+            statuses.each do |status_name, status|
+              if (current_time - Time.from_nsec_from_epoch(status.updated_at)) > CLEANUP_CYCLE_SECONDS
+                status.destroy
+              end
+            end
+          end
+        end
+
+        sleep(MONITOR_CYCLE_SECONDS)
+      end
+    end
+
+  end
+end