openc3 5.2.0 → 5.4.0

data/lib/openc3/utilities/metric.rb

@@ -1,6 +1,6 @@
 # encoding: ascii-8bit
 
-# Copyright 2022 Ball Aerospace & Technologies Corp.
+# Copyright 2022 OpenC3, Inc.
 # All Rights Reserved.
 #
 # This program is free software; you can modify and/or redistribute it
@@ -12,133 +12,109 @@
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
-
-# Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
-# All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
+#
+# Note: This file has been completely reimplemented post version 5.3.0
 
 require 'openc3/models/metric_model'
 require 'thread'
 
 module OpenC3
   class Metric
-    # This class is designed to output metrics to the openc3-cosmos-cmd-tlm-api
-    # InternalMetricsController. Output format can be read about here
-    # https://prometheus.io/docs/concepts/data_model/
-    #
-    # Warning contains some sorcery.
-    #
-    # examples:
-    #    TYPE foobar histogram
-    #    HELP foobar internal metric generated from openc3/utilities/metric.rb
-    #    foobar{code="200",method="get",path="/metrics"} 5.0
-    #
-    # items = {"name|labels" => [value_array], ...}
-
-    attr_reader :items
-    attr_accessor :size
-    attr_reader :scope
+    # The update interval. How often in seconds metrics are updated by this process
+    UPDATE_INTERVAL = 5
+
+    # Mutex protecting class variables
+    @@mutex = Mutex.new
+
+    # Array of instances used to keep track of metrics
+    @@instances = []
+
+    # Thread used to post metrics across all classes
+    @@update_thread = nil
+
+    # Sleeper used to delay update thread
+    @@update_sleeper = nil
+
     attr_reader :microservice
+    attr_reader :scope
+    attr_reader :data
+    attr_reader :mutex
 
     def initialize(microservice:, scope:)
-      if microservice.include? '|' or scope.include? '|'
-        raise ArgumentError.new('invalid input must not contain: |')
-      end
-
-      @items = {}
       @scope = scope
       @microservice = microservice
-      @size = 5000
+      @data = {}
       @mutex = Mutex.new
-    end
 
-    def add_sample(name:, value:, labels:)
-      # add a value to the metric to report out later or a seperate thread
-      # name is a string often function_name_duration_seconds
-      #    name: debug_duration_seconds
-      # value is a numerical value to add to a round robin array.
-      #    value: 0.1211
-      # labels is a hash of values that could have an effect on the value.
-      #    labels: {"code"=>200,"method"=>"get","path"=>"/metrics"}
-      # internal:
-      # the internal items hash is used as a lookup table to store unique
-      # varients of a similar metric. these varients are values that could
-      # cause a difference in run time to add context to the metric. the
-      # microservice and scope are added to the labels the labels are
-      # converted to a string and joined with the name to create a unique
-      # metric item. this is looked up in the items hash and if not found
-      # the key is created and an array of size @size is allocated. then
-      # the value is added to @items and the count of the value is increased
-      # if the count of the values exceed the size of the array it sets the
-      # count back to zero and the array will over write older data.
-      @mutex.synchronize do
-        key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
-        if not @items.has_key?(key)
-          Logger.debug("new data for #{@scope}, #{key}")
-          @items[key] = { 'values' => Array.new(@size), 'count' => 0 }
+      # Always make sure there is a update thread
+      @@mutex.synchronize do
+        @@instances << self
+
+        unless @@update_thread
+          @@update_thread = OpenC3.safe_thread("Metrics") do
+            update_thread_body()
+          end
         end
-        count = @items[key]['count']
-        # Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
-        @items[key]['values'][count] = value
-        @items[key]['count'] = count + 1 >= @size ? 0 : count + 1
       end
     end
 
-    def percentile(sorted_values, percentile)
-      # get the percentile out of an ordered array
-      len = sorted_values.length
-      return sorted_values.first if len == 1
+    def set(name:, value:, type: nil, unit: nil, help: nil, labels: nil, time_ms: nil)
+      @mutex.synchronize do
+        @data[name] ||= {}
+        @data[name]['value'] = value
+        @data[name]['type'] = type if type
+        @data[name]['unit'] = unit if unit
+        @data[name]['help'] = help if help
+        @data[name]['labels'] = labels if labels
+        @data[name]['time_ms'] = time_ms if time_ms
+      end
+    end
 
-      k = ((percentile / 100.0) * (len - 1) + 1).floor - 1
-      f = ((percentile / 100.0) * (len - 1) + 1).modulo(1)
-      return sorted_values[k] + (f * (sorted_values[k + 1] - sorted_values[k]))
+    def set_multiple(data)
+      @mutex.synchonize do
+        @data.merge!(data)
+      end
     end
 
-    def output
-      # Output percentile based metrics to Redis under the key of the
-      # #{@scope}__openc3__metric we will use hset with a subkey.
-      # internal:
-      # loop over the key value pairs within the @items hash, remove nil
-      # and sort the values added via the add_sample method. calculate the
-      # different percentiles. the labels are still only contained in the
-      # key of the @items hash to extract these you split the key on | the
-      # name|labels then to make the labels back into a hash we split the ,
-      # into an array ["foo=bar", ...] and split again this time on the =
-      # into [["foo","bar"], ...] to map the internal array into a hash
-      # [{"foo"=>"bar"}, ...] finally reducing the array into a single hash
-      # to add the percentile and percentile value. this hash is added to an
-      # array. to store the array as the value with the metric name again joined
-      # with the @microservice and @scope.
-      Logger.debug("#{@microservice} #{@scope} sending metrics to redis, #{@items.length}") if @items.length > 0
-      @mutex.synchronize do
-        @items.each do |key, values|
-          label_list = []
-          name, labels = key.split('|')
-          metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
-          sorted_values = values['values'].compact.sort
-          for percentile_value in [10, 50, 90, 95, 99]
-            percentile_result = percentile(sorted_values, percentile_value)
-            labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
-            labels['percentile'] = percentile_value
-            labels['metric__value'] = percentile_result
-            label_list.append(labels)
-          end
-          begin
-            Logger.debug("sending metrics summary to redis key: #{@microservice}")
-            metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
-            metric.create(force: true)
-          rescue RuntimeError
-            Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
+    def update_thread_body
+      @@update_sleeper = Sleeper.new
+      while true
+        start_time = Time.now
+        @@mutex.synchronize do
+          @@instances.each do |instance|
+            instance.mutex.synchronize do
+              json = {}
+              json['name'] = instance.microservice
+              values = instance.data
+              json['values'] = values
+              MetricModel.set(json, scope: instance.scope) if values.length > 0
+            end
           end
         end
+
+        # Only check whether to update at a set interval
+        run_time = Time.now - start_time
+        sleep_time = UPDATE_INTERVAL - run_time
+        sleep_time = 0 if sleep_time < 0
+        break if @@update_sleeper.sleep(sleep_time)
+      end
+    end
+
+    def shutdown
+      @@mutex.synchronize do
+        @@instances.delete(self)
+        if @@instances.length <= 0
+          @@update_sleeper.cancel if @@update_sleeper
+          OpenC3.kill_thread(self, @@update_thread) if @@update_thread
+          @@update_thread = nil
+        end
       end
     end
 
-    def destroy
-      MetricModel.destroy(scope: @scope, name: @microservice)
+    def graceful_kill
     end
   end
 end

data/lib/openc3/utilities/redis_secrets.rb

@@ -0,0 +1,46 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/models/secret_model'
+require 'openc3/utilities/secrets'
+
+module OpenC3
+  class RedisSecrets < Secrets
+    def keys(scope:)
+      SecretModel.names(scope: scope)
+    end
+
+    def get(key, scope:)
+      data = SecretModel.get(name: key, scope: scope)
+      if data
+        return data['value']
+      else
+        return nil
+      end
+    end
+
+    def set(key, value, scope:)
+      SecretModel.set( {name: key, value: value.to_s }, scope: scope)
+    end
+
+    def delete(key, scope:)
+      model = SecretModel.get_model(name: key, scope: scope)
+      model.destroy if model
+    end
+  end
+end

data/lib/openc3/utilities/s3_autoload.rb

@@ -1,11 +1,21 @@
 require 'aws-sdk-s3'
 
-Aws.config.update(
-  s3: {
-    endpoint: ENV['OPENC3_BUCKET_URL'] || (ENV['OPENC3_DEVEL'] ? 'http://127.0.0.1:9000' : 'http://openc3-minio:9000'),
-    access_key_id: ENV['OPENC3_BUCKET_USERNAME'],
-    secret_access_key: ENV['OPENC3_BUCKET_PASSWORD'],
-    force_path_style: true,
-    region: 'us-east-1'
-  }
-)
+if ENV['OPENC3_CLOUD'] == 'local'
+  Aws.config.update(
+    s3: {
+      endpoint: ENV['OPENC3_BUCKET_URL'] || (ENV['OPENC3_DEVEL'] ? 'http://127.0.0.1:9000' : 'http://openc3-minio:9000'),
+      access_key_id: ENV['OPENC3_BUCKET_USERNAME'],
+      secret_access_key: ENV['OPENC3_BUCKET_PASSWORD'],
+      force_path_style: true,
+      region: 'us-east-1'
+    }
+  )
+else # AWS
+  Aws.config.update(
+    s3: {
+      endpoint: "https://s3.#{ENV['AWS_REGION']}.amazonaws.com",
+      force_path_style: true,
+      region: ENV['AWS_REGION']
+    }
+  )
+end

data/lib/openc3/utilities/secrets.rb

@@ -0,0 +1,63 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+ENV['OPENC3_SECRET_BACKEND'] ||= 'redis'
+
+module OpenC3
+  class Secrets
+    def initialize
+      @local_secrets = {}
+    end
+
+    def self.getClient
+      raise 'OPENC3_SECRET_BACKEND environment variable is required' unless ENV['OPENC3_SECRET_BACKEND']
+      secrets_class = ENV['OPENC3_SECRET_BACKEND'].capitalize + 'Secrets'
+      klass = OpenC3.require_class('openc3/utilities/' + secrets_class.class_name_to_filename)
+      klass.new
+    end
+
+    def keys(scope:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def get(key, scope:)
+      return @local_secrets[key]
+    end
+
+    def set(key, value, scope:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def delete(key, scope:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def setup(secrets)
+      secrets.each do |type, key, data|
+        case type
+        when 'ENV'
+          @local_secrets[key] = ENV[data]
+        when 'FILE'
+          @local_secrets[key] = File.read(data)
+        else
+          raise "Unknown secret type: #{type}"
+        end
+      end
+    end
+  end
+end

data/lib/openc3/utilities/target_file.rb

@@ -97,7 +97,9 @@ module OpenC3
         bucket: ENV['OPENC3_CONFIG_BUCKET'],
         prefix: "#{scope}/targets_modified/#{TEMP_FOLDER}",
       )
+      files = []
       resp.each do |object|
+        files << object.key
         bucket.delete_object(
           bucket: ENV['OPENC3_CONFIG_BUCKET'],
           key: object.key,
@@ -106,7 +108,7 @@ module OpenC3
           OpenC3::LocalMode.delete_local(object.key)
         end
       end
-      true
+      files
     end
 
     def self.body(scope, name)

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '5.2.0'
+OPENC3_VERSION = '5.4.0'
 module OpenC3
   module Version
     MAJOR = '5'
-    MINOR = '2'
+    MINOR = '4'
     PATCH = '0'
     OTHER = ''
-    BUILD = '49c62512b75f45a8cbfc20a6fb5ed114bab35e23'
+    BUILD = 'f10d7e7ecbe3c6917e4f87eeac1f5eb0ddcf3d67'
   end
-  VERSION = '5.2.0'
-  GEM_VERSION = '5.2.0'
+  VERSION = '5.4.0'
+  GEM_VERSION = '5.4.0'
 end

data/templates/plugin-template/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright 2023 <COPYRIGHT HOLDER>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/templates/plugin-template/README.md

@@ -1,10 +1,10 @@
-# OpenC3 Configuration
+# OpenC3 COSMOS Plugin
 
 See the [OpenC3](https://openc3.com) documentation for all things OpenC3.
 
 ## Building the plugin
 
-1. <Path to OpenC3 installation>\openc3.bat openc3 rake VERSION=X.Y.Z
+1. <Path to COSMOS installation>\openc3.bat cli rake build VERSION=X.Y.Z
    - VERSION is required
    - gem file will be built locally
 
@@ -12,4 +12,5 @@ See the [OpenC3](https://openc3.com) documentation for all things OpenC3.
 
 1. Go to localhost:2900/tools/admin
 1. Click the paperclip icon and choose your plugin.gem file
-1. Click Upload
+1. Fill out plugin parameters
+1. Click Install

data/templates/plugin-template/plugin.gemspec

@@ -7,17 +7,17 @@ spec = Gem::Specification.new do |s|
   s.description = <<-EOF
     <%= plugin_name %> plugin for deployment to OpenC3
   EOF
-  s.license = 'AGPL-3.0-only'
+  s.license = 'MIT'
   s.authors = ['Anonymous']
   s.email = ['name@domain.com']
   s.homepage = 'https://github.com/OpenC3/openc3'
   s.platform = Gem::Platform::RUBY
 
-  time = Time.now.strftime("%Y%m%d%H%M%S")
   if ENV['VERSION']
-    s.version = ENV['VERSION'].dup + ".#{time}"
+    s.version = ENV['VERSION'].dup
   else
+    time = Time.now.strftime("%Y%m%d%H%M%S")
     s.version = '0.0.0' + ".#{time}"
   end
-  s.files = Dir.glob("{targets,lib,tools,microservices}/**/*") + %w(Rakefile README.md plugin.txt)
+  s.files = Dir.glob("{targets,lib,tools,microservices}/**/*") + %w(Rakefile README.md LICENSE.txt plugin.txt)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openc3
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.4.0
 platform: ruby
 authors:
 - Ryan Melton
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-13 00:00:00.000000000 Z
+date: 2023-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -333,6 +333,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: mqtt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: opentelemetry-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -653,7 +667,6 @@ files:
 - data/config/_id_items.yaml
 - data/config/_id_params.yaml
 - data/config/_interfaces.yaml
-- data/config/_interfaces.yaml.err
 - data/config/_items.yaml
 - data/config/_params.yaml
 - data/config/cmd_tlm_server.yaml
@@ -718,6 +731,7 @@ files:
 - lib/openc3/api/config_api.rb
 - lib/openc3/api/interface_api.rb
 - lib/openc3/api/limits_api.rb
+- lib/openc3/api/metrics_api.rb
 - lib/openc3/api/offline_access_api.rb
 - lib/openc3/api/router_api.rb
 - lib/openc3/api/settings_api.rb
@@ -768,6 +782,7 @@ files:
 - lib/openc3/interfaces.rb
 - lib/openc3/interfaces/interface.rb
 - lib/openc3/interfaces/linc_interface.rb
+- lib/openc3/interfaces/mqtt_interface.rb
 - lib/openc3/interfaces/protocols/burst_protocol.rb
 - lib/openc3/interfaces/protocols/crc_protocol.rb
 - lib/openc3/interfaces/protocols/fixed_protocol.rb
@@ -849,6 +864,7 @@ files:
 - lib/openc3/models/router_model.rb
 - lib/openc3/models/router_status_model.rb
 - lib/openc3/models/scope_model.rb
+- lib/openc3/models/secret_model.rb
 - lib/openc3/models/settings_model.rb
 - lib/openc3/models/sorted_model.rb
 - lib/openc3/models/stash_model.rb
@@ -951,8 +967,10 @@ files:
 - lib/openc3/utilities/open_telemetry.rb
 - lib/openc3/utilities/process_manager.rb
 - lib/openc3/utilities/quaternion.rb
+- lib/openc3/utilities/redis_secrets.rb
 - lib/openc3/utilities/ruby_lex_utils.rb
 - lib/openc3/utilities/s3_autoload.rb
+- lib/openc3/utilities/secrets.rb
 - lib/openc3/utilities/simulated_target.rb
 - lib/openc3/utilities/sleeper.rb
 - lib/openc3/utilities/store.rb
@@ -965,6 +983,7 @@ files:
 - lib/openc3/win32/win32_main.rb
 - tasks/gemfile_stats.rake
 - tasks/spec.rake
+- templates/plugin-template/LICENSE.txt
 - templates/plugin-template/README.md
 - templates/plugin-template/Rakefile
 - templates/plugin-template/plugin.gemspec