openc3 5.0.10 → 5.1.0

data/lib/openc3/utilities/local_mode.rb

@@ -12,36 +12,39 @@
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'fileutils'
 require 'json'
 require 'openc3/core_ext/file'
 # require 'openc3/models/gem_model' # These are used but also create circular dependency
 # require 'openc3/models/plugin_model' # These are used but also create circular dependency
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket'
 
 module OpenC3
   module LocalMode
     OPENC3_LOCAL_MODE_PATH = ENV['OPENC3_LOCAL_MODE_PATH'] || "/plugins"
 
     DEFAULT_PLUGINS = [
-      'openc3-tool-admin',
-      'openc3-tool-autonomic',
-      'openc3-tool-base',
-      'openc3-tool-calendar',
-      'openc3-tool-cmdsender',
-      'openc3-tool-cmdtlmserver',
-      'openc3-tool-dataextractor',
-      'openc3-tool-dataviewer',
-      'openc3-tool-handbooks',
-      'openc3-tool-limitsmonitor',
-      'openc3-tool-packetviewer',
-      'openc3-tool-scriptrunner',
-      'openc3-tool-tablemanager',
-      'openc3-tool-tlmgrapher',
-      'openc3-tool-tlmviewer',
+      'openc3-cosmos-tool-admin',
+      'openc3-cosmos-tool-autonomic',
+      'openc3-cosmos-tool-calendar',
+      'openc3-cosmos-tool-cmdsender',
+      'openc3-cosmos-tool-cmdtlmserver',
+      'openc3-cosmos-tool-dataextractor',
+      'openc3-cosmos-tool-dataviewer',
+      'openc3-cosmos-tool-handbooks',
+      'openc3-cosmos-tool-limitsmonitor',
+      'openc3-cosmos-tool-packetviewer',
+      'openc3-cosmos-tool-scriptrunner',
+      'openc3-cosmos-tool-tablemanager',
+      'openc3-cosmos-tool-tlmgrapher',
+      'openc3-cosmos-tool-tlmviewer',
+      'openc3-cosmos-enterprise-tool-admin',
       'openc3-enterprise-tool-base',
-      'openc3-enterprise-tool-admin',
+      'openc3-tool-base',
     ]
 
     # Install plugins from local plugins folder
@@ -277,7 +280,7 @@ module OpenC3
       temp_dir = Dir.mktmpdir
       begin
         unless File.exists?(plugin_file_path)
-          plugin_file_path = OpenC3::GemModel.get(temp_dir, plugin_file_path)
+          plugin_file_path = OpenC3::GemModel.get(plugin_file_path)
         end
         File.open(File.join(full_folder_path, File.basename(plugin_file_path)), 'wb') do |file|
           data = File.read(plugin_file_path)
@@ -318,18 +321,10 @@ module OpenC3
 
     def self.sync_targets_modified
       if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
-        rubys3_client = Aws::S3::Client.new
-
-        # Ensure config bucket exists
-        begin
-          rubys3_client.head_bucket(bucket: 'config')
-        rescue Aws::S3::Errors::NotFound
-          rubys3_client.create_bucket(bucket: 'config')
-        end
-
+        bucket = Bucket.getClient()
         scopes = ScopeModel.names()
         scopes.each do |scope|
-          sync_with_minio(rubys3_client, scope: scope)
+          sync_with_bucket(bucket, scope: scope)
         end
       end
     end
@@ -389,15 +384,22 @@ module OpenC3
 
     def self.open_local_file(path, scope:)
       full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{path}"
-      return File.open(full_path, 'rb') if File.exist?(full_path)
-      return nil
+      return File.open(full_path, 'rb')
+    rescue Errno::ENOENT
+      nil
     end
 
-    def self.local_target_files(scope:, path_matchers:)
+    def self.local_target_files(scope:, path_matchers:, include_temp: false)
       files = []
       local_catalog = build_local_catalog(scope: scope)
       local_catalog.each do |key, size|
         split_key = key.split('/')
+        # DEFAULT/targets_modified/__TEMP__/YYYY_MM_DD_HH_MM_SS_mmm_temp.rb
+        # See target_file.rb TEMP_FOLDER
+        if split_key[2] === '__TEMP__'
+          files << split_key[2..-1].join('/') if include_temp
+          next
+        end
         found = false
         path_matchers.each do |path|
           if split_key.include?(path)
@@ -413,16 +415,16 @@ module OpenC3
 
     # Helper methods
 
-    def self.sync_remote_to_local(rubys3_client, key)
+    def self.sync_remote_to_local(bucket, key)
       local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
       FileUtils.mkdir_p(File.dirname(local_path))
-      rubys3_client.get_object(bucket: 'config', key: key, response_target: local_path)
+      bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, path: local_path)
     end
 
-    def self.sync_local_to_remote(rubys3_client, key)
+    def self.sync_local_to_remote(bucket, key)
       local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
       File.open(local_path, 'rb') do |read_file|
-        rubys3_client.put_object(bucket: 'config', key: key, body: read_file)
+        bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: read_file)
       end
     end
 
@@ -432,8 +434,8 @@ module OpenC3
       nil
     end
 
-    def self.delete_remote(rubys3_client, key)
-      rubys3_client.delete_object(bucket: 'config', key: key)
+    def self.delete_remote(bucket, key)
+      bucket.delete_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key)
     end
 
     # Returns equivalent names and sizes to remote catalog
@@ -453,32 +455,23 @@ module OpenC3
 
     # Returns keys and sizes from remote catalog
     # {"scope/targets_modified/target_name/file" => size}
-    def self.build_remote_catalog(rubys3_client, scope:)
+    def self.build_remote_catalog(bucket, scope:)
       remote_catalog = {}
-      bucket = 'config'
       prefix = "#{scope}/targets_modified"
-      token = nil
-      while true
-        resp = rubys3_client.list_objects_v2({
-          bucket: bucket,
-          max_keys: 1000,
-          prefix: prefix,
-          continuation_token: token
-        })
-
-        resp.contents.each do |item|
-          remote_catalog[item.key] = item.size
-        end
-        break unless resp.is_truncated
-        token = resp.next_continuation_token
+      resp = bucket.list_objects(
+        bucket: ENV['OPENC3_CONFIG_BUCKET'],
+        prefix: prefix,
+      )
+      resp.each do |item|
+        remote_catalog[item.key] = item.size
       end
       return remote_catalog
     end
 
-    def self.sync_with_minio(rubys3_client, scope:)
+    def self.sync_with_bucket(bucket, scope:)
       # Build catalogs
       local_catalog = build_local_catalog(scope: scope)
-      remote_catalog = build_remote_catalog(rubys3_client, scope: scope)
+      remote_catalog = build_remote_catalog(bucket, scope: scope)
 
       # Find and Handle Differences
       local_catalog.each do |key, size|
@@ -486,9 +479,9 @@ module OpenC3
         if remote_size
           # Both files exist
           if ENV['OPENC3_LOCAL_MODE_SECONDARY']
-            sync_remote_to_local(rubys3_client, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
+            sync_remote_to_local(bucket, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
           else
-            sync_local_to_remote(rubys3_client, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
+            sync_local_to_remote(bucket, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
           end
         else
           # Remote is missing local file
@@ -496,7 +489,7 @@ module OpenC3
             delete_local(key)
           else
             # Go ahead and copy up to get in sync
-            sync_local_to_remote(rubys3_client, key)
+            sync_local_to_remote(bucket, key)
           end
         end
       end
@@ -508,14 +501,13 @@ module OpenC3
         else
           # Local is missing remote file
           if not ENV['OPENC3_LOCAL_MODE_SECONDARY'] and ENV['OPENC3_LOCAL_MODE_SYNC_REMOVE']
-            delete_remote(rubys3_client, key)
+            delete_remote(bucket, key)
           else
             # Go ahead and copy down to get in sync
-            sync_remote_to_local(rubys3_client, key)
+            sync_remote_to_local(bucket, key)
           end
         end
       end
     end
-
   end
-end
+end

data/lib/openc3/utilities/logger.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/core_ext/class'
 require 'openc3/core_ext/time'
@@ -171,7 +174,10 @@ module OpenC3
         end
         data[:container_name] = @container_name
         data[:log] = message
-        puts data.as_json(:allow_nan => true).to_json(:allow_nan => true) if @stdout
+        if @stdout
+          puts data.as_json(:allow_nan => true).to_json(:allow_nan => true)
+          $stdout.flush
+        end
         unless @no_store
           if scope
             Topic.write_topic("#{scope}__openc3_log_messages", data)

data/lib/openc3/utilities/message_log.rb

@@ -16,11 +16,14 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/config/config_parser'
 require 'openc3/system/system'
 require 'fileutils'
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket_utilities'
 
 module OpenC3
   # Handles writing message logs to a file
@@ -59,14 +62,14 @@ module OpenC3
     end
 
     # Closes the message log and marks it read only
-    def stop(take_mutex = true, s3_object_metadata: {})
+    def stop(take_mutex = true, metadata: {})
       @mutex.lock if take_mutex
       if @file and not @file.closed?
         @file.close
         File.chmod(0444, @filename)
-        s3_key = File.join(@remote_log_directory, @start_day, File.basename(@filename))
+        bucket_key = File.join(@remote_log_directory, @start_day, File.basename(@filename))
         begin
-          thread = S3Utilities.move_log_file_to_s3(@filename, s3_key, metadata: s3_object_metadata)
+          thread = BucketUtilities.move_log_file_to_bucket(@filename, bucket_key, metadata: metadata)
           thread.join
         rescue StandardError => e
           Logger.error e.formatted

data/lib/openc3/utilities/metric.rb

@@ -16,13 +16,16 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/models/metric_model'
 require 'thread'
 
 module OpenC3
   class Metric
-    # This class is designed to output metrics to the openc3-cmd-tlm-api
+    # This class is designed to output metrics to the openc3-cosmos-cmd-tlm-api
     # InternalMetricsController. Output format can be read about here
     # https://prometheus.io/docs/concepts/data_model/
     #

data/lib/openc3/utilities/open_telemetry.rb

@@ -0,0 +1,96 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
+
+require 'openc3/utilities/bucket'
+
+module OpenC3
+  @otel_enabled = false
+
+  def self.otel_enabled
+    @otel_enabled
+  end
+
+  def self.inject_context(hash)
+    if @otel_enabled
+      OpenTelemetry.propagation.inject(hash)
+    end
+  end
+
+  def self.with_context(hash)
+    if @otel_enabled
+      extracted_context = OpenTelemetry.propagation.extract(hash)
+      OpenTelemetry::Context.with_current(extracted_context) do
+        yield
+      end
+    else
+      yield
+    end
+  end
+
+  def self.in_span(span_name, tracer_name = 'openc3-tracer')
+    if @otel_enabled
+      tracer = OpenTelemetry.tracer_provider.tracer(tracer_name)
+      tracer.in_span(span_name) do |span|
+        yield span
+      end
+    else
+      yield nil
+    end
+  end
+
+  def self.setup_open_telemetry(service_name, support_rails = false)
+    if ENV['OTEL_EXPORTER_OTLP_ENDPOINT']
+      split_services = ENV['OPENC3_OTEL'].to_s.split(',')
+      @otel_enabled = true if split_services.include?(service_name) or split_services.include?('ALL')
+
+      if @otel_enabled
+        require 'redis'
+        require 'httpclient'
+        require 'openc3/utilities/bucket'
+        # Load the bucket client code so the instrumentation works later
+        Bucket.getClient()
+        require 'opentelemetry/sdk'
+        require 'opentelemetry/exporter/otlp'
+        require 'opentelemetry/instrumentation/redis'
+        require 'opentelemetry/instrumentation/http_client'
+        require 'opentelemetry/instrumentation/aws_sdk'
+        if support_rails
+          require 'opentelemetry/instrumentation/rack'
+          require 'opentelemetry/instrumentation/action_pack'
+        end
+        OpenTelemetry::SDK.configure do |c|
+          c.service_name = service_name
+          if support_rails
+            c.use('OpenTelemetry::Instrumentation::Rack')
+            c.use('OpenTelemetry::Instrumentation::ActionPack', { enable_recognize_route: true })
+          end
+          c.use 'OpenTelemetry::Instrumentation::Redis', {
+            # The obfuscation of arguments in the db.statement attribute is enabled by default.
+            # To include the full query, set db_statement to :include.
+            # To obfuscate, set db_statement to :obfuscate.
+            # To omit the attribute, set db_statement to :omit.
+            db_statement: :include,
+          }
+          c.use 'OpenTelemetry::Instrumentation::HttpClient'
+          c.use 'OpenTelemetry::Instrumentation::AwsSdk'
+          # TODO: Add in additional cloud SDKs
+        end
+      end
+    end
+  end
+end

data/lib/openc3/utilities/process_manager.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/operators/operator'
 require 'openc3/models/process_status_model'

data/lib/openc3/utilities/quaternion.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/core_ext/matrix'
 

data/lib/openc3/utilities/ruby_lex_utils.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'irb/ruby-lex'
 require 'stringio'

data/lib/openc3/utilities/s3_autoload.rb

@@ -1,9 +1,11 @@
 require 'aws-sdk-s3'
 
 Aws.config.update(
-  endpoint: ENV['OPENC3_S3_URL'] || (ENV['OPENC3_DEVEL'] ? 'http://127.0.0.1:9000' : 'http://openc3-minio:9000'),
-  access_key_id: ENV['OPENC3_MINIO_USERNAME'],
-  secret_access_key: ENV['OPENC3_MINIO_PASSWORD'],
-  force_path_style: true,
-  region: 'us-east-1'
-)
+  s3: {
+    endpoint: ENV['OPENC3_BUCKET_URL'] || (ENV['OPENC3_DEVEL'] ? 'http://127.0.0.1:9000' : 'http://openc3-minio:9000'),
+    access_key_id: ENV['OPENC3_BUCKET_USERNAME'],
+    secret_access_key: ENV['OPENC3_BUCKET_PASSWORD'],
+    force_path_style: true,
+    region: 'us-east-1'
+  }
+)

data/lib/openc3/utilities/simulated_target.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet'
 require 'openc3/packets/structure'

data/lib/openc3/utilities/sleeper.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   # Allows for a breakable sleep implementation using the self-pipe trick

data/lib/openc3/utilities/store.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   autoload(:Store, "openc3/utilities/store_autoload.rb")

data/lib/openc3/utilities/store_autoload.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'redis'
 require 'json'
@@ -23,9 +26,9 @@ require 'connection_pool'
 
 if ENV['OPENC3_REDIS_CLUSTER']
   require 'openc3-enterprise/utilities/store'
-  $openc3_enterprise = true
+  $openc3_redis_cluster = true
 else
-  $openc3_enterprise = false
+  $openc3_redis_cluster = false
 end
 
 module OpenC3
@@ -73,7 +76,6 @@ module OpenC3
     end
 
     def initialize(pool_size = 10)
-      Redis.exists_returns_integer = true
       @redis_username = ENV['OPENC3_REDIS_USERNAME']
       @redis_key = ENV['OPENC3_REDIS_PASSWORD']
       @redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
@@ -81,7 +83,7 @@ module OpenC3
       @topic_offsets = {}
     end
 
-    unless $openc3_enterprise
+    unless $openc3_redis_cluster
       def build_redis
         return Redis.new(url: @redis_url, username: @redis_username, password: @redis_key)
       end
@@ -154,22 +156,26 @@ module OpenC3
       return offsets
     end
 
-    unless $openc3_enterprise
+    unless $openc3_redis_cluster
       def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
-        # Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
-        @redis_pool.with do |redis|
-          offsets = update_topic_offsets(topics) unless offsets
-          result = redis.xread(topics, offsets, block: timeout_ms, count: count)
-          if result and result.length > 0
-            result.each do |topic, messages|
-              messages.each do |msg_id, msg_hash|
-                @topic_offsets[topic] = msg_id
-                yield topic, msg_id, msg_hash, redis if block_given?
+        begin
+          # Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
+          @redis_pool.with do |redis|
+            offsets = update_topic_offsets(topics) unless offsets
+            result = redis.xread(topics, offsets, block: timeout_ms, count: count)
+            if result and result.length > 0
+              result.each do |topic, messages|
+                messages.each do |msg_id, msg_hash|
+                  @topic_offsets[topic] = msg_id
+                  yield topic, msg_id, msg_hash, redis if block_given?
+                end
               end
             end
+            # Logger.debug "result:#{result}" if result and result.length > 0
+            return result
           end
-          # Logger.debug "result:#{result}" if result and result.length > 0
-          return result
+        rescue Redis::TimeoutError
+          return {} # Should return an empty hash not array - xread returns a hash
         end
       end
     end
@@ -180,7 +186,7 @@ module OpenC3
     # @example Without options
     #   store.write_topic('MANGO__TOPIC', {'message' => 'something'})
     # @example With options
-    #   store.write_topic('MANGO__TOPIC', {'message' => 'something'}, id: '0-0', maxlen: 1000, approximate: true)
+    #   store.write_topic('MANGO__TOPIC', {'message' => 'something'}, id: '0-0', maxlen: 1000, approximate: 'true')
     #
     # @param topic [String] the stream / topic
     # @param msg_hash [Hash]   one or multiple field-value pairs
@@ -188,10 +194,10 @@ module OpenC3
     # @option opts [String]  :id          the entry id, default value is `*`, it means auto generation,
     #   if `nil` id is passed it will be changed to `*`
     # @option opts [Integer] :maxlen      max length of entries, default value is `nil`, it means will grow forever
-    # @option opts [Boolean] :approximate whether to add `~` modifier of maxlen or not, default value is `true`
+    # @option opts [String] :approximate whether to add `~` modifier of maxlen or not, default value is 'true'
     #
     # @return [String] the entry id
-    def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = true)
+    def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
       id = '*' if id.nil?
       # Logger.debug "write_topic topic:#{topic} id:#{id} hash:#{msg_hash}"
       @redis_pool.with do |redis|
@@ -205,14 +211,15 @@ module OpenC3
     # @example Without options
     #   store.trim_topic('MANGO__TOPIC', 1000)
     # @example With options
-    #   store.trim_topic('MANGO__TOPIC', 1000, approximate: true, limit: 0)
+    #   store.trim_topic('MANGO__TOPIC', 1000, approximate: 'true', limit: 0)
     #
     # @param topic  [String]  the stream key
     # @param minid  [Integer] mid id length of entries to trim
+    # @param approximate [Boolean] whether to add `~` modifier of maxlen or not
     # @param limit  [Boolean] whether to add `~` modifier of maxlen or not
     #
     # @return [Integer] the number of entries actually deleted
-    def trim_topic(topic, minid, approximate = true, limit: 0)
+    def trim_topic(topic, minid, approximate = 'true', limit: 0)
       @redis_pool.with do |redis|
         return redis.xtrim_minid(topic, minid, approximate: approximate, limit: limit)
       end
@@ -229,9 +236,9 @@ module OpenC3
 end
 
 class Redis
-  def xtrim_minid(key, minid, approximate: true, limit: nil)
+  def xtrim_minid(key, minid, approximate: 'true', limit: nil)
     args = [:xtrim, key, :MINID, (approximate ? '~' : nil), minid].compact
     args.concat([:LIMIT, limit]) if limit
-    synchronize { |client| client.call(args) }
+    send_command(args)
   end
 end