openc3 5.0.10 → 5.1.0

data/lib/openc3/models/target_model.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'openc3/top_level'
 require 'openc3/models/model'
@@ -25,7 +28,7 @@ require 'openc3/topics/limits_event_topic'
 require 'openc3/topics/config_topic'
 require 'openc3/system'
 require 'openc3/utilities/local_mode'
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket'
 require 'openc3/utilities/zip'
 require 'fileutils'
 require 'tmpdir'
@@ -94,29 +97,13 @@ module OpenC3
           targets[target_name]['modified'] = true if targets[target_name]
         end
       else
-        rubys3_client = Aws::S3::Client.new
-        token = nil
-        while true
-          resp = rubys3_client.list_objects_v2({
-            bucket: 'config',
-            max_keys: 1000,
-            # The trailing slash is important!
-            prefix: "#{scope}/targets_modified/",
-            delimiter: '/',
-            continuation_token: token
-          })
-          resp.common_prefixes.each do |item|
-            # Results look like DEFAULT/targets_modified/INST/
-            # so split on '/' and pull out the last value
-            target_name = item.prefix.split('/')[-1]
-            # A target could have been deleted without removing the modified files
-            # Thus we have to check for the existance of the target_name key
-            if targets.has_key?(target_name)
-              targets[target_name]['modified'] = true
-            end
+        modified_targets = Bucket.getClient().list_directories(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "DEFAULT/targets_modified/")
+        modified_targets.each do |target_name|
+          # A target could have been deleted without removing the modified files
+          # Thus we have to check for the existance of the target_name key
+          if targets.has_key?(target_name)
+            targets[target_name]['modified'] = true
           end
-          break unless resp.is_truncated
-          token = resp.next_continuation_token
         end
       end
       # Sort (which turns hash to array) and return hash
@@ -131,23 +118,15 @@ module OpenC3
       if ENV['OPENC3_LOCAL_MODE']
         modified = OpenC3::LocalMode.modified_files(target_name, scope: scope)
       else
-        rubys3_client = Aws::S3::Client.new
-        token = nil
-        while true
-          resp = rubys3_client.list_objects_v2({
-            bucket: 'config',
-            max_keys: 1000,
-            # The trailing slash is important!
-            prefix: "#{scope}/targets_modified/#{target_name}/",
-            continuation_token: token
-          })
-          resp.contents.each do |item|
-            # Results look like DEFAULT/targets_modified/INST/procedures/new.rb
-            # so split on '/' and ignore the first two values
-            modified << item.key.split('/')[2..-1].join('/')
-          end
-          break unless resp.is_truncated
-          token = resp.next_continuation_token
+        resp = Bucket.getClient().list_objects(
+          bucket: ENV['OPENC3_CONFIG_BUCKET'],
+          # The trailing slash is important!
+          prefix: "#{scope}/targets_modified/#{target_name}/",
+        )
+        resp.each do |item|
+          # Results look like DEFAULT/targets_modified/INST/procedures/new.rb
+          # so split on '/' and ignore the first two values
+          modified << item.key.split('/')[2..-1].join('/')
         end
       end
       # Sort to enable a consistent listing of the modified files
@@ -158,23 +137,15 @@ module OpenC3
       if ENV['OPENC3_LOCAL_MODE']
         OpenC3::LocalMode.delete_modified(target_name, scope: scope)
       end
-
+      bucket = Bucket.getClient()
       # Delete the remote files as well
-      rubys3_client = Aws::S3::Client.new
-      token = nil
-      while true
-        resp = rubys3_client.list_objects_v2({
-          bucket: 'config',
-          max_keys: 1000,
-          # The trailing slash is important!
-          prefix: "#{scope}/targets_modified/#{target_name}/",
-          continuation_token: token
-        })
-        resp.contents.each do |item|
-          rubys3_client.delete_object(bucket: 'config', key: item.key)
-        end
-        break unless resp.is_truncated
-        token = resp.next_continuation_token
+      resp = bucket.list_objects(
+        bucket: ENV['OPENC3_CONFIG_BUCKET'],
+        # The trailing slash is important!
+        prefix: "#{scope}/targets_modified/#{target_name}/",
+      )
+      resp.each do |item|
+        bucket.delete_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: item.key)
       end
     end
 
@@ -187,28 +158,21 @@ module OpenC3
       if ENV['OPENC3_LOCAL_MODE']
         OpenC3::LocalMode.zip_target(target_name, zip, scope: scope)
       else
-        rubys3_client = Aws::S3::Client.new
-        token = nil
+        bucket = Bucket.getClient()
         # The trailing slash is important!
         prefix = "#{scope}/targets_modified/#{target_name}/"
-        while true
-          resp = rubys3_client.list_objects_v2({
-            bucket: 'config',
-            max_keys: 1000,
-            prefix: prefix,
-            continuation_token: token
-          })
-          resp.contents.each do |item|
-            # item.key looks like DEFAULT/targets_modified/INST/screens/blah.txt
-            base_path = item.key.sub(prefix, '') # remove prefix
-            local_path = File.join(tmp_dir, base_path)
-            # Ensure dir structure exists, get_object fails if not
-            FileUtils.mkdir_p(File.dirname(local_path))
-            rubys3_client.get_object(bucket: 'config', key: item.key, response_target: local_path)
-            zip.add(base_path, local_path)
-          end
-          break unless resp.is_truncated
-          token = resp.next_continuation_token
+        resp = bucket.list_objects(
+          bucket: ENV['OPENC3_CONFIG_BUCKET'],
+          prefix: prefix,
+        )
+        resp.each do |item|
+          # item.key looks like DEFAULT/targets_modified/INST/screens/blah.txt
+          base_path = item.key.sub(prefix, '') # remove prefix
+          local_path = File.join(tmp_dir, base_path)
+          # Ensure dir structure exists, get_object fails if not
+          FileUtils.mkdir_p(File.dirname(local_path))
+          bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: item.key, path: local_path)
+          zip.add(base_path, local_path)
         end
       end
       zip.close
@@ -384,6 +348,7 @@ module OpenC3
       @reduced_day_log_retain_time = reduced_day_log_retain_time
       @cleanup_poll_time = cleanup_poll_time
       @needs_dependencies = needs_dependencies
+      @bucket = Bucket.getClient()
     end
 
     def as_json(*a)
@@ -506,7 +471,6 @@ module OpenC3
     end
 
     def deploy(gem_path, variables, validate_only: false)
-      rubys3_client = Aws::S3::Client.new
       variables["target_name"] = @name
       start_path = "/targets/#{@folder_name}/"
       temp_dir = Dir.mktmpdir
@@ -534,7 +498,7 @@ module OpenC3
           FileUtils.mkdir_p(File.dirname(local_path))
           File.open(local_path, 'wb') { |file| file.write(data) }
           found = true
-          rubys3_client.put_object(bucket: 'config', key: key, body: data) unless validate_only
+          @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: data) unless validate_only
         end
         raise "No target files found at #{target_path}" unless found
 
@@ -546,7 +510,7 @@ module OpenC3
           system.packet_config.to_xtce(variables["xtce_output"])
         end
         unless validate_only
-          build_target_archive(rubys3_client, temp_dir, target_folder)
+          build_target_archive(temp_dir, target_folder)
           system = update_store(system)
           deploy_microservices(gem_path, variables, system)
           ConfigTopic.write({ kind: 'created', type: 'target', name: @name, plugin: @plugin }, scope: @scope)
@@ -557,10 +521,9 @@ module OpenC3
     end
 
     def undeploy
-      rubys3_client = Aws::S3::Client.new
       prefix = "#{@scope}/targets/#{@name}/"
-      rubys3_client.list_objects(bucket: 'config', prefix: prefix).contents.each do |object|
-        rubys3_client.delete_object(bucket: 'config', key: object.key)
+      @bucket.list_objects(bucket: ENV['OPENC3_CONFIG_BUCKET'], prefix: prefix).each do |object|
+        @bucket.delete_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: object.key)
       end
 
       self.class.get_model(name: @name, scope: @scope).limits_groups.each do |group|
@@ -573,9 +536,12 @@ module OpenC3
       self.class.packets(@name, scope: @scope).each do |packet|
         Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}")
         Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__REDUCED_MINUTE__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__REDUCED_HOUR__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__REDUCED_DAY__{#{@name}}__#{packet['packet_name']}")
         CvtModel.del(target_name: @name, packet_name: packet['packet_name'], scope: @scope)
-        LimitsEventTopic.delete(@name, packet['packet_name'], scope: @scope)
       end
+      LimitsEventTopic.delete(@name, scope: @scope)
       Store.del("#{@scope}__openc3tlm__#{@name}")
       Store.del("#{@scope}__openc3cmd__#{@name}")
 
@@ -618,14 +584,14 @@ module OpenC3
       end
     end
 
-    def build_target_archive(rubys3_client, temp_dir, target_folder)
+    def build_target_archive(temp_dir, target_folder)
       target_files = []
       Find.find(target_folder) { |file| target_files << file }
       target_files.sort!
       hash = OpenC3.hash_files(target_files, nil, 'SHA256').hexdigest
       File.open(File.join(target_folder, 'target_id.txt'), 'wb') { |file| file.write(hash) }
       key = "#{@scope}/targets/#{@name}/target_id.txt"
-      rubys3_client.put_object(bucket: 'config', key: key, body: hash)
+      @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: hash)
 
       # Create target archive zip file
       prefix = File.dirname(target_folder) + '/'
@@ -642,14 +608,14 @@ module OpenC3
         end
       end
 
-      # Write Target Archive to S3 Bucket
+      # Write Target Archive to bucket
       File.open(output_file, 'rb') do |file|
-        s3_key = key = "#{@scope}/target_archives/#{@name}/#{@name}_current.zip"
-        rubys3_client.put_object(bucket: 'config', key: s3_key, body: file)
+        bucket_key = key = "#{@scope}/target_archives/#{@name}/#{@name}_current.zip"
+        @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: bucket_key, body: file)
       end
       File.open(output_file, 'rb') do |file|
-        s3_key = key = "#{@scope}/target_archives/#{@name}/#{@name}_#{hash}.zip"
-        rubys3_client.put_object(bucket: 'config', key: s3_key, body: file)
+        bucket_key = key = "#{@scope}/target_archives/#{@name}/#{@name}_#{hash}.zip"
+        @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: bucket_key, body: file)
       end
     end
 
@@ -722,6 +688,7 @@ module OpenC3
       decom_command_topic_list = []
       packet_topic_list = []
       decom_topic_list = []
+      reduced_topic_list = []
       begin
         system.commands.packets(@name).each do |packet_name, packet|
           command_topic_list << "#{@scope}__COMMAND__{#{@name}}__#{packet_name}"
@@ -734,6 +701,9 @@ module OpenC3
         system.telemetry.packets(@name).each do |packet_name, packet|
           packet_topic_list << "#{@scope}__TELEMETRY__{#{@name}}__#{packet_name}"
           decom_topic_list  << "#{@scope}__DECOM__{#{@name}}__#{packet_name}"
+          reduced_topic_list << "#{@scope}__REDUCED_MINUTE__{#{@name}}__#{packet_name}"
+          reduced_topic_list << "#{@scope}__REDUCED_HOUR__{#{@name}}__#{packet_name}"
+          reduced_topic_list << "#{@scope}__REDUCED_DAY__{#{@name}}__#{packet_name}"
         end
       rescue
         # No telemetry packets for this target
@@ -743,6 +713,7 @@ module OpenC3
       Topic.initialize_streams(decom_command_topic_list)
       Topic.initialize_streams(packet_topic_list)
       Topic.initialize_streams(decom_topic_list)
+      Topic.initialize_streams(reduced_topic_list)
 
       unless command_topic_list.empty?
         # CommandLog Microservice
@@ -759,7 +730,6 @@ module OpenC3
             ["CYCLE_SIZE", @cmd_log_cycle_size]
           ],
           topics: command_topic_list,
-          target_names: [@name],
           plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
@@ -782,7 +752,6 @@ module OpenC3
             ["CYCLE_SIZE", @cmd_decom_log_cycle_size]
           ],
           topics: decom_command_topic_list,
-          target_names: [@name],
           plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
@@ -807,7 +776,6 @@ module OpenC3
             ["CYCLE_SIZE", @tlm_log_cycle_size]
           ],
           topics: packet_topic_list,
-          target_names: [@name],
           plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
@@ -830,7 +798,6 @@ module OpenC3
             ["CYCLE_SIZE", @tlm_decom_log_cycle_size]
           ],
           topics: decom_topic_list,
-          target_names: [@name],
           plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope

data/lib/openc3/models/timeline_model.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
 require 'openc3/models/activity_model'
@@ -66,7 +69,7 @@ module OpenC3
 
     # @return [TimelineModel] Model generated from the passed JSON
     def self.from_json(json, name:, scope:)
-      json = JSON.parse(json) if String === json
+      json = JSON.parse(json, :allow_nan => true, :create_additions => true) if String === json
       raise "json data is nil" if json.nil?
 
       json.transform_keys!(&:to_sym)

data/lib/openc3/models/tool_config_model.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   class ToolConfigModel

data/lib/openc3/models/tool_model.rb

@@ -16,10 +16,14 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
 require 'openc3/models/scope_model'
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket'
+require 'openc3/utilities/bucket_utilities'
 require 'rack'
 
 module OpenC3
@@ -215,9 +219,6 @@ module OpenC3
     def deploy(gem_path, variables, validate_only: false)
       return unless @folder_name
 
-      # Ensure tools bucket exists
-      OpenC3::S3Utilities.ensure_public_bucket('tools') unless validate_only
-
       variables["tool_name"] = @name
       start_path = "/tools/#{@folder_name}/"
       Dir.glob(gem_path + start_path + "**/*") do |filename|
@@ -231,8 +232,9 @@ module OpenC3
         data = File.read(filename, mode: "rb")
         data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
         unless validate_only
-          cache_control = OpenC3::S3Utilities.get_cache_control(filename)
-          Aws::S3::Client.new.put_object(bucket: 'tools', content_type: content_type, cache_control: cache_control, key: key, body: data)
+          client = Bucket.getClient()
+          cache_control = BucketUtilities.get_cache_control(filename)
+          client.put_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], content_type: content_type, cache_control: cache_control, key: key, body: data)
           ConfigTopic.write({ kind: 'created', type: 'tool', name: @folder_name, plugin: @plugin }, scope: @scope)
         end
       end
@@ -240,10 +242,10 @@ module OpenC3
 
     def undeploy
       if @folder_name and @folder_name.to_s.length > 0
-        rubys3_client = Aws::S3::Client.new
+        bucket = Bucket.getClient
         prefix = "#{@folder_name}/"
-        rubys3_client.list_objects(bucket: 'tools', prefix: prefix).contents.each do |object|
-          rubys3_client.delete_object(bucket: 'tools', key: object.key)
+        bucket.list_objects(bucket: ENV['OPENC3_TOOLS_BUCKET'], prefix: prefix).each do |object|
+          bucket.delete_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], key: object.key)
           ConfigTopic.write({ kind: 'deleted', type: 'tool', name: @folder_name, plugin: @plugin }, scope: @scope)
         end
       end

data/lib/openc3/models/trigger_group_model.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
 require 'openc3/models/microservice_model'

data/lib/openc3/models/trigger_model.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
 require 'openc3/models/microservice_model'

data/lib/openc3/models/widget_model.rb

@@ -16,11 +16,15 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/top_level'
 require 'openc3/models/model'
 require 'openc3/models/scope_model'
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket'
+require 'openc3/utilities/bucket_utilities'
 
 module OpenC3
   class WidgetModel < Model
@@ -29,7 +33,7 @@ module OpenC3
     attr_accessor :name
     attr_accessor :full_name
     attr_accessor :filename
-    attr_accessor :s3_key
+    attr_accessor :bucket_key
     attr_accessor :needs_dependencies
 
     # NOTE: The following three class methods are used by the ModelController
@@ -86,7 +90,7 @@ module OpenC3
       super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at, scope: scope)
       @full_name = @name.capitalize + 'Widget'
       @filename = @full_name + '.umd.min.js'
-      @s3_key = 'widgets/' + @full_name + '/' + @filename
+      @bucket_key = 'widgets/' + @full_name + '/' + @filename
       @needs_dependencies = needs_dependencies
     end
 
@@ -110,7 +114,11 @@ module OpenC3
 
     def deploy(gem_path, variables, validate_only: false)
       # Ensure tools bucket exists
-      OpenC3::S3Utilities.ensure_public_bucket('tools') unless validate_only
+      bucket = nil
+      unless validate_only
+        bucket = Bucket.getClient()
+        bucket.ensure_public(ENV['OPENC3_TOOLS_BUCKET'])
+      end
 
       filename = gem_path + "/tools/widgets/" + @full_name + '/' + @filename
 
@@ -120,19 +128,18 @@ module OpenC3
         data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
       end
       unless validate_only
-        cache_control = OpenC3::S3Utilities.get_cache_control(@filename)
+        cache_control = BucketUtilities.get_cache_control(@filename)
         # TODO: support widgets that aren't just a single js file (and its associated map file)
-        rubys3_client = Aws::S3::Client.new
-        rubys3_client.put_object(bucket: 'tools', content_type: 'application/javascript', cache_control: cache_control, key: @s3_key, body: data)
+        bucket.put_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], content_type: 'application/javascript', cache_control: cache_control, key: @bucket_key, body: data)
         data = File.read(filename + '.map', mode: "rb")
-        rubys3_client.put_object(bucket: 'tools', content_type: 'application/json', cache_control: cache_control, key: @s3_key + '.map', body: data)
+        bucket.put_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], content_type: 'application/json', cache_control: cache_control, key: @bucket_key + '.map', body: data)
       end
     end
 
     def undeploy
-      rubys3_client = Aws::S3::Client.new
-      rubys3_client.delete_object(bucket: 'tools', key: @s3_key)
-      rubys3_client.delete_object(bucket: 'tools', key: @s3_key + '.map')
+      bucket = Bucket.getClient()
+      bucket.delete_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], key: @bucket_key)
+      bucket.delete_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], key: @bucket_key + '.map')
     end
   end
 end

data/lib/openc3/operators/microservice_operator.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3'
 require 'openc3/models/microservice_model'