openc3 5.5.0 → 5.5.2.pre.beta0

data/lib/openc3/api/settings_api.rb

@@ -20,7 +20,7 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
-require 'openc3/models/settings_model'
+require 'openc3/models/setting_model'
 
 module OpenC3
   module Api
@@ -35,41 +35,35 @@ module OpenC3
 
     def list_settings(scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', scope: scope, token: token)
-      SettingsModel.names(scope: scope)
+      SettingModel.names(scope: scope)
     end
 
     def get_all_settings(scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', scope: scope, token: token)
-      SettingsModel.all(scope: scope)
+      SettingModel.all(scope: scope)
     end
 
     def get_setting(name, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', scope: scope, token: token)
-      setting = SettingsModel.get(name: name, scope: scope)
+      setting = SettingModel.get(name: name, scope: scope)
       if setting
-        return setting["data"]
+        setting['data']
       else
-        return nil
+        nil
       end
     end
 
-    def get_settings(*args, scope: $openc3_scope, token: $openc3_token)
+    def get_settings(*settings, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', scope: scope, token: token)
-      ret = []
-      args.each do |name|
-        setting = SettingsModel.get(name: name, scope: scope)
-        if setting
-          ret << setting["data"]
-        else
-          ret << nil
-        end
-      end
-      return ret
+      result = []
+      settings.each { |name| result << get_setting(name, scope: scope, token: token) }
+      result
     end
 
     def save_setting(name, data, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'admin', scope: scope, token: token)
-      SettingsModel.set({ name: name,  data: data }, scope: scope)
+      SettingModel.set({ name: name, data: data }, scope: scope)
+      LocalMode.save_setting(scope, name, data)
     end
   end
 end

data/lib/openc3/models/interface_model.rb

@@ -257,12 +257,15 @@ module OpenC3
         @log_raw = true
 
       when 'SECRET'
-        parser.verify_num_parameters(3, 4, "#{keyword} <Secret Type: ENV or FILE> <Secret Name> <Environment Variable Name or File Path> <Option Name (Optional)>")
+        parser.verify_num_parameters(3, 5, "#{keyword} <Secret Type: ENV or FILE> <Secret Name> <Environment Variable Name or File Path> <Option Name (Optional)> <Secret Store Name (Optional)>")
         @secrets << parameters[0..2]
-        if parameters[3]
+        if ConfigParser.handle_nil(parameters[3])
           # Option Name, Secret Name
           @secret_options << [parameters[3], parameters[1]]
         end
+        if ConfigParser.handle_nil(parameters[4])
+          @secrets[-1] << parameters[4]
+        end
 
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Interface/Router: #{keyword} #{parameters.join(" ")}")

data/lib/openc3/models/microservice_model.rb

@@ -23,7 +23,6 @@
 require 'openc3/top_level'
 require 'openc3/models/model'
 require 'openc3/models/metric_model'
-require 'openc3/models/traefik_model'
 require 'openc3/utilities/bucket'
 
 module OpenC3
@@ -193,8 +192,12 @@ module OpenC3
         parser.verify_num_parameters(1, 1, "#{keyword} <Container Image Name>")
         @container = parameters[0]
       when 'SECRET'
-        parser.verify_num_parameters(3, 3, "#{keyword} <Secret Type: ENV or FILE> <Secret Name> <Environment Variable Name or File Path>")
-        @secrets << parameters.dup
+        parser.verify_num_parameters(3, 4, "#{keyword} <Secret Type: ENV or FILE> <Secret Name> <Environment Variable Name or File Path> <Secret Store Name (Optional)>")
+        if ConfigParser.handle_nil(parameters[3])
+          @secrets << parameters.dup
+        else
+          @secrets << parameters[0..2]
+        end
       when 'ROUTE_PREFIX'
         parser.verify_num_parameters(1, 1, "#{keyword} <Route Prefix>")
         @prefix = parameters[0]
@@ -225,7 +228,6 @@ module OpenC3
         end
       end
       unless validate_only
-        TraefikModel.register_route(microservice_name: @name, port: @ports[0][0], prefix: @prefix) if @ports[0] and ports[0][0] and @prefix
         ConfigTopic.write({ kind: 'created', type: 'microservice', name: @name, plugin: @plugin }, scope: @scope)
       end
     end
@@ -235,7 +237,6 @@ module OpenC3
       @bucket.list_objects(bucket: ENV['OPENC3_CONFIG_BUCKET'], prefix: prefix).each do |object|
         @bucket.delete_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: object.key)
       end
-      TraefikModel.unregister_route(microservice_name: @name, port: @ports[0][0], prefix: @prefix) if @ports[0] and ports[0][0] and @prefix
       ConfigTopic.write({ kind: 'deleted', type: 'microservice', name: @name, plugin: @plugin }, scope: @scope)
     rescue Exception => error
       Logger.error("Error undeploying microservice model #{@name} in scope #{@scope} due to #{error}")

data/lib/openc3/models/plugin_model.rb

@@ -44,7 +44,7 @@ module OpenC3
   class PluginModel < Model
     PRIMARY_KEY = 'openc3_plugins'
     # Reserved VARIABLE names. See local_mode.rb: update_local_plugin()
-    RESERVED_VARIABLE_NAMES = ['target_name', 'microservice_name']
+    RESERVED_VARIABLE_NAMES = ['target_name', 'microservice_name', 'scope']
 
     attr_accessor :variables
     attr_accessor :plugin_txt_lines
@@ -198,6 +198,8 @@ module OpenC3
           tf.close
           plugin_txt_path = tf.path
           variables = plugin_hash['variables']
+          variables ||= {}
+          variables['scope'] = scope
           if File.exist?(plugin_txt_path)
             parser = OpenC3::ConfigParser.new("https://openc3.com")
 

data/lib/openc3/models/scope_model.rb

@@ -24,7 +24,7 @@ require 'openc3/version'
 require 'openc3/models/model'
 require 'openc3/models/plugin_model'
 require 'openc3/models/microservice_model'
-require 'openc3/models/settings_model'
+require 'openc3/models/setting_model'
 
 module OpenC3
   class ScopeModel < Model
@@ -261,8 +261,10 @@ module OpenC3
     end
 
     def seed_database
-      setting = SettingsModel.get(name: 'source_url')
-      SettingsModel.set({ name: 'source_url', data: 'https://github.com/OpenC3/cosmos' }, scope: @scope) unless setting
+      setting = SettingModel.get(name: 'source_url')
+      SettingModel.set({ name: 'source_url', data: 'https://github.com/OpenC3/cosmos' }, scope: @scope) unless setting
+      setting = SettingModel.get(name: 'rubygems_url')
+      SettingModel.set({ name: 'rubygems_url', data: 'https://rubygems.org' }, scope: @scope) unless setting
     end
   end
 end

data/lib/openc3/models/{settings_model.rb → setting_model.rb}

@@ -23,7 +23,7 @@
 require 'openc3/models/model'
 
 module OpenC3
-  class SettingsModel < Model
+  class SettingModel < Model
     PRIMARY_KEY = 'openc3__settings'
 
     # NOTE: The following three class methods are used by the ModelController

data/lib/openc3/models/tool_config_model.rb

@@ -17,9 +17,11 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
+require 'openc3/utilities/local_mode'
+
 module OpenC3
   class ToolConfigModel
     def self.list_configs(tool, scope: $openc3_scope)
@@ -30,12 +32,14 @@ module OpenC3
       Store.hget("#{scope}__config__#{tool}", name)
     end
 
-    def self.save_config(tool, name, data, scope: $openc3_scope)
+    def self.save_config(tool, name, data, scope: $openc3_scope, local_mode: true)
       Store.hset("#{scope}__config__#{tool}", name, data)
+      LocalMode.save_tool_config(scope, tool, name, data) if local_mode
     end
 
     def self.delete_config(tool, name, scope: $openc3_scope)
       Store.hdel("#{scope}__config__#{tool}", name)
+      LocalMode.delete_tool_config(scope, tool, name)
     end
   end
 end

data/lib/openc3/operators/microservice_operator.rb

@@ -60,8 +60,8 @@ module OpenC3
       # Setup secrets for microservice
       secrets = microservice_config["secrets"]
       if secrets
-        secrets.each do |type, secret_name, env_name_or_path|
-          secret_value = @secrets.get(secret_name, scope: scope)
+        secrets.each do |type, secret_name, env_name_or_path, secret_store|
+          secret_value = @secrets.get(secret_name, secret_store: secret_store, scope: scope)
           if secret_value
             case type
             when 'ENV'

data/lib/openc3/script/api_shared.rb

@@ -501,6 +501,10 @@ module OpenC3
       return start(procedure_name)
     end
     def require_utility(procedure_name)
+      # Ensure require_utility works like require where you don't need the .rb extension
+      if File.extname(procedure_name) != '.rb'
+        procedure_name += '.rb'
+      end
       @require_utility_cache ||= {}
       if @require_utility_cache[procedure_name]
         return false

data/lib/openc3/utilities/authentication.rb

@@ -144,7 +144,7 @@ module OpenC3
 
     # Make the post request to keycloak
     def _make_request(headers, data)
-      uri = URI("#{@url}/auth/realms/openc3/protocol/openid-connect/token")
+      uri = URI("#{@url}/realms/openc3/protocol/openid-connect/token")
       @log[0] = "request uri: #{uri.to_s} header: #{headers.to_s} body: #{data.to_s}"
       STDOUT.puts @log[0] if JsonDRb.debug?
       saved_verbose = $VERBOSE; $VERBOSE = nil

data/lib/openc3/utilities/aws_bucket.rb

@@ -129,7 +129,7 @@ module OpenC3
     end
 
     # Lists the files under a specified path
-    def list_files(bucket:, path:, only_directories: false, include_head: false)
+    def list_files(bucket:, path:, only_directories: false, metadata: false)
       # Trailing slash is important in AWS S3 when listing files
       # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Types/ListObjectsV2Output.html#common_prefixes-instance_method
       if path[-1] != '/'
@@ -163,8 +163,8 @@ module OpenC3
             item['name'] = aws_item.key.split('/')[-1]
             item['modified'] = aws_item.last_modified
             item['size'] = aws_item.size
-            if include_head
-              item['head'] = head_object(bucket: bucket, key: aws_item.key)
+            if metadata
+              item['metadata'] = head_object(bucket: bucket, key: aws_item.key)
             end
             files << item
           end
@@ -180,12 +180,12 @@ module OpenC3
 
     # get metadata for a specific object
     def head_object(bucket:, key:)
-      head = @client.head_object({
+      @client.head_object({
         bucket: bucket,
         key: key
       })
-    rescue Aws::S3::Errors::NotFound => error
-      raise NotFound, "Object: #{bucket}/#{key}"
+    rescue Aws::S3::Errors::NotFound
+      raise NotFound, "Object '#{bucket}/#{key}' does not exist."
     end
 
     # put_object fires off the request to store but does not confirm

data/lib/openc3/utilities/bucket.rb

@@ -61,7 +61,7 @@ module OpenC3
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 
-    def list_files(bucket:, path:, only_directories: false)
+    def list_files(bucket:, path:, only_directories: false, metadata: false)
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 

data/lib/openc3/utilities/local_mode.rb

@@ -58,7 +58,10 @@ module OpenC3
           puts "Local init found scope: #{scope_dir}"
           Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}") do |plugin_dir|
             full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}/#{plugin_dir}"
-            next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
+            if plugin_dir == "targets_modified" || plugin_dir == "tool_config" ||
+               plugin_dir == "settings" || !File.directory?(full_folder_path)
+              next
+            end
             puts "Local init found plugin_dir: #{full_folder_path}"
             gems, plugin_instance = scan_plugin_dir(full_folder_path)
 
@@ -79,6 +82,8 @@ module OpenC3
           end
         end
         sync_targets_modified()
+        sync_tool_config()
+        sync_settings()
         puts "Local init complete"
       else
         puts "Local init canceled: Local mode not enabled or #{OPENC3_LOCAL_MODE_PATH} does not exist"
@@ -322,12 +327,10 @@ module OpenC3
     end
 
     def self.sync_targets_modified
-      if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
-        bucket = Bucket.getClient()
-        scopes = ScopeModel.names()
-        scopes.each do |scope|
-          sync_with_bucket(bucket, scope: scope)
-        end
+      bucket = Bucket.getClient()
+      scopes = ScopeModel.names()
+      scopes.each do |scope|
+        sync_with_bucket(bucket, scope: scope)
       end
     end
 
@@ -417,6 +420,58 @@ module OpenC3
       return files.sort
     end
 
+    def self.sync_tool_config()
+      scopes = ScopeModel.names()
+      scopes.each do |scope|
+        Dir["#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/**/*.json"].each do |config|
+          parts = config.split('/')
+          puts "Syncing tool_config #{parts[-2]} #{File.basename(config)}"
+          data = File.read(config)
+          begin
+            # Parse just to ensure we have valid JSON
+            JSON.parse(data, :allow_nan => true, :create_additions => true)
+            # Only save if the parse was successful
+            ToolConfigModel.save_config(parts[-2], File.basename(config), data, scope: scope, local_mode: false)
+          rescue JSON::ParserError => error
+            puts "Unable to initialize tool config due to #{error.message}"
+          end
+        end
+      end
+    end
+
+    def self.save_tool_config(scope, tool, name, data)
+      json = JSON.parse(data, :allow_nan => true, :create_additions => true)
+      config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/#{tool}/#{name}.json"
+      FileUtils.mkdir_p(File.dirname(config_path))
+      File.open(config_path, 'w') do |file|
+        file.write(JSON.pretty_generate(json, :allow_nan => true))
+      end
+    end
+
+    def self.delete_tool_config(scope, tool, name)
+      FileUtils.rm_f("#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/#{tool}/#{name}.json")
+    end
+
+    def self.sync_settings()
+      scopes = ScopeModel.names()
+      scopes.each do |scope|
+        Dir["#{OPENC3_LOCAL_MODE_PATH}/#{scope}/settings/*.json"].each do |config|
+          name = File.basename(config, ".json")
+          puts "Syncing setting #{name}"
+          # Anything can be stored in settings so read and set directly
+          data = File.read(config)
+          SettingModel.set({ name: name, data: data }, scope: scope)
+        end
+      end
+    end
+
+    def self.save_setting(scope, name, data)
+      config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/settings/#{name}.json"
+      FileUtils.mkdir_p(File.dirname(config_path))
+      # Anything can be stored as a setting so write it out directly
+      File.write(config_path, data)
+    end
+
     # Helper methods
 
     def self.sync_remote_to_local(bucket, key)

data/lib/openc3/utilities/redis_secrets.rb

@@ -21,11 +21,11 @@ require 'openc3/utilities/secrets'
 
 module OpenC3
   class RedisSecrets < Secrets
-    def keys(scope:)
+    def keys(secret_store: nil, scope:)
       SecretModel.names(scope: scope)
     end
 
-    def get(key, scope:)
+    def get(key, secret_store: nil, scope:)
       data = SecretModel.get(name: key, scope: scope)
       if data
         return data['value']
@@ -34,11 +34,11 @@ module OpenC3
       end
     end
 
-    def set(key, value, scope:)
+    def set(key, value, secret_store: nil, scope:)
       SecretModel.set( {name: key, value: value.to_s }, scope: scope)
     end
 
-    def delete(key, scope:)
+    def delete(key, secret_store: nil, scope:)
       model = SecretModel.get_model(name: key, scope: scope)
       model.destroy if model
     end

data/lib/openc3/utilities/secrets.rb

@@ -31,24 +31,24 @@ module OpenC3
       klass.new
     end
 
-    def keys(scope:)
+    def keys(secret_store: nil, scope:)
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 
-    def get(key, scope:)
+    def get(key, secret_store: nil, scope:)
       return @local_secrets[key]
     end
 
-    def set(key, value, scope:)
+    def set(key, value, secret_store: nil, scope:)
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 
-    def delete(key, scope:)
+    def delete(key, secret_store: nil, scope:)
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 
     def setup(secrets)
-      secrets.each do |type, key, data|
+      secrets.each do |type, key, data, secret_store|
         case type
         when 'ENV'
           @local_secrets[key] = ENV[data]

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '5.5.0'
+OPENC3_VERSION = '5.5.2-beta0'
 module OpenC3
   module Version
     MAJOR = '5'
     MINOR = '5'
-    PATCH = '0'
-    OTHER = ''
-    BUILD = '6e9e3efe6170bd7adca14b849620f03fb622338d'
+    PATCH = '2'
+    OTHER = 'pre.beta0'
+    BUILD = '417d0e4ab762ecc0ada17fd8e230b394721f9cd4'
   end
-  VERSION = '5.5.0'
-  GEM_VERSION = '5.5.0'
+  VERSION = '5.5.2-beta0'
+  GEM_VERSION = '5.5.2.pre.beta0'
 end

data/templates/target/targets/TARGET/lib/target.rb

@@ -1,5 +1,5 @@
 # This class can be used in your scripts like so:
-#   require '<%= target_lib_filename %>'
+#   require_utility '<%= target_class.upcase %>/lib/<%= target_lib_filename %>'
 #   <%= target_object %> = <%= target_class %>.new
 #   <%= target_object %>.utility
 # For more information see the OpenC3 scripting guide

data/templates/target/targets/TARGET/screens/status.txt

@@ -2,8 +2,7 @@ SCREEN AUTO AUTO 1.0
 
 TITLE "Status"
 
-VERTICALBOX
-  SECTIONHEADER "Information"
+VERTICALBOX "Information"
   LABELVALUE <%= target_name %> STATUS VALUE
   LABELVALUE <%= target_name %> STATUS BOOL
 END

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openc3
 version: !ruby/object:Gem::Version
-  version: 5.5.0
+  version: 5.5.2.pre.beta0
 platform: ruby
 authors:
 - Ryan Melton
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-23 00:00:00.000000000 Z
+date: 2023-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -692,6 +692,8 @@ files:
 - bin/openc3cli
 - bin/rubysloc
 - data/config/_array_params.yaml
+- data/config/_canvas_values.yaml
+- data/config/_graph_params.yaml
 - data/config/_id_items.yaml
 - data/config/_id_params.yaml
 - data/config/_interfaces.yaml
@@ -700,6 +702,7 @@ files:
 - data/config/command.yaml
 - data/config/command_modifiers.yaml
 - data/config/command_telemetry.yaml
+- data/config/graph_settings.yaml
 - data/config/interface_modifiers.yaml
 - data/config/item_modifiers.yaml
 - data/config/microservice.yaml
@@ -708,6 +711,7 @@ files:
 - data/config/plugins.yaml
 - data/config/protocols.yaml
 - data/config/screen.yaml
+- data/config/settings.yaml
 - data/config/table_manager.yaml
 - data/config/table_parameter_modifiers.yaml
 - data/config/target.yaml
@@ -892,14 +896,13 @@ files:
 - lib/openc3/models/router_status_model.rb
 - lib/openc3/models/scope_model.rb
 - lib/openc3/models/secret_model.rb
-- lib/openc3/models/settings_model.rb
+- lib/openc3/models/setting_model.rb
 - lib/openc3/models/sorted_model.rb
 - lib/openc3/models/stash_model.rb
 - lib/openc3/models/target_model.rb
 - lib/openc3/models/timeline_model.rb
 - lib/openc3/models/tool_config_model.rb
 - lib/openc3/models/tool_model.rb
-- lib/openc3/models/traefik_model.rb
 - lib/openc3/models/trigger_group_model.rb
 - lib/openc3/models/trigger_model.rb
 - lib/openc3/models/widget_model.rb
@@ -1045,9 +1048,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.3.14
 signing_key:

data/lib/openc3/models/traefik_model.rb

@@ -1,47 +0,0 @@
-# encoding: ascii-8bit
-
-# Copyright 2023 OpenC3, Inc.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# This file may also be used under the terms of a commercial license
-# if purchased from OpenC3, Inc.
-
-require 'openc3/utilities/store'
-
-module OpenC3
-  class TraefikModel
-    def self.register_route(microservice_name:, port:, prefix:, priority: 20)
-      prefix = '/' + prefix unless prefix[0] == '/'
-      if ENV['KUBERNETES_SERVICE_HOST']
-        url = "http://#{microservice_name.gsub('__', '-')}:#{port}"
-      else
-        url = "http://openc3-operator:#{port}"
-      end
-      service_name = microservice_name
-      router_name = microservice_name
-      Store.set("traefik/http/services/#{service_name}/loadbalancer/servers/0/url", url)
-      Store.set("traefik/http/routers/#{router_name}/service", service_name)
-      Store.set("traefik/http/routers/#{router_name}/priority", priority)
-      Store.set("traefik/http/routers/#{router_name}/rule", "PathPrefix(`#{prefix}`)")
-    end
-
-    def self.unregister_route(microservice_name:)
-      service_name = microservice_name
-      router_name = microservice_name
-      Store.del("traefik/http/routers/#{router_name}/rule")
-      Store.del("traefik/http/routers/#{router_name}/priority")
-      Store.del("traefik/http/routers/#{router_name}/service")
-      Store.del("traefik/http/services/#{service_name}/loadbalancer/servers/0/url")
-    end
-  end
-end