cosmos 5.0.2 → 5.0.5

data/lib/cosmos/models/model.rb

@@ -27,12 +27,16 @@ module Cosmos
     attr_accessor :plugin
     attr_accessor :scope
 
+    def self.store
+      Store
+    end
+
     # NOTE: The following three methods must be reimplemented by Model subclasses
     # without primary_key to support other class methods.
 
     # @return [Hash|nil] Hash of this model or nil if name not found under primary_key
     def self.get(primary_key, name:)
-      json = Store.hget(primary_key, name)
+      json = store.hget(primary_key, name)
       if json
         return JSON.parse(json)
       else
@@ -42,12 +46,12 @@ module Cosmos
 
     # @return [Array<String>] All the names stored under the primary key
     def self.names(primary_key)
-      Store.hkeys(primary_key).sort
+      store.hkeys(primary_key).sort
     end
 
     # @return [Array<Hash>] All the models (as Hash objects) stored under the primary key
     def self.all(primary_key)
-      hash = Store.hgetall(primary_key)
+      hash = store.hgetall(primary_key)
       hash.each do |key, value|
         hash[key] = JSON.parse(value)
       end
@@ -117,16 +121,6 @@ module Cosmos
       raise "must be implemented by subclass"
     end
 
-    # TODO: Not used
-    # def self.from_config(primary_key, filename)
-    #   model = nil
-    #   parser = ConfigParser.new
-    #   parser.parse_file(filename) do |keyword, parameters|
-    #     model = self.handle_config(primary_key, parser, model, keyword, parameters)
-    #   end
-    #   model
-    # end
-
     # Store the primary key and keyword arguments
     def initialize(primary_key, **kw_args)
       @primary_key = primary_key
@@ -134,13 +128,14 @@ module Cosmos
       @updated_at = kw_args[:updated_at]
       @plugin = kw_args[:plugin]
       @scope = kw_args[:scope]
+      @destroyed = false
     end
 
     # Update the Redis hash at primary_key and set the field "name"
     # to the JSON generated via calling as_json
     def create(update: false, force: false)
       unless force
-        existing = Store.hget(@primary_key, @name)
+        existing = self.class.store.hget(@primary_key, @name)
         if existing
           raise "#{@primary_key}:#{@name} already exists at create" unless update
         else
@@ -148,7 +143,7 @@ module Cosmos
         end
       end
       @updated_at = Time.now.to_nsec_from_epoch
-      Store.hset(@primary_key, @name, JSON.generate(self.as_json))
+      self.class.store.hset(@primary_key, @name, JSON.generate(self.as_json))
     end
 
     # Alias for create(update: true)
@@ -169,8 +164,14 @@ module Cosmos
 
     # Delete the model from the Store
     def destroy
+      @destroyed = true
       undeploy()
-      Store.hdel(@primary_key, @name)
+      self.class.store.hdel(@primary_key, @name)
+    end
+
+    # Indicate if destroy has been called
+    def destroyed?
+      @destroyed
     end
 
     # @return [Hash] JSON encoding of this model
@@ -186,4 +187,10 @@ module Cosmos
       ""
     end
   end
+
+  class EphemeralModel < Model
+    def self.store
+      EphemeralStore
+    end
+  end
 end

data/lib/cosmos/models/note_model.rb

@@ -0,0 +1,122 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
+
+# https://www.rubydoc.info/gems/redis/Redis/Commands/SortedSets
+
+require 'cosmos/models/sorted_model'
+
+module Cosmos
+  class NoteModel < SortedModel
+    NOTE_TYPE = 'note'.freeze
+    PRIMARY_KEY = '__NOTE'.freeze
+
+    def self.pk(scope)
+      "#{scope}#{PRIMARY_KEY}"
+    end
+
+    attr_reader :stop, :color, :description, :type
+
+    # @param [String] scope - Cosmos scope to track event to
+    # @param [Integer] start - start of the event in seconds from Epoch
+    # @param [Integer] stop - stop of the event in seconds from Epoch
+    # @param [String] color - The event color
+    # @param [String] description - What the event is about
+    def initialize(
+      scope:,
+      start:,
+      stop:,
+      color: nil,
+      description:,
+      type: NOTE_TYPE,
+      updated_at: 0
+    )
+      super(start: start, scope: scope, updated_at: updated_at)
+      @start = start
+      @stop = stop
+      @color = color
+      @description = description
+      @type = type # For the as_json, from_json round trip
+    end
+
+    # Validates the instance variables: @start, @stop, @color, @description
+    def validate(update: false)
+      validate_start(update: update)
+      validate_stop()
+      validate_color()
+    end
+
+    def validate_stop()
+      unless @stop.is_a?(Integer)
+        raise SortedInputError.new "stop must be integer: #{@stop}"
+      end
+      if @stop.to_i < @start
+        raise SortedInputError.new "stop: #{@stop} must be >= start: #{@start}"
+      end
+      @stop = @stop.to_i
+    end
+
+    def validate_color()
+      if @color.nil?
+        @color = '#%06x' % (rand * 0xffffff)
+      end
+      unless @color =~ /(#*)([0-9,a-f,A-f]{6})/
+        raise SortedInputError.new "invalid color, must be in hex format, e.g. #FF0000"
+      end
+      @color = "##{@color}" unless @color.start_with?('#')
+    end
+
+    # Update the Redis hash at primary_key based on the initial passed start
+    # The member is set to the JSON generated via calling as_json
+    def create(update: false)
+      validate(update: update)
+      @updated_at = Time.now.to_nsec_from_epoch
+      NoteModel.destroy(scope: @scope, start: update) if update
+      Store.zadd(@primary_key, @start, JSON.generate(as_json()))
+      if update
+        notify(kind: 'updated')
+      else
+        notify(kind: 'created')
+      end
+    end
+
+    # Update the Redis hash at primary_key
+    def update(start:, stop:, color:, description:)
+      orig_start = @start
+      @start = start
+      @stop = stop
+      @color = color
+      @description = description
+      create(update: orig_start)
+    end
+
+    # @return [Hash] generated from the NoteModel
+    def as_json
+      return {
+        'scope' => @scope,
+        'start' => @start,
+        'stop' => @stop,
+        'color' => @color,
+        'description' => @description,
+        'type' => NOTE_TYPE,
+        'updated_at' => @updated_at,
+      }
+    end
+    alias to_s as_json
+  end
+end

data/lib/cosmos/models/ping_model.rb

@@ -24,7 +24,8 @@ module Cosmos
     # @return String ['UP' or 'DOWN']
     def self.get()
       response = Store.ping()
-      if response
+      response2 = EphemeralStore.ping()
+      if response and response2
         return 'UP'
       else
         return 'DOWN'

data/lib/cosmos/models/plugin_model.rb

@@ -32,6 +32,7 @@ require 'cosmos/models/tool_model'
 require 'cosmos/models/widget_model'
 require 'cosmos/models/microservice_model'
 require 'tmpdir'
+require 'tempfile'
 
 module Cosmos
   # Represents a COSMOS plugin that can consist of targets, interfaces, routers
@@ -41,6 +42,9 @@ module Cosmos
     PRIMARY_KEY = 'cosmos_plugins'
 
     attr_accessor :variables
+    attr_accessor :plugin_txt_lines
+    attr_accessor :needs_dependencies
+
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -58,74 +62,103 @@ module Cosmos
 
     # Called by the PluginsController to parse the plugin variables
     # Doesn't actaully create the plugin during the phase
-    def self.install_phase1(gem_file_path, existing_variables = nil, scope:)
-      gem_filename = File.basename(gem_file_path)
-
-      # Load gem to internal gem server
-      Cosmos::GemModel.put(gem_file_path, gem_install: false, scope: scope)
-
-      # Extract gem and process plugin.txt to determine what VARIABLEs need to be filled in
-      pkg = Gem::Package.new(gem_file_path)
+    def self.install_phase1(gem_file_path, existing_variables: nil, existing_plugin_txt_lines: nil, process_existing: false, scope:, validate_only: false)
+      gem_name = File.basename(gem_file_path).split("__")[0]
 
       temp_dir = Dir.mktmpdir
+      tf = nil
       begin
-        pkg.extract_files(temp_dir)
-        plugin_txt_path = File.join(temp_dir, 'plugin.txt')
-        if File.exist?(plugin_txt_path)
-          parser = Cosmos::ConfigParser.new("http://cosmosc2.com")
-
-          # Phase 1 Gather Variables
-          variables = {}
-          parser.parse_file(plugin_txt_path,
-                            false,
-                            true,
-                            false) do |keyword, params|
-            case keyword
-            when 'VARIABLE'
-              usage = "#{keyword} <Variable Name> <Default Value>"
-              parser.verify_num_parameters(2, nil, usage)
-              variable_name = params[0]
-              value = params[1..-1].join(" ")
-              variables[variable_name] = value
-              if existing_variables && existing_variables.key?(variable_name)
-                variables[variable_name] = existing_variables[variable_name]
-              end
-              # Ignore everything else during phase 1
-            end
+        if File.exists?(gem_file_path)
+          # Load gem to internal gem server
+          Cosmos::GemModel.put(gem_file_path, gem_install: false, scope: scope) unless validate_only
+        else
+          gem_file_path = Cosmos::GemModel.get(temp_dir, gem_name)
+        end
+
+        # Extract gem and process plugin.txt to determine what VARIABLEs need to be filled in
+        pkg = Gem::Package.new(gem_file_path)
+
+        if existing_plugin_txt_lines and process_existing
+          # This is only used in cosmos load when everything is known
+          plugin_txt_lines = existing_plugin_txt_lines
+          file_data = existing_plugin_txt_lines.join("\n")
+          tf = Tempfile.new("plugin.txt")
+          tf.write(file_data)
+          tf.close
+          plugin_txt_path = tf.path
+        else
+          # Otherwise we always process the new and return both
+          pkg.extract_files(temp_dir)
+          plugin_txt_path = File.join(temp_dir, 'plugin.txt')
+          plugin_text = File.read(plugin_txt_path)
+          plugin_txt_lines = []
+          plugin_text.each_line do |line|
+            plugin_txt_lines << line.chomp
           end
+        end
+
+        parser = Cosmos::ConfigParser.new("http://cosmosc2.com")
 
-          model = PluginModel.new(name: gem_filename, variables: variables, scope: scope)
-          return model.as_json
+        # Phase 1 Gather Variables
+        variables = {}
+        parser.parse_file(plugin_txt_path,
+                          false,
+                          true,
+                          false) do |keyword, params|
+          case keyword
+          when 'VARIABLE'
+            usage = "#{keyword} <Variable Name> <Default Value>"
+            parser.verify_num_parameters(2, nil, usage)
+            variable_name = params[0]
+            value = params[1..-1].join(" ")
+            variables[variable_name] = value
+            if existing_variables && existing_variables.key?(variable_name)
+              variables[variable_name] = existing_variables[variable_name]
+            end
+            # Ignore everything else during phase 1
+          end
         end
+
+        model = PluginModel.new(name: gem_name, variables: variables, plugin_txt_lines: plugin_txt_lines, scope: scope)
+        result = model.as_json
+        result['existing_plugin_txt_lines'] = existing_plugin_txt_lines if existing_plugin_txt_lines and not process_existing and existing_plugin_txt_lines != result['plugin_txt_lines']
+        return result
       ensure
         FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
+        tf.unlink if tf
       end
     end
 
     # Called by the PluginsController to create the plugin
     # Because this uses ERB it must be run in a seperate process from the API to
     # prevent corruption and single require problems in the current proces
-    def self.install_phase2(name, variables, scope:)
+    def self.install_phase2(plugin_hash, scope:, validate_only: false)
       rubys3_client = Aws::S3::Client.new
 
       # Ensure config bucket exists
-      begin
-        rubys3_client.head_bucket(bucket: 'config')
-      rescue Aws::S3::Errors::NotFound
-        rubys3_client.create_bucket(bucket: 'config')
+      unless validate_only
+        begin
+          rubys3_client.head_bucket(bucket: 'config')
+        rescue Aws::S3::Errors::NotFound
+          rubys3_client.create_bucket(bucket: 'config')
+        end
       end
 
       # Register plugin to aid in uninstall if install fails
-      plugin_model = PluginModel.new(name: name, variables: variables, scope: scope)
-      plugin_model.create
+      plugin_hash.delete("existing_plugin_txt_lines")
+      plugin_model = PluginModel.new(**(plugin_hash.transform_keys(&:to_sym)), scope: scope)
+      plugin_model.create unless validate_only
 
       temp_dir = Dir.mktmpdir
       begin
+        tf = nil
+
         # Get the gem from local gem server
-        gem_file_path = Cosmos::GemModel.get(temp_dir, name)
+        gem_name = plugin_hash['name'].split("__")[0]
+        gem_file_path = Cosmos::GemModel.get(temp_dir, gem_name)
 
         # Actually install the gem now (slow)
-        Cosmos::GemModel.install(gem_file_path)
+        Cosmos::GemModel.install(gem_file_path, scope: scope)
 
         # Extract gem contents
         gem_path = File.join(temp_dir, "gem")
@@ -133,6 +166,11 @@ module Cosmos
         pkg = Gem::Package.new(gem_file_path)
         needs_dependencies = pkg.spec.runtime_dependencies.length > 0
         pkg.extract_files(gem_path)
+        needs_dependencies = true if Dir.exist?(File.join(gem_path, 'lib'))
+        if needs_dependencies
+          plugin_model.needs_dependencies = true
+          plugin_model.update
+        end
 
         # Temporarily add all lib folders from the gem to the end of the load path
         load_dirs = []
@@ -145,7 +183,12 @@ module Cosmos
           end
 
           # Process plugin.txt file
-          plugin_txt_path = File.join(gem_path, 'plugin.txt')
+          file_data = plugin_hash['plugin_txt_lines'].join("\n")
+          tf = Tempfile.new("plugin.txt")
+          tf.write(file_data)
+          tf.close
+          plugin_txt_path = tf.path
+          variables = plugin_hash['variables']
           if File.exist?(plugin_txt_path)
             parser = Cosmos::ConfigParser.new("http://cosmosc2.com")
 
@@ -156,8 +199,8 @@ module Cosmos
                 # Ignore during phase 2
               when 'TARGET', 'INTERFACE', 'ROUTER', 'MICROSERVICE', 'TOOL', 'WIDGET'
                 if current_model
-                  current_model.create
-                  current_model.deploy(gem_path, variables)
+                  current_model.create unless validate_only
+                  current_model.deploy(gem_path, variables, validate_only: validate_only)
                   current_model = nil
                 end
                 current_model = Cosmos.const_get((keyword.capitalize + 'Model').intern).handle_config(parser, keyword, params, plugin: plugin_model.name, needs_dependencies: needs_dependencies, scope: scope)
@@ -170,8 +213,8 @@ module Cosmos
               end
             end
             if current_model
-              current_model.create
-              current_model.deploy(gem_path, variables)
+              current_model.create unless validate_only
+              current_model.deploy(gem_path, variables, validate_only: validate_only)
               current_model = nil
             end
           end
@@ -182,21 +225,28 @@ module Cosmos
         end
       rescue => err
         # Install failed - need to cleanup
-        plugin_model.destroy
+        plugin_model.destroy unless validate_only
         raise err
       ensure
         FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
+        tf.unlink if tf
       end
+
+      return plugin_model.as_json
     end
 
     def initialize(
       name:,
       variables: {},
+      plugin_txt_lines: [],
+      needs_dependencies: false,
       updated_at: nil,
       scope:
     )
       super("#{scope}__#{PRIMARY_KEY}", name: name, updated_at: updated_at, scope: scope)
       @variables = variables
+      @plugin_txt_lines = plugin_txt_lines
+      @needs_dependencies = ConfigParser.handle_true_false(needs_dependencies)
     end
 
     def create(update: false, force: false)
@@ -208,6 +258,8 @@ module Cosmos
       {
         'name' => @name,
         'variables' => @variables,
+        'plugin_txt_lines' => @plugin_txt_lines,
+        'needs_dependencies' => @needs_dependencies,
         'updated_at' => @updated_at
       }
     end
@@ -220,5 +272,13 @@ module Cosmos
         end
       end
     end
+
+    # Reinstall
+    def restore
+      plugin_hash = self.as_json
+      plugin_hash['name'] = plugin_hash['name'].split("__")[0]
+      Cosmos::PluginModel.install_phase2(plugin_hash, scope: @scope)
+      @destroyed = false
+    end
   end
 end

data/lib/cosmos/models/process_status_model.rb

@@ -21,7 +21,7 @@ require 'cosmos/models/model'
 
 module Cosmos
   # Stores the status about an process.
-  class ProcessStatusModel < Model
+  class ProcessStatusModel < EphemeralModel
     PRIMARY_KEY = 'cosmos_process_status'
 
     attr_accessor :state

data/lib/cosmos/models/scope_model.rb

@@ -20,6 +20,7 @@
 require 'cosmos/version'
 require 'cosmos/models/model'
 require 'cosmos/models/microservice_model'
+require 'cosmos/models/settings_model'
 
 module Cosmos
   class ScopeModel < Model
@@ -51,23 +52,7 @@ module Cosmos
     def deploy(gem_path, variables)
       seed_database()
 
-      # Cleanup Microservice
-      microservice_name = "#{@scope}__CLEANUP__S3"
-      microservice = MicroserviceModel.new(
-        name: microservice_name,
-        cmd: ["ruby", "cleanup_microservice.rb", microservice_name],
-        work_dir: '/cosmos/lib/cosmos/microservices',
-        options: [
-          ["SIZE", "20_000_000_000"], # Max Size to keep in S3
-          ["DELAY", "300"], # Delay between size checks
-          ["BUCKET", "logs"], # Bucket to monitor
-          ["PREFIX", @scope + "/"], # Path into bucket to monitor
-        ],
-        scope: @scope
-      )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      Logger.info "Configured microservice #{microservice_name}"
+      ConfigTopic.initialize_stream(@scope)
 
       # COSMOS Log Microservice
       microservice_name = "#{@scope}__COSMOS__LOG"
@@ -105,7 +90,7 @@ module Cosmos
       Logger.info "Configured microservice #{microservice_name}"
 
       # UNKNOWN CommandLog Microservice
-      Store.initialize_streams(["#{@scope}__COMMAND__{UNKNOWN}__UNKNOWN"])
+      Topic.initialize_streams(["#{@scope}__COMMAND__{UNKNOWN}__UNKNOWN"])
       microservice_name = "#{@scope}__COMMANDLOG__UNKNOWN"
       microservice = MicroserviceModel.new(
         name: microservice_name,
@@ -125,7 +110,7 @@ module Cosmos
       Logger.info "Configured microservice #{microservice_name}"
 
       # UNKNOWN PacketLog Microservice
-      Store.initialize_streams(["#{@scope}__TELEMETRY__{UNKNOWN}__UNKNOWN"])
+      Topic.initialize_streams(["#{@scope}__TELEMETRY__{UNKNOWN}__UNKNOWN"])
       microservice_name = "#{@scope}__PACKETLOG__UNKNOWN"
       microservice = MicroserviceModel.new(
         name: microservice_name,
@@ -146,8 +131,6 @@ module Cosmos
     end
 
     def undeploy
-      model = MicroserviceModel.get_model(name: "#{@scope}__CLEANUP__S3", scope: @scope)
-      model.destroy if model
       model = MicroserviceModel.get_model(name: "#{@scope}__COSMOS__LOG", scope: @scope)
       model.destroy if model
       model = MicroserviceModel.get_model(name: "#{@scope}__NOTIFICATION__LOG", scope: @scope)
@@ -159,10 +142,12 @@ module Cosmos
     end
 
     def seed_database
-      # Set default values for items in the db that should be set
-      # Use the "nx" (not-exists) variant of redis calls here to not overwrite things the user has already set
-      Cosmos::Store.hsetnx('cosmos__settings', 'source_url', 'https://github.com/BallAerospace/COSMOS')
-      Cosmos::Store.hsetnx('cosmos__settings', 'version', ENV['COSMOS_VERSION'] || COSMOS_VERSION)
+      setting = SettingsModel.get(name: 'source_url')
+      SettingsModel.set({ name: 'source_url', data: 'https://github.com/BallAerospace/COSMOS' }, scope: @scope) unless setting
+    end
+
+    def self.limits_set(scope:)
+      Store.hget("#{scope}__cosmos_system", 'limits_set').intern
     end
   end
 end

data/lib/cosmos/models/settings_model.rb

@@ -0,0 +1,55 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
+
+require 'cosmos/models/model'
+
+module Cosmos
+  class SettingsModel < Model
+    PRIMARY_KEY = 'cosmos__settings'
+
+    # NOTE: The following three class methods are used by the ModelController
+    # and are reimplemented to enable various Model class methods to work
+    def self.get(name:, scope: nil)
+      super(PRIMARY_KEY, name: name)
+    end
+
+    def self.names(scope: nil)
+      super(PRIMARY_KEY)
+    end
+
+    def self.all(scope: nil)
+      super(PRIMARY_KEY)
+    end
+    # END NOTE
+
+    def initialize(name:, scope: nil, data:)
+      super(PRIMARY_KEY, name: name, scope: scope)
+      @data = data
+    end
+
+    # @return [Hash] JSON encoding of this model
+    def as_json
+      {
+        'name' => @name,
+        'data' => @data,
+        'updated_at' => @updated_at
+      }
+    end
+  end
+end