cosmos 5.0.4 → 5.0.5

data/lib/cosmos/logs/log_writer.rb

@@ -32,9 +32,41 @@ module Cosmos
     # @return [true/false] Whether logging is enabled
     attr_reader :logging_enabled
 
+    # @return cycle_time [Integer] The amount of time in seconds before creating
+    #   a new log file. This can be combined with cycle_size but is better used
+    #   independently.
+    attr_reader :cycle_time
+
+    # @return cycle_hour [Integer] The time at which to cycle the log. Combined with
+    #   cycle_minute to cycle the log daily at the specified time. If nil, the log
+    #   will be cycled hourly at the specified cycle_minute.
+    attr_reader :cycle_hour
+
+    # @return cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
+    #   for more information.
+    attr_reader :cycle_minute
+
+    # @return [Time] Time that the current log file started
+    attr_reader :start_time
+
+    # @return [Mutex] Instance mutex protecting file
+    attr_reader :mutex
+
     # The cycle time interval. Cycle times are only checked at this level of
     # granularity.
-    CYCLE_TIME_INTERVAL = 2
+    CYCLE_TIME_INTERVAL = 10
+
+    # Mutex protecting class variables
+    @@mutex = Mutex.new
+
+    # Array of instances used to keep track of cycling logs
+    @@instances = []
+
+    # Thread used to cycle logs across all log writers
+    @@cycle_thread = nil
+
+    # Sleeper used to delay cycle thread
+    @@cycle_sleeper = nil
 
     # @param remote_log_directory [String] The s3 path to store the log files
     # @param logging_enabled [Boolean] Whether to start with logging enabled
@@ -89,11 +121,15 @@ module Cosmos
       # each time we create an entry which we do a LOT!
       @entry = String.new
 
-      @cycle_thread = nil
       if @cycle_time or @cycle_hour or @cycle_minute
-        @cycle_sleeper = Sleeper.new
-        @cycle_thread = Cosmos.safe_thread("Log cycle") do
-          cycle_thread_body()
+        @@mutex.synchronize do
+          @@instances << self
+
+          unless @@cycle_thread
+            @@cycle_thread = Cosmos.safe_thread("Log cycle") do
+              cycle_thread_body()
+            end
+          end
         end
       end
     end
@@ -113,10 +149,13 @@ module Cosmos
     # Stop all logging, close the current log file, and kill the logging threads.
     def shutdown
       stop()
-      if @cycle_thread
-        @cycle_sleeper.cancel
-        Cosmos.kill_thread(self, @cycle_thread)
-        @cycle_thread = nil
+      @@mutex.synchronize do
+        @@instances.delete(self)
+        if @@instances.length <= 0
+          @@cycle_sleeper.cancel if @@cycle_sleeper
+          Cosmos.kill_thread(self, @@cycle_thread) if @@cycle_thread
+          @@cycle_thread = nil
+        end
       end
     end
 
@@ -143,28 +182,38 @@ module Cosmos
     end
 
     def cycle_thread_body
+      @@cycle_sleeper = Sleeper.new
       while true
-        # The check against start_time needs to be mutex protected to prevent a packet coming in between the check
-        # and closing the file
-        @mutex.synchronize do
-          utc_now = Time.now.utc
-          # Logger.debug("start:#{@start_time.to_f} now:#{utc_now.to_f} cycle:#{@cycle_time} new:#{(utc_now - @start_time) > @cycle_time}")
-          if @logging_enabled and
-             (
-               # Cycle based on total time logging
-               (@cycle_time and (utc_now - @start_time) > @cycle_time) or
-
-               # Cycle daily at a specific time
-               (@cycle_hour and @cycle_minute and utc_now.hour == @cycle_hour and utc_now.min == @cycle_minute and @start_time.yday != utc_now.yday) or
-
-               # Cycle hourly at a specific time
-               (@cycle_minute and not @cycle_hour and utc_now.min == @cycle_minute and @start_time.hour != utc_now.hour)
-             )
-            close_file(false)
+        start_time = Time.now
+        @@mutex.synchronize do
+          @@instances.each do |instance|
+            # The check against start_time needs to be mutex protected to prevent a packet coming in between the check
+            # and closing the file
+            instance.mutex.synchronize do
+              utc_now = Time.now.utc
+              # Logger.debug("start:#{@start_time.to_f} now:#{utc_now.to_f} cycle:#{@cycle_time} new:#{(utc_now - @start_time) > @cycle_time}")
+              if instance.logging_enabled and
+                (
+                  # Cycle based on total time logging
+                  (instance.cycle_time and (utc_now - instance.start_time) > instance.cycle_time) or
+
+                  # Cycle daily at a specific time
+                  (instance.cycle_hour and instance.cycle_minute and utc_now.hour == instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.yday != utc_now.yday) or
+
+                  # Cycle hourly at a specific time
+                  (instance.cycle_minute and not instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.hour != utc_now.hour)
+                )
+                instance.close_file(false)
+              end
+            end
           end
         end
+
         # Only check whether to cycle at a set interval
-        break if @cycle_sleeper.sleep(CYCLE_TIME_INTERVAL)
+        run_time = Time.now - start_time
+        sleep_time = CYCLE_TIME_INTERVAL - run_time
+        sleep_time = 0 if sleep_time < 0
+        break if @@cycle_sleeper.sleep(sleep_time)
       end
     end
 

data/lib/cosmos/microservices/interface_microservice.rb

@@ -83,11 +83,6 @@ module Cosmos
             @interface.write_raw(msg_hash['raw'])
             next 'SUCCESS'
           end
-          if msg_hash['inject_tlm']
-            Logger.info "#{@interface.name}: Inject telemetry" if msg_hash['log']
-            @tlm.inject_tlm(msg_hash)
-            next 'SUCCESS'
-          end
           if msg_hash.key?('log_raw')
             if msg_hash['log_raw'] == 'true'
               Logger.info "#{@interface.name}: Enable raw logging"
@@ -412,16 +407,6 @@ module Cosmos
       TelemetryTopic.write_packet(packet, scope: @scope)
     end
 
-    def inject_tlm(hash)
-      packet = System.telemetry.packet(hash['target_name'], hash['packet_name']).clone
-      if hash['item_hash']
-        JSON.parse(hash['item_hash']).each do |item, value|
-          packet.write(item.to_s, value, hash['type'].to_sym)
-        end
-      end
-      handle_packet(packet)
-    end
-
     def handle_connection_failed(connect_error)
       @error = connect_error
       Logger.error "#{@interface.name}: Connection Failed: #{connect_error.formatted(false, false)}"

data/lib/cosmos/models/gem_model.rb

@@ -81,7 +81,13 @@ module Cosmos
         else
           gem_file_path = get(temp_dir, name_or_path)
         end
-        rubygems_url = get_setting('rubygems_url', scope: scope)
+        begin
+          rubygems_url = get_setting('rubygems_url', scope: scope)
+        rescue
+          # If Redis isn't running try the ENV, then simply rubygems.org
+          rubygems_url = ENV['RUBYGEMS_URL']
+          rubygems_url ||= 'https://rubygems.org'
+        end
         Gem.sources = [rubygems_url] if rubygems_url
         Gem.done_installing_hooks.clear
         Gem.install(gem_file_path, "> 0.pre", :build_args => ['--no-document'], :prerelease => true)

data/lib/cosmos/models/interface_model.rb

@@ -237,7 +237,7 @@ module Cosmos
     end
 
     # Creates a MicroserviceModel to deploy the Interface/Router
-    def deploy(gem_path, variables)
+    def deploy(gem_path, variables, validate_only: false)
       type = self.class._get_type
       microservice_name = "#{@scope}__#{type}__#{@name}"
       microservice = MicroserviceModel.new(
@@ -249,9 +249,12 @@ module Cosmos
         needs_dependencies: @needs_dependencies,
         scope: @scope
       )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      Logger.info "Configured #{type.downcase} microservice #{microservice_name}"
+      unless validate_only
+        microservice.create
+        microservice.deploy(gem_path, variables)
+        ConfigTopic.write({ kind: 'created', type: type.downcase, name: @name, plugin: @plugin }, scope: @scope)
+        Logger.info "Configured #{type.downcase} microservice #{microservice_name}"
+      end
       microservice
     end
 
@@ -259,9 +262,15 @@ module Cosmos
     # should should trigger the operator to kill the microservice that in turn
     # will destroy the InterfaceStatusModel when a stop is called.
     def undeploy
-      model = MicroserviceModel.get_model(name: "#{@scope}__#{self.class._get_type}__#{@name}", scope: @scope)
-      model.destroy if model
-      if self.class._get_type == 'INTERFACE'
+      type = self.class._get_type
+      name = "#{@scope}__#{type}__#{@name}"
+      model = MicroserviceModel.get_model(name: name, scope: @scope)
+      if model
+        model.destroy
+        ConfigTopic.write({ kind: 'deleted', type: type.downcase, name: @name, plugin: @plugin }, scope: @scope)
+      end
+
+      if type == 'INTERFACE'
         status_model = InterfaceStatusModel.get_model(name: @name, scope: @scope)
       else
         status_model = RouterStatusModel.get_model(name: @name, scope: @scope)

data/lib/cosmos/models/metadata_model.rb

@@ -30,17 +30,19 @@ module Cosmos
       "#{scope}#{PRIMARY_KEY}"
     end
 
-    attr_reader :color, :metadata, :type
+    attr_reader :color, :metadata, :constraints, :type
 
-    # @param [Integer] start - time metadata is active in seconds from Epoch
+    # @param [Integer] start - Time metadata is active in seconds from Epoch
     # @param [String] color - The event color
-    # @param [String] metadata - Key value pair object to link to name
+    # @param [Hash] metadata - Hash of metadata values
+    # @param [Hash] constraints - Constraints to apply to the metadata
     # @param [String] scope - Cosmos scope to track event to
     def initialize(
       scope:,
       start:,
       color: nil,
       metadata:,
+      constraints: nil,
       type: METADATA_TYPE,
       updated_at: 0
     )
@@ -48,6 +50,7 @@ module Cosmos
       @start = start
       @color = color
       @metadata = metadata
+      @constraints = constraints if constraints
       @type = type # For the as_json, from_json round trip
     end
 
@@ -56,6 +59,7 @@ module Cosmos
       validate_start(update: update)
       validate_color()
       validate_metadata()
+      validate_constraints() if @constraints
     end
 
     def validate_color()
@@ -72,6 +76,26 @@ module Cosmos
       unless @metadata.is_a?(Hash)
         raise SortedInputError.new "Metadata must be a hash/object: #{@metadata}"
       end
+      # Convert keys to strings. This isn't quite as efficient as symbols
+      # but we store as JSON which is all strings and it makes comparisons easier.
+      @metadata = @metadata.transform_keys(&:to_s)
+    end
+
+    def validate_constraints()
+      unless @constraints.is_a?(Hash)
+        raise SortedInputError.new "Constraints must be a hash/object: #{@constraints}"
+      end
+      # Convert keys to strings. This isn't quite as efficient as symbols
+      # but we store as JSON which is all strings and it makes comparisons easier.
+      @constraints = @constraints.transform_keys(&:to_s)
+      unless (@constraints.keys - @metadata.keys).empty?
+        raise SortedInputError.new "Constraints keys must be subset of metadata: #{@constraints.keys} subset #{@metadata.keys}"
+      end
+      @constraints.each do |key, constraint|
+        unless constraint.include?(@metadata[key])
+          raise SortedInputError.new "Constraint violation! key:#{key} value:#{@metadata[key]} constraint:#{constraint}"
+        end
+      end
     end
 
     # Update the Redis hash at primary_key based on the initial passed start
@@ -79,6 +103,7 @@ module Cosmos
     def create(update: false)
       validate(update: update)
       @updated_at = Time.now.to_nsec_from_epoch
+      MetadataModel.destroy(scope: @scope, start: update) if update
       Store.zadd(@primary_key, @start, JSON.generate(as_json()))
       if update
         notify(kind: 'updated')
@@ -87,29 +112,28 @@ module Cosmos
       end
     end
 
-    # Update the Redis hash at primary_key
-    def update(start:, color:, metadata:)
-      @start = start
-      @color = color
-      @metadata = metadata
-      create(update: true)
+    # Update the model. All arguments are optional, only those set will be updated.
+    def update(start: nil, color: nil, metadata: nil, constraints: nil)
+      orig_start = @start
+      @start = start if start
+      @color = color if color
+      @metadata = metadata if metadata
+      @constraints = constraints if constraints
+      create(update: orig_start)
     end
 
     # @return [Hash] generated from the MetadataModel
     def as_json
-      return {
+      {
         'scope' => @scope,
         'start' => @start,
         'color' => @color,
         'metadata' => @metadata,
+        'constraints' => @constraints,
         'type' => METADATA_TYPE,
         'updated_at' => @updated_at,
       }
     end
-
-    # @return [String] string view of metadata
-    def to_s
-      return "<MetadataModel s: #{@start}, c: #{@color}, m: #{@metadata}>"
-    end
+    alias to_s as_json
   end
 end

data/lib/cosmos/models/microservice_model.rb

@@ -171,11 +171,10 @@ module Cosmos
       return nil
     end
 
-    def deploy(gem_path, variables)
+    def deploy(gem_path, variables, validate_only: false)
       return unless @folder_name
 
       variables["microservice_name"] = @name
-      rubys3_client = Aws::S3::Client.new
       start_path = "/microservices/#{@folder_name}/"
       Dir.glob(gem_path + start_path + "**/*") do |filename|
         next if filename == '.' or filename == '..' or File.directory?(filename)
@@ -188,7 +187,10 @@ module Cosmos
         Cosmos.set_working_dir(File.dirname(filename)) do
           data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
         end
-        rubys3_client.put_object(bucket: 'config', key: key, body: data)
+        unless validate_only
+          Aws::S3::Client.new.put_object(bucket: 'config', key: key, body: data)
+          ConfigTopic.write({ kind: 'created', type: 'microservice', name: @name, plugin: @plugin }, scope: @scope)
+        end
       end
     end
 
@@ -198,6 +200,7 @@ module Cosmos
       rubys3_client.list_objects(bucket: 'config', prefix: prefix).contents.each do |object|
         rubys3_client.delete_object(bucket: 'config', key: object.key)
       end
+      ConfigTopic.write({ kind: 'deleted', type: 'microservice', name: @name, plugin: @plugin }, scope: @scope)
     end
   end
 end

data/lib/cosmos/models/model.rb

@@ -128,6 +128,7 @@ module Cosmos
       @updated_at = kw_args[:updated_at]
       @plugin = kw_args[:plugin]
       @scope = kw_args[:scope]
+      @destroyed = false
     end
 
     # Update the Redis hash at primary_key and set the field "name"
@@ -163,10 +164,16 @@ module Cosmos
 
     # Delete the model from the Store
     def destroy
+      @destroyed = true
       undeploy()
       self.class.store.hdel(@primary_key, @name)
     end
 
+    # Indicate if destroy has been called
+    def destroyed?
+      @destroyed
+    end
+
     # @return [Hash] JSON encoding of this model
     def as_json
       { 'name' => @name,

data/lib/cosmos/models/note_model.rb

@@ -86,6 +86,7 @@ module Cosmos
     def create(update: false)
       validate(update: update)
       @updated_at = Time.now.to_nsec_from_epoch
+      NoteModel.destroy(scope: @scope, start: update) if update
       Store.zadd(@primary_key, @start, JSON.generate(as_json()))
       if update
         notify(kind: 'updated')
@@ -96,11 +97,12 @@ module Cosmos
 
     # Update the Redis hash at primary_key
     def update(start:, stop:, color:, description:)
+      orig_start = @start
       @start = start
       @stop = stop
       @color = color
       @description = description
-      create(update: true)
+      create(update: orig_start)
     end
 
     # @return [Hash] generated from the NoteModel
@@ -115,10 +117,6 @@ module Cosmos
         'updated_at' => @updated_at,
       }
     end
-
-    # @return [String] string view of NoteModel
-    def to_s
-      return "<NoteModel s: #{@start}, x: #{@stop}, c: #{@color}, d: #{@description}>"
-    end
+    alias to_s as_json
   end
 end