cosmos 5.0.2.pre.beta2 → 5.0.4

data/lib/cosmos/microservices/reducer_microservice.rb

@@ -96,9 +96,8 @@ module Cosmos
         ReducerModel
           .all_files(type: :DECOM, target: @target_name, scope: @scope)
           .each do |file|
-            if process_file(file, 'minute', MINUTE_ENTRY_SECS, MINUTE_FILE_SECS)
-              ReducerModel.rm_file(file)
-            end
+            process_file(file, 'minute', MINUTE_ENTRY_SECS, MINUTE_FILE_SECS)
+            ReducerModel.rm_file(file)
           end
       end
     end
@@ -108,9 +107,8 @@ module Cosmos
         ReducerModel
           .all_files(type: :MINUTE, target: @target_name, scope: @scope)
           .each do |file|
-            if process_file(file, 'hour', HOUR_ENTRY_SECS, HOUR_FILE_SECS)
-              ReducerModel.rm_file(file)
-            end
+            process_file(file, 'hour', HOUR_ENTRY_SECS, HOUR_FILE_SECS)
+            ReducerModel.rm_file(file)
           end
       end
     end
@@ -120,9 +118,8 @@ module Cosmos
         ReducerModel
           .all_files(type: :HOUR, target: @target_name, scope: @scope)
           .each do |file|
-            if process_file(file, 'day', DAY_ENTRY_SECS, DAY_FILE_SECS)
-              ReducerModel.rm_file(file)
-            end
+            process_file(file, 'day', DAY_ENTRY_SECS, DAY_FILE_SECS)
+            ReducerModel.rm_file(file)
           end
       end
     end
@@ -250,7 +247,12 @@ module Cosmos
       )
       true
     rescue => e
-      Logger.error("Reducer Error #{e}\n#{e.backtrace}")
+      if file.local_path and File.exist?(file.local_path)
+        Logger.error("Reducer Error: #{filename}:#{File.size(file.local_path)} bytes: \n#{e.formatted}")
+      else
+        Logger.error("Reducer Error: #{filename}:(Not Retrieved): \n#{e.formatted}")
+      end
+      false
     end
 
     def reduce(type, data_keys, reduced)

data/lib/cosmos/models/cvt_model.rb

@@ -40,12 +40,12 @@ module Cosmos
 
     # Delete the current value table for a target
     def self.del(target_name:, packet_name:, scope:)
-      Store.hdel("#{scope}__tlm__#{target_name}", packet_name)
+      EphemeralStore.hdel("#{scope}__tlm__#{target_name}", packet_name)
     end
 
     # Set the current value table for a target, packet
     def self.set(hash, target_name:, packet_name:, scope:)
-      Store.hset("#{scope}__tlm__#{target_name}", packet_name, JSON.generate(hash.as_json))
+      EphemeralStore.hset("#{scope}__tlm__#{target_name}", packet_name, JSON.generate(hash.as_json))
     end
 
     # Set an item in the current value table
@@ -62,9 +62,9 @@ module Cosmos
       else
         raise "Unknown type '#{type}' for #{target_name} #{packet_name} #{item_name}"
       end
-      hash = JSON.parse(Store.hget("#{scope}__tlm__#{target_name}", packet_name))
+      hash = JSON.parse(EphemeralStore.hget("#{scope}__tlm__#{target_name}", packet_name))
       hash[field] = value
-      Store.hset("#{scope}__tlm__#{target_name}", packet_name, JSON.generate(hash.as_json))
+      EphemeralStore.hset("#{scope}__tlm__#{target_name}", packet_name, JSON.generate(hash.as_json))
     end
 
     # Get an item from the current value table
@@ -86,7 +86,7 @@ module Cosmos
       else
         raise "Unknown type '#{type}' for #{target_name} #{packet_name} #{item_name}"
       end
-      hash = JSON.parse(Store.hget("#{scope}__tlm__#{target_name}", packet_name))
+      hash = JSON.parse(EphemeralStore.hget("#{scope}__tlm__#{target_name}", packet_name))
       hash.values_at(*types).each do |result|
         return result if result
       end
@@ -106,7 +106,7 @@ module Cosmos
 
       lookups.each do |target_packet_key, target_name, packet_name, packet_values|
         unless packet_lookup[target_packet_key]
-          packet = Store.hget("#{scope}__tlm__#{target_name}", packet_name)
+          packet = EphemeralStore.hget("#{scope}__tlm__#{target_name}", packet_name)
           raise "Packet '#{target_name} #{packet_name}' does not exist" unless packet
           packet_lookup[target_packet_key] = JSON.parse(packet)
         end

data/lib/cosmos/models/gem_model.rb

@@ -73,7 +73,7 @@ module Cosmos
       return nil
     end
 
-    def self.install(name_or_path)
+    def self.install(name_or_path, scope:)
       temp_dir = Dir.mktmpdir
       begin
         if File.exist?(name_or_path)
@@ -81,10 +81,10 @@ module Cosmos
         else
           gem_file_path = get(temp_dir, name_or_path)
         end
-        rubygems_url = get_setting('rubygems_url')
+        rubygems_url = get_setting('rubygems_url', scope: scope)
         Gem.sources = [rubygems_url] if rubygems_url
         Gem.done_installing_hooks.clear
-        Gem.install(gem_file_path, {:build_args => '--no-document'})
+        Gem.install(gem_file_path, "> 0.pre", :build_args => ['--no-document'], :prerelease => true)
       rescue => err
         message = "Gem file #{gem_file_path} error installing to /gems\n#{err.formatted}"
         Logger.error message

data/lib/cosmos/models/info_model.rb

@@ -24,7 +24,7 @@ module Cosmos
 
     # @return Hash information and statistics about the redis server
     def self.get()
-      return Store.info()
+      return [Store.info(), EphemeralStore.info]
     end
 
   end

data/lib/cosmos/models/interface_status_model.rb

@@ -23,7 +23,7 @@ module Cosmos
   # Stores the status about an interface. This class also implements logic
   # to handle status for a router since the functionality is identical
   # (only difference is the Redis key used).
-  class InterfaceStatusModel < Model
+  class InterfaceStatusModel < EphemeralModel
     INTERFACES_PRIMARY_KEY = 'cosmos_interface_status'
     ROUTERS_PRIMARY_KEY = 'cosmos_router_status'
 

data/lib/cosmos/models/metadata_model.rb

@@ -19,271 +19,97 @@
 
 # https://www.rubydoc.info/gems/redis/Redis/Commands/SortedSets
 
-require 'cosmos/topics/calendar_topic'
+require 'cosmos/models/sorted_model'
 
 module Cosmos
-
-  class MetadataError < StandardError; end
-
-  class MetadataInputError < MetadataError; end
-
-  class MetadataOverlapError < MetadataError; end
-
-  class MetadataModel < Model
-
-    CHRONICLE_TYPE = 'metadata'.freeze
-    CURRENT_VALUE = '__current.metadata.value'.freeze
+  class MetadataModel < SortedModel
+    METADATA_TYPE = 'metadata'.freeze
     PRIMARY_KEY = '__METADATA'.freeze
 
     def self.pk(scope)
-      return "#{scope}#{PRIMARY_KEY}"
-    end
-
-    # @return [String|nil] String of the saved json or nil if score not found under current value
-    def self.get_current_value(target:, scope:)
-      json = Store.hget("#{scope}#{CURRENT_VALUE}", target)
-      return nil unless json
-      return self.from_json(JSON.parse(json), scope: scope)
-    end
-
-    # @return [Array|nil] Array up to 100 of this model or empty array
-    def self.get(start:, stop:, scope:, limit: 100)
-      if start > stop
-        raise MetadataInputError.new "start: #{start} must be before stop: #{stop}"
-      end
-      pk = self.pk(scope)
-      array = Store.zrangebyscore(pk, start, stop, :limit => [0, limit])
-      ret_array = Array.new
-      array.each do |value|
-        ret_array << JSON.parse(value)
-      end
-      return ret_array
-    end
-
-    # @return [Array<Hash>] Array up to the limit of the models (as Hash objects) stored under the primary key
-    def self.all(scope:, limit: 100)
-      pk = self.pk(scope)
-      array = Store.zrange(pk, 0, -1, :limit => [0, limit])
-      ret_array = Array.new
-      array.each do |value|
-        ret_array << JSON.parse(value)
-      end
-      return ret_array
-    end
-
-    # @return [Integer] count of the members stored under the primary key
-    def self.count(scope:)
-      return Store.zcard(self.pk(scope))
+      "#{scope}#{PRIMARY_KEY}"
     end
 
-    # @return [String|nil] String of the saved json or nil if score not found under primary_key
-    def self.score(score:, scope:)
-      pk = self.pk(scope)
-      array = Store.zrangebyscore(pk, score, score, :limit => [0, 1])
-      array.each do |value|
-        return JSON.parse(value)
-      end
-      return nil
-    end
-
-    # Remove member from a sorted set based on the score.
-    # @return [Integer] count of the members removed
-    def self.destroy(scope:, score:)
-      pk = self.pk(scope)
-      Store.zremrangebyscore(pk, score, score)
-    end
-
-    # Remove members from min to max of the sorted set.
-    # @return [Integer] count of the members removed
-    def self.range_destroy(scope:, min:, max:)
-      pk = self.pk(scope)
-      Store.zremrangebyscore(pk, min, max)
-    end
-
-    # @return [MetadataModel] Model generated from the passed JSON
-    def self.from_json(json, scope:)
-      json = JSON.parse(json) if String === json
-      raise "json data is nil" if json.nil?
-
-      json.transform_keys!(&:to_sym)
-      self.new(**json, scope: scope)
-    end
-
-    attr_reader :target, :start, :color, :metadata, :type
+    attr_reader :color, :metadata, :type
 
-    # @param [String] target - should be the target but can be anything
     # @param [Integer] start - time metadata is active in seconds from Epoch
     # @param [String] color - The event color
     # @param [String] metadata - Key value pair object to link to name
     # @param [String] scope - Cosmos scope to track event to
     def initialize(
-      target:,
+      scope:,
       start:,
       color: nil,
       metadata:,
-      scope:,
-      type: CHRONICLE_TYPE,
+      type: METADATA_TYPE,
       updated_at: 0
     )
-      super(MetadataModel.pk(scope), name: start.to_s, scope: scope)
-      set_input(start: start, color: color, metadata: metadata)
-      @target = target
-      @type = type
-      @updated_at = updated_at
-    end
-
-    # validate color
-    def validate_color(color)
-      if color.nil?
-        color = '#%06x' % (rand * 0xffffff)
-      end
-      valid_color = color =~ /(#*)([0-9,a-f,A-f]{6})/
-      if valid_color.nil?
-        raise MetadataInputError.new "invalid color, must be in hex format, e.g. #FF0000"
-      end
-
-      color = "##{color}" unless color.start_with?('#')
-      return color
-    end
-
-    # validate the input to the rules we have created for timelines.
-    # - An entry's start MUST be valid.
-    # - An entry's start MUST NOT be in the future.
-    # - An entry's metadata MUST a hash/object.
-    def validate_input(start:, color:, metadata:)
-      if start.is_a?(Integer) == false
-        raise MetadataInputError.new "failed validation input must be integer: #{start}"
-      end
-      now = Time.now.strftime('%s%3N').to_i
-      if start > now
-        raise MetadataInputError.new "start can not be in the future: #{start} > #{now}"
-      end
-      validate_color(color)
-      if metadata.is_a?(Hash) == false
-        raise MetadataInputError.new "Metadata must be a hash/object: #{metadata}"
-      end
-    end
-
-    # Set the values of the instance, @start, @stop, @metadata...
-    def set_input(start:, color:, metadata:)
-      if start.is_a?(Integer) == false
-        raise MetadataInputError.new "start input must be integer: #{start}"
-      end
+      super(start: start, scope: scope, updated_at: updated_at)
       @start = start
       @color = color
       @metadata = metadata
+      @type = type # For the as_json, from_json round trip
     end
 
-    # validate_time will be called on create and update this will validate
-    # that no other chronicle event or metadata had been saved for that time.
-    # One event or metadata per second to ensure data can be updated.
-    #
-    # @param [Integer] ignore_score - should be nil unless you want to ignore
-    #   a time when doing an update
-    def validate_time(ignore_score: nil)
-      array = Store.zrangebyscore(@primary_key, @start, @start, :limit => [0, 1])
-      array.each do |value|
-        entry = JSON.parse(value)
-        if ignore_score == entry['start']
-          next
-        else
-          return entry
-        end
-      end
-      return nil
+    # Validates the instance variables: @start, @color, @metadata
+    def validate(update: false)
+      validate_start(update: update)
+      validate_color()
+      validate_metadata()
     end
 
-    # Update the Redis hash at primary_key and set the score equal to the start Epoch time
-    # the member is set to the JSON generated via calling as_json
-    def create
-      validate_input(start: @start, color: @color, metadata: @metadata)
-      collision = validate_time()
-      unless collision.nil?
-        raise MetadataOverlapError.new "no chronicle can overlap, collision: #{collision}"
+    def validate_color()
+      if @color.nil?
+        @color = '#%06x' % (rand * 0xffffff)
       end
-
-      @updated_at = Time.now.to_nsec_from_epoch
-      Store.zadd(@primary_key, @start, JSON.generate(as_json()))
-      update_current_value()
-      notify(kind: 'created')
-    end
-
-    # Update the Redis hash at primary_key and remove the current activity at the current score
-    # and update the score to the new score equal to the start Epoch time this uses a multi
-    # to execute both the remove and create. The member via the JSON generated via calling as_json
-    def update(start:, color:, metadata:)
-      validate_input(start: start, color: color, metadata: metadata)
-      old_start = @start
-      @updated_at = Time.now.to_nsec_from_epoch
-      set_input(start: start, color: color, metadata: metadata)
-      # copy of create
-      collision = validate_time(ignore_score: old_start)
-      unless collision.nil?
-        raise MetadataOverlapError.new "failed to update #{old_start}, no chronicles can overlap, collision: #{collision}"
+      unless @color =~ /(#*)([0-9,a-f,A-f]{6})/
+        raise SortedInputError.new "invalid color, must be in hex format, e.g. #FF0000"
       end
+      @color = "##{@color}" unless @color.start_with?('#')
+    end
 
-      Store.multi do |multi|
-        multi.zremrangebyscore(@primary_key, old_start, old_start)
-        multi.zadd(@primary_key, @start, JSON.generate(as_json()))
+    def validate_metadata()
+      unless @metadata.is_a?(Hash)
+        raise SortedInputError.new "Metadata must be a hash/object: #{@metadata}"
       end
-      update_current_value(old_start: old_start)
-      notify(kind: 'updated', extra: old_start)
-      return @start
     end
 
-    # Update the Redis hash at primary_key and check if this metadata instance
-    # is newer than the current instance stored in the hash. If the hash does
-    # NOT contain an instance or this metadata instance is newer it will update
-    # the current hash.
-    def update_current_value(old_start: nil)
-      update = true
-      json = Store.hget("#{@scope}#{CURRENT_VALUE}", @target)
-      unless json.nil?
-        model = MetadataModel.from_json(JSON.parse(json), scope: @scope)
-        update = model.start <= @start || model.start == old_start
-      end
+    # Update the Redis hash at primary_key based on the initial passed start
+    # The member is set to the JSON generated via calling as_json
+    def create(update: false)
+      validate(update: update)
+      @updated_at = Time.now.to_nsec_from_epoch
+      Store.zadd(@primary_key, @start, JSON.generate(as_json()))
       if update
-        return Store.hset("#{@scope}#{CURRENT_VALUE}", @target, JSON.generate(as_json()))
+        notify(kind: 'updated')
+      else
+        notify(kind: 'created')
       end
     end
 
-    # destroy the activity from the redis database
-    def destroy
-      Store.zremrangebyscore(@primary_key, @start, @start)
-      notify(kind: 'deleted')
-    end
-
-    # @return [] update the redis stream / timeline topic that something has changed
-    def notify(kind:, extra: nil)
-      notification = {
-        'data' => JSON.generate(as_json()),
-        'kind' => kind,
-        'type' => 'calendar',
-      }
-      notification['extra'] = extra unless extra.nil?
-      begin
-        CalendarTopic.write_entry(notification, scope: @scope)
-      rescue StandardError => e
-        raise MetadataError.new "Failed to write to stream: #{notification}, #{e}"
-      end
+    # Update the Redis hash at primary_key
+    def update(start:, color:, metadata:)
+      @start = start
+      @color = color
+      @metadata = metadata
+      create(update: true)
     end
 
     # @return [Hash] generated from the MetadataModel
     def as_json
       return {
-        'target' => @target,
         'scope' => @scope,
-        'updated_at' => @updated_at,
         'start' => @start,
         'color' => @color,
         'metadata' => @metadata,
-        'type' => CHRONICLE_TYPE,
+        'type' => METADATA_TYPE,
+        'updated_at' => @updated_at,
       }
     end
 
     # @return [String] string view of metadata
     def to_s
-      return "<MetadataModel t: #{@target}, s: #{@start}, c: #{@color}, m: #{@metadata}>"
+      return "<MetadataModel s: #{@start}, c: #{@color}, m: #{@metadata}>"
     end
   end
 end

data/lib/cosmos/models/metric_model.rb

@@ -20,7 +20,7 @@
 require 'cosmos/models/model'
 
 module Cosmos
-  class MetricModel < Model
+  class MetricModel < EphemeralModel
     PRIMARY_KEY = '__cosmos__metric'.freeze
 
     # NOTE: The following three class methods are used by the ModelController
@@ -38,7 +38,7 @@ module Cosmos
     end
 
     def self.destroy(scope:, name:)
-      Store.hdel("#{scope}#{PRIMARY_KEY}", name)
+      EphemeralStore.hdel("#{scope}#{PRIMARY_KEY}", name)
     end
 
     def initialize(name:, scope:, metric_name:, label_list:)

data/lib/cosmos/models/microservice_model.rb

@@ -186,7 +186,7 @@ module Cosmos
         # Load microservice files
         data = File.read(filename, mode: "rb")
         Cosmos.set_working_dir(File.dirname(filename)) do
-          data = ERB.new(data).result(binding.set_variables(variables)) if data.is_printable?
+          data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
         end
         rubys3_client.put_object(bucket: 'config', key: key, body: data)
       end

data/lib/cosmos/models/microservice_status_model.rb

@@ -20,7 +20,7 @@
 require 'cosmos/models/model'
 
 module Cosmos
-  class MicroserviceStatusModel < Model
+  class MicroserviceStatusModel < EphemeralModel
     PRIMARY_KEY = 'cosmos_microservice_status'
 
     attr_accessor :state

data/lib/cosmos/models/model.rb

@@ -27,12 +27,16 @@ module Cosmos
     attr_accessor :plugin
     attr_accessor :scope
 
+    def self.store
+      Store
+    end
+
     # NOTE: The following three methods must be reimplemented by Model subclasses
     # without primary_key to support other class methods.
 
     # @return [Hash|nil] Hash of this model or nil if name not found under primary_key
     def self.get(primary_key, name:)
-      json = Store.hget(primary_key, name)
+      json = store.hget(primary_key, name)
       if json
         return JSON.parse(json)
       else
@@ -42,12 +46,12 @@ module Cosmos
 
     # @return [Array<String>] All the names stored under the primary key
     def self.names(primary_key)
-      Store.hkeys(primary_key).sort
+      store.hkeys(primary_key).sort
     end
 
     # @return [Array<Hash>] All the models (as Hash objects) stored under the primary key
     def self.all(primary_key)
-      hash = Store.hgetall(primary_key)
+      hash = store.hgetall(primary_key)
       hash.each do |key, value|
         hash[key] = JSON.parse(value)
       end
@@ -117,16 +121,6 @@ module Cosmos
       raise "must be implemented by subclass"
     end
 
-    # TODO: Not used
-    # def self.from_config(primary_key, filename)
-    #   model = nil
-    #   parser = ConfigParser.new
-    #   parser.parse_file(filename) do |keyword, parameters|
-    #     model = self.handle_config(primary_key, parser, model, keyword, parameters)
-    #   end
-    #   model
-    # end
-
     # Store the primary key and keyword arguments
     def initialize(primary_key, **kw_args)
       @primary_key = primary_key
@@ -140,7 +134,7 @@ module Cosmos
     # to the JSON generated via calling as_json
     def create(update: false, force: false)
       unless force
-        existing = Store.hget(@primary_key, @name)
+        existing = self.class.store.hget(@primary_key, @name)
         if existing
           raise "#{@primary_key}:#{@name} already exists at create" unless update
         else
@@ -148,7 +142,7 @@ module Cosmos
         end
       end
       @updated_at = Time.now.to_nsec_from_epoch
-      Store.hset(@primary_key, @name, JSON.generate(self.as_json))
+      self.class.store.hset(@primary_key, @name, JSON.generate(self.as_json))
     end
 
     # Alias for create(update: true)
@@ -170,7 +164,7 @@ module Cosmos
     # Delete the model from the Store
     def destroy
       undeploy()
-      Store.hdel(@primary_key, @name)
+      self.class.store.hdel(@primary_key, @name)
     end
 
     # @return [Hash] JSON encoding of this model
@@ -186,4 +180,10 @@ module Cosmos
       ""
     end
   end
+
+  class EphemeralModel < Model
+    def self.store
+      EphemeralStore
+    end
+  end
 end