cosmos 5.0.2.pre.beta2 → 5.0.4

data/lib/cosmos/topics/command_topic.rb

@@ -31,17 +31,17 @@ module Cosmos
                    received_count: packet.received_count,
                    stored: packet.stored,
                    buffer: packet.buffer(false) }
-      Store.write_topic(topic, msg_hash)
+      Topic.write_topic(topic, msg_hash)
     end
 
     # @param command [Hash] Command hash structure read to be written to a topic
     def self.send_command(command, scope:)
       ack_topic = "{#{scope}__ACKCMD}TARGET__#{command['target_name']}"
-      Store.update_topic_offsets([ack_topic])
+      Topic.update_topic_offsets([ack_topic])
       # Save the existing cmd_params Hash and JSON generate before writing to the topic
       cmd_params = command['cmd_params']
       command['cmd_params'] = JSON.generate(command['cmd_params'].as_json)
-      cmd_id = Store.write_topic("{#{scope}__CMD}TARGET__#{command['target_name']}", command, '*', 100)
+      cmd_id = Topic.write_topic("{#{scope}__CMD}TARGET__#{command['target_name']}", command, '*', 100)
       # TODO: This timeout is fine for most but can we get the write_timeout from the interface here?
       time = Time.now
       while (Time.now - time) < COMMAND_ACK_TIMEOUT_S
@@ -66,7 +66,7 @@ module Cosmos
     ###########################################################################
 
     def self.raise_hazardous_error(msg_hash, target_name, cmd_name, cmd_params)
-      _, description, _ = msg_hash["result"].split("\n")
+      _, description, formatted = msg_hash["result"].split("\n")
       # Create and populate a new HazardousError and raise it up
       # The _cmd method in script/commands.rb rescues this and calls prompt_for_hazardous
       error = HazardousError.new
@@ -74,6 +74,8 @@ module Cosmos
       error.cmd_name = cmd_name
       error.cmd_params = cmd_params
       error.hazardous_description = description
+      error.formatted = formatted
+
       # No Logger.info because the error is already logged by the Logger.info "Ack Received ...
       raise error
     end

data/lib/cosmos/topics/interface_topic.rb

@@ -34,38 +34,38 @@ module Cosmos
 
     def self.receive_commands(interface, scope:)
       while true
-        Store.read_topics(InterfaceTopic.topics(interface, scope: scope)) do |topic, msg_id, msg_hash, redis|
+        Topic.read_topics(InterfaceTopic.topics(interface, scope: scope)) do |topic, msg_id, msg_hash, redis|
           result = yield topic, msg_hash
           ack_topic = topic.split("__")
           ack_topic[1] = 'ACK' + ack_topic[1]
           ack_topic = ack_topic.join("__")
-          Store.write_topic(ack_topic, { 'result' => result, 'id' => msg_id }, '*', 100)
+          Topic.write_topic(ack_topic, { 'result' => result, 'id' => msg_id }, '*', 100)
         end
       end
     end
 
     def self.write_raw(interface_name, data, scope:)
-      Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'raw' => data }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'raw' => data }, '*', 100)
     end
 
     def self.connect_interface(interface_name, scope:)
-      Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'connect' => 'true' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'connect' => 'true' }, '*', 100)
     end
 
     def self.disconnect_interface(interface_name, scope:)
-      Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'disconnect' => 'true' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'disconnect' => 'true' }, '*', 100)
     end
 
     def self.start_raw_logging(interface_name, scope:)
-      Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'log_raw' => 'true' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'log_raw' => 'true' }, '*', 100)
     end
 
     def self.stop_raw_logging(interface_name, scope:)
-      Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'log_raw' => 'false' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'log_raw' => 'false' }, '*', 100)
     end
 
     def self.shutdown(interface, scope:)
-      Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface.name}", { 'shutdown' => 'true' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface.name}", { 'shutdown' => 'true' }, '*', 100)
       sleep 1 # Give some time for the interface to shutdown
       InterfaceTopic.clear_topics(InterfaceTopic.topics(interface, scope: scope))
     end

data/lib/cosmos/topics/limits_event_topic.rb

@@ -52,13 +52,13 @@ module Cosmos
         raise "Invalid limits event type '#{event[:type]}'"
       end
 
-      Store.write_topic("#{scope}__cosmos_limits_events", event, '*', 1000)
+      Topic.write_topic("#{scope}__cosmos_limits_events", event, '*', 1000)
     end
 
     def self.read(offset = nil, count: 100, scope:)
       topic = "#{scope}__cosmos_limits_events"
       if offset
-        result = Store.xread(topic, offset, count: count)
+        result = Topic.read_topics([topic], [offset], nil, count)
         if result.empty?
           [] # We want to return an empty array rather than an empty hash
         else
@@ -67,7 +67,9 @@ module Cosmos
           result[topic]
         end
       else
-        Store.xrevrange(topic, count: 1)
+        result = Topic.get_newest_message(topic)
+        return [result] if result
+        return []
       end
     end
 

data/lib/cosmos/topics/notifications_topic.rb

@@ -22,7 +22,7 @@ require 'cosmos/topics/topic'
 module Cosmos
   class NotificationsTopic < Topic
     def self.write_notification(notification, scope:)
-      Store.write_topic("#{scope}__cosmos_notifications", notification)
+      Topic.write_topic("#{scope}__cosmos_notifications", notification)
     end
   end
 end

data/lib/cosmos/topics/router_topic.rb

@@ -36,13 +36,13 @@ module Cosmos
 
     def self.receive_telemetry(router, scope:)
       while true
-        Store.read_topics(RouterTopic.topics(router, scope: scope)) do |topic, msg_id, msg_hash, redis|
+        Topic.read_topics(RouterTopic.topics(router, scope: scope)) do |topic, msg_id, msg_hash, redis|
           result = yield topic, msg_hash
           if /CMD}ROUTER/.match?(topic)
             ack_topic = topic.split("__")
             ack_topic[1] = 'ACK' + ack_topic[1]
             ack_topic = ack_topic.join("__")
-            Store.write_topic(ack_topic, { 'result' => result }, msg_id, 100)
+            Topic.write_topic(ack_topic, { 'result' => result }, msg_id, 100)
           end
         end
       end
@@ -51,33 +51,33 @@ module Cosmos
     def self.route_command(packet, target_names, scope:)
       if packet.identified?
         topic = "{#{scope}__CMD}TARGET__#{packet.target_name}"
-        Store.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => packet.packet_name, 'cmd_buffer' => packet.buffer(false) }, '*', 100)
+        Topic.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => packet.packet_name, 'cmd_buffer' => packet.buffer(false) }, '*', 100)
       elsif target_names.length == 1
         topic = "{#{scope}__CMD}TARGET__#{target_names[0]}"
-        Store.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => 'UNKNOWN', 'cmd_buffer' => packet.buffer(false) }, '*', 100)
+        Topic.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => 'UNKNOWN', 'cmd_buffer' => packet.buffer(false) }, '*', 100)
       else
         raise "No route for command: #{packet.target_name} #{packet.packet_name}"
       end
     end
 
     def self.connect_router(router_name, scope:)
-      Store.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => true }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => true }, '*', 100)
     end
 
     def self.disconnect_router(router_name, scope:)
-      Store.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'disconnect' => true }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'disconnect' => true }, '*', 100)
     end
 
     def self.start_raw_logging(router_name, scope:)
-      Store.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_raw' => 'true' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_raw' => 'true' }, '*', 100)
     end
 
     def self.stop_raw_logging(router_name, scope:)
-      Store.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_raw' => 'false' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_raw' => 'false' }, '*', 100)
     end
 
     def self.shutdown(router, scope:)
-      Store.write_topic("{#{scope}__CMD}ROUTER__#{router.name}", { 'shutdown' => 'true' }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router.name}", { 'shutdown' => 'true' }, '*', 100)
       sleep 1 # Give some time for the interface to shutdown
       RouterTopic.clear_topics(RouterTopic.topics(router, scope: scope))
     end

data/lib/cosmos/topics/telemetry_decom_topic.rb

@@ -21,6 +21,10 @@ require 'cosmos/topics/topic'
 
 module Cosmos
   class TelemetryDecomTopic < Topic
+    def self.topics(scope:)
+      super(scope, 'DECOM')
+    end
+
     def self.write_packet(packet, id: nil, scope:)
       # Need to build a JSON hash of the decommutated data
       # Support "downward typing"
@@ -39,7 +43,7 @@ module Cosmos
         :received_count => packet.received_count,
         :json_data => JSON.generate(json_hash.as_json),
       }
-      Store.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id)
+      Topic.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id)
       # Also update the current value table with the latest decommutated data
       CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, scope: scope)
     end

data/lib/cosmos/topics/telemetry_topic.rb

@@ -30,7 +30,7 @@ module Cosmos
         :received_count => packet.received_count,
         :buffer => packet.buffer(false),
       }
-      Store.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
+      Topic.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
     end
   end
 end

data/lib/cosmos/topics/timeline_topic.rb

@@ -39,7 +39,7 @@ module Cosmos
     #  }
     # ```
     def self.write_activity(activity, scope:)
-      Store.write_topic("#{scope}#{PRIMARY_KEY}", activity, '*', 1000)
+      Topic.write_topic("#{scope}#{PRIMARY_KEY}", activity, '*', 1000)
     end
   end
 end

data/lib/cosmos/topics/topic.rb

@@ -21,18 +21,33 @@ require 'cosmos/utilities/store'
 
 module Cosmos
   class Topic
-    def self.initialize_streams(topics)
-      Store.initialize_streams(topics)
+    if RUBY_VERSION < "3"
+      # Delegate all unknown class methods to delegate to the EphemeralStore
+      def self.method_missing(message, *args, &block)
+        EphemeralStore.public_send(message, *args, &block)
+      end
+    else
+      # Delegate all unknown class methods to delegate to the EphemeralStore
+      def self.method_missing(message, *args, **kwargs, &block)
+        EphemeralStore.public_send(message, *args, **kwargs, &block)
+      end
     end
 
-    def self.read_topics(topics, offsets = nil, timeout_ms = 1000, &block)
-      Store.read_topics(topics, offsets, timeout_ms, &block)
+    def self.clear_topics(topics, maxlen = 0)
+      topics.each { |topic| EphemeralStore.xtrim(topic, maxlen) }
     end
 
-    def self.clear_topics(topics, maxlen = 0)
-      topics.each do |topic|
-        Store.xtrim(topic, maxlen)
-      end
+    def self.topics(scope, key)
+      EphemeralStore
+        .scan_each(match: "#{scope}__#{key}__*", type: 'stream', count: 100)
+        .to_a # Change the enumerator into an array
+        .uniq # Scan can return duplicates so ensure unique
+        .sort # Sort not entirely necessary but nice
+    end
+
+    def self.get_cnt(topic)
+      _, packet = EphemeralStore.get_newest_message(topic)
+      packet ? packet["received_count"].to_i : 0
     end
   end
 end

data/lib/cosmos/utilities/logger.rb

@@ -19,10 +19,11 @@
 
 require 'cosmos/core_ext/class'
 require 'cosmos/core_ext/time'
-require 'cosmos/utilities/store'
+require 'cosmos/topics/topic'
 require 'socket'
 require 'logger'
 require 'time'
+require 'json'
 
 module Cosmos
   # Supports different levels of logging and only writes if the level
@@ -173,11 +174,11 @@ module Cosmos
         puts data.to_json if @stdout
         unless @no_store
           if scope
-            Store.write_topic("#{scope}__cosmos_log_messages", data)
+            Topic.write_topic("#{scope}__cosmos_log_messages", data)
           else
             # The base cosmos_log_messages doesn't have an associated logger
             # so it must be limited to prevent unbounded stream growth
-            Store.write_topic("cosmos_log_messages", data, '*', 1000)
+            Topic.write_topic("cosmos_log_messages", data, '*', 1000)
           end
         end
       end

data/lib/cosmos/utilities/metric.rb

@@ -18,6 +18,7 @@
 # copyright holder
 
 require 'cosmos/models/metric_model'
+require 'thread'
 
 module Cosmos
   class Metric
@@ -48,6 +49,7 @@ module Cosmos
       @scope = scope
       @microservice = microservice
       @size = 5000
+      @mutex = Mutex.new
     end
 
     def add_sample(name:, value:, labels:)
@@ -69,15 +71,17 @@ module Cosmos
       # the value is added to @items and the count of the value is increased
       # if the count of the values exceed the size of the array it sets the
       # count back to zero and the array will over write older data.
-      key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
-      if not @items.has_key?(key)
-        Logger.debug("new data for #{@scope}, #{key}")
-        @items[key] = { 'values' => Array.new(@size), 'count' => 0 }
+      @mutex.synchronize do
+        key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
+        if not @items.has_key?(key)
+          Logger.debug("new data for #{@scope}, #{key}")
+          @items[key] = { 'values' => Array.new(@size), 'count' => 0 }
+        end
+        count = @items[key]['count']
+        # Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
+        @items[key]['values'][count] = value
+        @items[key]['count'] = count + 1 >= @size ? 0 : count + 1
       end
-      count = @items[key]['count']
-      # Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
-      @items[key]['values'][count] = value
-      @items[key]['count'] = count + 1 >= @size ? 0 : count + 1
     end
 
     def percentile(sorted_values, percentile)
@@ -106,24 +110,26 @@ module Cosmos
       # array. to store the array as the value with the metric name again joined
       # with the @microservice and @scope.
       Logger.debug("#{@microservice} #{@scope} sending metrics to redis, #{@items.length}") if @items.length > 0
-      @items.each do |key, values|
-        label_list = []
-        name, labels = key.split('|')
-        metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
-        sorted_values = values['values'].compact.sort
-        for percentile_value in [10, 50, 90, 95, 99]
-          percentile_result = percentile(sorted_values, percentile_value)
-          labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
-          labels['percentile'] = percentile_value
-          labels['metric__value'] = percentile_result
-          label_list.append(labels)
-        end
-        begin
-          Logger.debug("sending metrics summary to redis key: #{@microservice}")
-          metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
-          metric.create(force: true)
-        rescue RuntimeError
-          Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
+      @mutex.synchronize do
+        @items.each do |key, values|
+          label_list = []
+          name, labels = key.split('|')
+          metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
+          sorted_values = values['values'].compact.sort
+          for percentile_value in [10, 50, 90, 95, 99]
+            percentile_result = percentile(sorted_values, percentile_value)
+            labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
+            labels['percentile'] = percentile_value
+            labels['metric__value'] = percentile_result
+            label_list.append(labels)
+          end
+          begin
+            Logger.debug("sending metrics summary to redis key: #{@microservice}")
+            metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
+            metric.create(force: true)
+          rescue RuntimeError
+            Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
+          end
         end
       end
     end

data/lib/cosmos/utilities/s3.rb

@@ -22,6 +22,67 @@ require 'cosmos/models/reducer_model'
 
 module Cosmos
   class S3Utilities
+    def self.list_files_before_time(bucket, prefix, time)
+      rubys3_client = Aws::S3::Client.new
+      oldest_list = []
+      total_size = 0
+
+      # Return nothing if bucket doesn't exist (it won't at the very beginning)
+      begin
+        rubys3_client.head_bucket(bucket: bucket)
+      rescue Aws::S3::Errors::NotFound
+        return total_size, oldest_list
+      end
+
+      # Get List of Packet Names - Assumes prefix gets us to a folder of packet names
+      token = nil
+      folder_list = []
+      while true
+        resp = rubys3_client.list_objects_v2({
+          bucket: bucket,
+          max_keys: 1000,
+          prefix: prefix,
+          delimiter: '/',
+          continuation_token: token
+        })
+
+        resp.common_prefixes.each do |item|
+          folder_list << item.prefix
+        end
+        break unless resp.is_truncated
+        token = resp.next_continuation_token
+      end
+
+      # Go through each folder and keep files that end before time
+      folder_list.each do |folder|
+        token = nil
+        next_folder = false
+        while true
+          resp = rubys3_client.list_objects_v2({
+                                                bucket: bucket,
+                                                max_keys: 1000,
+                                                prefix: folder,
+                                                continuation_token: token
+                                              })
+          resp.contents.each do |item|
+            t = item.key.split('__')[1]
+            file_end_time = Time.utc(t[0..3], t[4..5], t[6..7], t[8..9], t[10..11], t[12..13])
+            if file_end_time < time
+              oldest_list << item
+              total_size += item.size
+            else
+              next_folder = true
+              break
+            end
+          end
+          break if !resp.is_truncated or next_folder
+
+          token = resp.next_continuation_token
+        end
+      end
+      return total_size, oldest_list
+    end
+
     def self.get_total_size_and_oldest_list(bucket, prefix, max_list_length = 10000)
       rubys3_client = Aws::S3::Client.new
       oldest_list = []

data/lib/cosmos/utilities/s3_file_cache.rb

@@ -18,6 +18,7 @@
 # copyright holder
 
 require 'fileutils'
+require 'tmpdir'
 require 'cosmos'
 require 'cosmos/utilities/s3'
 
@@ -48,7 +49,7 @@ class S3File
 
   def retrieve
     local_path = "#{S3FileCache.instance.cache_dir}/#{File.basename(@s3_path)}"
-    Cosmos::Logger.info "Retrieving #{@s3_path} from logs bucket"
+    Cosmos::Logger.debug "Retrieving #{@s3_path} from logs bucket"
     @rubys3_client.get_object(bucket: "logs", key: @s3_path, response_target: local_path)
     if File.exist?(local_path)
       @size = File.size(local_path)
@@ -57,6 +58,7 @@ class S3File
   rescue => err
     @error = err
     Cosmos::Logger.error "Failed to retrieve #{@s3_path}\n#{err.formatted}"
+    raise err
   end
 
   def reserve
@@ -165,11 +167,11 @@ class S3FileCache
     end
 
     # Create local file cache location
-    @cache_dir = File.join(Dir.tmpdir, 'cosmos', 'file_cache', name)
+    @cache_dir = Dir.mktmpdir
     FileUtils.mkdir_p(@cache_dir)
-
-    # Clear out local file cache
-    FileUtils.rm_f Dir.glob("#{@cache_dir}/*")
+    at_exit do
+      FileUtils.remove_dir(@cache_dir, true)
+    end
 
     @cached_files = S3FileCollection.new
 
@@ -178,7 +180,11 @@ class S3FileCache
         file = @cached_files.get_next_to_retrieve
         # Cosmos::Logger.debug "Next file: #{file}"
         if file and (file.size + @cached_files.current_disk_usage()) <= @max_disk_usage
-          file.retrieve
+          begin
+            file.retrieve
+          rescue
+            # Will be automatically retried
+          end
         else
           sleep(1)
         end

data/lib/cosmos/utilities/store.rb

@@ -19,4 +19,5 @@
 
 module Cosmos
   autoload(:Store, "cosmos/utilities/store_autoload.rb")
+  autoload(:EphemeralStore, "cosmos/utilities/store_autoload.rb")
 end