openc3 5.0.6

data/lib/openc3/utilities/s3.rb

@@ -0,0 +1,202 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+autoload(:Aws, 'openc3/utilities/s3_autoload.rb')
+require 'openc3/models/reducer_model'
+
+module OpenC3
+  class S3Utilities
+    def self.list_files_before_time(bucket, prefix, time)
+      rubys3_client = Aws::S3::Client.new
+      oldest_list = []
+      total_size = 0
+
+      # Return nothing if bucket doesn't exist (it won't at the very beginning)
+      begin
+        rubys3_client.head_bucket(bucket: bucket)
+      rescue Aws::S3::Errors::NotFound
+        return total_size, oldest_list
+      end
+
+      # Get List of Packet Names - Assumes prefix gets us to a folder of packet names
+      token = nil
+      folder_list = []
+      while true
+        resp = rubys3_client.list_objects_v2({
+          bucket: bucket,
+          max_keys: 1000,
+          prefix: prefix,
+          delimiter: '/',
+          continuation_token: token
+        })
+
+        resp.common_prefixes.each do |item|
+          folder_list << item.prefix
+        end
+        break unless resp.is_truncated
+        token = resp.next_continuation_token
+      end
+
+      # Go through each folder and keep files that end before time
+      folder_list.each do |folder|
+        token = nil
+        next_folder = false
+        while true
+          resp = rubys3_client.list_objects_v2({
+                                                bucket: bucket,
+                                                max_keys: 1000,
+                                                prefix: folder,
+                                                continuation_token: token
+                                              })
+          resp.contents.each do |item|
+            t = item.key.split('__')[1]
+            file_end_time = Time.utc(t[0..3], t[4..5], t[6..7], t[8..9], t[10..11], t[12..13])
+            if file_end_time < time
+              oldest_list << item
+              total_size += item.size
+            else
+              next_folder = true
+              break
+            end
+          end
+          break if !resp.is_truncated or next_folder
+
+          token = resp.next_continuation_token
+        end
+      end
+      return total_size, oldest_list
+    end
+
+    def self.get_total_size_and_oldest_list(bucket, prefix, max_list_length = 10000)
+      rubys3_client = Aws::S3::Client.new
+      oldest_list = []
+      total_size = 0
+
+      # Return nothing if bucket doesn't exist (it won't at the very beginning)
+      begin
+        rubys3_client.head_bucket(bucket: bucket)
+      rescue Aws::S3::Errors::NotFound
+        return total_size, oldest_list
+      end
+
+      # Get List of Files from S3
+      token = nil
+      while true
+        resp = rubys3_client.list_objects_v2({
+                                               bucket: bucket,
+                                               max_keys: 1000,
+                                               prefix: prefix,
+                                               continuation_token: token
+                                             })
+        resp.contents.each do |item|
+          total_size += item.size
+        end
+        oldest_list.concat(resp.contents)
+        oldest_list.sort! { |a, b| File.basename(a.key) <=> File.basename(b.key) }
+        oldest_list = oldest_list[0..(max_list_length - 1)]
+        break unless resp.is_truncated
+
+        token = resp.next_continuation_token
+      end
+      return total_size, oldest_list
+    end
+
+    def self.move_log_file_to_s3(filename, s3_key, metadata: {})
+      Thread.new do
+        rubys3_client = Aws::S3::Client.new
+
+        # Ensure logs bucket exists
+        begin
+          rubys3_client.head_bucket(bucket: 'logs')
+        rescue Aws::S3::Errors::NotFound
+          rubys3_client.create_bucket(bucket: 'logs')
+        end
+
+        # Write to S3 Bucket
+        File.open(filename, 'rb') do |read_file|
+          rubys3_client.put_object(bucket: 'logs', key: s3_key, body: read_file, metadata: metadata)
+        end
+        Logger.debug "logs/#{s3_key} written to S3"
+        ReducerModel.add_file(s3_key) # Record the new file for data reduction
+
+        File.delete(filename)
+      rescue => err
+        Logger.error("Error saving log file to bucket: #{filename}\n#{err.formatted}")
+      end
+    end
+
+    def self.ensure_public_bucket(bucket_name)
+      rubys3_client = Aws::S3::Client.new
+      begin
+        rubys3_client.head_bucket(bucket: bucket_name)
+      rescue Aws::S3::Errors::NotFound
+        rubys3_client.create_bucket(bucket: bucket_name)
+
+        policy = <<~EOL
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Action": [
+                  "s3:GetBucketLocation",
+                  "s3:ListBucket"
+                ],
+                "Effect": "Allow",
+                "Principal": {
+                  "AWS": [
+                    "*"
+                  ]
+                },
+                "Resource": [
+                  "arn:aws:s3:::#{bucket_name}"
+                ],
+                "Sid": ""
+              },
+              {
+                "Action": [
+                  "s3:GetObject"
+                ],
+                "Effect": "Allow",
+                "Principal": {
+                  "AWS": [
+                    "*"
+                  ]
+                },
+                "Resource": [
+                  "arn:aws:s3:::#{bucket_name}/*"
+                ],
+                "Sid": ""
+              }
+            ]
+          }
+        EOL
+
+        rubys3_client.put_bucket_policy({ bucket: bucket_name, policy: policy })
+      end
+    end
+
+    def self.get_cache_control(filename)
+      # Allow caching for files that have a filename versioning strategy
+      has_version_number = /(-|_|\.)\d+(-|_|\.)\d+(-|_|\.)\d+\./.match(@filename)
+      has_content_hash = /\.[a-f0-9]{20}\./.match(@filename)
+      return nil if has_version_number or has_content_hash
+      return 'no-cache'
+    end
+  end
+end

data/lib/openc3/utilities/s3_autoload.rb

@@ -0,0 +1,9 @@
+require 'aws-sdk-s3'
+
+Aws.config.update(
+  endpoint: ENV['OPENC3_S3_URL'] || (ENV['OPENC3_DEVEL'] ? 'http://127.0.0.1:9000' : 'http://openc3-minio:9000'),
+  access_key_id: ENV['OPENC3_MINIO_USERNAME'],
+  secret_access_key: ENV['OPENC3_MINIO_PASSWORD'],
+  force_path_style: true,
+  region: 'us-east-1'
+)

data/lib/openc3/utilities/s3_file_cache.rb

@@ -0,0 +1,274 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'fileutils'
+require 'tmpdir'
+require 'openc3'
+require 'openc3/utilities/s3'
+
+class S3File
+  attr_reader :s3_path
+  attr_reader :local_path
+  attr_reader :reservation_count
+  attr_reader :size
+  attr_reader :error
+  attr_accessor :priority
+
+  def initialize(s3_path, size = 0, priority = 0)
+    @rubys3_client = Aws::S3::Client.new
+    begin
+      @rubys3_client.head_bucket(bucket: 'logs')
+    rescue Aws::S3::Errors::NotFound
+      @rubys3_client.create_bucket(bucket: 'logs')
+    end
+
+    @s3_path = s3_path
+    @local_path = nil
+    @reservation_count = 0
+    @size = size
+    @priority = priority
+    @error = nil
+    @mutex = Mutex.new
+  end
+
+  def retrieve
+    local_path = "#{S3FileCache.instance.cache_dir}/#{File.basename(@s3_path)}"
+    OpenC3::Logger.debug "Retrieving #{@s3_path} from logs bucket"
+    @rubys3_client.get_object(bucket: "logs", key: @s3_path, response_target: local_path)
+    if File.exist?(local_path)
+      @size = File.size(local_path)
+      @local_path = local_path
+    end
+  rescue => err
+    @error = err
+    OpenC3::Logger.error "Failed to retrieve #{@s3_path}\n#{err.formatted}"
+    raise err
+  end
+
+  def reserve
+    @mutex.synchronize do
+      @reservation_count += 1
+    end
+  end
+
+  def unreserve
+    @mutex.synchronize do
+      @reservation_count -= 1
+      delete() if @reservation_count <= 0
+    end
+  end
+
+  # private
+
+  def delete
+    if @local_path and File.exist?(local_path)
+      File.delete(@local_path)
+      @local_path = nil
+    end
+  end
+end
+
+class S3FileCollection
+  def initialize
+    @array = []
+    @mutex = Mutex.new
+  end
+
+  def add(s3_path, size, priority)
+    @mutex.synchronize do
+      @array.each do |file|
+        if file.s3_path == s3_path
+          file.priority = priority if priority < file.priority
+          @array.sort! {|a,b| a.priority <=> b.priority}
+          return file
+        end
+      end
+      file = S3File.new(s3_path, size, priority)
+      @array << file
+      @array.sort! {|a,b| a.priority <=> b.priority}
+      return file
+    end
+  end
+
+  def length
+    @array.length
+  end
+
+  def get(local_path)
+    @mutex.synchronize do
+      @array.each do |file|
+        return file if file.local_path == local_path
+      end
+    end
+    return nil
+  end
+
+  def get_next_to_retrieve
+    @mutex.synchronize do
+      @array.each do |file|
+        return file unless file.local_path
+      end
+    end
+    return nil
+  end
+
+  def current_disk_usage
+    @mutex.synchronize do
+      total_size = 0
+      @array.each do |file|
+        total_size += file.size if file.local_path
+      end
+      return total_size
+    end
+  end
+end
+
+class S3FileCache
+  MAX_DISK_USAGE = 20_000_000_000 # 20 GB
+  TIMESTAMP_FORMAT = "%Y%m%d%H%M%S%N" # TODO: get from different class?
+
+  attr_reader :cache_dir
+
+  @@instance = nil
+  @@mutex = Mutex.new
+
+  def self.instance
+    return @@instance if @@instance
+    @@mutex.synchronize do
+      @@instance ||= S3FileCache.new
+    end
+    @@instance
+  end
+
+  def initialize(name = 'default', max_disk_usage = MAX_DISK_USAGE)
+    @max_disk_usage = max_disk_usage
+
+    @rubys3_client = Aws::S3::Client.new
+    begin
+      @rubys3_client.head_bucket(bucket: 'logs')
+    rescue Aws::S3::Errors::NotFound
+      @rubys3_client.create_bucket(bucket: 'logs')
+    end
+
+    # Create local file cache location
+    @cache_dir = Dir.mktmpdir
+    FileUtils.mkdir_p(@cache_dir)
+    at_exit do
+      FileUtils.remove_dir(@cache_dir, true)
+    end
+
+    @cached_files = S3FileCollection.new
+
+    @thread = Thread.new do
+      while true
+        file = @cached_files.get_next_to_retrieve
+        # OpenC3::Logger.debug "Next file: #{file}"
+        if file and (file.size + @cached_files.current_disk_usage()) <= @max_disk_usage
+          begin
+            file.retrieve
+          rescue
+            # Will be automatically retried
+          end
+        else
+          sleep(1)
+        end
+      end
+    rescue => err
+      OpenC3::Logger.error "S3FileCache thread unexpectedly died\n#{err.formatted}"
+    end
+  end
+
+  def reserve_file(cmd_or_tlm, target_name, packet_name, start_time_nsec, end_time_nsec, type = :DECOM, timeout = 60, scope:)
+    # OpenC3::Logger.debug "reserve_file #{cmd_or_tlm}:#{target_name}:#{packet_name} start:#{start_time_nsec / 1_000_000_000} end:#{end_time_nsec / 1_000_000_000} type:#{type} timeout:#{timeout}"
+    # Get List of Files from S3
+    total_resp = []
+    token = nil
+    dates = []
+    cur_date = Time.at(start_time_nsec / Time::NSEC_PER_SECOND).beginning_of_day
+    end_date = Time.at(end_time_nsec / Time::NSEC_PER_SECOND).beginning_of_day
+    cur_date -= 1.day # start looking in the folder for the day before because log files can span across midnight
+    while cur_date <= end_date
+      dates << cur_date.strftime("%Y%m%d")
+      cur_date += 1.day
+    end
+    prefixes = []
+    dates.each do |date|
+      while true
+        prefixes << "#{scope}/#{type.to_s.downcase}_logs/#{cmd_or_tlm.to_s.downcase}/#{target_name}/#{packet_name}/#{date}"
+        resp = @rubys3_client.list_objects_v2({
+          bucket: "logs",
+          max_keys: 1000,
+          prefix: prefixes[-1],
+          continuation_token: token
+        })
+        total_resp.concat(resp.contents)
+        break unless resp.is_truncated
+        token = resp.next_continuation_token
+      end
+    end
+
+    # Add to needed files
+    files = []
+    total_resp.each_with_index do |item, index|
+      s3_path = item.key
+      if file_in_time_range(s3_path, start_time_nsec, end_time_nsec)
+        file = @cached_files.add(s3_path, item.size, index)
+        files << file
+      end
+    end
+
+    # Wait for first file retrieval
+    if files.length > 0
+      wait_start = Time.now
+      file = files[0]
+      file.reserve
+      while (Time.now - wait_start) < timeout
+        return file.local_path if file.local_path
+        sleep(1)
+      end
+      # Remove reservations if we timeout
+      file.unreserve
+    else
+      OpenC3::Logger.info "No files found for #{prefixes}"
+    end
+
+    return nil
+  end
+
+  def unreserve_file(filename)
+    @@mutex.synchronize do
+      file = @cached_files.get(filename)
+      file.unreserve if file
+    end
+  end
+
+  # private
+
+  def file_in_time_range(s3_path, start_time_nsec, end_time_nsec)
+    basename = File.basename(s3_path)
+    file_start_timestamp, file_end_timestamp, other = basename.split("__")
+    file_start_time_nsec = DateTime.strptime(file_start_timestamp, TIMESTAMP_FORMAT).to_f * Time::NSEC_PER_SECOND
+    file_end_time_nsec = DateTime.strptime(file_end_timestamp, TIMESTAMP_FORMAT).to_f * Time::NSEC_PER_SECOND
+    if (start_time_nsec < file_end_time_nsec) and (end_time_nsec >= file_start_time_nsec)
+      return true
+    else
+      return false
+    end
+  end
+end

data/lib/openc3/utilities/simulated_target.rb

@@ -0,0 +1,117 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'openc3/packets/packet'
+require 'openc3/packets/structure'
+require 'openc3/system/system'
+
+module OpenC3
+  class Packet < Structure
+    attr_accessor :packet_rate
+  end
+
+  # Base class for all virtual OpenC3 targets which must be implemented by
+  # a subclass. Provides a framework and helper methods to implement a
+  # virtual target which can cycle telemetry values and emit telemetry
+  # packets.
+  class SimulatedTarget
+    attr_accessor :tlm_packets
+
+    def initialize(target_name)
+      @tlm_packets = {}
+
+      # Generate copy of telemetry packets for this target
+      System.telemetry.packets(target_name).each do |name, packet|
+        @tlm_packets[name] = packet.clone
+        @tlm_packets[name].enable_method_missing
+      end
+
+      # Set id values
+      @tlm_packets.each do |name, packet|
+        ids = packet.id_items
+        ids.each do |id|
+          packet.public_send((id.name + '=').to_sym, id.id_value)
+        end
+      end
+
+      @current_cycle_delta = {}
+    end
+
+    def set_rates
+      raise "Error: set_rates must be implemented by subclass"
+    end
+
+    def write(packet)
+      raise "Error: write must be implemented by subclass"
+    end
+
+    def read(count_100hz, time)
+      raise "Error: read must be implemented by subclass"
+    end
+
+    def tick_period_seconds
+      return 0.01 # Override this method to optimize
+    end
+
+    def tick_increment
+      return 1 # Override this method to optimize
+    end
+
+    protected
+
+    def set_rate(packet_name, rate)
+      packet = @tlm_packets[packet_name.upcase]
+      packet.packet_rate = rate if packet
+    end
+
+    def get_pending_packets(count_100hz)
+      pending_packets = []
+
+      # Determine if packets are due to be sent and add to pending
+      @tlm_packets.each do |name, packet|
+        if packet.packet_rate
+          if (count_100hz % packet.packet_rate) == 0
+            pending_packets << packet
+          end
+        end
+      end
+
+      pending_packets
+    end
+
+    def cycle_tlm_item(packet, item_name, min, max, first_delta)
+      packet_name = packet.packet_name
+      @current_cycle_delta[packet_name] ||= {}
+      @current_cycle_delta[packet_name][item_name] ||= first_delta
+
+      current_delta = @current_cycle_delta[packet_name][item_name]
+      current_value = packet.read(item_name)
+      updated_value = current_value + current_delta
+      if updated_value < min
+        updated_value = min
+        @current_cycle_delta[packet_name][item_name] = -current_delta
+      elsif updated_value > max
+        updated_value = max
+        @current_cycle_delta[packet_name][item_name] = -current_delta
+      end
+      packet.write(item_name, updated_value)
+      updated_value
+    end
+  end
+end

data/lib/openc3/utilities/sleeper.rb

@@ -0,0 +1,51 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+module OpenC3
+  # Allows for a breakable sleep implementation using the self-pipe trick
+  # See http://www.sitepoint.com/the-self-pipe-trick-explained/
+  class Sleeper
+    def initialize
+      @pipe_reader, @pipe_writer = IO.pipe
+      @readers = [@pipe_reader]
+      @canceled = false
+    end
+
+    # Breakable version of sleep
+    # @param seconds Number of seconds to sleep
+    # @return true if the sleep was broken by someone calling cancel
+    #   otherwise returns false
+    def sleep(seconds)
+      read_ready, _ = IO.select(@readers, nil, nil, seconds)
+      if read_ready && read_ready.include?(@pipe_reader)
+        return true
+      else
+        return false
+      end
+    end
+
+    # Break sleeping - Once canceled a sleeper cannot be used again
+    def cancel
+      if !@canceled
+        @canceled = true
+        @pipe_writer.write('.')
+      end
+    end
+  end
+end

data/lib/openc3/utilities/store.rb

@@ -0,0 +1,23 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+module OpenC3
+  autoload(:Store, "openc3/utilities/store_autoload.rb")
+  autoload(:EphemeralStore, "openc3/utilities/store_autoload.rb")
+end