openc3 5.0.11 → 5.1.0

data/lib/openc3/utilities/aws_bucket.rb

@@ -0,0 +1,199 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
+
+require 'openc3/utilities/bucket'
+module Aws
+  autoload(:S3, 'openc3/utilities/s3_autoload.rb')
+end
+
+module OpenC3
+  class AwsBucket < Bucket
+    CREATE_CHECK_COUNT = 100 # 10 seconds
+
+    def initialize
+      @client = Aws::S3::Client.new
+    end
+
+    def create(bucket)
+      unless exist?(bucket)
+        @client.create_bucket({ bucket: bucket })
+        count = 0
+        until exist?(bucket) or count > CREATE_CHECK_COUNT
+          sleep(0.1)
+          count += 1
+        end
+      end
+      bucket
+    end
+
+    def ensure_public(bucket)
+      policy = <<~EOL
+      {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Action": [
+              "s3:GetBucketLocation",
+              "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+              "AWS": [
+                "*"
+              ]
+            },
+            "Resource": [
+              "arn:aws:s3:::#{bucket}"
+            ],
+            "Sid": ""
+          },
+          {
+            "Action": [
+              "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+              "AWS": [
+                "*"
+              ]
+            },
+            "Resource": [
+              "arn:aws:s3:::#{bucket}/*"
+            ],
+            "Sid": ""
+          }
+        ]
+      }
+      EOL
+      @client.put_bucket_policy({ bucket: bucket, policy: policy })
+    end
+
+    def exist?(bucket)
+      @client.head_bucket({ bucket: bucket })
+      true
+    rescue Aws::S3::Errors::NotFound
+      false
+    end
+
+    def delete(bucket)
+      if exist?(bucket)
+        @client.delete_bucket({ bucket: bucket })
+      end
+    end
+
+    def get_object(bucket:, key:, path: nil)
+      if path
+        @client.get_object(bucket: bucket, key: key, response_target: path)
+      else
+        @client.get_object(bucket: bucket, key: key)
+      end
+    # If the key is not found return nil
+    rescue Aws::S3::Errors::NoSuchKey
+      nil
+    end
+
+    def list_objects(bucket:, prefix: nil)
+      token = nil
+      result = []
+      while true
+        resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: 1000)
+        result.concat(resp.contents)
+        break unless resp.is_truncated
+        token = resp.next_continuation_token
+      end
+      # Array of objects with key and size methods
+      result
+    end
+
+    # Lists the directories under a specified path
+    def list_directories(bucket:, path:)
+      # Trailing slash is important in AWS S3 when listing files
+      # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Types/ListObjectsV2Output.html#common_prefixes-instance_method
+      if path[-1] != '/'
+        path += '/'
+      end
+      token = nil
+      result = []
+      while true
+        resp = @client.list_objects_v2({
+          bucket: bucket,
+          max_keys: 1000,
+          prefix: path,
+          delimiter: '/',
+          continuation_token: token
+        })
+        resp.common_prefixes.each do |item|
+          # If path was DEFAULT/targets_modified/ then the
+          # results look like DEFAULT/targets_modified/INST/
+          result << item.prefix.split('/')[-1]
+        end
+        break unless resp.is_truncated
+        token = resp.next_continuation_token
+      end
+      result
+    end
+
+    # put_object fires off the request to store but does not confirm
+    def put_object(bucket:, key:, body:, content_type: nil, cache_control: nil, metadata: nil)
+      @client.put_object(bucket: bucket, key: key, body: body,
+        content_type: content_type, cache_control: cache_control, metadata: metadata)
+    end
+
+    # @returns [Boolean] Whether the file exists
+    def check_object(bucket:, key:)
+      @client.wait_until(:object_exists,
+        {
+          bucket: bucket,
+          key: key
+        },
+        {
+          max_attempts: 30,
+          delay: 0.1, # seconds
+        }
+      )
+      true
+    rescue Aws::Waiters::Errors::TooManyAttemptsError
+      false
+    end
+
+    def delete_object(bucket:, key:)
+      @client.delete_object(bucket: bucket, key: key)
+    end
+
+    def delete_objects(bucket:, keys:)
+      @client.delete_objects(bucket: bucket, delete: { objects: keys.map {|key| { key: key } } })
+    end
+
+    def presigned_request(bucket:, key:, method:, internal: true)
+      s3_presigner = Aws::S3::Presigner.new
+
+      if internal
+        prefix = '/'
+      else
+        prefix = '/files/'
+      end
+
+      url, headers = s3_presigner.presigned_request(method, bucket: bucket, key: key)
+      return {
+        :url => prefix + url.split('/')[3..-1].join('/'),
+        :headers => headers,
+        :method => method.to_s.split('_')[0],
+      }
+    end
+  end
+end

data/lib/openc3/utilities/bucket.rb

@@ -0,0 +1,82 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
+
+# Interface class implemented by each cloud provider: AWS, GCS, Azure
+module OpenC3
+  class Bucket
+    def self.getClient
+      raise 'OPENC3_CLOUD environment variable is required' unless ENV['OPENC3_CLOUD']
+      # Base is AwsBucket which works with MINIO, Enterprise implements additional
+      bucket_class = ENV['OPENC3_CLOUD'].capitalize + 'Bucket'
+      klass = OpenC3.require_class('openc3/utilities/'+bucket_class.class_name_to_filename)
+      klass.new
+    end
+
+    def initialize
+      # Setup the client instance
+    end
+
+    def create(bucket)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def ensure_public(bucket)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def exist?(bucket)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def delete(bucket)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def get_object(bucket:, key:, path: nil)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def list_objects(bucket:, prefix: nil)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def list_directories(bucket:, path:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def put_object(bucket:, key:, body:, content_type: nil, cache_control: nil, metadata: nil)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def check_object(bucket:, key:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def delete_object(bucket:, key:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def delete_objects(bucket:, keys:)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    def presigned_request(bucket:, key:, method:, internal: true)
+      raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+    end
+  end
+end

data/lib/openc3/utilities/bucket_file_cache.rb

@@ -0,0 +1,264 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
+
+require 'fileutils'
+require 'tmpdir'
+require 'openc3'
+require 'openc3/utilities/bucket_utilities'
+require 'openc3/utilities/bucket'
+
+class BucketFile
+  MAX_AGE_SECONDS = 3600 * 4 # 4 hours
+
+  attr_reader :bucket_path
+  attr_reader :local_path
+  attr_reader :reservation_count
+  attr_reader :size
+  attr_reader :error
+  attr_reader :topic_prefix
+  attr_reader :init_time
+  attr_accessor :priority
+
+  def initialize(bucket_path, client = nil)
+    @bucket = client
+    unless @bucket
+      @bucket = OpenC3::Bucket.getClient()
+    end
+    @bucket_path = bucket_path
+    @local_path = nil
+    @reservation_count = 0
+    @size = 0
+    @priority = 0
+    @error = nil
+    @init_time = Time.now
+    @mutex = Mutex.new
+    path_split = @bucket_path.split("/")
+    scope = path_split[0].to_s.upcase
+    stream_mode = path_split[1].to_s.split("_")[0].to_s.upcase
+    if stream_mode == 'REDUCED'
+      stream_mode << '_' << path_split[1].to_s.split("_")[1].to_s.upcase
+    end
+    cmd_or_tlm = path_split[2].to_s.upcase
+    target_name = path_split[3].to_s.upcase
+    if stream_mode == 'RAW'
+      type = (@cmd_or_tlm == 'CMD') ? 'COMMAND' : 'TELEMETRY'
+    else
+      if stream_mode == 'DECOM'
+        type = (@cmd_or_tlm == 'CMD') ? 'DECOMCMD' : 'DECOM'
+      else
+        type = stream_mode # REDUCED_MINUTE, REDUCED_HOUR, or REDUCED_DAY
+      end
+    end
+    @topic_prefix = "#{scope}__#{type}__{#{target_name}}"
+  end
+
+  def retrieve(client = @bucket, uncompress = true)
+    @mutex.synchronize do
+      local_path = "#{BucketFileCache.instance.cache_dir}/#{File.basename(@bucket_path)}"
+      unless File.exist?(local_path)
+        OpenC3::Logger.debug "Retrieving #{@bucket_path} from logs bucket"
+        client.get_object(bucket: "logs", key: @bucket_path, path: local_path)
+        if File.exist?(local_path)
+          basename = File.basename(local_path)
+          if uncompress and File.extname(basename) == ".gz"
+            uncompressed = OpenC3::BucketUtilities.uncompress_file(local_path)
+            File.delete(local_path)
+            local_path = uncompressed
+          end
+          @size = File.size(local_path)
+          @local_path = local_path
+          return true
+        end
+      end
+      return false
+    end
+  rescue => err
+    @error = err
+    OpenC3::Logger.error "Failed to retrieve #{@bucket_path}\n#{err.formatted}"
+    raise err
+  end
+
+  def reserve
+    @mutex.synchronize { @reservation_count += 1 }
+    return retrieve()
+  end
+
+  def unreserve
+    @mutex.synchronize do
+      @reservation_count -= 1
+      delete() if @reservation_count <= 0
+      return @reservation_count
+    end
+  end
+
+  def age_check
+    @mutex.synchronize do
+      if (Time.now - @init_time) > MAX_AGE_SECONDS and @reservation_count <= 0
+        delete()
+        return true
+      else
+        return false
+      end
+    end
+  end
+
+  # private
+
+  def delete
+    if @local_path and File.exist?(@local_path)
+      File.delete(@local_path)
+      @local_path = nil
+    end
+  end
+end
+
+class BucketFileCache
+  MAX_DISK_USAGE = (ENV['OPENC3_BUCKET_FILE_CACHE_SIZE'] || 20_000_000_000).to_i # Default 20 GB
+  CHECK_TIME_SECONDS = 3600
+
+  attr_reader :cache_dir
+
+  @@instance = nil
+  @@mutex = Mutex.new
+
+  def self.instance
+    return @@instance if @@instance
+    @@mutex.synchronize do
+      @@instance ||= BucketFileCache.new
+    end
+    @@instance
+  end
+
+  def initialize
+    @bucket = OpenC3::Bucket.getClient()
+
+    # Create local file cache location
+    @cache_dir = Dir.mktmpdir
+    FileUtils.mkdir_p(@cache_dir)
+    at_exit do
+      FileUtils.remove_dir(@cache_dir, true)
+    end
+
+    @current_disk_usage = 0
+    @queued_bucket_files = []
+    @bucket_file_hash = {}
+
+    @thread = Thread.new do
+      client = OpenC3::Bucket.getClient()
+      while true
+        check_time = Time.now + CHECK_TIME_SECONDS # Check for aged out files periodically
+        if @queued_bucket_files.length > 0 and @current_disk_usage < MAX_DISK_USAGE
+          @@mutex.synchronize do
+            bucket_file = @queued_bucket_files.shift
+          end
+          begin
+            retrieved = bucket_file.retrieve(client)
+            @@mutex.synchronize do
+              @current_disk_usage += bucket_file.size if retrieved
+            end
+          rescue
+            # Might have been deleted
+          end
+          sleep(0.01) # Small throttle
+        else
+          # Nothing to do or disk full
+          if Time.now > check_time
+            check_time = Time.now + CHECK_TIME_SECONDS
+            # Delete any files that aren't reserved and are old
+            removed_files = []
+            @bucket_file_hash.each do |bucket_path, bucket_file|
+              deleted = bucket_file.age_check
+              if deleted
+                removed_files << bucket_path
+                @current_disk_usage -= bucket_file.size
+              end
+            end
+            removed_files.each do |bucket_path|
+              @bucket_file_hash.delete(bucket_path)
+            end
+          end
+          sleep(1)
+        end
+      end
+    rescue => err
+      OpenC3::Logger.error "BucketFileCache thread unexpectedly died\n#{err.formatted}"
+    end
+  end
+
+  def self.hint(bucket_paths)
+    return instance().hint(bucket_paths)
+  end
+
+  def self.reserve(bucket_path)
+    return instance().reserve(bucket_path)
+  end
+
+  def self.unreserve(bucket_path)
+    return instance().unreserve(bucket_path)
+  end
+
+  def hint(bucket_paths)
+    @@mutex.synchronize do
+      bucket_paths.each_with_index do |bucket_path, index|
+        bucket_file = create_bucket_file(bucket_path)
+        bucket_file.priority = index
+      end
+      @queued_bucket_files.sort! {|file1, file2| file1.priority <=> file2.priority}
+    end
+  end
+
+  def reserve(bucket_path)
+    @@mutex.synchronize do
+      bucket_file = create_bucket_file(bucket_path)
+      retrieved = bucket_file.reserve
+      @current_disk_usage += bucket_file.size if retrieved
+      @queued_bucket_files.delete(bucket_file)
+      return bucket_file
+    end
+  end
+
+  def unreserve(bucket_path)
+    @@mutex.synchronize do
+      bucket_file = @bucket_file_hash[bucket_path]
+      if bucket_file
+        bucket_file.unreserve
+        if bucket_file.reservation_count <= 0 and !@queued_bucket_files.include?(bucket_file)
+          @current_disk_usage -= bucket_file.size
+          @bucket_file_hash.delete(bucket_file)
+        end
+      end
+    end
+  end
+
+  # Private
+
+  def create_bucket_file(bucket_path)
+    bucket_file = @bucket_file_hash[bucket_path]
+    unless bucket_file
+      bucket_file = BucketFile.new(bucket_path)
+      @queued_bucket_files << bucket_file
+      @bucket_file_hash[bucket_path] = bucket_file
+    end
+    return bucket_file
+  end
+
+end

data/lib/openc3/utilities/bucket_utilities.rb

@@ -0,0 +1,109 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
+
+require 'openc3/utilities/bucket'
+require 'openc3/models/reducer_model'
+require 'zlib'
+
+module OpenC3
+  class BucketUtilities
+    def self.list_files_before_time(bucket, prefix, time)
+      client = Bucket.getClient
+      oldest_list = []
+
+      # Return nothing if bucket doesn't exist (it won't at the very beginning)
+      unless client.exist?(bucket)
+        return oldest_list
+      end
+
+      next_folder = false
+      resp = client.list_objects(bucket: bucket, prefix: prefix)
+      resp.each do |item|
+        t = File.basename(item.key).split('__')[1]
+        file_end_time = Time.utc(t[0..3], t[4..5], t[6..7], t[8..9], t[10..11], t[12..13])
+        if file_end_time < time
+          oldest_list << item.key
+        else
+          break
+        end
+      end
+      return oldest_list
+    end
+
+    def self.move_log_file_to_bucket(filename, bucket_key, metadata: {})
+      Thread.new do
+        client = Bucket.getClient
+
+        zipped = compress_file(filename)
+        bucket_key = bucket_key + '.gz'
+        File.open(zipped, 'rb') do |read_file|
+          client.put_object(bucket: ENV['OPENC3_LOGS_BUCKET'], key: bucket_key, body: read_file, metadata: metadata)
+        end
+        Logger.debug "wrote #{ENV['OPENC3_LOGS_BUCKET']}/#{bucket_key}"
+        ReducerModel.add_file(bucket_key) # Record the new file for data reduction
+
+        File.delete(zipped)
+        File.delete(filename)
+      rescue => err
+        Logger.error("Error saving log file to bucket: #{filename}\n#{err.formatted}")
+      end
+    end
+
+    def self.get_cache_control(filename)
+      # Allow caching for files that have a filename versioning strategy
+      has_version_number = /(-|_|\.)\d+(-|_|\.)\d+(-|_|\.)\d+\./.match(filename)
+      has_content_hash = /\.[a-f0-9]{20}\./.match(filename)
+      return nil if has_version_number or has_content_hash
+      return 'no-cache'
+    end
+
+    def self.compress_file(filename, chunk_size = 50_000_000)
+      zipped = "#{filename}.gz"
+
+      Zlib::GzipWriter.open(zipped) do |gz|
+        gz.mtime = File.mtime(filename)
+        gz.orig_name = filename
+        File.open(filename, 'rb') do |file|
+          while chunk = file.read(chunk_size) do
+            gz.write(chunk)
+          end
+        end
+      end
+
+      return zipped
+    end
+
+    def self.uncompress_file(filename, chunk_size = 50_000_000)
+      unzipped = filename[0..-4] # Drop .gz
+
+      Zlib::GzipReader.open(filename) do |gz|
+        File.open(unzipped, 'wb') do |file|
+          while chunk = gz.read(chunk_size)
+            file.write(chunk)
+          end
+        end
+      end
+
+      return unzipped
+    end
+  end
+end

data/lib/openc3/utilities/crc.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/ext/crc' if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']
 

data/lib/openc3/utilities/csv.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'csv'
 

data/lib/openc3/utilities/local_bucket.rb

@@ -0,0 +1,28 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
+
+require 'openc3/utilities/aws_bucket'
+
+module OpenC3
+  # This class exists simply to enable the following code in bucket.rb
+  #   bucket_class = ENV['OPENC3_CLOUD'].capitalize + 'Bucket'
+  # So when the OPENC3_CLOUD var is set to 'local' this file is used
+  # The local code uses Minio which is identical to the Aws APIs
+  class LocalBucket < AwsBucket
+  end
+end