openc3 5.0.6

data/lib/openc3/io/win32_serial_driver.rb

@@ -0,0 +1,196 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'openc3/win32/win32'
+require 'timeout' # For Timeout::Error
+
+module OpenC3
+  # Serial driver for use on Windows serial ports
+  class Win32SerialDriver
+    # (see SerialDriver#initialize)
+    def initialize(port_name = 'COM1',
+                   baud_rate = 9600,
+                   parity = :NONE,
+                   stop_bits = 1,
+                   write_timeout = 10.0,
+                   read_timeout = nil,
+                   read_polling_period = 0.01,
+                   read_max_length = 1000,
+                   flow_control = :NONE,
+                   data_bits = 8)
+
+      # Verify Parameters
+      port_name = '\\\\.\\' + port_name if /^COM[0-9]{2,3}$/.match?(port_name)
+
+      raise(ArgumentError, "Invalid baud rate: #{baud_rate}") unless baud_rate.between?(Win32::BAUD_RATES[0], Win32::BAUD_RATES[-1])
+      raise(ArgumentError, "Invalid data bits: #{data_bits}") unless [5, 6, 7, 8].include?(data_bits)
+      raise(ArgumentError, "Invalid parity: #{parity}") if parity and !SerialDriver::VALID_PARITY.include?(parity)
+
+      case parity
+      when SerialDriver::ODD
+        parity = Win32::ODDPARITY
+      when SerialDriver::EVEN
+        parity = Win32::EVENPARITY
+      when SerialDriver::NONE
+        parity = Win32::NOPARITY
+      end
+
+      raise(ArgumentError, "Invalid stop bits: #{stop_bits}") unless [1, 2].include?(stop_bits)
+
+      if stop_bits == 1
+        stop_bits = Win32::ONESTOPBIT
+      else
+        stop_bits = Win32::TWOSTOPBITS
+      end
+
+      @write_timeout = write_timeout
+      @read_timeout = read_timeout
+      @read_polling_period = read_polling_period
+      @read_max_length = read_max_length
+
+      # Open the Comm Port
+      @handle = Win32.create_file(port_name,
+                                  Win32::GENERIC_READ | Win32::GENERIC_WRITE,
+                                  0,
+                                  Win32::NULL,
+                                  Win32::OPEN_EXISTING,
+                                  Win32::FILE_ATTRIBUTE_NORMAL)
+
+      @mutex = Mutex.new
+
+      # Configure the Comm Port - See: https://msdn.microsoft.com/en-us/library/windows/desktop/aa363214(v=vs.85).aspx
+      dcb = Win32.get_comm_state(@handle)
+      dcb.write('BaudRate', baud_rate)
+      dcb.write('ByteSize', data_bits)
+      dcb.write('Parity',   parity)
+      dcb.write('StopBits', stop_bits)
+      if flow_control == :RTSCTS
+        # Monitor CTS
+        dcb.write('fOutxCtsFlow', 1)
+
+        # 0x00 - RTS_CONTROL_DISABLE - Disables the RTS line when the device is opened and leaves it disabled.
+        # 0x01 - RTS_CONTROL_ENABLE - Enables the RTS line when the device is opened and leaves it on.
+        # 0x02 - RTS_CONTROL_HANDSHAKE - Enables RTS handshaking. The driver raises the RTS line when the "type-ahead" (input) buffer is less than one-half full and lowers the RTS line when the buffer is more than three-quarters full. If handshaking is enabled, it is an error for the application to adjust the line by using the EscapeCommFunction function.
+        # 0x03 - RTS_CONTROL_TOGGLE - Specifies that the RTS line will be high if bytes are available for transmission. After all buffered bytes have been sent, the RTS line will be low.
+        dcb.write('fRtsControl', 0x03)
+      end
+      Win32.set_comm_state(@handle, dcb)
+
+      # Configure Timeouts, the WinAPI structure is COMMTIMEOUTS:
+      #   DWORD ReadIntervalTimeout;
+      #   DWORD ReadTotalTimeoutMultiplier;
+      #   DWORD ReadTotalTimeoutConstant;
+      #   DWORD WriteTotalTimeoutMultiplier;
+      #   DWORD WriteTotalTimeoutConstant;
+      # 0xFFFFFFFF, 0, 0 specifies that the read operation is to return immediately
+      # with the bytes that have already been received, even if no bytes have been received.
+      # The WriteTotalTimeoutMultiplier is multiplied by the number of bytes to be written
+      # and the WriteTotalTimeoutConstant is added to that total (both are in milliseconds).
+      bits_per_symbol = data_bits + 1 # 1 start bit
+      case stop_bits
+      when Win32::ONESTOPBIT
+        bits_per_symbol += 1
+      when Win32::TWOSTOPBITS
+        bits_per_symbol += 2
+      end
+      case parity
+      when Win32::ODDPARITY, Win32::EVENPARITY
+        bits_per_symbol += 1
+      end
+      delay = (1000.0 / (baud_rate / bits_per_symbol.to_f)).ceil
+      Win32.set_comm_timeouts(@handle, 0xFFFFFFFF, 0, 0, delay, 1000)
+    end
+
+    # (see SerialDriver#close)
+    def close
+      if @handle
+        # Close the Comm Port
+        Win32.close_handle(@handle)
+        @mutex.synchronize do
+          @handle = nil
+        end
+      end
+    end
+
+    # (see SerialDriver#closed?)
+    def closed?
+      if @handle
+        false
+      else
+        true
+      end
+    end
+
+    # (see SerialDriver#write)
+    def write(data)
+      # Write the data
+      time = Time.now.sys
+      bytes_to_write = data.length
+      while bytes_to_write > 0
+        bytes_written = Win32.write_file(@handle, data, data.length)
+        raise "Error writing to comm port" if bytes_written <= 0
+
+        bytes_to_write -= bytes_written
+        data = data[bytes_written..-1]
+        raise Timeout::Error, "Write Timeout" if @write_timeout and (Time.now.sys - time > @write_timeout) and bytes_to_write > 0
+      end
+    end
+
+    # (see SerialDriver#read)
+    def read
+      data = ''
+      sleep_time = 0.0
+
+      loop do
+        loop do
+          buffer = nil
+          @mutex.synchronize do
+            break unless @handle
+
+            buffer = Win32.read_file(@handle, @read_max_length - data.length)
+          end
+          break unless buffer
+
+          data << buffer
+          break if buffer.length <= 0 or data.length >= @read_max_length or !@handle
+        end
+        break if data.length > 0 or !@handle
+        if @read_timeout and sleep_time >= @read_timeout
+          raise Timeout::Error, "Read Timeout"
+        end
+
+        sleep(@read_polling_period)
+        sleep_time += @read_polling_period
+      end
+
+      data
+    end
+
+    # (see SerialDriver#read_nonblock)
+    def read_nonblock
+      data = ''
+      loop do
+        buffer = Win32.read_file(@handle, @read_max_length - data.length)
+        data << buffer
+        break if buffer.length <= 0 or data.length >= @read_max_length
+      end
+      data
+    end
+  end
+end

data/lib/openc3/logs/log_writer.rb

@@ -0,0 +1,302 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+require 'thread'
+require 'openc3/config/config_parser'
+require 'openc3/topics/topic'
+require 'openc3/utilities/s3'
+
+module OpenC3
+  # Creates a log. Can automatically cycle the log based on an elasped
+  # time period or when the log file reaches a predefined size.
+  class LogWriter
+    # @return [String] The filename of the packet log
+    attr_reader :filename
+
+    # @return [true/false] Whether logging is enabled
+    attr_reader :logging_enabled
+
+    # @return cycle_time [Integer] The amount of time in seconds before creating
+    #   a new log file. This can be combined with cycle_size but is better used
+    #   independently.
+    attr_reader :cycle_time
+
+    # @return cycle_hour [Integer] The time at which to cycle the log. Combined with
+    #   cycle_minute to cycle the log daily at the specified time. If nil, the log
+    #   will be cycled hourly at the specified cycle_minute.
+    attr_reader :cycle_hour
+
+    # @return cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
+    #   for more information.
+    attr_reader :cycle_minute
+
+    # @return [Time] Time that the current log file started
+    attr_reader :start_time
+
+    # @return [Mutex] Instance mutex protecting file
+    attr_reader :mutex
+
+    # The cycle time interval. Cycle times are only checked at this level of
+    # granularity.
+    CYCLE_TIME_INTERVAL = 10
+
+    # Mutex protecting class variables
+    @@mutex = Mutex.new
+
+    # Array of instances used to keep track of cycling logs
+    @@instances = []
+
+    # Thread used to cycle logs across all log writers
+    @@cycle_thread = nil
+
+    # Sleeper used to delay cycle thread
+    @@cycle_sleeper = nil
+
+    # @param remote_log_directory [String] The s3 path to store the log files
+    # @param logging_enabled [Boolean] Whether to start with logging enabled
+    # @param cycle_time [Integer] The amount of time in seconds before creating
+    #   a new log file. This can be combined with cycle_size but is better used
+    #   independently.
+    # @param cycle_size [Integer] The size in bytes before creating a new log
+    #   file. This can be combined with cycle_time but is better used
+    #   independently.
+    # @param cycle_hour [Integer] The time at which to cycle the log. Combined with
+    #   cycle_minute to cycle the log daily at the specified time. If nil, the log
+    #   will be cycled hourly at the specified cycle_minute.
+    # @param cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
+    #   for more information.
+    # @param redis_topic [String] The key of the Redis stream to trim when files are
+    #   moved to S3
+    def initialize(
+      remote_log_directory,
+      logging_enabled = true,
+      cycle_time = nil,
+      cycle_size = 1000000000,
+      cycle_hour = nil,
+      cycle_minute = nil,
+      redis_topic: nil
+    )
+      @remote_log_directory = remote_log_directory
+      @logging_enabled = ConfigParser.handle_true_false(logging_enabled)
+      @cycle_time = ConfigParser.handle_nil(cycle_time)
+      if @cycle_time
+        @cycle_time = Integer(@cycle_time)
+        raise "cycle_time must be >= #{CYCLE_TIME_INTERVAL}" if @cycle_time < CYCLE_TIME_INTERVAL
+      end
+      @cycle_size = ConfigParser.handle_nil(cycle_size)
+      @cycle_size = Integer(@cycle_size) if @cycle_size
+      @cycle_hour = ConfigParser.handle_nil(cycle_hour)
+      @cycle_hour = Integer(@cycle_hour) if @cycle_hour
+      @cycle_minute = ConfigParser.handle_nil(cycle_minute)
+      @cycle_minute = Integer(@cycle_minute) if @cycle_minute
+      @mutex = Mutex.new
+      @file = nil
+      @file_size = 0
+      @filename = nil
+      @start_time = Time.now.utc
+      @first_time = nil
+      @last_time = nil
+      @cancel_threads = false
+      @last_offset = nil
+      @previous_file_redis_offset = nil
+      @redis_topic = redis_topic
+
+      # This is an optimization to avoid creating a new entry object
+      # each time we create an entry which we do a LOT!
+      @entry = String.new
+
+      if @cycle_time or @cycle_hour or @cycle_minute
+        @@mutex.synchronize do
+          @@instances << self
+
+          unless @@cycle_thread
+            @@cycle_thread = OpenC3.safe_thread("Log cycle") do
+              cycle_thread_body()
+            end
+          end
+        end
+      end
+    end
+
+    # Starts a new log file by closing the existing log file. New log files are
+    # not created until packets are written by {#write} so this does not
+    # immediately create a log file on the filesystem.
+    def start
+      @mutex.synchronize { close_file(false); @logging_enabled = true }
+    end
+
+    # Stops all logging and closes the current log file.
+    def stop
+      @mutex.synchronize { @logging_enabled = false; close_file(false) }
+    end
+
+    # Stop all logging, close the current log file, and kill the logging threads.
+    def shutdown
+      stop()
+      @@mutex.synchronize do
+        @@instances.delete(self)
+        if @@instances.length <= 0
+          @@cycle_sleeper.cancel if @@cycle_sleeper
+          OpenC3.kill_thread(self, @@cycle_thread) if @@cycle_thread
+          @@cycle_thread = nil
+        end
+      end
+    end
+
+    def graceful_kill
+      @cancel_threads = true
+    end
+
+    # implementation details
+
+    def create_unique_filename(ext = extension)
+      # Create a filename that doesn't exist
+      attempt = nil
+      while true
+        filename_parts = [attempt]
+        filename_parts.unshift @label if @label
+        filename = File.join(Dir.tmpdir, File.build_timestamped_filename([@label, attempt], ext))
+        if File.exist?(filename)
+          attempt ||= 0
+          attempt += 1
+        else
+          return filename
+        end
+      end
+    end
+
+    def cycle_thread_body
+      @@cycle_sleeper = Sleeper.new
+      while true
+        start_time = Time.now
+        @@mutex.synchronize do
+          @@instances.each do |instance|
+            # The check against start_time needs to be mutex protected to prevent a packet coming in between the check
+            # and closing the file
+            instance.mutex.synchronize do
+              utc_now = Time.now.utc
+              # Logger.debug("start:#{@start_time.to_f} now:#{utc_now.to_f} cycle:#{@cycle_time} new:#{(utc_now - @start_time) > @cycle_time}")
+              if instance.logging_enabled and
+                (
+                  # Cycle based on total time logging
+                  (instance.cycle_time and (utc_now - instance.start_time) > instance.cycle_time) or
+
+                  # Cycle daily at a specific time
+                  (instance.cycle_hour and instance.cycle_minute and utc_now.hour == instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.yday != utc_now.yday) or
+
+                  # Cycle hourly at a specific time
+                  (instance.cycle_minute and not instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.hour != utc_now.hour)
+                )
+                instance.close_file(false)
+              end
+            end
+          end
+        end
+
+        # Only check whether to cycle at a set interval
+        run_time = Time.now - start_time
+        sleep_time = CYCLE_TIME_INTERVAL - run_time
+        sleep_time = 0 if sleep_time < 0
+        break if @@cycle_sleeper.sleep(sleep_time)
+      end
+    end
+
+    # Starting a new log file is a critical operation so the entire method is
+    # wrapped with a rescue and handled with handle_critical_exception
+    # Assumes mutex has already been taken
+    def start_new_file
+      close_file(false)
+
+      # Start log file
+      @filename = create_unique_filename()
+      @file = File.new(@filename, 'wb')
+      @file_size = 0
+
+      @start_time = Time.now.utc
+      @first_time = nil
+      @last_time = nil
+      Logger.debug "Log File Opened : #{@filename}"
+    rescue => err
+      Logger.error "Error starting new log file: #{err.formatted}"
+      @logging_enabled = false
+      OpenC3.handle_critical_exception(err)
+    end
+
+    def prepare_write(time_nsec_since_epoch, data_length, redis_offset)
+      # This check includes logging_enabled again because it might have changed since we acquired the mutex
+      if @logging_enabled and (!@file or (@cycle_size and (@file_size + data_length) > @cycle_size))
+        start_new_file()
+      end
+      @last_offset = redis_offset # This is needed for the redis offset marker entry at the end of the log file
+    end
+
+    # Closing a log file isn't critical so we just log an error. NOTE: This also trims the Redis stream
+    # to keep a full file's worth of data in the stream. This is what prevents continuous stream growth.
+    def close_file(take_mutex = true)
+      @mutex.lock if take_mutex
+      begin
+        if @file
+          begin
+            @file.close unless @file.closed?
+            Logger.debug "Log File Closed : #{@filename}"
+            date = first_timestamp[0..7] # YYYYMMDD
+            s3_key = File.join(@remote_log_directory, date, s3_filename)
+            S3Utilities.move_log_file_to_s3(@filename, s3_key)
+            # Now that the file is in S3, trim the Redis stream up until the previous file.
+            # This keeps one file worth of data in Redis as a safety buffer
+            Topic.trim_topic(@redis_topic, @previous_file_redis_offset) if @redis_topic and @previous_file_redis_offset
+            @previous_file_redis_offset = @last_offset
+          rescue Exception => err
+            Logger.instance.error "Error closing #{@filename} : #{err.formatted}"
+          end
+
+          @file = nil
+          @file_size = 0
+          @filename = nil
+        end
+      ensure
+        @mutex.unlock if take_mutex
+      end
+    end
+
+    def s3_filename
+      "#{first_timestamp}__#{last_timestamp}" + extension
+    end
+
+    def extension
+      '.log'.freeze
+    end
+
+    def first_time
+      Time.from_nsec_from_epoch(@first_time)
+    end
+
+    def last_time
+      Time.from_nsec_from_epoch(@last_time)
+    end
+
+    def first_timestamp
+      first_time().to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
+    end
+
+    def last_timestamp
+      last_time().to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
+    end
+  end
+end

data/lib/openc3/logs/packet_log_constants.rb

@@ -0,0 +1,62 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# Modified by OpenC3, Inc.
+# All changes Copyright 2022, OpenC3, Inc.
+# All Rights Reserved
+
+module OpenC3
+  module PacketLogConstants
+    # Constants to detect old file formats
+    COSMOS2_FILE_HEADER = 'COSMOS2_'.freeze
+    COSMOS4_FILE_HEADER = 'COSMOS4_'.freeze
+
+    # OPENC3 5 Constants
+    OPENC3_FILE_HEADER = 'COSMOS5_'.freeze
+    OPENC3_INDEX_HEADER = 'COSIDX5_'.freeze
+    OPENC3_HEADER_LENGTH = OPENC3_FILE_HEADER.length
+    # Flags which are bit masked into file entries
+    OPENC3_ENTRY_TYPE_MASK = 0xF000
+    OPENC3_TARGET_DECLARATION_ENTRY_TYPE_MASK = 0x1000
+    OPENC3_PACKET_DECLARATION_ENTRY_TYPE_MASK = 0x2000
+    OPENC3_RAW_PACKET_ENTRY_TYPE_MASK = 0x3000
+    OPENC3_JSON_PACKET_ENTRY_TYPE_MASK = 0x4000
+    OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK = 0x5000
+    OPENC3_ID_FLAG_MASK = 0x0200
+    OPENC3_STORED_FLAG_MASK = 0x0400
+    OPENC3_CMD_FLAG_MASK = 0x0800
+
+    OPENC3_ID_FIXED_SIZE = 32
+    OPENC3_MAX_PACKET_INDEX = 65535
+    OPENC3_MAX_TARGET_INDEX = 65535
+
+    OPENC3_PRIMARY_FIXED_SIZE = 2
+    OPENC3_TARGET_DECLARATION_SECONDARY_FIXED_SIZE = 0
+    OPENC3_TARGET_DECLARATION_PACK_DIRECTIVE = 'Nn'.freeze
+    OPENC3_TARGET_DECLARATION_PACK_ITEMS = 2 # Useful for testing
+
+    OPENC3_PACKET_DECLARATION_SECONDARY_FIXED_SIZE = 2
+    OPENC3_PACKET_DECLARATION_PACK_DIRECTIVE = 'Nnn'.freeze
+    OPENC3_PACKET_DECLARATION_PACK_ITEMS = 3 # Useful for testing
+
+    OPENC3_OFFSET_MARKER_SECONDARY_FIXED_SIZE = 0
+    OPENC3_OFFSET_MARKER_PACK_DIRECTIVE = 'Nn'.freeze
+    OPENC3_OFFSET_MARKER_PACK_ITEMS = 2 # Useful for testing
+
+    OPENC3_PACKET_SECONDARY_FIXED_SIZE = 10
+    OPENC3_PACKET_PACK_DIRECTIVE = 'NnnQ>'.freeze
+    OPENC3_PACKET_PACK_ITEMS = 4 # Useful for testing
+  end
+end