openc3 5.10.1 → 5.11.0

data/lib/openc3/migrations/20230915000002_no_scope_log_messages.rb

@@ -0,0 +1,44 @@
+require 'openc3/utilities/migration'
+require 'openc3/models/microservice_model'
+
+module OpenC3
+  class NoScopeLogMessages < Migration
+    def self.run
+      # Add NOSCOPE topic to log message microservice for DEFAULT scope
+      model = MicroserviceModel.get_model(name: "DEFAULT__OPENC3__LOG", scope: 'DEFAULT')
+      if model
+        model.topics = ["DEFAULT__openc3_log_messages", "NOSCOPE__openc3_log_messages"]
+        model.update
+      end
+
+      ScopeModel.get_all_models(scope: nil).each do |scope, scope_model|
+        parent = "#{scope}__SCOPEMULTI__#{scope}"
+
+        # Remove NOTIFICATION log microservice from scopes
+        model = MicroserviceModel.get_model(name: "#{scope}__NOTIFICATION__LOG", scope: scope)
+        if model
+          model.destroy
+        end
+
+        # Add Scope Cleanup microservice to scopes
+        model = MicroserviceModel.get_model(name: "#{scope}__SCOPECLEANUP__#{scope}", scope: scope)
+        unless model
+          scope_model.deploy_scopecleanup_microservice("", {}, parent)
+        end
+
+        model = MicroserviceModel.get_model(name: parent, scope: scope)
+        if model
+          model.cmd.delete("#{scope}__NOTIFICATION__LOG")
+          unless model.cmd.include?("#{scope}__SCOPECLEANUP__#{scope}")
+            model.cmd << "#{scope}__SCOPECLEANUP__#{scope}"
+          end
+          model.update
+        end
+      end
+    end
+  end
+end
+
+unless ENV['OPENC3_NO_MIGRATE']
+  OpenC3::NoScopeLogMessages.run
+end

data/lib/openc3/models/microservice_model.rb

@@ -221,7 +221,7 @@ module OpenC3
         # Load microservice files
         data = File.read(filename, mode: "rb")
         OpenC3.set_working_dir(File.dirname(filename)) do
-          data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_'
+          data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_'
         end
         unless validate_only
           @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: data)

data/lib/openc3/models/scope_model.rb

@@ -32,6 +32,11 @@ module OpenC3
     PRIMARY_KEY = 'openc3_scopes'
 
     attr_accessor :children
+    attr_accessor :text_log_cycle_time
+    attr_accessor :text_log_cycle_size
+    attr_accessor :text_log_retain_time
+    attr_accessor :tool_log_retain_time
+    attr_accessor :cleanup_poll_time
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -62,8 +67,31 @@ module OpenC3
       end
     end
 
-    def initialize(name:, updated_at: nil, scope: nil)
-      super(PRIMARY_KEY, name: name, scope: name, updated_at: updated_at)
+    def initialize(name:,
+      text_log_cycle_time: 600,
+      text_log_cycle_size: 50_000_000,
+      text_log_retain_time: nil,
+      tool_log_retain_time: nil,
+      cleanup_poll_time: 900,
+      updated_at: nil,
+      scope: nil
+    )
+      super(
+        PRIMARY_KEY,
+        name: name,
+        text_log_cycle_time: text_log_cycle_time,
+        text_log_cycle_size: text_log_cycle_size,
+        text_log_retain_time: text_log_retain_time,
+        tool_log_retain_time: tool_log_retain_time,
+        cleanup_poll_time: cleanup_poll_time,
+        updated_at: updated_at,
+        scope: name
+      )
+      @text_log_cycle_time = text_log_cycle_time
+      @text_log_cycle_size = text_log_cycle_size
+      @text_log_retain_time = text_log_retain_time
+      @tool_log_retain_time = tool_log_retain_time
+      @cleanup_poll_time = cleanup_poll_time
       @children = []
     end
 
@@ -89,41 +117,31 @@ module OpenC3
 
     def as_json(*a)
       { 'name' => @name,
-        'updated_at' => @updated_at }
+        'updated_at' => @updated_at,
+        'text_log_cycle_time' => @text_log_cycle_time,
+        'text_log_cycle_size' => @text_log_cycle_size,
+        'text_log_retain_time' => @text_log_retain_time,
+        'tool_log_retain_time' => @tool_log_retain_time,
+        'cleanup_poll_time' => @cleanup_poll_time,
+       }
     end
 
     def deploy_openc3_log_messages_microservice(gem_path, variables, parent)
       microservice_name = "#{@scope}__OPENC3__LOG"
+      topics = ["#{@scope}__openc3_log_messages"]
+      # Also log the NOSCOPE messages with this microservice for the DEFAULT scope
+      if @scope == 'DEFAULT'
+        topics << "NOSCOPE__openc3_log_messages"
+      end
       microservice = MicroserviceModel.new(
         name: microservice_name,
         cmd: ["ruby", "text_log_microservice.rb", microservice_name],
         work_dir: '/openc3/lib/openc3/microservices',
         options: [
-          # The following options are optional (600 and 50_000_000 are the defaults)
-          # ["CYCLE_TIME", "600"], # Keep at most 10 minutes per log
-          # ["CYCLE_SIZE", "50_000_000"] # Keep at most ~50MB per log
-        ],
-        topics: ["#{@scope}__openc3_log_messages"],
-        parent: parent,
-        scope: @scope
-      )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      @children << microservice_name if parent
-      Logger.info "Configured microservice #{microservice_name}"
-    end
-
-    def deploy_openc3_notifications_microservice(gem_path, variables, parent)
-      microservice_name = "#{@scope}__NOTIFICATION__LOG"
-      microservice = MicroserviceModel.new(
-        name: microservice_name,
-        cmd: ["ruby", "text_log_microservice.rb", microservice_name],
-        work_dir: '/openc3/lib/openc3/microservices',
-        options: [
-          # The following options are optional (600 and 50_000_000 are the defaults)
-          ["CYCLE_TIME", "3600"], # Keep at most 1 hour per log
+          ["CYCLE_TIME", @text_log_cycle_time],
+          ["CYCLE_SIZE", @text_log_cycle_size],
         ],
-        topics: ["#{@scope}__openc3_notifications"],
+        topics: topics,
         parent: parent,
         scope: @scope
       )
@@ -192,6 +210,21 @@ module OpenC3
       Logger.info "Configured microservice #{microservice_name}"
     end
 
+    def deploy_scopecleanup_microservice(gem_path, variables, parent)
+      microservice_name = "#{@scope}__SCOPECLEANUP__#{@scope}"
+      microservice = MicroserviceModel.new(
+        name: microservice_name,
+        cmd: ["ruby", "scope_cleanup_microservice.rb", microservice_name],
+        work_dir: '/openc3/lib/openc3/microservices',
+        parent: parent,
+        scope: @scope
+      )
+      microservice.create
+      microservice.deploy(gem_path, variables)
+      @children << microservice_name if parent
+      Logger.info "Configured microservice #{microservice_name}"
+    end
+
     def deploy_scopemulti_microservice(gem_path, variables)
       microservice_name = "#{@scope}__SCOPEMULTI__#{@scope}"
       microservice = MicroserviceModel.new(
@@ -220,17 +253,12 @@ module OpenC3
       # Create UNKNOWN target for display of unknown data
       model = TargetModel.new(name: "UNKNOWN", scope: @scope)
       model.create
-      # Not deployed - we only want raw packet logging for UNKNOWN
-      # TODO: Cleanup support
 
       @parent = "#{@scope}__SCOPEMULTI__#{@scope}"
 
       # OpenC3 Log Microservice
       deploy_openc3_log_messages_microservice(gem_path, variables, @parent)
 
-      # Notification Log Microservice
-      deploy_openc3_notifications_microservice(gem_path, variables, @parent)
-
       # UNKNOWN CommandLog Microservice
       deploy_unknown_commandlog_microservice(gem_path, variables, @parent)
 
@@ -240,6 +268,9 @@ module OpenC3
       # Periodic Microservice
       deploy_periodic_microservice(gem_path, variables, @parent)
 
+      # Scope Cleanup Microservice
+      deploy_scopecleanup_microservice(gem_path, variables, @parent)
+
       # Multi Microservice to parent other scope microservices
       deploy_scopemulti_microservice(gem_path, variables)
     end
@@ -247,9 +278,9 @@ module OpenC3
     def undeploy
       model = MicroserviceModel.get_model(name: "#{@scope}__SCOPEMULTI__#{@scope}", scope: @scope)
       model.destroy if model
-      model = MicroserviceModel.get_model(name: "#{@scope}__OPENC3__LOG", scope: @scope)
+      model = MicroserviceModel.get_model(name: "#{@scope}__SCOPECLEANUP__#{@scope}", scope: @scope)
       model.destroy if model
-      model = MicroserviceModel.get_model(name: "#{@scope}__NOTIFICATION__LOG", scope: @scope)
+      model = MicroserviceModel.get_model(name: "#{@scope}__OPENC3__LOG", scope: @scope)
       model.destroy if model
       model = MicroserviceModel.get_model(name: "#{@scope}__COMMANDLOG__UNKNOWN", scope: @scope)
       model.destroy if model

data/lib/openc3/models/target_model.rb

@@ -44,6 +44,7 @@ module OpenC3
   class TargetModel < Model
     PRIMARY_KEY = 'openc3_targets'
     VALID_TYPES = %i(CMD TLM)
+    ERB_EXTENSIONS = %w(.txt .rb .py .json .yaml .yml)
     ITEM_MAP_CACHE_TIMEOUT = 10.0
     @@item_map_cache = {}
 
@@ -346,6 +347,8 @@ module OpenC3
       cleanup_poll_time: 900,
       needs_dependencies: false,
       target_microservices: {'REDUCER' => [[]]},
+      reducer_disable: false,
+      reducer_max_cpu_utilization: 30.0,
       scope:
     )
       super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at,
@@ -360,6 +363,7 @@ module OpenC3
         reduced_minute_log_retain_time: reduced_minute_log_retain_time,
         reduced_hour_log_retain_time: reduced_hour_log_retain_time, reduced_day_log_retain_time: reduced_day_log_retain_time,
         cleanup_poll_time: cleanup_poll_time, needs_dependencies: needs_dependencies, target_microservices: target_microservices,
+        reducer_disable: reducer_disable, reducer_max_cpu_utilization: reducer_max_cpu_utilization,
         scope: scope)
       @folder_name = folder_name
       @requires = requires
@@ -390,6 +394,8 @@ module OpenC3
       @cleanup_poll_time = cleanup_poll_time
       @needs_dependencies = needs_dependencies
       @target_microservices = target_microservices
+      @reducer_disable = reducer_disable
+      @reducer_max_cpu_utilization = reducer_max_cpu_utilization
       @bucket = Bucket.getClient()
       @children = []
     end
@@ -427,7 +433,9 @@ module OpenC3
         'reduced_day_log_retain_time' => @reduced_day_log_retain_time,
         'cleanup_poll_time' => @cleanup_poll_time,
         'needs_dependencies' => @needs_dependencies,
-        'target_microservices' => @target_microservices.as_json(:allow_nan => true)
+        'target_microservices' => @target_microservices.as_json(:allow_nan => true),
+        'reducer_disable' => @reducer_disable,
+        'reducer_max_cpu_utilization' => @reducer_max_cpu_utilization
       }
     end
 
@@ -509,6 +517,11 @@ module OpenC3
           @reduced_hour_log_retain_time = reduced_log_retain_time.to_i
           @reduced_day_log_retain_time = reduced_log_retain_time.to_i
         end
+      when 'REDUCER_DISABLE', 'REDUCER_DISABLED' # Handle typos
+        @reducer_disable = true
+      when 'REDUCER_MAX_CPU_UTILIZATION', 'REDUCED_MAX_CPU_UTILIZATION' # Handle typos
+        parser.verify_num_parameters(1, 1, "#{keyword} <Max cpu utilization to allocate to the reducer microservice - 0.0 to 100.0>")
+        @reducer_max_cpu_utilization = Float(parameters[0])
       when 'CLEANUP_POLL_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Cleanup polling period in seconds>")
         @cleanup_poll_time = parameters[0].to_i
@@ -556,7 +569,9 @@ module OpenC3
           data = File.read(filename, mode: "rb")
           begin
             OpenC3.set_working_dir(File.dirname(filename)) do
-              data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_'
+              if ERB_EXTENSIONS.include?(File.extname(filename).downcase) and File.basename(filename)[0] != '_'
+                data = ERB.new(data.force_encoding("UTF-8").comment_erb(), trim_mode: "-").result(binding.set_variables(variables))
+              end
             end
           rescue => error
             # ERB error parsing a screen is just a logger error because life can go on
@@ -666,7 +681,7 @@ module OpenC3
 
       begin
         OpenC3.set_working_dir(File.dirname(path)) do
-          return ERB.new(File.read(path), trim_mode: "-").result(b)
+          return ERB.new(File.read(path).force_encoding("UTF-8").comment_erb(), trim_mode: "-").result(b)
         end
       rescue => error
         raise "ERB error parsing: #{path}: #{error.formatted}"
@@ -909,6 +924,9 @@ module OpenC3
         folder_name: @folder_name,
         cmd: ["ruby", "reducer_microservice.rb", microservice_name],
         work_dir: '/openc3/lib/openc3/microservices',
+        options: [
+          ["MAX_CPU_UTILIZATION", @reducer_max_cpu_utilization],
+        ],
         topics: topics,
         plugin: @plugin,
         parent: parent,
@@ -1053,8 +1071,10 @@ module OpenC3
         end
 
         # Reducer Microservice
-        deploy_target_microservices('REDUCER', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent|
-          deploy_reducer_microservice(gem_path, variables, topics, instance, parent)
+        unless @reducer_disable
+          deploy_target_microservices('REDUCER', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent|
+            deploy_reducer_microservice(gem_path, variables, topics, instance, parent)
+          end
         end
       end
 

data/lib/openc3/models/tool_model.rb

@@ -155,8 +155,19 @@ module OpenC3
     end
 
     def create(update: false, force: false)
+      tools = self.class.all(scope: @scope)
+
+      # Make sure a tool with this folder_name doesn't already exist
+      unless update
+        tools.each do |_tool_name, tool|
+          if tool['folder_name'] == @folder_name
+            raise "Tool with folder_name #{@folder_name} already exists at create"
+          end
+        end
+      end
+
+      # Autoset tool position
       unless @position
-        tools = self.class.all(scope: @scope)
         _, tool = tools.max_by { |_tool_name, tool| tool['position'] }
         if tool
           @position = tool['position'] + 1
@@ -164,6 +175,7 @@ module OpenC3
           @position = 0
         end
       end
+
       super(update: update, force: force)
     end
 
@@ -228,7 +240,7 @@ module OpenC3
 
         # Load tool files
         data = File.read(filename, mode: "rb")
-        data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
+        data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
         unless validate_only
           client = Bucket.getClient()
           cache_control = BucketUtilities.get_cache_control(filename)

data/lib/openc3/models/widget_model.rb

@@ -124,7 +124,7 @@ module OpenC3
       # Load widget file
       data = File.read(filename, mode: "rb")
       OpenC3.set_working_dir(File.dirname(filename)) do
-        data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
+        data = ERB.new(data.comment_erb(), trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
       end
       unless validate_only
         cache_control = BucketUtilities.get_cache_control(@filename)

data/lib/openc3/packets/json_packet.rb

@@ -32,13 +32,21 @@ module OpenC3
     attr_accessor :packet_time
     attr_accessor :stored
     attr_accessor :json_hash
+    attr_accessor :received_time
+    attr_accessor :extra
 
-    def initialize(cmd_or_tlm, target_name, packet_name, time_nsec_from_epoch, stored, json_data, key_map = nil)
+    def initialize(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data, key_map = nil, received_time_nsec_since_epoch: nil, extra: nil)
       @cmd_or_tlm = cmd_or_tlm.intern
       @target_name = target_name
       @packet_name = packet_name
-      @packet_time = ::Time.from_nsec_from_epoch(time_nsec_from_epoch)
+      @packet_time = ::Time.from_nsec_from_epoch(time_nsec_since_epoch)
       @stored = ConfigParser.handle_true_false(stored)
+      if received_time_nsec_since_epoch
+        @received_time = ::Time.from_nsec_from_epoch(received_time_nsec_since_epoch)
+      else
+        @received_time = @packet_time
+      end
+      @extra = extra
       @json_hash = json_data
       @json_hash = JSON.parse(json_data, :allow_nan => true, :create_additions => true) if String === json_data
       if key_map

data/lib/openc3/packets/packet.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2023, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -127,6 +127,7 @@ module OpenC3
         @extra = nil
         @cmd_or_tlm = nil
         @template = nil
+        @packet_time = nil
       end
 
       # Sets the target name this packet is associated with. Unidentified packets
@@ -267,10 +268,18 @@ module OpenC3
       if item
         return read_item(item, :CONVERTED, @buffer)
       else
-        return @received_time
+        if @packet_time
+          return @packet_time
+        else
+          return @received_time
+        end
       end
     end
 
+    def packet_time=(time)
+      @packet_time = time
+    end
+
     # Calculates a unique hashing sum that changes if the parts of the packet configuration change that could affect
     # the "shape" of the packet.  This value is cached and that packet should not be changed if this method is being used
     def config_name
@@ -294,9 +303,7 @@ module OpenC3
         begin
           internal_buffer_equals(buffer)
         rescue RuntimeError
-          if BinaryAccessor == @accessor
-            Logger.instance.error "#{@target_name} #{@packet_name} received with actual packet length of #{buffer.length} but defined length of #{@defined_length}"
-          end
+          Logger.instance.error "#{@target_name} #{@packet_name} received with actual packet length of #{buffer.length} but defined length of #{@defined_length}"
         end
         @read_conversion_cache.clear if @read_conversion_cache
         process()
@@ -709,10 +716,10 @@ module OpenC3
         if item.write_conversion
           value = item.write_conversion.call(value, self, buffer)
         else
-          raise "Cannot write DERIVED item #{item.name} without a write conversion" if item.data_type == :DERIVED
+          raise "Cannot write DERIVED item #{item.name} without a write conversion" if item.data_type == :DERIVED and @accessor.enforce_derived_write_conversion(item)
         end
         begin
-          super(item, value, :RAW, buffer) unless item.data_type == :DERIVED
+          super(item, value, :RAW, buffer)
         rescue ArgumentError => err
           if item.states and String === value and err.message =~ /invalid value for/
             raise "Unknown state #{value} for #{item.name}"
@@ -1051,7 +1058,8 @@ module OpenC3
       config['messages_disabled'] = true if @messages_disabled
       config['disabled'] = true if @disabled
       config['hidden'] = true if @hidden
-      config['accessor'] = @accessor.to_s
+      config['accessor'] = @accessor.class.to_s
+      config['accessor_args'] = @accessor.args
       config['template'] = Base64.encode64(@template) if @template
 
       if @processors
@@ -1092,7 +1100,12 @@ module OpenC3
       packet.hidden = hash['hidden']
       if hash['accessor']
         begin
-          packet.accessor = OpenC3::const_get(hash['accessor'])
+          accessor = OpenC3::const_get(hash['accessor'])
+          if hash['accessor_args'] and hash['accessor_args'].length > 0
+            packet.accessor = accessor.new(packet, *hash['accessor_args'])
+          else
+            packet.accessor = accessor.new(packet)
+          end
         rescue => error
           Logger.instance.error "#{packet.target_name} #{packet.packet_name} accessor of #{hash['accessor']} could not be found due to #{error}"
         end
@@ -1110,6 +1123,13 @@ module OpenC3
       # Read all the RAW at once because this could be optimized by the accessor
       json_hash = read_items(@sorted_items)
 
+      # Decom extra into the values (all raw)
+      if @extra
+        @extra.each do |key, value|
+          json_hash[key.upcase] = value
+        end
+      end
+
       # Now read all other value types - no accessor required
       @sorted_items.each do |item|
         given_raw = json_hash[item.name]
@@ -1119,6 +1139,7 @@ module OpenC3
         limits_state = item.limits.state
         json_hash["#{item.name}__L"] = limits_state if limits_state
       end
+
       json_hash
     end
 

data/lib/openc3/packets/packet_config.rb

@@ -433,10 +433,14 @@ module OpenC3
         @current_packet.disabled = true
       when 'ACCESSOR'
         usage = "#{keyword} <Accessor class name>"
-        parser.verify_num_parameters(1, 1, usage)
+        parser.verify_num_parameters(1, nil, usage)
         begin
           klass = OpenC3.require_class(params[0])
-          @current_packet.accessor = klass
+          if params.length > 1
+            @current_packet.accessor = klass.new(@current_packet, *params[1..-1])
+          else
+            @current_packet.accessor = klass.new(@current_packet)
+          end
         rescue Exception => err
           raise parser.error(err)
         end

data/lib/openc3/packets/parsers/packet_item_parser.rb

@@ -14,10 +14,10 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2023, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/packets/packet_item'
@@ -158,6 +158,7 @@ module OpenC3
       return nil if data_type == :STRING or data_type == :BLOCK
 
       index = append? ? 3 : 4
+      return nil if @parser.parameters[index] == 'nil'
       min = ConfigParser.handle_defined_constants(
         @parser.parameters[index].convert_to_value, get_data_type(), get_bit_size()
       )
@@ -183,9 +184,13 @@ module OpenC3
           return @parser.parameters[index]
         end
       else
-        return ConfigParser.handle_defined_constants(
-          @parser.parameters[index + 2].convert_to_value, get_data_type(), get_bit_size()
-        )
+        if data_type != :DERIVED
+          return ConfigParser.handle_defined_constants(
+            @parser.parameters[index + 2].convert_to_value, data_type, get_bit_size()
+          )
+        else
+          return @parser.parameters[index + 2].convert_to_value
+        end
       end
     end
 
@@ -248,7 +253,7 @@ module OpenC3
           data_type = get_data_type()
         rescue
           # If the data type could not be determined set something
-          data_type == :INT
+          data_type = :INT
         end
         # STRING and BLOCK types do not have min or max values
         if data_type == :STRING || data_type == :BLOCK

data/lib/openc3/packets/structure.rb

@@ -54,7 +54,7 @@ module OpenC3
     #   required data size is allowed.
     attr_accessor :short_buffer_allowed
 
-    # @return [Accessor] Class used to access raw data of structure from buffer
+    # @return [Accessor] Instance of class used to access raw data of structure from buffer
     attr_reader :accessor
 
     if RUBY_ENGINE != 'ruby' or ENV['OPENC3_NO_EXT']
@@ -92,7 +92,7 @@ module OpenC3
           @fixed_size = true
           @short_buffer_allowed = false
           @mutex = nil
-          @accessor = BinaryAccessor
+          @accessor = BinaryAccessor.new(self)
         else
           raise(ArgumentError, "Unknown endianness '#{default_endianness}', must be :BIG_ENDIAN or :LITTLE_ENDIAN")
         end
@@ -107,8 +107,6 @@ module OpenC3
       # @return Value based on the item definition. This could be a string, integer,
       #   float, or array of values.
       def read_item(item, value_type = :RAW, buffer = @buffer)
-        return nil if item.data_type == :DERIVED
-
         buffer = allocate_buffer_if_needed() unless buffer
         return @accessor.read_item(item, buffer)
       end
@@ -141,7 +139,10 @@ module OpenC3
     # @param accessor [Accessor] The class to use as an accessor
     def accessor=(accessor)
       @accessor = accessor
-      @short_buffer_allowed = true if @accessor != BinaryAccessor
+      if @accessor.enforce_short_buffer_allowed
+        @short_buffer_allowed = true
+      end
+      return accessor
     end
 
     # Read a list of items in the structure
@@ -493,6 +494,8 @@ module OpenC3
       # Use instance_variable_set since we have overriden buffer= to do
       # additional work that isn't neccessary here
       structure.instance_variable_set("@buffer".freeze, @buffer.clone) if @buffer
+      # Need to update reference packet in the Accessor
+      structure.accessor.packet = structure
       return structure
     end
     alias dup clone
@@ -567,13 +570,17 @@ module OpenC3
       raise ArgumentError, "Buffer class is #{buffer.class} but must be String" unless String === buffer
 
       @buffer = buffer.dup
-      @buffer.force_encoding('ASCII-8BIT'.freeze)
-      if @buffer.length != @defined_length
-        if @buffer.length < @defined_length
-          resize_buffer()
-          raise "Buffer length less than defined length" unless @short_buffer_allowed
-        elsif @fixed_size and @defined_length != 0
-          raise "Buffer length greater than defined length"
+      if @accessor.enforce_encoding
+        @buffer.force_encoding(@accessor.enforce_encoding)
+      end
+      if @accessor.enforce_length
+        if @buffer.length != @defined_length
+          if @buffer.length < @defined_length
+            resize_buffer()
+            raise "Buffer length less than defined length" unless @short_buffer_allowed
+          elsif @fixed_size and @defined_length != 0
+            raise "Buffer length greater than defined length"
+          end
         end
       end
     end

data/lib/openc3/script/storage.rb

@@ -104,7 +104,7 @@ module OpenC3
           if part == "targets_modified" and ENV['OPENC3_LOCAL_MODE']
             local_file = OpenC3::LocalMode.open_local_file(path, scope: scope)
             if local_file
-              OpenC3::Logger.info "Reading local #{scope}/#{path}"
+              OpenC3::Logger.info "Reading local #{scope}/#{part}/#{path}"
               file = Tempfile.new('target', binmode: true)
               file.filename = path
               file.write(local_file.read)